From 6e0b084d65774d5a973687410a3ebbf64f55bf95 Mon Sep 17 00:00:00 2001 From: Ekaterina Dimitrova Date: Fri, 16 Jul 2021 10:50:10 -0400 Subject: [PATCH] Add config.yml.MIDRES for older Cassandra versions patch by Ekaterina Dimitrova; review by Andres de la Pena for CASSANDRA-16804 --- .circleci/config-2_1.yml.mid_res.patch | 75 ++ .circleci/config.yml.MIDRES | 1634 ++++++++++++++++++++++++ .circleci/generate.sh | 6 + .circleci/readme.md | 16 +- 4 files changed, 1728 insertions(+), 3 deletions(-) create mode 100644 .circleci/config-2_1.yml.mid_res.patch create mode 100644 .circleci/config.yml.MIDRES diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config-2_1.yml.mid_res.patch new file mode 100644 index 0000000000..c24cebfd82 --- /dev/null +++ b/.circleci/config-2_1.yml.mid_res.patch @@ -0,0 +1,75 @@ +diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml +index 03e4ae9e8f..cf2b758187 100644 +--- a/.circleci/config-2_1.yml ++++ b/.circleci/config-2_1.yml +@@ -93,24 +93,40 @@ j8_par_executor: &j8_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge +- parallelism: 4 ++ exec_resource_class: medium ++ parallelism: 25 + + j8_small_par_executor: &j8_small_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge +- parallelism: 1 ++ exec_resource_class: large ++ parallelism: 10 + + j8_medium_par_executor: &j8_medium_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge +- parallelism: 1 ++ exec_resource_class: large ++ parallelism: 4 ++ ++j8_large_par_executor: &j8_large_par_executor ++ executor: ++ name: java8-executor ++ exec_resource_class: large ++ parallelism: 50 ++ ++j8_very_large_par_executor: &j8_very_large_par_executor ++ executor: ++ name: java8-executor ++ exec_resource_class: xlarge ++ parallelism: 100 + + j8_seq_executor: &j8_seq_executor + executor: + name: java8-executor + #exec_resource_class: xlarge ++ exec_resource_class: medium + parallelism: 1 # sequential, single container tests: no parallelism benefits + + with_dtests_jobs: &with_dtest_jobs +@@ -308,7 +324,7 @@ jobs: + target: testclasslist-compression + + j8_dtests-with-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -322,7 +338,7 @@ jobs: + pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' + + j8_dtests-no-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -336,7 +352,7 @@ jobs: + pytest_extra_args: '--skip-resource-intensive-tests' + + j8_upgradetests-no-vnodes: +- <<: *j8_par_executor ++ <<: *j8_very_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES new file mode 100644 index 0000000000..4eff27a726 --- /dev/null +++ b/.circleci/config.yml.MIDRES @@ -0,0 +1,1634 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: 2 +jobs: + j8_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + repeated_jvm_upgrade_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=test-jvm-dtest-some + class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + repeated_upgrade_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTEST_NAME}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTEST_NAME} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if ${REPEATED_DTEST_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if ${REPEATED_DTEST_VNODES}; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra Repository (via git) + command: | + git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + - run: + name: Build Cassandra + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + no_output_timeout: 15m + - run: + name: Run eclipse-warnings + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + ant eclipse-warnings + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgradetests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_upgradetests_without_vnodes) + no_output_timeout: 5m + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + cd cassandra-dtest + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then + export RUN_STATIC_UPGRADE_MATRIX=true + fi + + echo "***Collected DTests (j8_upgradetests_without_vnodes)***" + set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra + if [ -z '^upgrade_tests' ]; then + mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes + else + grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; } + fi + set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt + cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt + cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt + - run: + name: Run dtests (j8_upgradetests_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then + export RUN_STATIC_UPGRADE_MATRIX=true + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_upgradetests_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_long: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (long-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_compression: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist-compression) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_UTEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_UTEST_TARGET} + class_path=${REPEATED_UTEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_UTEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 +workflows: + version: 2 + build_and_run_tests: + jobs: + - build + - j8_unit_tests: + requires: + - build + - j8_jvm_dtests: + requires: + - build + - start_utests_long: + type: approval + requires: + - build + - utests_long: + requires: + - start_utests_long + - start_utests_compression: + type: approval + requires: + - build + - utests_compression: + requires: + - start_utests_compression + - start_j8_dtest_jars_build: + type: approval + - j8_dtest_jars_build: + requires: + - build + - start_j8_dtest_jars_build + - start_jvm_upgrade_dtest: + type: approval + - j8_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtest + - j8_dtest_jars_build + - start_j8_dtests: + type: approval + requires: + - build + - j8_dtests-with-vnodes: + requires: + - start_j8_dtests + - j8_dtests-no-vnodes: + requires: + - start_j8_dtests + - start_upgrade_tests: + type: approval + requires: + - build + - j8_upgradetests-no-vnodes: + requires: + - start_upgrade_tests + - start_j8_repeated_utest: + type: approval + - j8_repeated_utest: + requires: + - start_j8_repeated_utest + - build + - start_j8_repeated_dtest: + type: approval + - j8_repeated_dtest: + requires: + - start_j8_repeated_dtest + - build + - start_repeated_upgrade_dtest: + type: approval + - repeated_upgrade_dtest: + requires: + - start_repeated_upgrade_dtest + - build + - start_repeated_jvm_upgrade_dtest: + type: approval + - repeated_jvm_upgrade_dtest: + requires: + - start_repeated_jvm_upgrade_dtest + - j8_dtest_jars_build diff --git a/.circleci/generate.sh b/.circleci/generate.sh index 59a1a9fabd..832c63f69b 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -24,6 +24,12 @@ circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml.LOWRES rm $BASEDIR/config.yml.LOWRES.tmp +# setup midres +patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch +circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp +cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml.MIDRES +rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp + # setup highres patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp diff --git a/.circleci/readme.md b/.circleci/readme.md index c128c9cc07..3c4d9631d2 100644 --- a/.circleci/readme.md +++ b/.circleci/readme.md @@ -26,14 +26,17 @@ between them by copying the correct file to config.yml and committing the result `cp .circleci/config.yml.HIGHRES .circleci/config.yml` +config.yml.LOWRES is the default config. MIDRES and HIGHRES are custom configs for those who +have access to premium CircleCI resources. + Make sure you never edit the config.yml manually. ## Updating the config master -To update the config (other than just swapping high/low resources) you need to install +To update the config (other than just swapping high/mid/low resources) you need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install). -The directory contains `config-2_1.yml` which is then converted to the actual HIGH/LOW -resource files. There is a script called `generate.sh` which creates the LOWRES and +The directory contains `config-2_1.yml` which is then converted to the actual HIGH/MID/LOW +resource files. There is a script called `generate.sh` which creates the LOWRES, MIDRES, and HIGHRES files, read below for details how to do it manually; 1. make your edits to config-2_1.yml - let it stay at lowres settings @@ -53,4 +56,11 @@ HIGHRES files, read below for details how to do it manually; 1. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES` 1. add the Apache license header to the newly created HIGHRES file: `cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv config.yml.HIGHRES.new config.yml.HIGHRES` +1. repeat the last steps to generate the MIDRES file: + ``` + patch -o config-2_1.yml.MIDRES config-2_1.yml config-2_1.yml.mid_res.patch + circleci config process config-2_1.yml.MIDRES > config.yml.MIDRES + rm config-2_1.yml.MIDRES + cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv config.yml.MIDRES.new config.yml.MIDRES + ```