cassandra/.build/run-tests.sh

422 lines
18 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Wrapper script for running a split or regexp of tests (excluding python dtests)
#
[ $DEBUG ] && set -x
set -o errexit
set -o pipefail
[ $DEBUG ] && set -x
# variables, with defaults
[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)"
[ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build"
# pre-conditions
command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
command -v uuidgen >/dev/null 2>&1 || test -f /proc/sys/kernel/random/uuid || { error 1 "uuidgen needs to be installed"; }
[ -d "${CASSANDRA_DIR}" ] || { error 1 "Directory ${CASSANDRA_DIR} must exist"; }
[ -f "${CASSANDRA_DIR}/build.xml" ] || { error 1 "${CASSANDRA_DIR}/build.xml must exist"; }
[ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; }
error() {
echo >&2 $2;
set -x
exit $1
}
print_help() {
echo "Usage: $0 [-a|-t|-c|-e|-i|-b|-s|-h]"
echo " -a Test target type: test, test-compression, test-cdc, ..."
echo " -t Test name regexp to run."
echo " -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks."
echo " -b Specify the base git branch for comparison when determining changed tests to"
echo " repeat. Defaults to ${BASE_BRANCH}. Note that this option is not used when"
echo " the '-a' option is specified."
echo " -s Skip automatic detection of changed tests. Useful when you need to repeat a few ones,"
echo " or when there are too many changed tests the CI env to handle."
echo " -e <key=value> Environment variables to be used in the repeated runs:"
echo " -e REPEATED_TESTS_STOP_ON_FAILURE=false"
echo " -e REPEATED_TESTS_COUNT=500"
echo " If you want to specify multiple environment variables simply add multiple -e options."
echo " -i Ignore unknown environment variables"
echo " -h Print help"
}
# legacy argument handling
case ${1} in
"build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
test_type="-a ${1}"
if [[ -z ${2} ]]; then
test_list=""
elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then
test_list="-c ${2}";
else
test_list="-t ${2}";
fi
echo "Using deprecated legacy arguments. Please update to new parameter format: ${test_type} ${test_list}"
$0 ${test_type} ${test_list}
exit $?
esac
env_vars=""
has_env_vars=false
check_env_vars=true
detect_changed_tests=true
while getopts "a:t:c:e:ib:shj:" opt; do
case $opt in
a ) test_target="$OPTARG"
;;
t ) test_name_regexp="$OPTARG"
;;
c ) chunk="$OPTARG"
;;
e ) if (! ($has_env_vars)); then
env_vars="$OPTARG"
else
env_vars="$env_vars|$OPTARG"
fi
has_env_vars=true
;;
b ) BASE_BRANCH="$OPTARG"
;;
i ) check_env_vars=false
;;
s ) detect_changed_tests=false
;;
h ) print_help
exit 0
;;
j ) ;; # To avoid failing on java_version param from docker/run_tests.sh
\?) error 1 "Invalid option: -$OPTARG"
;;
esac
done
shift $((OPTIND-1))
if [ "$#" -ne 0 ]; then
error 1 "Unexpected arguments"
fi
# validate environment variables
if $has_env_vars && $check_env_vars; then
for entry in $(echo $env_vars | tr "|" "\n"); do
key=$(echo $entry | tr "=" "\n" | sed -n 1p)
case $key in
"REPEATED_TESTS_STOP_ON_FAILURE" | "REPEATED_TESTS_COUNT" )
[[ ${test_target} == *"-repeat" ]] || { error 1 "'-e REPEATED_*' variables only valid against *-repeat target types"; }
;;
*)
error 1 "unrecognized environment variable name: $key"
;;
esac
done
fi
# print debug information on versions
ant -version
git --version
java -version 2>&1
javac -version 2>&1
# set the OFFLINE env var (to anything) to allow running jvm-dtest-upgrade offline
[ "x" != "x${OFFLINE}" ] && echo "WARNING: running in offline mode. jvm-dtest-upgrade results may be stale."
# lists all tests for the specific test type
_list_tests() {
local -r classlistprefix="$1"
find "test/${classlistprefix}" -name '*Test.java' | sed "s;^test/${classlistprefix}/;;g" | sort
}
_split_tests() {
local -r _split_chunk="$1"
split_cmd=split
if [[ "${_split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then
( split --help 2>&1 ) | grep -q "r/K/N" || split_cmd=gsplit
command -v ${split_cmd} >/dev/null 2>&1 || { error 1 "${split_cmd} needs to be installed"; }
${split_cmd} -n r/${_split_chunk}
elif [[ "x" != "x${_split_chunk}" ]] ; then
grep -e "${_split_chunk}"
else
echo
fi
}
_timeout_for() {
grep "name=\"${1}\"" build.xml | awk -F'"' '{print $4}'
}
_get_env_var() {
[[ ${env_vars} =~ ${1}=([^|]+) ]]
echo "${BASH_REMATCH[1]}"
}
_build_all_dtest_jars() {
# build the dtest-jar for the branch under test. remember to `ant clean` if you want a new dtest jar built
dtest_jar_version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
if [ -f "${DIST_DIR}/dtest-${dtest_jar_version}.jar" ] ; then
echo "Skipping dtest jar build for branch under test as ${DIST_DIR}/dtest-${dtest_jar_version}.jar already exists"
else
ant jar dtest-jar ${ANT_TEST_OPTS} -Dbuild.dir=${TMP_DIR}/cassandra-dtest-jars/build
cp "${TMP_DIR}/cassandra-dtest-jars/build/dtest-${dtest_jar_version}.jar" ${DIST_DIR}/
fi
if [ -d ${TMP_DIR}/cassandra-dtest-jars/.git ] && [ "https://github.com/apache/cassandra.git" == "$(git -C ${TMP_DIR}/cassandra-dtest-jars remote get-url origin)" ] ; then
echo "Reusing ${TMP_DIR}/cassandra-dtest-jars for past branch dtest jars"
if [ "x" == "x${OFFLINE}" ] ; then
until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git -C ${TMP_DIR}/cassandra-dtest-jars fetch failed… trying again… " ; done
fi
else
echo "Cloning cassandra to ${TMP_DIR}/cassandra-dtest-jars for past branch dtest jars"
rm -fR ${TMP_DIR}/cassandra-dtest-jars
pushd $TMP_DIR >/dev/null
until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
popd >/dev/null
fi
# cassandra-4 branches need CASSANDRA_USE_JDK11 to allow jdk11
[ "${java_version}" -eq 11 ] && export CASSANDRA_USE_JDK11=true
pushd ${TMP_DIR}/cassandra-dtest-jars >/dev/null
for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0 trunk ; do
git clean -qxdff && git reset --hard HEAD || echo "failed to reset/clean ${TMP_DIR}/cassandra-dtest-jars… continuing…"
git checkout --quiet $branch
dtest_jar_version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
if [ -f "${DIST_DIR}/dtest-${dtest_jar_version}.jar" ] ; then
echo "Skipping dtest jar build for branch ${branch} as ${DIST_DIR}/dtest-${dtest_jar_version}.jar already exists"
continue
fi
# redefine the build.dir to local build folder, rightmost definition wins with java command line system properties
ant realclean -Dbuild.dir=${TMP_DIR}/cassandra-dtest-jars/build
ant jar dtest-jar ${ANT_TEST_OPTS} -Dbuild.dir=${TMP_DIR}/cassandra-dtest-jars/build
cp "${TMP_DIR}/cassandra-dtest-jars/build/dtest-${dtest_jar_version}.jar" ${DIST_DIR}/
done
popd >/dev/null
ls -l ${DIST_DIR}/dtest*.jar
unset CASSANDRA_USE_JDK11
}
_run_testlist() {
local _target_prefix=$1
local _testlist_target=$2
local _test_name_regexp=$3
local _split_chunk=$4
local _test_timeout=$5
local _test_iterations=${6:-1}
# are we running ${_test_name_regexp} or ${_split_chunk}
if [ -n "${_test_name_regexp}" ]; then
echo "Running tests: ${_test_name_regexp} (${_test_iterations} times)"
# test regexp can come in csv
for i in ${_test_name_regexp//,/ }; do
[ -n "${testlist}" ] && testlist="${testlist}"$'\n'
testlist="${testlist}$( _list_tests "${_target_prefix}" | _split_tests "${i}")"
done
[[ -z "${testlist}" ]] && error 1 "No tests found in test name regexp: ${_test_name_regexp}"
else
[ -n "${_split_chunk}" ] || { error 1 "Neither name regexp or split chunk defined"; }
echo "Running split: ${_split_chunk}"
testlist="$( _list_tests "${_target_prefix}" | _split_tests "${_split_chunk}")"
if [[ -z "${testlist}" ]]; then
# something has to run in the split to generate a junit xml result
echo "Hacking ${_target_prefix} ${_testlist_target} to run only first test found as no tests in split ${_split_chunk} were found"
testlist="$( _list_tests "${_target_prefix}" | sed -n 1p)"
fi
fi
local -r _results_uuid="$(command -v uuidgen >/dev/null 2>&1 && uuidgen || cat /proc/sys/kernel/random/uuid)"
local failures=0
for ((i=0; i < _test_iterations; i++)); do
[ "${_test_iterations}" -eq 1 ] || printf " run ${i}\n"
set +o errexit
ant "$_testlist_target" -Dtest.classlistprefix="${_target_prefix}" -Dtest.classlistfile=<(echo "${testlist}") -Dtest.timeout="${_test_timeout}" ${ANT_TEST_OPTS}
ant_status=$?
set -o errexit
if [[ $ant_status -ne 0 ]]; then
echo "failed ${_target_prefix} ${_testlist_target} ${split_chunk} ${_test_name_regexp}"
# Only store logs for failed tests on repeats to save up space
if [ "${_test_iterations}" -gt 1 ]; then
# Get this test results and rename file with iteration and 'fail'
find "${DIST_DIR}"/test/output/ -type f -name "*.xml" -not -name "*fail.xml" -print0 | while read -r -d $'\0' file; do
mv "${file}" "${file%.xml}-${_results_uuid}-${i}-fail.xml"
done
find "${DIST_DIR}"/test/logs/ -type f -name "*.log" -not -name "*fail.log" -print0 | while read -r -d $'\0' file; do
mv "${file}" "${file%.log}-${_results_uuid}-${i}-fail.log"
done
if [ "$(_get_env_var 'REPEATED_TESTS_STOP_ON_FAILURE')" == true ]; then
error 0 "fail fast, after ${i} successful runs"
fi
let failures+=1
fi
fi
done
[ "${_test_iterations}" -eq 1 ] || printf "\nfailure rate: ${failures}/${_test_iterations}\n"
}
_main() {
# parameters
local -r target="${test_target/-repeat/}"
local -r split_chunk="${chunk:-'1/1'}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
# check split_chunk is compatible with target (if not a regexp)
if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
case ${target} in
"stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
error 1 "Target ${target} does not suport splits."
;;
*)
;;
esac
fi
# "-repeat" is a reserved suffix on target types
if [[ ${test_target} == *"-repeat" ]] ; then
[[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] && { error 1 "Repeated tests not valid with splits"; }
if [[ -z "${test_name_regexp}" ]] ; then
error 1 "Repeated tests requires use of -t option"
fi
local -r repeat_count="$(_get_env_var 'REPEATED_TESTS_COUNT')"
else
test_name_regexp="${test_name_regexp:-}"
fi
pushd ${CASSANDRA_DIR}/ >/dev/null
# jdk check
local -r java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
local -r version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
local -r java_version_default=`grep 'property\s*name="java.default"' build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
if [ "${java_version}" -eq 17 ] && [[ "${target}" == "jvm-dtest-upgrade" ]] ; then
error 1 "Invalid JDK${java_version}. Only overlapping supported JDKs can be used when upgrading, as the same jdk must be used over the upgrade path."
fi
# check project is already built. no cleaning is done, so jenkins unstash works, beware.
[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { error 1 "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; }
# check if dist artifacts exist, this breaks the dtests
[[ -d "${DIST_DIR}/dist" ]] && { error 1 "tests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; }
# ant test setup
export TMP_DIR="${DIST_DIR}/tmp"
[ -d ${TMP_DIR} ] || mkdir -p "${TMP_DIR}"
export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR} -Dbuild.test.output.dir=${DIST_DIR}/test/output/${target}"
# fresh virtualenv and test logs results everytime
[[ "/" == "${DIST_DIR}" ]] || rm -rf "${DIST_DIR}/test/{html,output,logs,reports}"
# cheap trick to ensure dependency libraries are in place. allows us to stash only project specific build artifacts.
# also recreate some of the non-build files we need
ant -quiet -silent resolver-dist-lib _createVersionPropFile
case ${target} in
"stress-test")
# hard fail on test compilation, but dont fail the test run as unstable test reports are processed
ant stress-build-test ${ANT_TEST_OPTS}
ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
;;
"fqltool-test")
# hard fail on test compilation, but dont fail the test run so unstable test reports are processed
ant fqltool-build-test ${ANT_TEST_OPTS}
ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
;;
"microbench")
ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true
;;
"test")
_run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-cdc")
_run_testlist "unit" "testclasslist-cdc" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-compression")
_run_testlist "unit" "testclasslist-compression" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-oa")
_run_testlist "unit" "testclasslist-oa" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-system-keyspace-directory")
_run_testlist "unit" "testclasslist-system-keyspace-directory" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-latest")
_run_testlist "unit" "testclasslist-latest" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"
;;
"test-burn")
_run_testlist "burn" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.burn.timeout')" "${repeat_count}"
;;
"long-test")
_run_testlist "long" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.long.timeout')" "${repeat_count}"
;;
"simulator-dtest")
ant test-simulator-dtest ${ANT_TEST_OPTS} || echo "failed ${target}"
;;
"jvm-dtest" | "jvm-dtest-novnode")
[ "jvm-dtest-novnode" == "${target}" ] || ANT_TEST_OPTS="${ANT_TEST_OPTS} -Dcassandra.dtest.num_tokens=16"
if [[ -z "${test_name_regexp}" ]] ; then
test_name_regexp=$( _list_tests "distributed" | grep -v "upgrade" | _split_tests "${split_chunk}")
if [[ -z "${test_name_regexp}" ]]; then
[[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] || { error 1 "No tests match ${test_name_regexp}"; }
# something has to run in the split to generate a junit xml result
echo "Hacking jvm-dtest to run only first test found as no tests in split ${split_chunk} were found"
test_name_regexp="$( _list_tests "distributed" | grep -v "upgrade" | sed -n 1p)"
fi
fi
_run_testlist "distributed" "testclasslist" "${test_name_regexp}" "" "$(_timeout_for 'test.distributed.timeout')" "${repeat_count}"
;;
"build_dtest_jars")
_build_all_dtest_jars
;;
"jvm-dtest-upgrade" | "jvm-dtest-upgrade-novnode")
_build_all_dtest_jars
[ "jvm-dtest-upgrade-novnode" == "${target}" ] || ANT_TEST_OPTS="${ANT_TEST_OPTS} -Dcassandra.dtest.num_tokens=16"
if [[ -z "${test_name_regexp}" ]] ; then
test_name_regexp=$( _list_tests "distributed" | grep "upgrade" | _split_tests "${split_chunk}")
if [[ -z "${test_name_regexp}" ]]; then
[[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] || { error 1 "No tests match ${test_name_regexp}"; }
# something has to run in the split to generate a junit xml result
echo "Hacking jvm-dtest-upgrade to run only first test found as no tests in split ${split_chunk} were found"
test_name_regexp="$( _list_tests "distributed" | grep "upgrade" | sed -n 1p)"
fi
fi
_run_testlist "distributed" "testclasslist" "${test_name_regexp}" "" "$(_timeout_for 'test.distributed.timeout')" "${repeat_count}"
;;
"cqlsh-test")
./pylib/cassandra-cqlsh-tests.sh $(pwd)
;;
*)
error 1 "unrecognized test type \"${target}\""
;;
esac
# merge all unit xml files into one, and print summary test numbers
ant -quiet -silent generate-test-report
popd >/dev/null
}
_main "$@"