diff --git a/.build/build-artifacts.sh b/.build/build-artifacts.sh index a1fe16399a..3214f273f8 100755 --- a/.build/build-artifacts.sh +++ b/.build/build-artifacts.sh @@ -16,7 +16,7 @@ # limitations under the License. # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" # pre-conditions command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } diff --git a/.build/build-bench.xml b/.build/build-bench.xml index 65854c6875..4ea0ff571e 100644 --- a/.build/build-bench.xml +++ b/.build/build-bench.xml @@ -41,11 +41,22 @@ - + - + + + + + + + + + + + + @@ -83,6 +94,10 @@ + + + + @@ -97,8 +112,9 @@ + - + diff --git a/.build/build-jars.sh b/.build/build-jars.sh index cd60e5a130..cf2b91b082 100755 --- a/.build/build-jars.sh +++ b/.build/build-jars.sh @@ -18,7 +18,7 @@ # temporary between CASSANDRA-18133 and CASSANDRA-18594 # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" # pre-conditions command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } diff --git a/.build/check-code.sh b/.build/check-code.sh index 60c96dc9ad..28bc5c576a 100755 --- a/.build/check-code.sh +++ b/.build/check-code.sh @@ -16,7 +16,7 @@ # limitations under the License. # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(dirname "$0")/.."; } +[ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(dirname -- "$0")/.."; } # pre-conditions command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } diff --git a/.build/ci/generate-ci-summary.sh b/.build/ci/generate-ci-summary.sh index 01de5a6d63..3828fa2c1e 100755 --- a/.build/ci/generate-ci-summary.sh +++ b/.build/ci/generate-ci-summary.sh @@ -28,7 +28,7 @@ # # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/../..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/../..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" # pre-conditions diff --git a/.build/ci/generate-test-report.sh b/.build/ci/generate-test-report.sh index 00672ecea5..81e2296419 100755 --- a/.build/ci/generate-test-report.sh +++ b/.build/ci/generate-test-report.sh @@ -24,7 +24,7 @@ # # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/../..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/../..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" # pre-conditions diff --git a/.build/docker/_build-debian.sh b/.build/docker/_build-debian.sh index c25d168b85..071c42a499 100755 --- a/.build/docker/_build-debian.sh +++ b/.build/docker/_build-debian.sh @@ -24,7 +24,7 @@ [ $DEBUG ] && set -x # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" # pre-conditions diff --git a/.build/docker/_build-redhat.sh b/.build/docker/_build-redhat.sh index ec58f17658..8acfb2b51a 100755 --- a/.build/docker/_build-redhat.sh +++ b/.build/docker/_build-redhat.sh @@ -22,7 +22,7 @@ ################################ # variables, w/ defaults, w/ checks -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" [ "x${RPM_BUILD_DIR}" != "x" ] || RPM_BUILD_DIR="$(mktemp -d /tmp/rpmbuild.XXXXXX)" diff --git a/.build/docker/_docker_run.sh b/.build/docker/_docker_run.sh index 4a2aa16c5b..1a67840bde 100755 --- a/.build/docker/_docker_run.sh +++ b/.build/docker/_docker_run.sh @@ -29,7 +29,7 @@ [ $DEBUG ] && set -x # variables, with defaults -[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)" +[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)" [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build" [ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository" [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; } diff --git a/.build/docker/build-artifacts.sh b/.build/docker/build-artifacts.sh index 85eec83005..65eb51397b 100755 --- a/.build/docker/build-artifacts.sh +++ b/.build/docker/build-artifacts.sh @@ -18,5 +18,5 @@ # # Creates the tarball artifact -$(dirname "$0")/_docker_run.sh bullseye-build.docker build-artifacts.sh $1 +$(dirname -- "$0")/_docker_run.sh bullseye-build.docker build-artifacts.sh $1 exit $? diff --git a/.build/docker/build-debian.sh b/.build/docker/build-debian.sh index e265fa5e8d..43f89b6620 100755 --- a/.build/docker/build-debian.sh +++ b/.build/docker/build-debian.sh @@ -32,5 +32,5 @@ echo # # Creates the debian package -$(dirname "$0")/_docker_run.sh bullseye-build.docker docker/_build-debian.sh $1 +$(dirname -- "$0")/_docker_run.sh bullseye-build.docker docker/_build-debian.sh $1 exit $? diff --git a/.build/docker/build-jars.sh b/.build/docker/build-jars.sh index b8039cb0a0..a23f2e6d59 100755 --- a/.build/docker/build-jars.sh +++ b/.build/docker/build-jars.sh @@ -18,5 +18,5 @@ # # Build the jars -$(dirname "$0")/_docker_run.sh bullseye-build.docker build-jars.sh $1 +$(dirname -- "$0")/_docker_run.sh bullseye-build.docker build-jars.sh $1 exit $? diff --git a/.build/docker/build-redhat.sh b/.build/docker/build-redhat.sh index 4163ce9153..1d11b25986 100755 --- a/.build/docker/build-redhat.sh +++ b/.build/docker/build-redhat.sh @@ -40,5 +40,5 @@ echo # # Creates the redhat package -$(dirname "$0")/_docker_run.sh almalinux-build.docker docker/_build-redhat.sh "${java_version}" +$(dirname -- "$0")/_docker_run.sh almalinux-build.docker docker/_build-redhat.sh "${java_version}" exit $? diff --git a/.build/docker/check-code.sh b/.build/docker/check-code.sh index 4afccaea98..5284c402d4 100755 --- a/.build/docker/check-code.sh +++ b/.build/docker/check-code.sh @@ -20,4 +20,4 @@ export CASSANDRA_DOCKER_ANT_OPTS="-Ddependency-check.home.base=/tmp" -$(dirname "$0")/_docker_run.sh bullseye-build.docker check-code.sh $1 +$(dirname -- "$0")/_docker_run.sh bullseye-build.docker check-code.sh $1 diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 0133ae3522..9b3229a87c 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -21,10 +21,24 @@ [ $DEBUG ] && set -x +# variables, with defaults +[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)" +[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest" +[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build" +[ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository" +[ "x${docker_timeout_hours}" != "x" ] || docker_timeout_hours="1" +[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; } +[ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; } + +# source TARGET_TYPES from the non-docker scripts +_target_test_types="$(grep '^TARGET_TYPES=' "${cassandra_dir}/.build/run-tests.sh" | cut -d'"' -f2)" +_target_dtest_types="$(cd ${cassandra_dir}; bash <(sed -n "/^TARGET_TYPES=/,/^done$/p" .build/run-python-dtests.sh; echo 'echo ${TARGET_TYPES}'))" +TARGET_TYPES="${_target_test_types} ${_target_dtest_types}" + print_help() { echo "" echo "Usage: $0 [-a|-t|-c|-j|-h] [extra arguments]" - echo " -a Test target type: test, test-compression, test-cdc, ..." + echo " -a Test target type: ${TARGET_TYPES}" echo " -t Test name regexp to run." echo " -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks." echo " -j Java version. Default java_version is what 'java.default' specifies in build.xml." @@ -39,21 +53,20 @@ error() { } # legacy argument handling -case ${1} in - "build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode") - test_type="-a ${1}" - if [[ -z ${2} ]]; then - test_list="" - elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then - test_list="-c ${2}"; - else - test_list="-t ${2}"; - fi - if [[ -n ${3} ]]; then java_version="-j ${3}"; else java_version=""; fi - echo "Using deprecated legacy arguments. Please update to new parameter format: ${test_type} ${test_list} ${java_version}" - $0 ${test_type} ${test_list} ${java_version} - exit $? -esac +if [[ " ${TARGET_TYPES} " =~ " ${1} " ]]; then + test_type="-a ${1}" + if [[ -z ${2} ]]; then + test_list="" + elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then + test_list="-c ${2}"; + else + test_list="-t ${2}"; + fi + if [[ -n ${3} ]]; then java_version="-j ${3}"; else java_version=""; fi + echo "Using deprecated legacy arguments. Please update to new parameter format: ${test_type} ${test_list} ${java_version}" + $0 ${test_type} ${test_list} ${java_version} + exit $? +fi env_vars="" while getopts ":a:t:c:e:hj:" opt; do @@ -61,6 +74,7 @@ while getopts ":a:t:c:e:hj:" opt; do # Invalid flags check disabled as we'll pass them to other scripts case $opt in a ) test_target="$OPTARG" + [[ " ${TARGET_TYPES} " =~ " ${test_target/-repeat/} " ]] || error 1 "Invalid test target type '${test_target}'. Valid types: ${TARGET_TYPES}" ;; t ) test_name_regexp="$OPTARG" ;; @@ -79,14 +93,6 @@ while getopts ":a:t:c:e:hj:" opt; do esac done -# variables, with defaults -[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)" -[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest" -[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build" -[ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository" -[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; } -[ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; } - # pre-conditions command -v docker >/dev/null 2>&1 || { error 1 "docker needs to be installed"; } command -v bc >/dev/null 2>&1 || { error 1 "bc needs to be installed"; } @@ -186,14 +192,14 @@ case ${test_target/-repeat/} in [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; } ;; # test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway - "microbench" | "test-burn" | "long-test" | "cqlsh-test" ) + "test-burn" | "long-test" | "cqlsh-test" ) [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; } ;; - "simulator-dtest") + "microbench" | "microbench-test" | "simulator-dtest") [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; } docker_flags="-m 15g --memory-swap 15g" ;; - "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-large-latest" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large") + "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-large-latest" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-large-novnode" | "dtest-large-novnode-latest") [ -f "${cassandra_dtest_dir}/dtest.py" ] || { error 1 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; } test_script="run-python-dtests.sh" docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest" @@ -204,7 +210,7 @@ case ${test_target/-repeat/} in [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; } ;; *) - error 1 "unrecognized test type \"${target}\"" + error 1 "docker resource limits unconfigured for test type \"${target}\"" ;; esac @@ -231,7 +237,7 @@ chmod -R ag+rwx "${build_dir}" # define testtag.extra so tests can be aggregated together. (jdk is already appended in build.xml) case "${target}" in - "cqlsh-test" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-large" | "dtest-upgrade-novnode" | "dtest-upgrade-novnode-large" ) + "cqlsh-test" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-large" | "dtest-upgrade-novnode" | "dtest-upgrade-large-novnode" ) ANT_OPTS="-Dtesttag.extra=_$(arch)_python${python_version/./-}" # intentionally not TMP_DIR DTEST_TMPDIR_LOCAL="$(mktemp -d ${build_dir}/run-python-dtest.XXXXXX)" @@ -286,7 +292,7 @@ docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh -a ${target} ${split_chunk_arg} ${test_name_regexp_arg} ${env_vars} ; exit \$?" # start the container, timeout after 4 hours -docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h) +docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep ${docker_timeout_hours}h) echo "Running container ${container_name} ${docker_id}" diff --git a/.build/run-ci b/.build/run-ci index 0d4f14dfef..04a257d858 100755 --- a/.build/run-ci +++ b/.build/run-ci @@ -267,7 +267,7 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ run_kubectl_command(kubeconfig, kubecontext, kube_ns, ["exec", DEFAULT_POD_NAME, "--", - "curl", "-sS", "https://svn.apache.org/repos/asf/comdev/project-logos/originals/cassandra-6.svg", + "curl", "-sS", "https://www.apache.org/logos/originals/cassandra-4.svg", "-o", "/var/jenkins_cache/war/images/svgs/logo.svg"]) if result.returncode != 0: diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh index 86652db0b8..f57f69668a 100755 --- a/.build/run-python-dtests.sh +++ b/.build/run-python-dtests.sh @@ -27,11 +27,24 @@ [ $DEBUG ] && set -x +# target types +TARGET_TYPES="dtest dtest-upgrade" +for base in ${TARGET_TYPES}; do + for large in "" "-large"; do + for novnode in "" "-novnode"; do + for latest in "" "-latest"; do + variant="${large}${novnode}${latest}" + [[ -n "${variant}" ]] && TARGET_TYPES="${TARGET_TYPES} ${base}${variant}" + done + done + done +done + # help if [ "$#" -lt 1 ] || [ "$1" == "-h" ]; then echo "" echo "Usage: $0 [-a|-t|-c|-j|-h]" - echo " -a Test target type: dtest, dtest-latest, ..." + echo " -a Test target type: ${TARGET_TYPES}" echo " -t Test name regexp to run." echo " -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks." echo "" @@ -46,6 +59,7 @@ DTEST_TARGET="dtest" while getopts "a:t:c:hj:" opt; do case $opt in a ) DTEST_TARGET="$OPTARG" + [[ " ${TARGET_TYPES} " =~ " ${DTEST_TARGET/-repeat/} " ]] || error 1 "Invalid test target type '${DTEST_TARGET}'. Valid types: ${TARGET_TYPES}" ;; t ) DTEST_SPLIT_CHUNK="$OPTARG" ;; @@ -65,7 +79,7 @@ if [ "$#" -ne 0 ]; then fi # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${CASSANDRA_DTEST_DIR}" != "x" ] || CASSANDRA_DTEST_DIR="$(readlink -f ${CASSANDRA_DIR}/../cassandra-dtest)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" [ "x${TMPDIR}" != "x" ] || { TMPDIR_SET=1 && export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)" ; } @@ -87,7 +101,7 @@ command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1 command -v virtualenv >/dev/null 2>&1 || { echo >&2 "virtualenv needs to be installed"; exit 1; } [ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } [ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; } -ALLOWED_DTEST_VARIANTS="novnode|large|latest|upgrade" +ALLOWED_DTEST_VARIANTS="large|latest|upgrade|novnode|latest" [[ "${DTEST_TARGET}" =~ ^dtest(-(${ALLOWED_DTEST_VARIANTS}))*$ ]] || { echo >&2 "Unknown dtest target: ${DTEST_TARGET}. Allowed variants are ${ALLOWED_DTEST_VARIANTS}"; exit 1; } java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}') diff --git a/.build/run-tests.sh b/.build/run-tests.sh index 9270922898..ee80aadd69 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -27,9 +27,12 @@ set -o pipefail [ $DEBUG ] && set -x # variables, with defaults -[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" +# target types +TARGET_TYPES="build_dtest_jars stress-test fqltool-test sstableloader-test microbench microbench-test test-burn long-test cqlsh-test simulator-dtest test test-cdc test-compression test-oa test-system-keyspace-directory test-latest jvm-dtest jvm-dtest-upgrade jvm-dtest-novnode jvm-dtest-upgrade-novnode" + # pre-conditions command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; } command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; } @@ -47,7 +50,7 @@ error() { print_help() { echo "Usage: $0 [-a|-t|-c|-e|-i|-b|-s|-h]" - echo " -a Test target type: test, test-compression, test-cdc, ..." + echo " -a Test target type: ${TARGET_TYPES}" echo " -t Test name regexp to run." echo " -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks." echo " -b Specify the base git branch for comparison when determining changed tests to" @@ -65,20 +68,19 @@ print_help() { # legacy argument handling -case ${1} in - "build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode") - test_type="-a ${1}" - if [[ -z ${2} ]]; then - test_list="" - elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then - test_list="-c ${2}"; - else - test_list="-t ${2}"; - fi - echo "Using deprecated legacy arguments. Please update to new parameter format: ${test_type} ${test_list}" - $0 ${test_type} ${test_list} - exit $? -esac +if [[ " ${TARGET_TYPES} " =~ " ${1} " ]]; then + test_type="-a ${1}" + if [[ -z ${2} ]]; then + test_list="" + elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then + test_list="-c ${2}"; + else + test_list="-t ${2}"; + fi + echo "Using deprecated legacy arguments. Please update to new parameter format: ${test_type} ${test_list}" + $0 ${test_type} ${test_list} + exit $? +fi env_vars="" @@ -88,6 +90,7 @@ detect_changed_tests=true while getopts "a:t:c:e:ib:shj:" opt; do case $opt in a ) test_target="$OPTARG" + [[ " ${TARGET_TYPES} " =~ " ${test_target/-repeat/} " ]] || error 1 "Invalid test target type '${test_target}'. Valid types: ${TARGET_TYPES}" ;; t ) test_name_regexp="$OPTARG" ;; @@ -349,8 +352,9 @@ _main() { ant sstableloader-build-test ${ANT_TEST_OPTS} ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" ;; - "microbench") - ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true + "microbench" | "microbench-test") + [[ "x${test_name_regexp}" != "x" ]] && test_name_regexp="-Dbenchmark.name=${test_name_regexp}" + ant $target ${ANT_TEST_OPTS} ${test_name_regexp} -Dmaven.test.failure.ignore=true ;; "test") _run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}" @@ -413,7 +417,7 @@ _main() { ./pylib/cassandra-cqlsh-tests.sh $(pwd) ;; *) - error 1 "unrecognized test type \"${target}\"" + error 1 "unconfigured build command for test type \"${target}\"" ;; esac diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 877cdc2850..0e5980bb19 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -29,21 +29,21 @@ // // This Jenkinsfile is expected to work on any Jenkins infrastructure. // The controller should have 4 cpu, 12GB ram (and be configured to use `-XX:+UseG1GC -Xmx8G`) -// It is required to have agents providing five labels, each that can provide docker and the following capabilities: -// - cassandra-amd64-small : 1 cpu, 1GB ram -// - cassandra-small : 1 cpu, 1GB ram (alias for above but for any arch) -// - cassandra-amd64-medium : 3 cpu, 5GB ram -// - cassandra-medium : 3 cpu, 5GB ram (alias for above but for any arch) -// - cassandra-amd64-large : 7 cpu, 16GB ram -// - cassandra-large : 7 cpu, 16GB ram +// +// It is required to have agents providing 6+ labels, each that can provide docker and the following capabilities: +// +// - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch) +// - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram +// - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram +// +// Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes. // // When running builds parameterised to other architectures the corresponding labels are expected. // For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large. // -// The built-in node must has the "controller" label. There must be more than two agents for each label. -// // Plugins required are: -// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact. +// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact, jmh-report. +// See .jenkins/k8s/jenkins-deployment.yaml for up to date list of plugins. // // Any functionality that depends upon ASF Infra ( i.e. the canonical ci-cassandra.a.o ) // will be ignored when run on other environments. @@ -68,6 +68,7 @@ pipeline { options { // must have: avoids agents waste in idle time on controller bottleneck durabilityHint('PERFORMANCE_OPTIMIZED') + disableResume() } parameters { string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository') @@ -140,9 +141,10 @@ def pipelineProfiles() { return [ 'packaging': ['artifacts', 'lint', 'debian', 'redhat'], 'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'], - 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'], - 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'], - 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'], + 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest', 'microbench-test'], + 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade', 'microbench-test'], + 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-large-novnode', 'microbench-test'], + 'performance': ['microbench'], 'custom': [] ] } @@ -204,13 +206,20 @@ def tasks() { 'dtest-upgrade': [splits: 160, size: 'large'], 'dtest-upgrade-novnode': [splits: 160, size: 'large'], 'dtest-upgrade-large': [splits: 40, size: 'large'], - 'dtest-upgrade-novnode-large': [splits: 40, size: 'large'], + 'dtest-upgrade-large-novnode': [splits: 40, size: 'large'], + 'microbench-test': [splits: 1, size: 'large', timeout_hours: 2], // TODO implement splits + // performance tests need 'cassandra-*large-dedicated' nodes + 'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true], ] testSteps.each() { it.value.put('type', 'test') if (!it.value['size']) { it.value.put('size', 'medium') } + if (!it.value['timeout_hours']) { + // default 1 hour + it.value.put('timeout_hours', 1) + } if (it.key.startsWith('dtest')) { it.value.put('python-dtest', true) } @@ -392,10 +401,11 @@ def test(command, cell) { script_vars = "${script_vars} cython=\'${cell.cython}\'" } script_vars = fetchDTestsSource(command, script_vars) - timeout(time: 1, unit: 'HOURS') { // best throughput with each cell at ~10 minutes + timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes def timer = System.currentTimeMillis() try { buildJVMDTestJars(cell, script_vars, logfile) + script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true dir("build") { archiveArtifacts artifacts: "${logfile}", fingerprint: true @@ -421,12 +431,14 @@ def test(command, cell) { find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';' find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';' """ - junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + if (!cell.step.startsWith("microbench")) { + junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + } sh """ find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l """ - archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz", fingerprint: true + archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } cleanAgent(cell.step) @@ -484,8 +496,15 @@ def fetchDockerImages(dockerfiles) { } def getNodeLabel(command, cell) { - echo "using node label: cassandra-${cell.arch}-${command.size}" - return "cassandra-${cell.arch}-${command.size}" + def label = "cassandra-${cell.arch}-${command.size}" + if (command.containsKey('benchmark') && command.benchmark) { + // to provide reliable results the "microbench" target + // expects to be running on baremetal jenkins agents configured with only one executor + // those jenkins agents need to be manually configured to have the "cassandra-amd64-large-dedicated" label + label = "${label}-dedicated" + } + echo "using node label: ${label}" + return label } def copyToNightlies(sourceFiles, remoteDirectory='') { @@ -508,7 +527,8 @@ def copyToNightlies(sourceFiles, remoteDirectory='') { } def cleanAgent(job_name) { - sh "hostname" + // get any public IP which is more helpful correlating back to the cloud instance + sh script: 'hostname; curl -sm 10 ifconfig.me', returnStatus: true if (isCanonical()) { def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/" cleanAgentDocker(job_name, agentScriptsUrl) @@ -558,7 +578,7 @@ def generateTestReports() { unzip -x -d build/test -q output.zip ) ${teeSuffix} """ } else { - copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true + copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true } if (fileExists('build/test/output')) { // merge splits for each target's test report, other axes are kept separate @@ -586,9 +606,12 @@ def generateTestReports() { dir('build/') { archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true - copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}') + copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json') } } + if (fileExists('build/test/jmh-result.json')) { + jmhReport('build/test/jmh-result.json') + } } } diff --git a/.jenkins/k8s/jenkins-deployment.yaml b/.jenkins/k8s/jenkins-deployment.yaml index 585a601b0d..46cc77fc3c 100644 --- a/.jenkins/k8s/jenkins-deployment.yaml +++ b/.jenkins/k8s/jenkins-deployment.yaml @@ -58,6 +58,7 @@ controller: - pipeline-rest-api - test-stability - copyartifact + - jmh-report node-selector: cassandra.jenkins.controller: true scriptApproval: @@ -294,7 +295,7 @@ agent: mountPath: /certs agent-dind-large: | - name: agent-dind-large - label: agent-dind cassandra-amd64-large + label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated nodeSelector: 'cassandra.jenkins.agent.large=true' activeDeadlineSeconds: '0' idleMinutes: 1 diff --git a/build.xml b/build.xml index b970240df4..832a9d86bb 100644 --- a/build.xml +++ b/build.xml @@ -1981,14 +1981,6 @@ - - - - - - - - diff --git a/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java b/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java index 87eb237f92..f2f88409ad 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java @@ -45,6 +45,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.cql3.BatchQueryOptions; +import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryHandler; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; @@ -75,7 +76,7 @@ public class BatchStatementBench static { - DatabaseDescriptor.toolInitialization(); + CQLTester.setUpClass(); // Partitioner is not set in client mode. if (DatabaseDescriptor.getPartitioner() == null) DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java index 393e088e51..8462d30090 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java @@ -80,7 +80,7 @@ public class CompactionBench extends CQLTester @Setup(Level.Trial) public void setup() throws Throwable { - CQLTester.prepareServer(); + CQLTester.setUpClass(); DatabaseDescriptor.setCursorCompactionEnabled(isCursor); DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(10*1024); // no rate limiting createSStables(); diff --git a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java index 170627cdbd..2adb0c2047 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java @@ -36,6 +36,7 @@ import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; +import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringBound; import org.apache.cassandra.db.ClusteringBoundary; @@ -70,6 +71,7 @@ public class MetadataCollectorBench @Setup public void setup() { + CQLTester.setUpClass(); TableMetadata.Builder tableMetadataBuilder = TableMetadata.builder("k", "t") .addPartitionKeyColumn("pk", LongType.instance) .addRegularColumn("rc", LongType.instance);