diff --git a/.build/docker/_copy_ccm_repositories.sh b/.build/docker/_copy_ccm_repositories.sh new file mode 100755 index 0000000000..7515211e23 --- /dev/null +++ b/.build/docker/_copy_ccm_repositories.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +if [ "$1" == "-h" ]; then + echo "$0 [-h]" + echo " this script is used internally by other scripts in the same directory to copy the image's ccm repositories into the dtest tmpdir" + exit 1 +fi + +command -v rsync >/dev/null 2>&1 || { echo >&2 "rsync needs to be installed"; exit 1; } + +################################ +# +# Main +# +################################ + + +# prepopulate a tmp ccm repository directory from whats already in the image +# +# the image has all versions and branches in ~/.ccm/repository already +# the container doesn't re-use ~/.ccm/repository so to avoid any writes in the containerfs +# so we rsync what's important from ~/.ccm/repository to the configured ${CCM_CONFIG_DIR} +# this appears to take a long time, but should be faster than ccm downloading +# rsync is used as it's the friendlier approach against open file ulimits +echo -n "prepopulating ${CCM_CONFIG_DIR}/repository/ for upgrade tests…" +mkdir -p "${CCM_CONFIG_DIR}/repository/" +rsync -rptgoL --include '[1-9]*' ${HOME}/.ccm/repository ${CCM_CONFIG_DIR}/ +rsync -rptgoL --include '_git_cache_apache' ${HOME}/.ccm/repository ${CCM_CONFIG_DIR}/ +rsync -rptgoL --include 'gitCOLON*' ${HOME}/.ccm/repository ${CCM_CONFIG_DIR}/ +echo " complete" diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 34b17fc4e9..9221fbff97 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -189,6 +189,8 @@ chmod -R ag+rwx "${build_dir}" case "${target}" in "cqlsh-test" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-large" | "dtest-upgrade-novnode" | "dtest-upgrade-novnode-large" ) ANT_OPTS="-Dtesttag.extra=_$(arch)_python${python_version/./-}" + # intentionally not TMP_DIR + DTEST_TMPDIR_LOCAL="$(mktemp -d ${build_dir}/run-python-dtest.XXXXXX)" ;; "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode" ) ANT_OPTS="-Dtesttag.extra=_$(arch)_novnode" @@ -208,6 +210,10 @@ fi # the docker container's env docker_envs="--env TEST_SCRIPT=${test_script} --env JAVA_VERSION=${java_version} --env PYTHON_VERSION=${python_version} --env cython=${cython} --env ANT_OPTS=\"${ANT_OPTS}\"" +if [ -n "${DTEST_TMPDIR_LOCAL}" ] ; then + DTEST_TMPDIR_REMOTE="$(sed "s:${build_dir}:/home/cassandra/cassandra/build:" <<< ${DTEST_TMPDIR_LOCAL})" + docker_envs="${docker_envs} --env TMPDIR=${DTEST_TMPDIR_REMOTE} --env CCM_CONFIG_DIR=${DTEST_TMPDIR_REMOTE}/.ccm" +fi [ $DEBUG ] && docker_envs="${docker_envs} --env DEBUG=1" split_str="0_0" @@ -241,6 +247,12 @@ echo "Running container ${container_name} ${docker_id}" docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile} docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile} +if [ -n "${DTEST_TMPDIR_LOCAL}" ] && [[ "${target}" =~ ^dtest-upgrade ]] ; then + # prepopulate a tmp ccm repository directory, if running dtest-upgrade tests + docker exec --user cassandra ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_copy_ccm_repositories.sh" | tee -a ${logfile} + trap 'nohup rm -rf "${DTEST_TMPDIR_LOCAL}/.ccm/repository" >/dev/null 2>&1 &' EXIT +fi + # capture logs and status set -o pipefail docker exec --user cassandra ${container_name} bash -c "${docker_command}" | tee -a ${logfile} @@ -266,5 +278,5 @@ xz -f ${logfile} 2>/dev/null popd >/dev/null popd >/dev/null -set -x +echo "+ exit ${status}" exit ${status} diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker index e55edf9b84..2c93719a35 100644 --- a/.build/docker/ubuntu2004_test.docker +++ b/.build/docker/ubuntu2004_test.docker @@ -51,7 +51,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3.8 python3.8-venv python3.8-dev \ python3.11 python3.11-venv python3.11-dev \ virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \ - vim lsof sudo libjemalloc2 dumb-init locales \ + vim lsof sudo libjemalloc2 dumb-init locales rsync \ openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh index 8a6995dcfb..80e15e8b73 100755 --- a/.build/run-python-dtests.sh +++ b/.build/run-python-dtests.sh @@ -43,19 +43,19 @@ DTEST_SPLIT_CHUNK="$2" # variables, with defaults [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" -[ "x${CASSANDRA_DTEST_DIR}" != "x" ] || CASSANDRA_DTEST_DIR="${CASSANDRA_DIR}/../cassandra-dtest" +[ "x${CASSANDRA_DTEST_DIR}" != "x" ] || CASSANDRA_DTEST_DIR="$(readlink -f ${CASSANDRA_DIR}/../cassandra-dtest)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" +[ "x${TMPDIR}" != "x" ] || { TMPDIR_SET=1 && export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)" ; } +[ "x${CCM_CONFIG_DIR}" != "x" ] && ls $CCM_CONFIG_DIR -export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)" export PYTHONIOENCODING="utf-8" export PYTHONUNBUFFERED=true export CASS_DRIVER_NO_EXTENSIONS=true export CASS_DRIVER_NO_CYTHON=true export CCM_MAX_HEAP_SIZE="1024M" export CCM_HEAP_NEWSIZE="512M" -export CCM_CONFIG_DIR="${TMPDIR}/.ccm" export NUM_TOKENS="16" -#Have Cassandra skip all fsyncs to improve test performance and reliability +# Have Cassandra skip all fsyncs to improve test performance and reliability export CASSANDRA_SKIP_SYNC=true unset CASSANDRA_HOME @@ -146,6 +146,12 @@ if [[ "${DTEST_SPLIT_CHUNK}" =~ ^[0-9]+/[0-9]+$ ]]; then ( split --help 2>&1 ) | grep -q "r/K/N" || split_cmd=gsplit command -v ${split_cmd} >/dev/null 2>&1 || { echo >&2 "${split_cmd} needs to be installed"; exit 1; } SPLIT_TESTS=$(${split_cmd} -n r/${DTEST_SPLIT_CHUNK} ${DIST_DIR}/test_list.txt) + if [[ -z "${SPLIT_TESTS}" ]]; then + # something has to run in the split to generate a nosetest xml result (and to not rerun all tests) + echo "Hacking ${DTEST_TARGET} to run only first test found as no tests in split ${DTEST_SPLIT_CHUNK} were found: " + SPLIT_TESTS="$( echo ${DIST_DIR}/test_list.txt | head -n1)" + echo " ${SPLIT_TESTS}" + fi SPLIT_STRING="_${DTEST_SPLIT_CHUNK//\//_}" elif [[ "x" != "x${DTEST_SPLIT_CHUNK}" ]] ; then SPLIT_TESTS=$(grep -e "${DTEST_SPLIT_CHUNK}" ${DIST_DIR}/test_list.txt) @@ -182,8 +188,10 @@ popd >/dev/null # ################################ -[[ "${TMPDIR}" == *"${DIST_DIR}/run-python-dtest."* ]] && rm -rf "${TMPDIR}" -unset TMPDIR +if [ ${TMPDIR_SET} ] ; then + [[ "${TMPDIR}" == *"${DIST_DIR}/run-python-dtest."* ]] && rm -rf "${TMPDIR}" + unset TMPDIR +fi deactivate # Exit cleanly for usable "Unstable" status diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index b0315c04f6..e44c2b31d4 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -178,8 +178,8 @@ def tasks() { 'dtest-large-novnode': [splits: 8, size: 'large'], 'dtest-upgrade': [splits: 64, size: 'large'], 'dtest-upgrade-novnode': [splits: 64, size: 'large'], - 'dtest-upgrade-large': [splits: 64, size: 'large'], - 'dtest-upgrade-novnode-large': [splits: 64, size: 'large'], + 'dtest-upgrade-large': [splits: 32, size: 'large'], + 'dtest-upgrade-novnode-large': [splits: 32, size: 'large'], ] testSteps.each() { it.value.put('type', 'test')