diff --git a/.build/docker/_set_java.sh b/.build/docker/_set_java.sh index 6770803c6c..e324f3350b 100755 --- a/.build/docker/_set_java.sh +++ b/.build/docker/_set_java.sh @@ -62,8 +62,4 @@ else sudo alternatives --set javac $(alternatives --display javac | grep "family java-${java_version}-openjdk" | cut -d' ' -f1) fi export JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::") -echo "Cassandra will be built with Java ${java_version}" - -# print debug information on versions -java -version 2>&1 -javac -version 2>&1 +echo "Using Java ${java_version}" \ No newline at end of file diff --git a/.build/docker/almalinux-build.docker b/.build/docker/almalinux-build.docker index c648eb581c..c850942620 100644 --- a/.build/docker/almalinux-build.docker +++ b/.build/docker/almalinux-build.docker @@ -28,9 +28,6 @@ ARG GID_ARG=1000 LABEL org.cassandra.buildenv=almalinux -VOLUME ${DIST_DIR} -VOLUME ${CASSANDRA_DIR} - RUN echo "Building with arguments:" \ && echo " - DIST_DIR=${DIST_DIR}" \ && echo " - BUILD_HOME=${BUILD_HOME}" \ diff --git a/.build/docker/bullseye-build.docker b/.build/docker/bullseye-build.docker index 7612a00290..7eb928bf5a 100644 --- a/.build/docker/bullseye-build.docker +++ b/.build/docker/bullseye-build.docker @@ -25,9 +25,6 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra LABEL org.cassandra.buildenv=bullseye -VOLUME ${DIST_DIR} -VOLUME ${CASSANDRA_DIR} - RUN echo "Building with arguments:" \ && echo " - DIST_DIR=${DIST_DIR}" \ && echo " - BUILD_HOME=${BUILD_HOME}" \ diff --git a/.build/docker/centos7-build.docker b/.build/docker/centos7-build.docker index 2f8aef98f2..9f9782ba80 100644 --- a/.build/docker/centos7-build.docker +++ b/.build/docker/centos7-build.docker @@ -29,9 +29,6 @@ ARG GID_ARG=1000 LABEL org.cassandra.buildenv=centos -VOLUME ${DIST_DIR} -VOLUME ${CASSANDRA_DIR} - RUN echo "Building with arguments:" \ && echo " - DIST_DIR=${DIST_DIR}" \ && echo " - BUILD_HOME=${BUILD_HOME}" \ diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index e18af803e9..ede5569834 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -69,8 +69,11 @@ if [[ ! "${java_version}" =~ $regx_java_version ]]; then exit 1 fi -python_version="3.6" -command -v python >/dev/null 2>&1 && python_version="$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')" +# allow python version override, otherwise default to current python version or 3.7 +if [ "x" == "x${python_version}" ] ; then + command -v python >/dev/null 2>&1 && python_version="$(python -V 2>&1 | awk '{print $2}' | awk -F'.' '{print $1"."$2}')" + python_version="${python_version:-3.7}" +fi # print debug information on versions docker --version @@ -82,6 +85,8 @@ dockerfile="ubuntu2004_test.docker" image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)" image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}" docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${HOME}/.m2/repository:/home/cassandra/.m2/repository" +# HACK hardlinks in overlay are buggy, the following mount prevents hardlinks from being used. ref $TMP_DIR in .build/run-tests.sh +docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/build/tmp" # Look for existing docker image, otherwise build if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then @@ -150,6 +155,7 @@ elif [[ "${target}" =~ dtest* ]] ; then else docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g" fi + docker_flags="${docker_flags} --env-file build/env.list -d --rm" # make sure build_dir is good diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker index 7035c5228f..1d21ae623c 100644 --- a/.build/docker/ubuntu2004_test.docker +++ b/.build/docker/ubuntu2004_test.docker @@ -25,9 +25,6 @@ ENV PYTHONUNBUFFERED=true LABEL org.cassandra.buildenv=ubuntu_2004 -VOLUME ${CASSANDRA_DIR} -VOLUME ${DIST_DIR} - RUN echo "Building with arguments:" \ && echo " - DIST_DIR=${DIST_DIR}" \ && echo " - BUILD_HOME=${BUILD_HOME}" \ @@ -41,7 +38,7 @@ RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf # install our python dependencies and some other stuff we need -# libev4 libev-dev are for the python driver / libssl-dev is for python3.6 +# libev4 libev-dev are for the python driver RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -51,20 +48,17 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ add-apt-repository -y ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y curl git-core python3-pip \ - python3.6 python3.6-venv python3.6-dev \ python3.7 python3.7-venv python3.7-dev \ python3.8 python3.8-venv python3.8-dev \ python3.11 python3.11-venv python3.11-dev \ - virtualenv net-tools libev4 libev-dev wget gcc libssl-dev libxml2 libxslt1-dev \ + virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \ vim lsof sudo libjemalloc2 dumb-init locales \ openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 5 -RUN python3.6 -m pip install --upgrade pip +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 3 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 4 RUN python3.7 -m pip install --upgrade pip RUN python3.8 -m pip install --upgrade pip @@ -109,15 +103,6 @@ ENV ANT_HOME=/usr/share/ant # included in the base image, the compiled objects are not updated by pip at run time, which can # cause errors if the tests rely on new driver functionality or bug fixes. -RUN virtualenv --python=python3.6 ${BUILD_HOME}/env3.6 -RUN chmod +x ${BUILD_HOME}/env3.6/bin/activate - -RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ - && source ${BUILD_HOME}/env3.6/bin/activate \ - && pip3 install --upgrade pip \ - && pip3 install -r /opt/requirements.txt \ - && pip3 freeze --user" - RUN virtualenv --python=python3.7 ${BUILD_HOME}/env3.7 RUN chmod +x ${BUILD_HOME}/env3.7/bin/activate @@ -146,11 +131,26 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ && pip3 freeze --user" # Initialize the CCM git repo as well as this also can fail to clone -RUN /bin/bash -c "source ${BUILD_HOME}/env3.6/bin/activate && \ - ccm create -n 1 -v git:trunk test && ccm remove test && \ +RUN /bin/bash -c "source ${BUILD_HOME}/env3.7/bin/activate && \ ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \ ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test" +# Initialize ccm versions. right side of each sequence needs to be updated with new releases. +# this can be checked with: +# `curl -s https://downloads.apache.org/cassandra/ | grep -oP '(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -V | uniq -w 3` +RUN bash -c 'source ~/env3.7/bin/activate && \ + for i in {1..11} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \ + for i in {1..3} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done' + +# 5+ requires java11 +RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture) + +# Initialize the CCM git repo as well as this also can fail to clone +RUN /bin/bash -c 'source ~/env3.7/bin/activate && \ + ccm create -n 1 -v git:trunk test && ccm remove test && \ + ccm create -n 1 -v git:cassandra-5.0 test && ccm remove test && \ + ccm create --quiet -n 1 -v binary:5.0-alpha1 test && ccm remove test ' + # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git # fix permissions, runtime user has different uid/gid diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh index 91b597be1c..4b791e267d 100755 --- a/.build/run-python-dtests.sh +++ b/.build/run-python-dtests.sh @@ -51,11 +51,10 @@ export CASS_DRIVER_NO_EXTENSIONS=true export CASS_DRIVER_NO_CYTHON=true export CCM_MAX_HEAP_SIZE="1024M" export CCM_HEAP_NEWSIZE="512M" -export CCM_CONFIG_DIR=${DIST_DIR}/.ccm export NUM_TOKENS="16" #Have Cassandra skip all fsyncs to improve test performance and reliability export CASSANDRA_SKIP_SYNC=true -export TMPDIR="$(mktemp -d /tmp/run-python-dtest.XXXXXX)" +export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)" unset CASSANDRA_HOME # pre-conditions @@ -73,7 +72,9 @@ if [ "${java_version}" -eq 17 ] && [[ "${target}" == "dtest-upgrade" ]] ; then exit 1 fi -python_version=$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}') +python_version=$(python -V 2>&1 | awk '{print $2}' | awk -F'.' '{print $1"."$2}') +python_regx_supported_versions="^(3.7|3.8|3.11)$" +[[ $python_version =~ $python_regx_supported_versions ]] || { echo "Python ${python_version} not supported."; exit 1; } # check project is already built. no cleaning is done, so jenkins unstash works, beware. [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; } @@ -163,8 +164,8 @@ fi # merge all unit xml files into one, and print summary test numbers pushd ${CASSANDRA_DIR}/ >/dev/null # remove wrapping elements. `ant generate-unified-test-report` doesn't like it` -sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > /tmp/nosetests.xml -cat /tmp/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml +sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > ${TMPDIR}/nosetests.xml +cat ${TMPDIR}/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml ant -quiet -silent generate-unified-test-report popd >/dev/null @@ -174,7 +175,7 @@ popd >/dev/null # ################################ -rm -rf "/tmp/run-python-dtest.${TMPDIR/\/tmp\/run-python-dtest./}" +[[ "${TMPDIR}" == *"${DIST_DIR}/run-python-dtest."* ]] && rm -rf "${TMPDIR}" unset TMPDIR deactivate diff --git a/.build/run-tests.sh b/.build/run-tests.sh index 4a0cecfa33..9e1291cf74 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -49,6 +49,9 @@ git --version java -version 2>&1 javac -version 2>&1 +# set the OFFLINE env var (to anything) to allow running jvm-dtest-upgrade offline +[ "x" != "x${OFFLINE}" ] && echo "WARNING: running in offline mode. jvm-dtest-upgrade results may be stale." + # lists all tests for the specific test type _list_tests() { local -r classlistprefix="$1" @@ -84,7 +87,9 @@ _build_all_dtest_jars() { fi if [ -d ${TMP_DIR}/cassandra-dtest-jars ] && [ "https://github.com/apache/cassandra.git" == "$(git -C ${TMP_DIR}/cassandra-dtest-jars remote get-url origin)" ] ; then - until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git pull failed… trying again… " ; done + if [ "x" == "x${OFFLINE}" ] ; then + until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git fetch failed… trying again… " ; done + fi else rm -fR ${TMP_DIR}/cassandra-dtest-jars pushd $TMP_DIR >/dev/null @@ -171,7 +176,7 @@ _main() { # ant test setup export TMP_DIR="${DIST_DIR}/tmp" - mkdir -p "${TMP_DIR}" || true + [ -d ${TMP_DIR} ] || mkdir -p "${TMP_DIR}" export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR}" # fresh virtualenv and test logs results everytime diff --git a/build.xml b/build.xml index b61e5a240f..de370b1154 100644 --- a/build.xml +++ b/build.xml @@ -624,7 +624,7 @@ - + diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java index c8624b96d4..0af3f612ed 100644 --- a/test/unit/org/apache/cassandra/db/ImportTest.java +++ b/test/unit/org/apache/cassandra/db/ImportTest.java @@ -43,6 +43,7 @@ import org.apache.cassandra.dht.BootStrapper; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.PathUtils; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.CacheService; @@ -245,6 +246,7 @@ public class ImportTest extends CQLTester } } } + PathUtils.deleteRecursiveOnExit(temp); return new File(backupdir); } diff --git a/test/unit/org/apache/cassandra/io/util/FileTest.java b/test/unit/org/apache/cassandra/io/util/FileTest.java index 0a7c478daf..718f321042 100644 --- a/test/unit/org/apache/cassandra/io/util/FileTest.java +++ b/test/unit/org/apache/cassandra/io/util/FileTest.java @@ -37,7 +37,6 @@ import org.apache.commons.lang3.RandomUtils; import org.junit.Assert; import org.junit.Test; -import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.assertj.core.api.Assertions; import org.psjava.util.Triple; @@ -45,13 +44,14 @@ import org.psjava.util.Triple; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR; +import static org.apache.cassandra.config.CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE; public class FileTest { private static final java.io.File dir; static { - CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE.setBoolean(false); + USE_NIX_RECURSIVE_DELETE.setBoolean(false); java.io.File parent = new java.io.File(JAVA_IO_TMPDIR.getString()); //checkstyle: permit this instantiation String dirName = Long.toHexString(ThreadLocalRandom.current().nextLong()); while (new java.io.File(parent, dirName).exists()) //checkstyle: permit this instantiation @@ -115,6 +115,9 @@ public class FileTest regularLink.delete(); regular.delete(); emptySubdir.delete(); + nonEmptySubdir.delete(); + nonEmptySubdirLink.delete(); + dir.setReadable(true); } private static String nonAbsolute(java.io.File file)