mirror of https://github.com/apache/cassandra
Merge branch 'cassandra-5.0' into trunk
* cassandra-5.0: Fixes for in-tree scripts: offline mode, maybe-build for fqltool-test, jvm-dtest-upgrade
This commit is contained in:
Mick Semb Wever
commit
302b272b14
|
|
@ -62,8 +62,4 @@ else
|
|||
sudo alternatives --set javac $(alternatives --display javac | grep "family java-${java_version}-openjdk" | cut -d' ' -f1)
|
||||
fi
|
||||
export JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::")
|
||||
echo "Cassandra will be built with Java ${java_version}"
|
||||
|
||||
# print debug information on versions
|
||||
java -version 2>&1
|
||||
javac -version 2>&1
|
||||
echo "Using Java ${java_version}"
|
||||
|
|
@ -28,9 +28,6 @@ ARG GID_ARG=1000
|
|||
|
||||
LABEL org.cassandra.buildenv=almalinux
|
||||
|
||||
VOLUME ${DIST_DIR}
|
||||
VOLUME ${CASSANDRA_DIR}
|
||||
|
||||
RUN echo "Building with arguments:" \
|
||||
&& echo " - DIST_DIR=${DIST_DIR}" \
|
||||
&& echo " - BUILD_HOME=${BUILD_HOME}" \
|
||||
|
|
|
|||
|
|
@ -25,9 +25,6 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
|
|||
|
||||
LABEL org.cassandra.buildenv=bullseye
|
||||
|
||||
VOLUME ${DIST_DIR}
|
||||
VOLUME ${CASSANDRA_DIR}
|
||||
|
||||
RUN echo "Building with arguments:" \
|
||||
&& echo " - DIST_DIR=${DIST_DIR}" \
|
||||
&& echo " - BUILD_HOME=${BUILD_HOME}" \
|
||||
|
|
|
|||
|
|
@ -29,9 +29,6 @@ ARG GID_ARG=1000
|
|||
|
||||
LABEL org.cassandra.buildenv=centos
|
||||
|
||||
VOLUME ${DIST_DIR}
|
||||
VOLUME ${CASSANDRA_DIR}
|
||||
|
||||
RUN echo "Building with arguments:" \
|
||||
&& echo " - DIST_DIR=${DIST_DIR}" \
|
||||
&& echo " - BUILD_HOME=${BUILD_HOME}" \
|
||||
|
|
|
|||
|
|
@ -69,8 +69,11 @@ if [[ ! "${java_version}" =~ $regx_java_version ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
python_version="3.6"
|
||||
command -v python >/dev/null 2>&1 && python_version="$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')"
|
||||
# allow python version override, otherwise default to current python version or 3.7
|
||||
if [ "x" == "x${python_version}" ] ; then
|
||||
command -v python >/dev/null 2>&1 && python_version="$(python -V 2>&1 | awk '{print $2}' | awk -F'.' '{print $1"."$2}')"
|
||||
python_version="${python_version:-3.7}"
|
||||
fi
|
||||
|
||||
# print debug information on versions
|
||||
docker --version
|
||||
|
|
@ -82,6 +85,8 @@ dockerfile="ubuntu2004_test.docker"
|
|||
image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
|
||||
image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
|
||||
docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${HOME}/.m2/repository:/home/cassandra/.m2/repository"
|
||||
# HACK hardlinks in overlay are buggy, the following mount prevents hardlinks from being used. ref $TMP_DIR in .build/run-tests.sh
|
||||
docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/build/tmp"
|
||||
|
||||
# Look for existing docker image, otherwise build
|
||||
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
|
||||
|
|
@ -150,6 +155,7 @@ elif [[ "${target}" =~ dtest* ]] ; then
|
|||
else
|
||||
docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
|
||||
fi
|
||||
|
||||
docker_flags="${docker_flags} --env-file build/env.list -d --rm"
|
||||
|
||||
# make sure build_dir is good
|
||||
|
|
|
|||
|
|
@ -25,9 +25,6 @@ ENV PYTHONUNBUFFERED=true
|
|||
|
||||
LABEL org.cassandra.buildenv=ubuntu_2004
|
||||
|
||||
VOLUME ${CASSANDRA_DIR}
|
||||
VOLUME ${DIST_DIR}
|
||||
|
||||
RUN echo "Building with arguments:" \
|
||||
&& echo " - DIST_DIR=${DIST_DIR}" \
|
||||
&& echo " - BUILD_HOME=${BUILD_HOME}" \
|
||||
|
|
@ -41,7 +38,7 @@ RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf
|
|||
RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
|
||||
|
||||
# install our python dependencies and some other stuff we need
|
||||
# libev4 libev-dev are for the python driver / libssl-dev is for python3.6
|
||||
# libev4 libev-dev are for the python driver
|
||||
|
||||
RUN export DEBIAN_FRONTEND=noninteractive && \
|
||||
apt-get update && \
|
||||
|
|
@ -51,20 +48,17 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
|
|||
add-apt-repository -y ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install -y curl git-core python3-pip \
|
||||
python3.6 python3.6-venv python3.6-dev \
|
||||
python3.7 python3.7-venv python3.7-dev \
|
||||
python3.8 python3.8-venv python3.8-dev \
|
||||
python3.11 python3.11-venv python3.11-dev \
|
||||
virtualenv net-tools libev4 libev-dev wget gcc libssl-dev libxml2 libxslt1-dev \
|
||||
virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
|
||||
vim lsof sudo libjemalloc2 dumb-init locales \
|
||||
openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
|
||||
|
||||
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 5
|
||||
RUN python3.6 -m pip install --upgrade pip
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 3
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 4
|
||||
RUN python3.7 -m pip install --upgrade pip
|
||||
RUN python3.8 -m pip install --upgrade pip
|
||||
|
||||
|
|
@ -109,15 +103,6 @@ ENV ANT_HOME=/usr/share/ant
|
|||
# included in the base image, the compiled objects are not updated by pip at run time, which can
|
||||
# cause errors if the tests rely on new driver functionality or bug fixes.
|
||||
|
||||
RUN virtualenv --python=python3.6 ${BUILD_HOME}/env3.6
|
||||
RUN chmod +x ${BUILD_HOME}/env3.6/bin/activate
|
||||
|
||||
RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
|
||||
&& source ${BUILD_HOME}/env3.6/bin/activate \
|
||||
&& pip3 install --upgrade pip \
|
||||
&& pip3 install -r /opt/requirements.txt \
|
||||
&& pip3 freeze --user"
|
||||
|
||||
RUN virtualenv --python=python3.7 ${BUILD_HOME}/env3.7
|
||||
RUN chmod +x ${BUILD_HOME}/env3.7/bin/activate
|
||||
|
||||
|
|
@ -146,11 +131,26 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
|
|||
&& pip3 freeze --user"
|
||||
|
||||
# Initialize the CCM git repo as well as this also can fail to clone
|
||||
RUN /bin/bash -c "source ${BUILD_HOME}/env3.6/bin/activate && \
|
||||
ccm create -n 1 -v git:trunk test && ccm remove test && \
|
||||
RUN /bin/bash -c "source ${BUILD_HOME}/env3.7/bin/activate && \
|
||||
ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
|
||||
ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test"
|
||||
|
||||
# Initialize ccm versions. right side of each sequence needs to be updated with new releases.
|
||||
# this can be checked with:
|
||||
# `curl -s https://downloads.apache.org/cassandra/ | grep -oP '(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -V | uniq -w 3`
|
||||
RUN bash -c 'source ~/env3.7/bin/activate && \
|
||||
for i in {1..11} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \
|
||||
for i in {1..3} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
|
||||
|
||||
# 5+ requires java11
|
||||
RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
|
||||
|
||||
# Initialize the CCM git repo as well as this also can fail to clone
|
||||
RUN /bin/bash -c 'source ~/env3.7/bin/activate && \
|
||||
ccm create -n 1 -v git:trunk test && ccm remove test && \
|
||||
ccm create -n 1 -v git:cassandra-5.0 test && ccm remove test && \
|
||||
ccm create --quiet -n 1 -v binary:5.0-alpha1 test && ccm remove test '
|
||||
|
||||
# the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
|
||||
RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git
|
||||
# fix permissions, runtime user has different uid/gid
|
||||
|
|
|
|||
|
|
@ -51,11 +51,10 @@ export CASS_DRIVER_NO_EXTENSIONS=true
|
|||
export CASS_DRIVER_NO_CYTHON=true
|
||||
export CCM_MAX_HEAP_SIZE="1024M"
|
||||
export CCM_HEAP_NEWSIZE="512M"
|
||||
export CCM_CONFIG_DIR=${DIST_DIR}/.ccm
|
||||
export NUM_TOKENS="16"
|
||||
#Have Cassandra skip all fsyncs to improve test performance and reliability
|
||||
export CASSANDRA_SKIP_SYNC=true
|
||||
export TMPDIR="$(mktemp -d /tmp/run-python-dtest.XXXXXX)"
|
||||
export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)"
|
||||
unset CASSANDRA_HOME
|
||||
|
||||
# pre-conditions
|
||||
|
|
@ -73,7 +72,9 @@ if [ "${java_version}" -eq 17 ] && [[ "${target}" == "dtest-upgrade" ]] ; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
python_version=$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')
|
||||
python_version=$(python -V 2>&1 | awk '{print $2}' | awk -F'.' '{print $1"."$2}')
|
||||
python_regx_supported_versions="^(3.7|3.8|3.11)$"
|
||||
[[ $python_version =~ $python_regx_supported_versions ]] || { echo "Python ${python_version} not supported."; exit 1; }
|
||||
|
||||
# check project is already built. no cleaning is done, so jenkins unstash works, beware.
|
||||
[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
|
||||
|
|
@ -163,8 +164,8 @@ fi
|
|||
# merge all unit xml files into one, and print summary test numbers
|
||||
pushd ${CASSANDRA_DIR}/ >/dev/null
|
||||
# remove <testsuites> wrapping elements. `ant generate-unified-test-report` doesn't like it`
|
||||
sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > /tmp/nosetests.xml
|
||||
cat /tmp/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml
|
||||
sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > ${TMPDIR}/nosetests.xml
|
||||
cat ${TMPDIR}/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml
|
||||
ant -quiet -silent generate-unified-test-report
|
||||
popd >/dev/null
|
||||
|
||||
|
|
@ -174,7 +175,7 @@ popd >/dev/null
|
|||
#
|
||||
################################
|
||||
|
||||
rm -rf "/tmp/run-python-dtest.${TMPDIR/\/tmp\/run-python-dtest./}"
|
||||
[[ "${TMPDIR}" == *"${DIST_DIR}/run-python-dtest."* ]] && rm -rf "${TMPDIR}"
|
||||
unset TMPDIR
|
||||
deactivate
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
#!/bin/bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
|
|
@ -49,6 +49,9 @@ git --version
|
|||
java -version 2>&1
|
||||
javac -version 2>&1
|
||||
|
||||
# set the OFFLINE env var (to anything) to allow running jvm-dtest-upgrade offline
|
||||
[ "x" != "x${OFFLINE}" ] && echo "WARNING: running in offline mode. jvm-dtest-upgrade results may be stale."
|
||||
|
||||
# lists all tests for the specific test type
|
||||
_list_tests() {
|
||||
local -r classlistprefix="$1"
|
||||
|
|
@ -85,7 +88,9 @@ _build_all_dtest_jars() {
|
|||
fi
|
||||
|
||||
if [ -d ${TMP_DIR}/cassandra-dtest-jars ] && [ "https://github.com/apache/cassandra.git" == "$(git -C ${TMP_DIR}/cassandra-dtest-jars remote get-url origin)" ] ; then
|
||||
until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git pull failed… trying again… " ; done
|
||||
if [ "x" == "x${OFFLINE}" ] ; then
|
||||
until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git fetch failed… trying again… " ; done
|
||||
fi
|
||||
else
|
||||
rm -fR ${TMP_DIR}/cassandra-dtest-jars
|
||||
pushd $TMP_DIR >/dev/null
|
||||
|
|
@ -172,7 +177,7 @@ _main() {
|
|||
|
||||
# ant test setup
|
||||
export TMP_DIR="${DIST_DIR}/tmp"
|
||||
mkdir -p "${TMP_DIR}" || true
|
||||
[ -d ${TMP_DIR} ] || mkdir -p "${TMP_DIR}"
|
||||
export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR}"
|
||||
|
||||
# fresh virtualenv and test logs results everytime
|
||||
|
|
|
|||
|
|
@ -624,7 +624,7 @@
|
|||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="fqltool-test" depends="fqltool-build-test, build-test" description="Runs fqltool tests">
|
||||
<target name="fqltool-test" depends="fqltool-build-test, maybe-build-test" description="Runs fqltool tests">
|
||||
<testmacro inputdir="${fqltool.test.src}"
|
||||
timeout="${test.timeout}">
|
||||
</testmacro>
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import org.apache.cassandra.dht.BootStrapper;
|
|||
import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.io.util.File;
|
||||
import org.apache.cassandra.io.util.PathUtils;
|
||||
import org.apache.cassandra.locator.InetAddressAndPort;
|
||||
import org.apache.cassandra.locator.TokenMetadata;
|
||||
import org.apache.cassandra.service.CacheService;
|
||||
|
|
@ -245,6 +246,7 @@ public class ImportTest extends CQLTester
|
|||
}
|
||||
}
|
||||
}
|
||||
PathUtils.deleteRecursiveOnExit(temp);
|
||||
return new File(backupdir);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ import org.apache.commons.lang3.RandomUtils;
|
|||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.config.CassandraRelevantProperties;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.psjava.util.Triple;
|
||||
|
|
@ -45,13 +44,14 @@ import org.psjava.util.Triple;
|
|||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
|
||||
import static org.apache.cassandra.config.CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE;
|
||||
|
||||
public class FileTest
|
||||
{
|
||||
private static final java.io.File dir;
|
||||
static
|
||||
{
|
||||
CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE.setBoolean(false);
|
||||
USE_NIX_RECURSIVE_DELETE.setBoolean(false);
|
||||
java.io.File parent = new java.io.File(JAVA_IO_TMPDIR.getString()); //checkstyle: permit this instantiation
|
||||
String dirName = Long.toHexString(ThreadLocalRandom.current().nextLong());
|
||||
while (new java.io.File(parent, dirName).exists()) //checkstyle: permit this instantiation
|
||||
|
|
@ -115,6 +115,9 @@ public class FileTest
|
|||
regularLink.delete();
|
||||
regular.delete();
|
||||
emptySubdir.delete();
|
||||
nonEmptySubdir.delete();
|
||||
nonEmptySubdirLink.delete();
|
||||
dir.setReadable(true);
|
||||
}
|
||||
|
||||
private static String nonAbsolute(java.io.File file)
|
||||
|
|
|
|||
Loading…
Reference in New Issue