diff --git a/.build/README.md b/.build/README.md new file mode 100644 index 0000000000..6bdb5a2c76 --- /dev/null +++ b/.build/README.md @@ -0,0 +1,130 @@ +Building and Testing with the helper sripts +------------------------------------------- + +Information on building and testing beyond the use of ant. +All scripts also print help if the first argument is `-h`. + +Code Checks and Lints +--------------------- + +Run in docker: + + .build/docker/check-code.sh + + +Run without docker: + + .build/check-code.sh + + +Run in docker with a specific jdk. +The following applies to all build scripts. + + .build/docker/check-code.sh 11 + + +Run in docker with a specific build path. +This permits parallel builds off the same source path. +The following applies to all build scripts. + + build_dir=/tmp/cass_Mtu462n .build/docker/check-code.sh + + +Building Artifacts (tarball and maven) +------------------------------------- + +Build with docker: + + .build/docker/build-artifacts.sh + + +Build without docker: + + .build/build-artifacts.sh + + +Build in docker with a specific jdk: + + .build/docker/build-artifacts.sh 11 + + +Building Debian and RedHat packages +----------------------------------- + +The packaging scripts are only intended to be used with docker. + +Build: + + .build/docker/build-debian.sh + .build/docker/build-redhat.sh + + +Build with a specific jdk: + + .build/docker/build-debian.sh 11 + .build/docker/build-redhat.sh rpm 11 + + +Build with centos7 and a specific jdk: + + .build/docker/build-redhat.sh noboolean 11 + + +Running Tests +------------- + +Running unit tests with docker: + + .build/docker/run-tests.sh test + + +Running unittests without docker: + + .build/run-tests.sh test + + +Running only a split of unittests, with docker: + + .build/docker/run-tests.sh test 1/64 + + +Running unittests with a specific jdk with docker: + + .build/docker/run-tests.sh test 1/64 11 + + +Running only unit tests matching a regexp, with docker: + + .build/docker/run-tests.sh test VerifyTest 11 + + +Running other types of tests with docker: + + .build/docker/run-tests.sh test + .build/docker/run-tests.sh stress-test + .build/docker/run-tests.sh fqltool-test + .build/docker/run-tests.sh microbench + .build/docker/run-tests.sh test-cdc + .build/docker/run-tests.sh test-compression + .build/docker/run-tests.sh test-burn + .build/docker/run-tests.sh long-test + .build/docker/run-tests.sh cqlsh-test + .build/docker/run-tests.sh jvm-dtest + .build/docker/run-tests.sh jvm-dtest-upgrade + .build/docker/run-tests.sh dtest + .build/docker/run-tests.sh dtest-novnode + .build/docker/run-tests.sh dtest-offheap + .build/docker/run-tests.sh dtest-large + .build/docker/run-tests.sh dtest-large-novnode + .build/docker/run-tests.sh dtest-upgrade + + +Running python dtests without docker: + + .build/run-python-dtests.sh dtest + + +Other test types without docker: + + .build/run-tests.sh test + diff --git a/.build/build-artifacts.sh b/.build/build-artifacts.sh new file mode 100755 index 0000000000..9ae577606f --- /dev/null +++ b/.build/build-artifacts.sh @@ -0,0 +1,28 @@ +#!/bin/sh -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } + +# execute +ant -f "${CASSANDRA_DIR}/build.xml" artifacts -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true +exit $? diff --git a/.build/build-git.xml b/.build/build-git.xml index 03b5ca7f5b..0ba6502a94 100644 --- a/.build/build-git.xml +++ b/.build/build-git.xml @@ -18,7 +18,8 @@ --> - + + diff --git a/.build/build-jars.sh b/.build/build-jars.sh new file mode 100755 index 0000000000..bce912a5ef --- /dev/null +++ b/.build/build-jars.sh @@ -0,0 +1,30 @@ +#!/bin/sh -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# temporary between CASSANDRA-18133 and CASSANDRA-18594 + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } + +# execute +ant -f "${CASSANDRA_DIR}/build.xml" jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Dno-javadoc=true +exit $? diff --git a/.build/build-owasp.xml b/.build/build-owasp.xml index a792730fb9..3aa0723dc7 100644 --- a/.build/build-owasp.xml +++ b/.build/build-owasp.xml @@ -18,14 +18,14 @@ --> - + + @@ -33,8 +33,10 @@ - + + + diff --git a/.build/build-rat.xml b/.build/build-rat.xml index f261a23cd1..dcb3b0681d 100644 --- a/.build/build-rat.xml +++ b/.build/build-rat.xml @@ -86,10 +86,10 @@ - + - + @@ -105,15 +105,4 @@ - - - - - - - - - - - diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml index e78bbf05e6..bd1f2f1ae2 100644 --- a/.build/build-resolver.xml +++ b/.build/build-resolver.xml @@ -46,8 +46,10 @@ Downloading Resolver ANT Tasks... - + + + @@ -175,15 +177,19 @@ - - - - - - - - - + + + + + + + + + + + + + @@ -198,44 +204,19 @@ - - - - + + + + + + - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - @@ -266,4 +247,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.build/check-code.sh b/.build/check-code.sh new file mode 100755 index 0000000000..662f5e337f --- /dev/null +++ b/.build/check-code.sh @@ -0,0 +1,28 @@ +#!/bin/sh -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(dirname "$0")/.."; } + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } + +# execute +ant -f "${CASSANDRA_DIR}/build.xml" rat-check eclipse-warnings checkstyle checkstyle-test dependency-check +exit $? diff --git a/.build/docker/_build-debian.sh b/.build/docker/_build-debian.sh new file mode 100755 index 0000000000..e8cfe373eb --- /dev/null +++ b/.build/docker/_build-debian.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +command -v git >/dev/null 2>&1 || { echo >&2 "git needs to be installed"; exit 1; } +command -v dch >/dev/null 2>&1 || { echo >&2 "dch needs to be installed"; exit 1; } +command -v dpkg-parsechangelog >/dev/null 2>&1 || { echo >&2 "dpkg-parsechangelog needs to be installed"; exit 1; } +command -v dpkg-buildpackage >/dev/null 2>&1 || { echo >&2 "dpkg-buildpackage needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } +[ -d "${DIST_DIR}" ] || mkdir -p "${DIST_DIR}" + +################################ +# +# Main +# +################################ + +set -e + +# note, this edits files in your working cassandra directory +pushd $CASSANDRA_DIR >/dev/null +export BUILD_DIR="$(realpath --relative-to=$CASSANDRA_DIR ${DIST_DIR})" + +# Used version for build will always depend on the git referenced used for checkout above +# Branches will always be created as snapshots, while tags are releases +tag=`git describe --tags --exact-match 2>/dev/null || true` +branch=`git symbolic-ref -q --short HEAD 2>/dev/null || true` + +is_tag=false +git_version='' + +# Parse version from build.xml so we can verify version against release tags and use the build.xml version +# for any branches. Truncate from snapshot suffix if needed. +buildxml_version=`grep 'property\s*name="base.version"' build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` +regx_snapshot="([0-9.]+)-SNAPSHOT$" +if [[ $buildxml_version =~ $regx_snapshot ]]; then + buildxml_version=${BASH_REMATCH[1]} +fi + +if [ "$tag" ]; then + is_tag=true + # Official release + regx_tag="cassandra-(([0-9.]+)(-(alpha|beta|rc)[0-9]+)?)$" + # Tentative release + regx_tag_tentative="(([0-9.]+)(-(alpha|beta|rc)[0-9]+)?)-tentative$" + if [[ $tag =~ $regx_tag ]] || [[ $tag =~ $regx_tag_tentative ]]; then + git_version=${BASH_REMATCH[1]} + else + echo "Error: could not recognize version from tag $tag">&2 + exit 2 + fi + CASSANDRA_VERSION=$git_version + CASSANDRA_REVISION='1' +else + regx_branch="cassandra-([0-9.]+)$" + if [[ $branch =~ $regx_branch ]]; then + git_version=${BASH_REMATCH[1]} + else + # This could be either trunk or any dev branch or SHA, so we won't be able to get the version + # from the branch name. In this case, fall back to debian change log version. + git_version=$(dpkg-parsechangelog | sed -ne 's/^Version: \(.*\).*/\1/p' | sed 's/~/-/') + if [ -z $git_version ]; then + echo "Error: could not recognize version from branch $branch">&2 + exit 2 + else + echo "Warning: could not recognize version from branch. dpkg version is $git_version" + fi + fi + # if CASSANDRA_VERSION is -alphaN, -betaN, -rcN, then rpmbuild fails on the '-' char; replace with '~' + CASSANDRA_VERSION=${buildxml_version/-/\~} + dt=`date +"%Y%m%d"` + ref=`git rev-parse --short HEAD` + CASSANDRA_REVISION="${dt}git${ref}" + dch -D unstable -v "${CASSANDRA_VERSION}-${CASSANDRA_REVISION}" --package "cassandra" "building ${CASSANDRA_VERSION}-${CASSANDRA_REVISION}" +fi + +# The version used for the deb build process will the current version in the debian/changelog file. +# See debian/rules for how the value is read. The only thing left for us to do here is to check if +# the changes file contains the correct version for the checked out git revision. The version value +# has to be updated manually by a committer and we only warn and abort on mismatches here. +changelog_version=$(dpkg-parsechangelog | sed -ne 's/^Version: \(.*\).*/\1/p' | sed 's/~/-/') +chl_expected="${buildxml_version}" +if [[ ! $changelog_version =~ $chl_expected ]]; then + echo "Error: changelog version (${changelog_version}) doesn't match expected (${chl_expected})">&2 + exit 3 +fi + +# Version (base.version) in build.xml must be set manually as well. Let's validate the set value. +if [ $buildxml_version != $git_version ]; then + echo "Warning: build.xml version ($buildxml_version) not matching git/dpkg derived version ($git_version)">&2 +fi + +# build package +dpkg-buildpackage -rfakeroot -uc -us -tc --source-option=--tar-ignore=.git + +# Move created artifacts to dist dir mapped to docker host directory (must have proper permissions) +mv ../cassandra[-_]*${CASSANDRA_VERSION}-* "${DIST_DIR}" +# clean build deps +rm -f cassandra-build-deps_* +# restore debian/changelog +git restore debian/changelog + +popd >/dev/null \ No newline at end of file diff --git a/.build/docker/_build-redhat.sh b/.build/docker/_build-redhat.sh new file mode 100755 index 0000000000..ff02931581 --- /dev/null +++ b/.build/docker/_build-redhat.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +# variables, w/ defaults, w/ checks +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" +[ "x${RPM_BUILD_DIR}" != "x" ] || RPM_BUILD_DIR="$(mktemp -d /tmp/rpmbuild.XXXXXX)" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +command -v git >/dev/null 2>&1 || { echo >&2 "git needs to be installed"; exit 1; } +command -v rpmbuild >/dev/null 2>&1 || { echo >&2 "rpm-build needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } +[ -d "${DIST_DIR}" ] || mkdir -p "${DIST_DIR}" +[ -d "${RPM_BUILD_DIR}/SOURCES" ] || mkdir -p ${RPM_BUILD_DIR}/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + + +if [ "$1" == "-h" ]; then + echo "$0 [-h] [dist_type]" + echo "dist types are [rpm, noboolean] and rpm is default" + exit 1 +fi + +RPM_DIST=$1 +[ "x${RPM_DIST}" != "x" ] || RPM_DIST="rpm" + +if [ "${RPM_DIST}" == "rpm" ]; then + RPM_SPEC="redhat/cassandra.spec" +elif [ "${RPM_DIST}" == "noboolean" ]; then # noboolean + RPM_SPEC="redhat/noboolean/cassandra.spec" +else + echo >&2 "Only rpm and noboolean are valid dist_type arguments. Got ${RPM_DIST}" + exit 1 +fi + +################################ +# +# Main +# +################################ + +set -e + +# note, this edits files in your working cassandra directory +pushd $CASSANDRA_DIR >/dev/null + +# Used version for build will always depend on the git referenced used for checkout above +# Branches will always be created as snapshots, while tags are releases +tag=`git describe --tags --exact-match 2> /dev/null || true` +branch=`git symbolic-ref -q --short HEAD 2> /dev/null || true` + +is_tag=false +git_version='' + +# Parse version from build.xml so we can verify version against release tags and use the build.xml version +# for any branches. Truncate from snapshot suffix if needed. +buildxml_version=`grep 'property\s*name="base.version"' build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` +regx_snapshot="([0-9.]+)-SNAPSHOT$" +if [[ $buildxml_version =~ $regx_snapshot ]]; then + buildxml_version=${BASH_REMATCH[1]} +fi + +if [ "$tag" ]; then + is_tag=true + # Official release + regx_tag="cassandra-(([0-9.]+)(-(alpha|beta|rc)[0-9]+)?)$" + # Tentative release + regx_tag_tentative="(([0-9.]+)(-(alpha|beta|rc)[0-9]+)?)-tentative$" + if [[ $tag =~ $regx_tag ]] || [[ $tag =~ $regx_tag_tentative ]]; then + git_version=${BASH_REMATCH[1]} + else + echo "Error: could not recognize version from tag $tag">&2 + exit 2 + fi + if [ $buildxml_version != $git_version ]; then + echo "Error: build.xml version ($buildxml_version) not matching git tag derived version ($git_version)">&2 + exit 4 + fi + CASSANDRA_VERSION=$git_version + CASSANDRA_REVISION='1' +else + # This could be either trunk or any dev branch or SHA, so we won't be able to get the version + # from the branch name. In this case, fall back to version specified in build.xml. + CASSANDRA_VERSION="${buildxml_version}" + dt=`date +"%Y%m%d"` + ref=`git rev-parse --short HEAD` + CASSANDRA_REVISION="${dt}git${ref}" +fi + +# Artifact will only be used internally for build process and won't be found with snapshot suffix +ant artifacts -Drelease=true -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true +cp ${DIST_DIR}/apache-cassandra-*-src.tar.gz ${RPM_BUILD_DIR}/SOURCES/ + +# if CASSANDRA_VERSION is -alphaN, -betaN, -rcN, then rpmbuild fails on the '-' char; replace with '~' +CASSANDRA_VERSION=${CASSANDRA_VERSION/-/\~} + +command -v python >/dev/null 2>&1 || alias python=/usr/bin/python3 +rpmbuild --define="version ${CASSANDRA_VERSION}" --define="revision ${CASSANDRA_REVISION}" --define="_topdir ${RPM_BUILD_DIR}" -ba ${RPM_SPEC} +cp ${RPM_BUILD_DIR}/SRPMS/*.rpm ${RPM_BUILD_DIR}/RPMS/noarch/*.rpm ${DIST_DIR} + +popd >/dev/null diff --git a/.build/docker/_create_user.sh b/.build/docker/_create_user.sh new file mode 100755 index 0000000000..2da9f4913e --- /dev/null +++ b/.build/docker/_create_user.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +if [ "$1" == "-h" ]; then + echo "$0 [-h] " + echo " this script is used internally by other scripts in the same directory to create a user with the running host user's same uid and gid" + exit 1 +fi + +# arguments +username=$1 +uid=$2 +gid=$3 + +################################ +# +# Main +# +################################ + +if grep "^ID=" /etc/os-release | grep -q 'debian\|ubuntu' ; then + adduser --quiet --disabled-login --no-create-home --uid $uid --gecos ${username} ${username} + groupmod --non-unique -g $gid $username + gpasswd -a ${username} sudo >/dev/null +else + adduser --no-create-home --uid $uid ${username} +fi + +# sudo priviledges +echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} +chmod 0440 /etc/sudoers.d/${username} +mkdir -p ${BUILD_HOME}/docker ${DIST_DIR} ${BUILD_HOME}/.ssh + +# we need to make SSH less strict to prevent various dtests from failing when they attempt to +# git clone a given commit/tag/etc +echo 'Host *\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no' > ${BUILD_HOME}/.ssh/config + +# proper permissions +chown ${username}:${username} ${BUILD_HOME} ${BUILD_HOME}/docker ${DIST_DIR} ${BUILD_HOME}/.ssh ${BUILD_HOME}/.ssh/config +chmod og+wx ${BUILD_HOME} ${DIST_DIR} +chmod 600 ${BUILD_HOME}/.ssh/config + +# disable git directory ownership checks +su ${username} -c "git config --global safe.directory '*'" diff --git a/.build/docker/_docker_init_tests.sh b/.build/docker/_docker_init_tests.sh new file mode 100755 index 0000000000..7c90833582 --- /dev/null +++ b/.build/docker/_docker_init_tests.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pre-conditions +[ "x" != "x${DIST_DIR}" ] || { echo "DIST_DIR must be defined"; exit 1 ; } +[ "x" != "x${TEST_SCRIPT}" ] || { echo "TEST_SCRIPT must be defined"; exit 1 ; } +[ "x" != "x${CASSANDRA_DIR}" ] || { echo "CASSANDRA_DIR must be defined"; exit 1 ; } + +# usage +if [ "$1" == "-h" ]; then + echo "$0 [-h] ..." + echo " this script is used by run-tests.sh (in the same directory) as a wrapper delegating the execution of the ${TEST_SCRIPT}. all arguments are passed through as-is to ${TEST_SCRIPT}" + exit 1 +fi + +pushd "${CASSANDRA_DIR}" >/dev/null + +echo "Running ${TEST_SCRIPT} $@" +.build/${TEST_SCRIPT} "$@" +status=$? +if [ -d "${DIST_DIR}/test/logs" ]; then + find "${DIST_DIR}/test/logs" -type f -name "*.log" | xargs xz -qq +fi +popd >/dev/null +exit ${status} \ No newline at end of file diff --git a/.build/docker/_docker_run.sh b/.build/docker/_docker_run.sh new file mode 100755 index 0000000000..13ecdaa8cf --- /dev/null +++ b/.build/docker/_docker_run.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Creates the artifacts, performing additional QA checks +# +# Usage: _docker_run.sh + +################################ +# +# Prep +# +################################ + +# variables, with defaults +[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)" +[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build" +[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; } + +java_version_default=`grep 'property\s*name="java.default"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` +java_version_supported=`grep 'property\s*name="java.supported"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` + +if [ "$1" == "-h" ]; then + echo "$0 [-h] []" + echo " this script is used by check|build*.sh scripts (in the same directory) as a wrapper delegating the container run of the and execution of the , and using [] is specified" + exit 1 +fi + +# arguments +dockerfile=$1 +run_script=$2 +java_version=$3 + +# pre-conditions +command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; } +command -v timeout >/dev/null 2>&1 || { echo >&2 "timeout needs to be installed"; exit 1; } +(docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; } +[ -f "${cassandra_dir}/build.xml" ] || { echo >&2 "${cassandra_dir}/build.xml must exist"; exit 1; } +[ -f "${cassandra_dir}/.build/docker/${dockerfile}" ] || { echo >&2 "${cassandra_dir}/.build/docker/${dockerfile} must exist"; exit 1; } +[ -f "${cassandra_dir}/.build/${run_script}" ] || { echo >&2 "${cassandra_dir}/.build/${run_script} must exist"; exit 1; } +[ "${build_dir:0:1}" == "/" ] || { echo >&2 "\$build_dir must be provided as an absolute path, was ${build_dir}"; exit 1; } + +if [ "x${java_version}" == "x" ] ; then + echo "Defaulting to java ${java_version_default}" + java_version="${java_version_default}" +fi + +regx_java_version="(${java_version_supported//,/|})" +if [[ ! "${java_version}" =~ $regx_java_version ]]; then + echo "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}" + exit 1 +fi + +# print debug information on versions +docker --version + +# make sure build_dir is good +chmod -R ag+rwx ${build_dir} + + +################################ +# +# Main +# +################################ + +pushd ${cassandra_dir}/.build >/dev/null + +image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)" +image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}" + +# Look for existing docker image, otherwise build +timeout -k 5 5 docker login >/dev/null +if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] || docker pull -q ${image_name} ) ; then + # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry + until docker build -t ${image_name} -f docker/${dockerfile} . ; do + echo "docker build failed… trying again in 10s… " + sleep 10 + done +fi + +# Run build script through docker +random_string="$(LC_ALL=C tr -dc A-Za-z0-9 /dev/null +popd >/dev/null +[ $RETURN -eq 0 ] && echo "Build directory found at ${build_dir}" +exit $RETURN diff --git a/.build/docker/_set_java.sh b/.build/docker/_set_java.sh new file mode 100755 index 0000000000..9d3d135546 --- /dev/null +++ b/.build/docker/_set_java.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(pwd)"; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } + +# pre-conditions +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } + +java_version_default=`grep 'property\s*name="java.default"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` +java_version_supported=`grep 'property\s*name="java.supported"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` + +if [ "$1" == "-h" ]; then + echo "$0 [-h] []" + echo " if Java version is not set, it is set to ${java_version_default} by default, valid ${java_version_supported}" + echo + echo " this script is used internally by other scripts in the same directory to ensure the correct java version is used inside the docker container" + exit 1 +fi + +# arguments +java_version=$1 + +[ "x${java_version}" != "x" ] || java_version="${java_version_default}" +regx_java_version="(${java_version_supported//,/|})" +if [[ ! "$java_version" =~ $regx_java_version ]]; then + echo "Error: Java version is not in ${java_version_supported}, it is set to $java_version" + exit 1 +fi + +# TODO – remove after CASSANDRA-18255 +if [ "$java_version" == "1.8" ]; then + echo "Error: Java 8 is not supported, ref CASSANDRA-18255" + exit 1 +fi + +################################ +# +# Main +# +################################ + +if grep "^ID=" /etc/os-release | grep -q 'debian\|ubuntu' ; then + sudo update-java-alternatives --set java-1.${java_version}.0-openjdk-$(dpkg --print-architecture) +else + sudo alternatives --set java $(alternatives --display java | grep "family java-${java_version}-openjdk" | cut -d' ' -f1) + sudo alternatives --set javac $(alternatives --display javac | grep "family java-${java_version}-openjdk" | cut -d' ' -f1) +fi +export JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::") +echo "Cassandra will be built with Java ${java_version}" + +# print debug information on versions +java -version 2>&1 +javac -version 2>&1 diff --git a/.build/docker/almalinux-build.docker b/.build/docker/almalinux-build.docker new file mode 100644 index 0000000000..c648eb581c --- /dev/null +++ b/.build/docker/almalinux-build.docker @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM almalinux:8 +MAINTAINER Apache Cassandra + +# CONTEXT is expected to be cassandra/.build + +ENV BUILD_HOME=/home/build +ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild +ENV DIST_DIR=/dist +ENV CASSANDRA_DIR=$BUILD_HOME/cassandra +ARG UID_ARG=1000 +ARG GID_ARG=1000 + +LABEL org.cassandra.buildenv=almalinux + +VOLUME ${DIST_DIR} +VOLUME ${CASSANDRA_DIR} + +RUN echo "Building with arguments:" \ + && echo " - DIST_DIR=${DIST_DIR}" \ + && echo " - BUILD_HOME=${BUILD_HOME}" \ + && echo " - RPM_BUILD_DIR=${RPM_BUILD_DIR}" \ + && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" \ + && echo " - UID_ARG=${UID_ARG}" \ + && echo " - GID_ARG=${GID_ARG}" + +# install deps +RUN yum -y install \ + ant \ + git \ + java-11-openjdk-devel \ + java-17-openjdk-devel \ + make \ + rpm-build \ + sudo \ + python3-pip \ + procps \ + rsync + +RUN until curl -f -S -s --retry 9 --retry-connrefused --retry-delay 1 http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm ; do echo "curl failed… trying again in 10s… " ; sleep 10 ; done + +RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm + +# python3 is needed for the gen-doc target +RUN pip3 install --upgrade pip diff --git a/.build/docker/build-artifacts.sh b/.build/docker/build-artifacts.sh new file mode 100755 index 0000000000..85eec83005 --- /dev/null +++ b/.build/docker/build-artifacts.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Creates the tarball artifact + +$(dirname "$0")/_docker_run.sh bullseye-build.docker build-artifacts.sh $1 +exit $? diff --git a/.build/docker/build-debian.sh b/.build/docker/build-debian.sh new file mode 100755 index 0000000000..ae2058013c --- /dev/null +++ b/.build/docker/build-debian.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ "$1" == "-h" ]; then + echo "$0 [-h] []" + echo " build debian packages" + exit 1 +fi + +echo +echo "===" +echo "WARNING: this script modifies local versioned files" +echo "===" +echo + +# +# Creates the debian package + +$(dirname "$0")/_docker_run.sh bullseye-build.docker docker/_build-debian.sh $1 +exit $? diff --git a/.build/docker/build-redhat.sh b/.build/docker/build-redhat.sh new file mode 100755 index 0000000000..35149855c7 --- /dev/null +++ b/.build/docker/build-redhat.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ "$1" == "-h" ]; then + echo "$0 [-h] [rpm|noboolean] []" + echo " build redhat packages, specify noboolean for centos7 compatibility" + exit 1 +fi + +# arguments +rpm_dist=$1 +java_version=$2 + +if [ "${rpm_dist}" == "noboolean" ]; then + dist_name="centos7" +else # noboolean + dist_name="almalinux" +fi + + +echo +echo "===" +echo "WARNING: this script modifies local versioned files" +echo "===" +echo + +# +# Creates the redhat package + +$(dirname "$0")/_docker_run.sh ${dist_name}-build.docker docker/_build-redhat.sh "${java_version}" ${rpm_dist} +exit $? diff --git a/.build/docker/bullseye-build.docker b/.build/docker/bullseye-build.docker new file mode 100644 index 0000000000..7612a00290 --- /dev/null +++ b/.build/docker/bullseye-build.docker @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:bullseye +MAINTAINER Apache Cassandra + +# CONTEXT is expected to be cassandra/.build + +ENV DIST_DIR=/dist +ENV BUILD_HOME=/home/build +ENV CASSANDRA_DIR=$BUILD_HOME/cassandra + +LABEL org.cassandra.buildenv=bullseye + +VOLUME ${DIST_DIR} +VOLUME ${CASSANDRA_DIR} + +RUN echo "Building with arguments:" \ + && echo " - DIST_DIR=${DIST_DIR}" \ + && echo " - BUILD_HOME=${BUILD_HOME}" \ + && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" + +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "99";' > /etc/apt/apt.conf.d/80-retries +RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf +RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf + +# install deps +RUN until apt-get update \ + && apt-get -y install ant build-essential curl devscripts git sudo \ + python3-pip rsync procps dh-python quilt bash-completion ; \ + do echo "apt failed… trying again in 10s… " ; sleep 10 ; done + +RUN echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list + +RUN until apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jdk openjdk-17-jdk ; \ + do echo "apt failed… trying again in 10s… " ; sleep 10 ; done + +RUN update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture) + +# python3 is needed for the gen-doc target +RUN pip install --upgrade pip diff --git a/.build/docker/centos7-build.docker b/.build/docker/centos7-build.docker new file mode 100644 index 0000000000..2f8aef98f2 --- /dev/null +++ b/.build/docker/centos7-build.docker @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:7 +MAINTAINER Apache Cassandra + +# CONTEXT is expected to be cassandra/.build + +ENV BUILD_HOME=/home/build +ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild +ENV DIST_DIR=/dist +ENV CASSANDRA_DIR=$BUILD_HOME/cassandra +ENV ANT_VERSION=1.10.12 +ARG UID_ARG=1000 +ARG GID_ARG=1000 + +LABEL org.cassandra.buildenv=centos + +VOLUME ${DIST_DIR} +VOLUME ${CASSANDRA_DIR} + +RUN echo "Building with arguments:" \ + && echo " - DIST_DIR=${DIST_DIR}" \ + && echo " - BUILD_HOME=${BUILD_HOME}" \ + && echo " - RPM_BUILD_DIR=${RPM_BUILD_DIR}" \ + && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" \ + && echo " - UID_ARG=${UID_ARG}" \ + && echo " - GID_ARG=${GID_ARG}" + +# install deps +RUN yum -y install \ + ant \ + git \ + java-11-openjdk-devel \ + make \ + rpm-build \ + sudo \ + python3-pip \ + rsync + +RUN until curl -f -S -s --retry 9 --retry-delay 1 http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm ; do echo "curl failed… trying again in 10s… " ; sleep 10 ; done + +RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm + +# try first downloads.a.o and then archive.a.o (as the latter has a 5GB per IP daily limit) +RUN until curl -f -S -s --retry 9 --retry-delay 1 https://downloads.apache.org/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz -o apache-ant-${ANT_VERSION}-bin.tar.gz || curl -f -S -s --retry 9 --retry-delay 1 http://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz -o apache-ant-${ANT_VERSION}-bin.tar.gz ; do echo "curl failed… trying again in 10s… " ; sleep 10 ; done + +RUN rm -rf /etc/ant.conf +RUN tar xvf apache-ant-${ANT_VERSION}-bin.tar.gz -C /opt +RUN ln -sfn /opt/apache-ant-${ANT_VERSION} /opt/ant +RUN sh -c 'echo ANT_HOME=/opt/ant >> /etc/environment' +RUN ln -sfn /opt/ant/bin/ant /usr/bin/ant + +# jdk17 latest on x64 +RUN if [ "$(arch)" == "x86_64" ] ; then \ + until curl -f -S -s --retry 9 --retry-delay 1 https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.tar.gz -o openjdk-17_bin.tar.gz ; do \ + echo "curl failed… trying again in 10s… " ; sleep 10 ; \ + done \ + fi + +# jdk17 latest on arm +RUN if [ "$(arch)" == "aarch64" ] ; then \ + until curl -f -S -s --retry 9 --retry-delay 1 https://download.oracle.com/java/17/latest/jdk-17_linux-aarch64_bin.tar.gz -o openjdk-17_bin.tar.gz ; do \ + echo "curl failed… trying again in 10s… " ; sleep 10 ; \ + done \ + fi + +RUN tar xvf openjdk-17_bin.tar.gz +RUN mv jdk-17* /opt/jdk-17 + +RUN alternatives --install /usr/bin/java java /opt/jdk-17/bin/java 3 --family java-17-openjdk.$(arch) +RUN alternatives --install /usr/bin/javac javac /opt/jdk-17/bin/javac 3 --family java-17-openjdk.$(arch) + +# python3 is needed for the gen-doc target +RUN pip3 install --upgrade pip + diff --git a/.build/docker/check-code.sh b/.build/docker/check-code.sh new file mode 100755 index 0000000000..4afccaea98 --- /dev/null +++ b/.build/docker/check-code.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Perform code checks + +export CASSANDRA_DOCKER_ANT_OPTS="-Ddependency-check.home.base=/tmp" + +$(dirname "$0")/_docker_run.sh bullseye-build.docker check-code.sh $1 diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh new file mode 100755 index 0000000000..d6ef16f7fb --- /dev/null +++ b/.build/docker/run-tests.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# +# A wrapper script to run-tests.sh (or dtest-python.sh) in docker. +# Can split (or grep) the test list into multiple docker runs, collecting results. +# +# Each split chunk may be further parallelised over docker containers based on the host's available cpu and memory (and the test type). +# Define env variable DISABLE_INNER_SPLITS to disable inner splitting. +# + +# help +if [ "$#" -lt 1 ] || [ "$#" -gt 3 ] || [ "$1" == "-h" ]; then + echo "" + echo "Usage: run-tests.sh target [split_chunk|test_regexp] [java_version]" + echo "" + echo " default split_chunk is 1/1" + echo " default java_version is what 'java.default' specifies in build.xml" + exit 1 +fi + +# variables, with defaults +[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)" +[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest" +[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build" +[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; } + +# pre-conditions +command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; } +command -v bc >/dev/null 2>&1 || { echo >&2 "bc needs to be installed"; exit 1; } +command -v timeout >/dev/null 2>&1 || { echo >&2 "timeout needs to be installed"; exit 1; } +(docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; } +[ -f "${cassandra_dir}/build.xml" ] || { echo >&2 "${cassandra_dir}/build.xml must exist"; exit 1; } +[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { echo >&2 "${cassandra_dir}/.build/run-tests.sh must exist"; exit 1; } + +# arguments +target=$1 +split_chunk="1/1" +[ "$#" -gt 1 ] && split_chunk=$2 +java_version=$3 + +test_script="run-tests.sh" +java_version_default=`grep 'property\s*name="java.default"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` +java_version_supported=`grep 'property\s*name="java.supported"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` + +if [ "x${java_version}" == "x" ] ; then + echo "Defaulting to java ${java_version_default}" + java_version="${java_version_default}" +fi + +regx_java_version="(${java_version_supported//,/|})" +if [[ ! "${java_version}" =~ $regx_java_version ]]; then + echo "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}" + exit 1 +fi + +python_version="3.6" +command -v python >/dev/null 2>&1 && python_version="$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')" + +# print debug information on versions +docker --version + +pushd ${cassandra_dir}/.build >/dev/null + +# build test image +dockerfile="ubuntu2004_test.docker" +image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)" +image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}" +docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${HOME}/.m2/repository:/home/cassandra/.m2/repository" + +# Look for existing docker image, otherwise build +timeout -k 5 5 docker login >/dev/null 2>/dev/null +if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] || docker pull -q ${image_name} ) >/dev/null 2>/dev/null ; then + # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry + until docker build -t ${image_name} -f docker/${dockerfile} . ; do + echo "docker build failed… trying again in 10s… " + sleep 10 + done +fi + +pushd ${cassandra_dir} >/dev/null + +# Optional lookup of Jenkins environment to see how many executors on this machine. `jenkins_executors=1` is used for anything non-jenkins. +jenkins_executors=1 +if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then + fetched_jenkins_executors=$(curl -s --retry 9 --retry-connrefused --retry-delay 1 "${JENKINS_URL}/computer/${NODE_NAME}/api/json?pretty=true" | grep 'numExecutors' | awk -F' : ' '{print $2}' | cut -d',' -f1) + # use it if we got a valid number (despite retry settings the curl above can still fail + [[ ${fetched_jenkins_executors} =~ '^[0-9]+$' ]] && jenkins_executors=${fetched_jenkins_executors} +fi + +# find host's available cores and mem +cores=1 +command -v nproc >/dev/null 2>&1 && cores=$(nproc --all) +mem=1 +# linux +command -v free >/dev/null 2>&1 && mem=$(free -b | grep Mem: | awk '{print $2}') +# macos +sysctl -n hw.memsize >/dev/null 2>&1 && mem=$(sysctl -n hw.memsize) + +# figure out resource limits, scripts, and mounts for the test type +case ${target} in + # test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway + "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" ) + [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "tests require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; } + ;; + "dtest" | "dtest-novnode" | "dtest-offheap" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" ) + [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py must exist"; exit 1; } + [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "dtests require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; } + test_script="run-python-dtests.sh" + docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest" + # check that ${cassandra_dtest_dir} is valid + [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; exit 1; } + ;; + "test"| "test-cdc" | "test-compression" | "jvm-dtest" | "jvm-dtest-upgrade") + [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "tests require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; } + max_docker_runs_by_cores=$( echo "sqrt( ${cores} / ${jenkins_executors} )" | bc ) + max_docker_runs_by_mem=$(( ${mem} / ( 5 * 1024 * 1024 * 1024 * ${jenkins_executors} ) )) + ;; + *) + echo "unrecognized \"${target}\"" + exit 1 + ;; +esac + +docker_cpus=$(echo "scale=2; ${cores} / ( ${jenkins_executors} )" | bc) + +# hack: long-test does not handle limited CPUs +if [ "${target}" == "long-test" ] ; then + docker_flags="-m 5g --memory-swap 5g" +elif [[ "${target}" =~ dtest* ]] ; then + docker_flags="--cpus=${docker_cpus} -m 15g --memory-swap 15g" +else + docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g" +fi +docker_flags="${docker_flags} --env-file build/env.list -d --rm" + +# make sure build_dir is good +mkdir -p ${build_dir}/tmp || true +mkdir -p ${build_dir}/test/logs || true +mkdir -p ${build_dir}/test/output || true +chmod -R ag+rwx ${build_dir} + +# cython can be used for cqlsh-test +if [ "$cython" == "yes" ]; then + [ "${target}" == "cqlsh-test" ] || { echo "cython is only supported for cqlsh-test"; exit 1; } +else + cython="no" +fi + +# the docker container's env +touch build/env.list +cat > build/env.list </dev/null 2>/dev/null & ) + +xz -f ${logfile} 2>/dev/null + +popd >/dev/null +popd >/dev/null +exit ${status} diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker new file mode 100644 index 0000000000..7035c5228f --- /dev/null +++ b/.build/docker/ubuntu2004_test.docker @@ -0,0 +1,163 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 +MAINTAINER Apache Cassandra + +# CONTEXT is expected to be cassandra/.build + +ENV BUILD_HOME=/home/cassandra +ENV CASSANDRA_DIR=$BUILD_HOME/cassandra +ENV DIST_DIR=$CASSANDRA_DIR/build +ENV LANG=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 +ENV PYTHONIOENCODING=utf-8 +ENV PYTHONUNBUFFERED=true + +LABEL org.cassandra.buildenv=ubuntu_2004 + +VOLUME ${CASSANDRA_DIR} +VOLUME ${DIST_DIR} + +RUN echo "Building with arguments:" \ + && echo " - DIST_DIR=${DIST_DIR}" \ + && echo " - BUILD_HOME=${BUILD_HOME}" \ + && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" \ + && echo " - UID_ARG=${UID_ARG}" \ + && echo " - GID_ARG=${GID_ARG}" + +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "99";' > /etc/apt/apt.conf.d/80-retries +RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf +RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf + +# install our python dependencies and some other stuff we need +# libev4 libev-dev are for the python driver / libssl-dev is for python3.6 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y --no-install-recommends software-properties-common apt-utils + +RUN export DEBIAN_FRONTEND=noninteractive && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y curl git-core python3-pip \ + python3.6 python3.6-venv python3.6-dev \ + python3.7 python3.7-venv python3.7-dev \ + python3.8 python3.8-venv python3.8-dev \ + python3.11 python3.11-venv python3.11-dev \ + virtualenv net-tools libev4 libev-dev wget gcc libssl-dev libxml2 libxslt1-dev \ + vim lsof sudo libjemalloc2 dumb-init locales \ + openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional + + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 5 +RUN python3.6 -m pip install --upgrade pip +RUN python3.7 -m pip install --upgrade pip +RUN python3.8 -m pip install --upgrade pip + +# generate locales for the standard en_US.UTF8 value we use for testing +RUN locale-gen en_US.UTF-8 + +# as we only need the requirements.txt file from the dtest repo, let's just get it from GitHub as a raw asset +# so we can avoid needing to clone the entire repo just to get this file +RUN curl https://raw.githubusercontent.com/apache/cassandra-dtest/trunk/requirements.txt --output /opt/requirements.txt +RUN chmod 0644 /opt/requirements.txt + +# now setup python via virtualenv with all of the python dependencies we need according to requirements.txt +RUN pip3 install virtualenv virtualenv-clone +RUN pip3 install --upgrade wheel + +# make Java 8 the default executable (we use to run all tests against Java 8) +RUN update-java-alternatives --set java-1.8.0-openjdk-$(dpkg --print-architecture) + +# enable legacy TLSv1 and TLSv1.1 (CASSANDRA-16848) +RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \; +RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE_CBC, TLSv1, TLSv1.1/' {} \; + +# create and change to cassandra-tmp user, use an rare uid to avoid collision later on +RUN adduser --disabled-login --uid 901743 --lastuid 901743 --gecos cassandra cassandra-tmp +RUN gpasswd -a cassandra-tmp sudo +RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build +RUN chmod 0440 /etc/sudoers.d/build + +# switch to the cassandra user +USER cassandra-tmp +ENV HOME ${BUILD_HOME} +WORKDIR ${BUILD_HOME} + +ENV ANT_HOME=/usr/share/ant + +# run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we +# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.6/3.7/3.8 +# Don't build cython extensions when installing cassandra-driver. During test execution the driver +# dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without +# requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by +# default in test jobs using the CASS_DRIVER_X env vars below. However, if the extensions are +# included in the base image, the compiled objects are not updated by pip at run time, which can +# cause errors if the tests rely on new driver functionality or bug fixes. + +RUN virtualenv --python=python3.6 ${BUILD_HOME}/env3.6 +RUN chmod +x ${BUILD_HOME}/env3.6/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.6/bin/activate \ + && pip3 install --upgrade pip \ + && pip3 install -r /opt/requirements.txt \ + && pip3 freeze --user" + +RUN virtualenv --python=python3.7 ${BUILD_HOME}/env3.7 +RUN chmod +x ${BUILD_HOME}/env3.7/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.7/bin/activate \ + && pip3 install --upgrade pip \ + && pip3 install -r /opt/requirements.txt \ + && pip3 freeze --user" + +RUN virtualenv --python=python3.8 ${BUILD_HOME}/env3.8 +RUN chmod +x ${BUILD_HOME}/env3.8/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.8/bin/activate \ + && pip3 install --upgrade pip \ + && pip3 install -r /opt/requirements.txt \ + && pip3 freeze --user" + +RUN virtualenv --python=python3.11 ${BUILD_HOME}/env3.11 +RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.11/bin/activate \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \ + && pip3 install -r /opt/requirements.txt \ + && pip3 freeze --user" + +# Initialize the CCM git repo as well as this also can fail to clone +RUN /bin/bash -c "source ${BUILD_HOME}/env3.6/bin/activate && \ + ccm create -n 1 -v git:trunk test && ccm remove test && \ + ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \ + ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test" + +# the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them +RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git +# fix permissions, runtime user has different uid/gid +RUN chmod -R og+wx ${BUILD_HOME}/.ccm ${BUILD_HOME}/env* ${BUILD_HOME}/.cache + +# mark "/tmp" as a volume so it will get mounted as an ext4 mount and not +# the stupid aufs/CoW stuff that the actual docker container mounts will have. +# we've been seeing 3+ minute hangs when calling sync on an aufs backed mount +# so it greatly makes tests flaky as things can hang basically anywhere +VOLUME ["/tmp"] diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh new file mode 100755 index 0000000000..14ad1ad578 --- /dev/null +++ b/.build/run-python-dtests.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Wrapper script for running a split chunk of a pytest run of cassandra-dtest +# +# Usage: dtest-python.sh target split_chunk +# split_chunk formatted as "K/N" for the Kth chunk of N chunks + +################################ +# +# Prep +# +################################ + +# Pass in target to run, defaults to dtest +DTEST_TARGET="${1:-dtest}" +# Optional: pass in chunk to test, formatted as "K/N" for the Kth chunk of N chunks +DTEST_SPLIT_CHUNK="$2" + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${CASSANDRA_DTEST_DIR}" != "x" ] || CASSANDRA_DTEST_DIR="${CASSANDRA_DIR}/../cassandra-dtest" +[ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" + +export PYTHONIOENCODING="utf-8" +export PYTHONUNBUFFERED=true +export CASS_DRIVER_NO_EXTENSIONS=true +export CASS_DRIVER_NO_CYTHON=true +export CCM_MAX_HEAP_SIZE="1024M" +export CCM_HEAP_NEWSIZE="512M" +export CCM_CONFIG_DIR=${DIST_DIR}/.ccm +export NUM_TOKENS="16" +#Have Cassandra skip all fsyncs to improve test performance and reliability +export CASSANDRA_SKIP_SYNC=true +export TMPDIR="$(mktemp -d /tmp/run-python-dtest.XXXXXX)" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +command -v virtualenv >/dev/null 2>&1 || { echo >&2 "virtualenv needs to be installed"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } +[ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; } + +java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}') +version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p') +java_version_default=`grep 'property\s*name="java.default"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` + +if [ "${java_version}" -eq 17 ] && [[ "${target}" == "dtest-upgrade" ]] ; then + echo "Invalid JDK${java_version}. Only overlapping supported JDKs can be used when upgrading, as the same jdk must be used over the upgrade path." + exit 1 +fi + +python_version=$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}') + +# check project is already built. no cleaning is done, so jenkins unstash works, beware. +[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; } + +# check if dist artifacts exist, this breaks the dtests +[[ -d "${DIST_DIR}/dist" ]] && { echo "dtests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; } + +# print debug information on versions +java -version +ant -version +python --version +virtualenv --version + +# cheap trick to ensure dependency libraries are in place. allows us to stash only project specific build artifacts. +ant -quiet -silent resolver-dist-lib + +# Set up venv with dtest dependencies +set -e # enable immediate exit if venv setup fails + +# fresh virtualenv and test logs results everytime +rm -rf ${DIST_DIR}/venv ${DIST_DIR}/test/{html,output,logs} + +# re-use when possible the pre-installed virtualenv found in the cassandra-ubuntu2004_test docker image +virtualenv-clone ${BUILD_HOME}/env${python_version} ${DIST_DIR}/venv || virtualenv --python=python${python_version} ${DIST_DIR}/venv +source ${DIST_DIR}/venv/bin/activate +pip3 install --exists-action w -r ${CASSANDRA_DTEST_DIR}/requirements.txt +pip3 freeze + +################################ +# +# Main +# +################################ + +cd ${CASSANDRA_DTEST_DIR} + +set +e # disable immediate exit from this point +if [ "${DTEST_TARGET}" = "dtest" ]; then + DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --skip-resource-intensive-tests --keep-failed-test-dir" +elif [ "${DTEST_TARGET}" = "dtest-novnode" ]; then + DTEST_ARGS="--skip-resource-intensive-tests --keep-failed-test-dir" +elif [ "${DTEST_TARGET}" = "dtest-offheap" ]; then + DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --use-off-heap-memtables --skip-resource-intensive-tests --keep-failed-test-dir" +elif [ "${DTEST_TARGET}" = "dtest-large" ]; then + DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --only-resource-intensive-tests --force-resource-intensive-tests --keep-failed-test-dir" +elif [ "${DTEST_TARGET}" = "dtest-large-novnode" ]; then + DTEST_ARGS="--only-resource-intensive-tests --force-resource-intensive-tests --keep-failed-test-dir" +elif [ "${DTEST_TARGET}" = "dtest-upgrade" ]; then + DTEST_ARGS="--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all" +else + echo "Unknown dtest target: ${DTEST_TARGET}" + exit 1 +fi + +touch ${DIST_DIR}/test_list.txt +./run_dtests.py --cassandra-dir=${CASSANDRA_DIR} ${DTEST_ARGS} --dtest-print-tests-only --dtest-print-tests-output=${DIST_DIR}/test_list.txt 2>&1 > ${DIST_DIR}/test_stdout.txt +[[ $? -eq 0 ]] || { cat ${DIST_DIR}/test_stdout.txt ; exit 1; } + +if [[ "${DTEST_SPLIT_CHUNK}" =~ ^[0-9]+/[0-9]+$ ]]; then + split_cmd=split + ( split --help 2>&1 ) | grep -q "r/K/N" || split_cmd=gsplit + command -v ${split_cmd} >/dev/null 2>&1 || { echo >&2 "${split_cmd} needs to be installed"; exit 1; } + SPLIT_TESTS=$(${split_cmd} -n r/${DTEST_SPLIT_CHUNK} ${DIST_DIR}/test_list.txt) +elif [[ "x" != "x${_split_chunk}" ]] ; then + SPLIT_TESTS=$(grep "${DTEST_SPLIT_CHUNK}" ${DIST_DIR}/test_list.txt) +else + SPLIT_TESTS=$(cat ${DIST_DIR}/test_list.txt) +fi + + +PYTEST_OPTS="-vv --log-cli-level=DEBUG --junit-xml=${DIST_DIR}/test/output/nosetests.xml --junit-prefix=${DTEST_TARGET} -s" +pytest ${PYTEST_OPTS} --cassandra-dir=${CASSANDRA_DIR} ${DTEST_ARGS} ${SPLIT_TESTS} 2>&1 | tee -a ${DIST_DIR}/test_stdout.txt + +# tar up any ccm logs for easy retrieval +if ls ${TMPDIR}/test/*/logs/* &>/dev/null ; then + mkdir -p ${DIST_DIR}/test/logs + tar -C ${TMPDIR} -cJf ${DIST_DIR}/test/logs/ccm_logs.tar.xz */test/*/logs/* +fi + +# merge all unit xml files into one, and print summary test numbers +pushd ${CASSANDRA_DIR}/ >/dev/null +# remove wrapping elements. `ant generate-unified-test-report` doesn't like it` +sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > /tmp/nosetests.xml +cat /tmp/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml +ant -quiet -silent generate-unified-test-report +popd >/dev/null + +################################ +# +# Clean +# +################################ + +rm -rf ${TMPDIR} +unset TMPDIR +deactivate + +# Exit cleanly for usable "Unstable" status +exit 0 diff --git a/.build/run-tests.sh b/.build/run-tests.sh new file mode 100755 index 0000000000..f0388250ce --- /dev/null +++ b/.build/run-tests.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Wrapper script for running a split chunk of an ant test target +# +# Usage: run-tests.sh target [split_chunk] +# split_chunk formatted as "K/N" for the Kth chunk of N chunks + +set -o errexit +set -o pipefail + +# variables, with defaults +[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" +[ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" + +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +command -v git >/dev/null 2>&1 || { echo >&2 "git needs to be installed"; exit 1; } +[ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; } +[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } +[ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; } + + +# help +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ "$1" == "-h" ]; then + echo "" + echo "Usage: run-tests.sh target [split_chunk|test_regexp]" + echo "" + echo " default split_chunk is 1/1" + exit 1 +fi + +# print debug information on versions +ant -version +git --version +java -version 2>&1 +javac -version 2>&1 + +# lists all tests for the specific test type +_list_tests() { + local -r classlistprefix="$1" + find "test/${classlistprefix}" -name '*Test.java' | sed "s;^test/${classlistprefix}/;;g" | sort +} + +_split_tests() { + local -r _split_chunk="$1" + split_cmd=split + if [[ "${_split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then + ( split --help 2>&1 ) | grep -q "r/K/N" || split_cmd=gsplit + command -v ${split_cmd} >/dev/null 2>&1 || { echo >&2 "${split_cmd} needs to be installed"; exit 1; } + ${split_cmd} -n r/${_split_chunk} + elif [[ "x" != "x${_split_chunk}" ]] ; then + grep ${_split_chunk} + else + echo + fi +} + +_timeout_for() { + grep "name=\"${1}\"" build.xml | awk -F'"' '{print $4}' +} + +_build_all_dtest_jars() { + pushd $TMP_DIR >/dev/null + rm -fR ${TMP_DIR}/cassandra-dtest-jars + until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done + + # cassandra-4 branches need CASSANDRA_USE_JDK11 to allow jdk11 + [ "${java_version}" -eq 11 ] && export CASSANDRA_USE_JDK11=true + + pushd ${TMP_DIR}/cassandra-dtest-jars >/dev/null + for branch in cassandra-4.0 cassandra-4.1 trunk ; do + git checkout $branch + dtest_jar_version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p') + if [ -f "${DIST_DIR}/dtest-${dtest_jar_version}.jar" ] ; then + echo "Skipping dtest jar build for branch ${branch} as ${DIST_DIR}/dtest-${dtest_jar_version}.jar already exists" + continue + fi + # redefine the build.dir to local build folder, rightmost definition wins with java command line system properties + ant realclean -Dbuild.dir=${TMP_DIR}/cassandra-dtest-jars/build + ant jar dtest-jar ${ANT_TEST_OPTS} -Dbuild.dir=${TMP_DIR}/cassandra-dtest-jars/build + cp "${TMP_DIR}/cassandra-dtest-jars/build/dtest-${dtest_jar_version}.jar" ${DIST_DIR}/ + done + popd >/dev/null + popd >/dev/null + ls -l ${DIST_DIR}/dtest*.jar + unset CASSANDRA_USE_JDK11 +} + +_run_testlist() { + local _target_prefix=$1 + local _testlist_target=$2 + local _split_chunk=$3 + local _test_timeout=$4 + testlist="$( _list_tests "${_target_prefix}" | _split_tests "${_split_chunk}")" + if [[ "${_split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then + if [[ -z "${testlist}" ]]; then + # something has to run in the split to generate a junit xml result + echo "Hacking ${_target_prefix} ${_testlist_target} to run only first test found as no tests in split ${_split_chunk} were found" + testlist="$( _list_tests "${_target_prefix}" | head -n1)" + fi + else + if [[ -z "${testlist}" ]]; then + echo "No tests match ${_split_chunk}" + exit 1 + fi + fi + ant $_testlist_target -Dtest.classlistprefix="${_target_prefix}" -Dtest.classlistfile=<(echo "${testlist}") -Dtest.timeout="${_test_timeout}" ${ANT_TEST_OPTS} || echo "failed ${_target_prefix} ${_testlist_target} ${split_chunk}" +} + +_main() { + # parameters + local -r target="${1:-}" + + local -r split_chunk="${2:-'1/1'}" # Optional: pass in chunk or regexp to test. Chunks formatted as "K/N" for the Kth chunk of N chunks + # check split_chunk is compatible with target (if not a regexp) + if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then + case ${target} in + "stress-test" | "fqltool-test" | "microbench" | "cqlsh-test") + echo "Target ${target} does not suport splits." + exit 1 + ;; + *) + ;; + esac + fi + + pushd ${CASSANDRA_DIR}/ >/dev/null + + # jdk check + local -r java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}') + local -r version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p') + local -r java_version_default=`grep 'property\s*name="java.default"' build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'` + + if [ "${java_version}" -eq 17 ] && [[ "${target}" == "jvm-dtest-upgrade" ]] ; then + echo "Invalid JDK${java_version}. Only overlapping supported JDKs can be used when upgrading, as the same jdk must be used over the upgrade path." + exit 1 + fi + + # check project is already built. no cleaning is done, so jenkins unstash works, beware. + [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; } + + # ant test setup + export TMP_DIR="${DIST_DIR}/tmp" + mkdir -p "${TMP_DIR}" || true + export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR} -Drat.skip=true -Dno-checkstyle=true -Dno-javadoc=true -Dant.gen-doc.skip=true" + + # fresh virtualenv and test logs results everytime + rm -rf ${DIST_DIR}/test/{html,output,logs} + + # cheap trick to ensure dependency libraries are in place. allows us to stash only project specific build artifacts. + ant -quiet -silent resolver-dist-lib + + case ${target} in + "stress-test") + # hard fail on test compilation, but dont fail the test run as unstable test reports are processed + ant stress-build-test ${ANT_TEST_OPTS} + ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" + ;; + "fqltool-test") + # hard fail on test compilation, but dont fail the test run so unstable test reports are processed + ant fqltool-build-test ${ANT_TEST_OPTS} + ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" + ;; + "microbench") + ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true + ;; + "test") + _run_testlist "unit" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.timeout')" + ;; + "test-cdc") + _run_testlist "unit" "testclasslist-cdc" "${split_chunk}" "$(_timeout_for 'test.timeout')" + ;; + "test-compression") + _run_testlist "unit" "testclasslist-compression" "${split_chunk}" "$(_timeout_for 'test.timeout')" + ;; + "test-burn") + _run_testlist "burn" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.burn.timeout')" + ;; + "long-test") + _run_testlist "long" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.long.timeout')" + ;; + "jvm-dtest") + testlist=$( _list_tests "distributed" | grep -v "upgrade" | _split_tests "${split_chunk}") + if [[ -z "$testlist" ]]; then + # something has to run in the split to generate a junit xml result + echo Hacking jvm-dtest to run only first test found as no tests in split ${split_chunk} were found + testlist="$( _list_tests "distributed" | grep -v "upgrade" | head -n1)" + fi + ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" + ;; + "jvm-dtest-upgrade") + _build_all_dtest_jars + testlist=$( _list_tests "distributed" | grep "upgrade" | _split_tests "${split_chunk}") + if [[ -z "${testlist}" ]]; then + # something has to run in the split to generate a junit xml result + echo Hacking jvm-dtest-upgrade to run only first test found as no tests in split ${split_chunk} were found + testlist="$( _list_tests "distributed" | grep "upgrade" | head -n1)" + fi + ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" + ;; + "cqlsh-test") + ./pylib/cassandra-cqlsh-tests.sh $(pwd) + ;; + *) + echo "unregconized \"${target}\"" + exit 1 + ;; + esac + + # merge all unit xml files into one, and print summary test numbers + ant -quiet -silent generate-unified-test-report + + popd >/dev/null +} + +_main "$@" diff --git a/.circleci/config.yml b/.circleci/config.yml index 57355ca59c..15e81d79d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,6 +215,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -3008,6 +3009,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8483,6 +8485,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -9857,6 +9860,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config.yml.FREE b/.circleci/config.yml.FREE index 57355ca59c..15e81d79d1 100644 --- a/.circleci/config.yml.FREE +++ b/.circleci/config.yml.FREE @@ -215,6 +215,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -3008,6 +3009,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8483,6 +8485,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -9857,6 +9860,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config.yml.PAID b/.circleci/config.yml.PAID index b690e4def1..4dcee96a99 100644 --- a/.circleci/config.yml.PAID +++ b/.circleci/config.yml.PAID @@ -215,6 +215,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -3008,6 +3009,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8483,6 +8485,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -9857,6 +9860,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config_11_and_17.yml b/.circleci/config_11_and_17.yml index 0701aa9ca3..cc72ded3ae 100644 --- a/.circleci/config_11_and_17.yml +++ b/.circleci/config_11_and_17.yml @@ -231,6 +231,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -581,6 +582,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -7651,6 +7653,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8631,6 +8634,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config_11_and_17.yml.FREE b/.circleci/config_11_and_17.yml.FREE index 0701aa9ca3..cc72ded3ae 100644 --- a/.circleci/config_11_and_17.yml.FREE +++ b/.circleci/config_11_and_17.yml.FREE @@ -231,6 +231,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -581,6 +582,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -7651,6 +7653,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8631,6 +8634,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config_11_and_17.yml.PAID b/.circleci/config_11_and_17.yml.PAID index 19a1054331..284c1516a7 100644 --- a/.circleci/config_11_and_17.yml.PAID +++ b/.circleci/config_11_and_17.yml.PAID @@ -231,6 +231,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -581,6 +582,7 @@ jobs: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -7651,6 +7653,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: @@ -8631,6 +8634,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: 15m - store_test_results: diff --git a/.circleci/config_template.yml b/.circleci/config_template.yml index 10b7aa2c61..5fb8062d23 100644 --- a/.circleci/config_template.yml +++ b/.circleci/config_template.yml @@ -3076,6 +3076,7 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: <> - store_test_results: @@ -3094,6 +3095,7 @@ commands: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: <> - store_test_results: diff --git a/.circleci/config_template_11_and_17.yml b/.circleci/config_template_11_and_17.yml index efd3b6c18f..29b49152fe 100644 --- a/.circleci/config_template_11_and_17.yml +++ b/.circleci/config_template_11_and_17.yml @@ -3042,6 +3042,7 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: <> - store_test_results: @@ -3060,6 +3061,7 @@ commands: export cython="yes" time mv ~/cassandra /tmp cd /tmp/cassandra/ + ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true ./pylib/cassandra-cqlsh-tests.sh $(pwd) no_output_timeout: <> - store_test_results: diff --git a/README.asc b/README.asc index 96c5395553..be26f9d97b 100644 --- a/README.asc +++ b/README.asc @@ -13,8 +13,9 @@ Issues should be reported on https://issues.apache.org/jira/projects/CASSANDRA/i Requirements ------------ -. Java >= 1.8 (OpenJDK and Oracle JVMS have been tested) -. Python 3.6+ (for cqlsh) +- Java: see supported versions in build.xml (search for property "java.supported"). +- Python: for `cqlsh`, see `bin/cqlsh` (search for function "is_supported_version"). + Getting started --------------- diff --git a/build.xml b/build.xml index bc361df079..1b204eb1a8 100644 --- a/build.xml +++ b/build.xml @@ -56,7 +56,7 @@ - + @@ -88,6 +88,7 @@ + @@ -480,7 +481,7 @@ - + @@ -498,7 +499,7 @@ - + @@ -528,12 +529,12 @@ - - + @@ -591,8 +592,8 @@ - - + + @@ -896,7 +897,7 @@ + description="Create Cassandra tarball and maven artifacts"> @@ -1112,6 +1113,9 @@ + + + @@ -1131,7 +1135,7 @@ - + @@ -1143,7 +1147,7 @@ --> - + @@ -1151,7 +1155,7 @@ - + @@ -1367,8 +1371,8 @@ - - + + @@ -1499,7 +1503,7 @@ - + @@ -1545,7 +1549,7 @@ - + @@ -1787,8 +1791,8 @@ --> - - + + @@ -1796,6 +1800,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -1919,10 +1948,10 @@ - + - + @@ -1938,12 +1967,12 @@ - - - - - - + + + + + + @@ -1980,7 +2009,7 @@ - + diff --git a/debian/rules b/debian/rules index 3d1babe916..827e2e73ad 100755 --- a/debian/rules +++ b/debian/rules @@ -7,16 +7,12 @@ include /usr/share/quilt/quilt.make ANT = /usr/bin/ant VERSION = $(shell dpkg-parsechangelog | sed -ne 's/^Version: \([^-|~|+]*\).*/\1/p') - -test: - dh_testdir - $(ANT) test +BUILD_DIR ?= "build" clean: unpatch dh_clean build-stamp dh_testdir dh_testroot - $(ANT) realclean rm -f build-stamp build.properties rm -rf bin/java find -name "*.pyc" -exec rm '{}' ';' @@ -30,7 +26,7 @@ build-stamp: $(QUILT_STAMPFN) printf "version=%s" $(VERSION) > build.properties $(ANT) generate-cql-html - $(ANT) jar + $(ANT) jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true cd pylib && python3 setup.py install --no-compile --install-layout deb \ --root $(CURDIR)/debian/cassandra @@ -43,15 +39,15 @@ install: build dh_install # Copy in the jar and symlink to something stable - dh_install build/apache-cassandra-$(VERSION).jar \ + dh_install $(BUILD_DIR)/apache-cassandra-$(VERSION).jar \ usr/share/cassandra # Copy stress jars - dh_install build/tools/lib/stress.jar \ + dh_install $(BUILD_DIR)/tools/lib/stress.jar \ usr/share/cassandra # Copy fqltool jars - dh_install build/tools/lib/fqltool.jar \ + dh_install $(BUILD_DIR)/tools/lib/fqltool.jar \ usr/share/cassandra dh_link usr/share/cassandra/apache-cassandra-$(VERSION).jar \ diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index f6b1f21654..71bb4a10ea 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -25,12 +25,8 @@ WORKSPACE=$1 -if [ "${WORKSPACE}" = "" ]; then - echo "Specify Cassandra source directory" - exit -fi - -PYTHON_VERSION=python3 +[ "x${WORKSPACE}" != "x" ] || WORKSPACE="$(readlink -f $(dirname "$0")/..)" +[ "x${BUILD_DIR}" != "x" ] || BUILD_DIR="${WORKSPACE}/build" export PYTHONIOENCODING="utf-8" export PYTHONUNBUFFERED=true @@ -38,43 +34,43 @@ export CASS_DRIVER_NO_EXTENSIONS=true export CASS_DRIVER_NO_CYTHON=true export CCM_MAX_HEAP_SIZE="2048M" export CCM_HEAP_NEWSIZE="200M" -export CCM_CONFIG_DIR=${WORKSPACE}/.ccm +export CCM_CONFIG_DIR=${BUILD_DIR}/.ccm export NUM_TOKENS="16" export CASSANDRA_DIR=${WORKSPACE} +export TMPDIR="$(mktemp -d /tmp/run-python-dtest.XXXXXX)" java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}') -export TESTSUITE_NAME="cqlshlib.${PYTHON_VERSION}.jdk${java_version}" +version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p') -ant -buildfile ${CASSANDRA_DIR}/build.xml realclean -# Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -for x in $(seq 1 3); do - ant -buildfile ${CASSANDRA_DIR}/build.xml jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi -done -# Exit, if we didn't build successfully -if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} -fi +python_version="3.6" +command -v python3 >/dev/null 2>&1 && python_version="$(python3 -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')" + +export TESTSUITE_NAME="cqlshlib.python${python_version}.jdk${java_version}" + +pushd ${CASSANDRA_DIR} >/dev/null + +# check project is already built. no cleaning is done, so jenkins unstash works, beware. +[[ -f "${BUILD_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${BUILD_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${BUILD_DIR} with: $(ls ${BUILD_DIR})"; exit 1; } # Set up venv with dtest dependencies set -e # enable immediate exit if venv setup fails -virtualenv --python=$PYTHON_VERSION venv -source venv/bin/activate -# 3.11 needs the newest pip -curl -sS https://bootstrap.pypa.io/get-pip.py | $PYTHON_VERSION -pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt +# fresh virtualenv and test logs results everytime +rm -fr ${DIST_DIR}/venv ${DIST_DIR}/test/{html,output,logs} + +# re-use when possible the pre-installed virtualenv found in the cassandra-ubuntu2004_test docker image +virtualenv-clone ${BUILD_HOME}/env${python_version} ${BUILD_DIR}/venv || virtualenv --python=python3 ${BUILD_DIR}/venv +source ${BUILD_DIR}/venv/bin/activate + +pip install --exists-action w -r ${CASSANDRA_DIR}/pylib/requirements.txt pip freeze if [ "$cython" = "yes" ]; then TESTSUITE_NAME="${TESTSUITE_NAME}.cython" pip install "Cython>=0.29.15,<3.0" - cd pylib/; python setup.py build_ext --inplace - cd ${WORKSPACE} + pushd pylib >/dev/null + python setup.py build_ext --inplace + popd >/dev/null else TESTSUITE_NAME="${TESTSUITE_NAME}.no_cython" fi @@ -108,14 +104,27 @@ esac ccm start --wait-for-binary-proto -cd ${CASSANDRA_DIR}/pylib/cqlshlib/ +pushd ${CASSANDRA_DIR}/pylib/cqlshlib/ >/dev/null set +e # disable immediate exit from this point -pytest --junitxml=${WORKSPACE}/cqlshlib.xml +pytest --junitxml=${BUILD_DIR}/test/output/cqlshlib.xml RETURN="$?" -sed -i "s/testsuite errors=\(\".*\"\) failures=\(\".*\"\) hostname=\(\".*\"\) name=\"pytest\"/testsuite errors=\1 failures=\2 hostname=\3 name=\"${TESTSUITE_NAME}\"/g" ${WORKSPACE}/cqlshlib.xml -sed -i "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" ${WORKSPACE}/cqlshlib.xml +# remove wrapping elements. `ant generate-unified-test-report` doesn't like it` +sed -r "s/<[\/]?testsuites>//g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml +cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml + +# don't do inline sed for linux+mac compat +sed "s/testsuite errors=\(\".*\"\) failures=\(\".*\"\) hostname=\(\".*\"\) name=\"pytest\"/testsuite errors=\1 failures=\2 hostname=\3 name=\"${TESTSUITE_NAME}\"/g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml +cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml +sed "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" ${BUILD_DIR}/test/output/cqlshlib.xml > /tmp/cqlshlib.xml +cat /tmp/cqlshlib.xml > ${BUILD_DIR}/test/output/cqlshlib.xml + +# tar up any ccm logs for easy retrieval +if ls ${TMPDIR}/test/*/logs/* &>/dev/null ; then + mkdir -p ${DIST_DIR}/test/logs + tar -C ${TMPDIR} -cJf ${DIST_DIR}/test/logs/ccm_logs.tar.xz */test/*/logs/* +fi ccm remove @@ -125,8 +134,12 @@ ccm remove # ################################ -# /virtualenv + +rm -rf ${TMPDIR} +unset TMPDIR deactivate +popd >/dev/null +popd >/dev/null # circleci needs non-zero exit on failures, jenkins need zero exit to process the test failures if ! command -v circleci >/dev/null 2>&1 diff --git a/redhat/cassandra.spec b/redhat/cassandra.spec index ae5ad4c308..b7257b90f0 100644 --- a/redhat/cassandra.spec +++ b/redhat/cassandra.spec @@ -32,6 +32,9 @@ %define upstream_version %(echo %{version} | sed -r 's/~/-/g') %define relname apache-cassandra-%{upstream_version} +# default DIST_DIR to build +%global _get_dist_dir %(echo "${DIST_DIR:-build}") + Name: cassandra Version: %{version} Release: %{revision} @@ -69,7 +72,7 @@ Cassandra is a distributed (peer-to-peer) system for the management and storage %build export LANG=en_US.UTF-8 export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8" -ant clean jar -Dversion=%{upstream_version} +ant jar -Dversion=%{upstream_version} -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true %install %{__rm} -rf %{buildroot} @@ -115,10 +118,10 @@ cp -p redhat/default %{buildroot}/%{_sysconfdir}/default/%{username} cp -pr lib/* %{buildroot}/usr/share/%{username}/lib/ # copy stress jar -cp -p build/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/ +cp -p %{_get_dist_dir}/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/ # copy fqltool jar -cp -p build/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/ +cp -p %{_get_dist_dir}/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/ # copy binaries mv bin/cassandra %{buildroot}/usr/sbin/ @@ -126,7 +129,7 @@ cp -p bin/* %{buildroot}/usr/bin/ cp -p tools/bin/* %{buildroot}/usr/bin/ # copy cassandra jar -cp build/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/ +cp %{_get_dist_dir}/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/ %clean %{__rm} -rf %{buildroot} @@ -207,6 +210,10 @@ This package contains extra tools for working with Cassandra clusters. %changelog +# packaging changes, not software changes +* Thu May 04 2023 Mick Semb Wever +- 5.0 +- RPM packaging brought in-tree. CASSANDRA-18133 * Mon Dec 05 2016 Michael Shuler - 2.1.17, 2.2.9, 3.0.11, 3.10 - Reintroduce RPM packaging diff --git a/redhat/noboolean/cassandra.spec b/redhat/noboolean/cassandra.spec index 2b6cdd4d6f..53e69dada5 100644 --- a/redhat/noboolean/cassandra.spec +++ b/redhat/noboolean/cassandra.spec @@ -32,6 +32,9 @@ %define upstream_version %(echo %{version} | sed -r 's/~/-/g') %define relname apache-cassandra-%{upstream_version} +# default DIST_DIR to build +%global _get_dist_dir %(echo "${DIST_DIR:-build}") + Name: cassandra Version: %{version} Release: %{revision} @@ -69,7 +72,7 @@ Cassandra is a distributed (peer-to-peer) system for the management and storage %build export LANG=en_US.UTF-8 export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8" -ant clean jar -Dversion=%{upstream_version} +ant jar -Dversion=%{upstream_version} -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true %install %{__rm} -rf %{buildroot} @@ -115,10 +118,10 @@ cp -p redhat/default %{buildroot}/%{_sysconfdir}/default/%{username} cp -pr lib/* %{buildroot}/usr/share/%{username}/lib/ # copy stress jar -cp -p build/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/ +cp -p %{_get_dist_dir}/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/ # copy fqltool jar -cp -p build/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/ +cp -p %{_get_dist_dir}/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/ # copy binaries mv bin/cassandra %{buildroot}/usr/sbin/ @@ -126,7 +129,7 @@ cp -p bin/* %{buildroot}/usr/bin/ cp -p tools/bin/* %{buildroot}/usr/bin/ # copy cassandra jar -cp build/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/ +cp %{_get_dist_dir}/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/ %clean %{__rm} -rf %{buildroot} @@ -207,6 +210,10 @@ This package contains extra tools for working with Cassandra clusters. %changelog +# packaging changes, not software changes +* Thu May 04 2023 Mick Semb Wever +- 5.0 +- RPM packaging brought in-tree. CASSANDRA-18133 * Mon Dec 05 2016 Michael Shuler - 2.1.17, 2.2.9, 3.0.11, 3.10 - Reintroduce RPM packaging diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index 707da494ce..590df6264e 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -234,7 +234,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance if (!f.exists()) f = new File(FileSystems.getDefault(), String.format("build/test/logs/%s/%s/%s/system.log", tag, clusterId, instanceId)); if (!f.exists()) - throw new AssertionError("Unable to locate system.log under " + new File("build/test/logs").absolutePath() + "; make sure ICluster.setup() is called or extend TestBaseImpl and do not define a static beforeClass function with @BeforeClass"); + throw new AssertionError("Unable to locate system.log under " + f.absolutePath() + "; make sure ICluster.setup() is called or extend TestBaseImpl and do not define a static beforeClass function with @BeforeClass"); return new FileLogAction(f); }