diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml index 6d47f4726f..2967328e46 100644 --- a/.build/build-resolver.xml +++ b/.build/build-resolver.xml @@ -245,6 +245,7 @@ + @@ -261,6 +262,7 @@ + diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 50c1d3bf9e..2357a77872 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -300,6 +300,7 @@ echo "Running container ${container_name} ${docker_id} using image ${image_name} docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile} docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile} +docker exec --user root ${container_name} update-alternatives --set python3 /usr/bin/python${python_version} | tee -a ${logfile} if [ -n "${DTEST_TMPDIR_LOCAL}" ] && [[ "${target}" =~ ^dtest-upgrade ]] ; then # prepopulate a tmp ccm repository directory, if running dtest-upgrade tests diff --git a/.build/docker/ubuntu-test.docker b/.build/docker/ubuntu-test.docker index 6eadf55e25..47b26307f0 100644 --- a/.build/docker/ubuntu-test.docker +++ b/.build/docker/ubuntu-test.docker @@ -49,15 +49,26 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y curl git-core python3-pip \ python3.8 python3.8-venv python3.8-dev \ + python3.10 python3.10-venv python3.10-dev \ python3.11 python3.11-venv python3.11-dev \ + python3.12 python3.12-venv python3.12-dev \ + python3.13 python3.13-venv python3.13-dev \ virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \ vim lsof sudo libjemalloc2 dumb-init locales rsync \ openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk openjdk-21-jdk ant ant-optional RUN update-alternatives --remove java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/jre/bin/java RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java 1081 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.13 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 1 RUN python3.8 -m pip install --upgrade pip # generate locales for the standard en_US.UTF8 value we use for testing @@ -101,7 +112,7 @@ RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository RUN cp -a /home/cassandra-tmp/.gradle /home/image-cache/.gradle # run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we -# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.8/3.11 +# setup the virtualenv for the cassandra user, not root) for Python 3.8-3.13 # Don't build cython extensions when installing cassandra-driver. During test execution the driver # dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without # requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by @@ -118,6 +129,16 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ && pip3 install -r /opt/requirements.txt \ && pip3 freeze --user" +RUN virtualenv --python=python3.10 ${BUILD_HOME}/env3.10 +RUN chmod +x ${BUILD_HOME}/env3.10/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.10/bin/activate \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \ + && pip3 install --upgrade \"pip<25.0\" \"setuptools==60.8.2\" wheel \ + && pip3 install --no-build-isolation -r /opt/requirements.txt \ + && pip3 freeze --user" + RUN python3.11 -m venv ${BUILD_HOME}/env3.11 RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate @@ -128,6 +149,28 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ && pip3 install --no-build-isolation -r /opt/requirements.txt \ && pip3 freeze --user" +RUN virtualenv --python=python3.12 ${BUILD_HOME}/env3.12 +RUN chmod +x ${BUILD_HOME}/env3.12/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.12/bin/activate \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \ + && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \ + && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.12/lib/python3.12/site-packages/pkg_resources/__init__.py \ + && pip3 install --no-build-isolation -r /opt/requirements.txt \ + && pip3 freeze --user" + +RUN virtualenv --python=python3.13 ${BUILD_HOME}/env3.13 +RUN chmod +x ${BUILD_HOME}/env3.13/bin/activate + +RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \ + && source ${BUILD_HOME}/env3.13/bin/activate \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 \ + && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \ + && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.13/lib/python3.13/site-packages/pkg_resources/__init__.py \ + && pip3 install --no-build-isolation -r /opt/requirements.txt \ + && pip3 freeze --user" + # 4* requires java8, sudo doesn't work on cross-platform builds USER root RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java diff --git a/.build/run-tests.sh b/.build/run-tests.sh index b56ebbf2a2..7d62231377 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -24,8 +24,6 @@ set -o errexit set -o pipefail -[ $DEBUG ] && set -x - # variables, with defaults [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index ea412aca0f..7f80fd98d3 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -59,7 +59,7 @@ /** CONSTANTS for both the pipeline and scripting **/ import groovy.transform.Field @Field List archsSupported = ["amd64", "arm64"] -@Field List pythonsSupported = ["3.8", "3.11"] +@Field List pythonsSupported = ["3.8", "3.11", "3.12", "3.13"] @Field String pythonDefault = "3.8" /** CONSTANTS end **********************************/ @@ -124,7 +124,7 @@ pipeline { } post { failure { - echo "ERROR pipeline failed – not all tests were run" + echo "ERROR pipeline failed – not all tests were run" } always { sendNotifications() @@ -257,6 +257,7 @@ def tasks() { (isStageEnabled(axis['step'])) && // skip disabled steps !(axis['python'] != pythonDefault && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test !(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others + !(axis['cython'] == 'yes' && (axis['python'] == '3.12' || axis['python'] == '3.13')) && // Skip cython for Python 3.12+ see CASSANDRA-21482 !(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11 // Disable splits for all but proper stages !(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) && diff --git a/CHANGES.txt b/CHANGES.txt index 3fa97b0d1c..3727b164b8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -53,6 +53,7 @@ Merged from 6.0: * Introduce minimum_threshold for data resurrection startup check (CASSANDRA-21293) * Synchronously publish changes to local gossip state following metadata updates (CASSANDRA-21239) Merged from 5.0: + * Support Python 3.12 and 3.13 in cqlsh (CASSANDRA-20997) * Make synchronization on VectorMemoryIndex inserts more granular (CASSANDRA-21160) * Fix RequestFailureReason serializer and nits in a few others (CASSANDRA-21437) * Remove golang dependency in gen-doc and replace with python implementation (CASSANDRA-21432) diff --git a/bin/cqlsh b/bin/cqlsh index 2a9651968b..25fa029b4f 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -63,7 +63,7 @@ is_supported_version() { major_version="${version%.*}" minor_version="${version#*.}" # python 3.8-3.11 are supported - if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 11 ]; then + if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 13 ]; then echo "supported" # python 3.6-3.7 are deprecated elif [ "$major_version" = 3 ] && [ "$minor_version" -ge 6 ] && [ "$minor_version" -le 7 ]; then @@ -88,7 +88,7 @@ run_if_supported_version() { exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" "$@" exit else - echo "Warning: unsupported version of Python, required 3.6-3.11 but found" "$version" >&2 + echo "Warning: unsupported version of Python, required 3.6-3.13 but found" "$version" >&2 fi fi } diff --git a/bin/cqlsh.py b/bin/cqlsh.py index 738f0aeeb7..5131f99062 100755 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -21,8 +21,8 @@ import platform import sys from glob import glob -if sys.version_info < (3, 6) or sys.version_info >= (3, 12): - sys.exit("\ncqlsh requires Python 3.6-3.11\n") +if sys.version_info < (3, 6) or sys.version_info >= (3, 14): + sys.exit("\ncqlsh requires Python 3.6-3.13\n") # see CASSANDRA-10428 if platform.python_implementation().startswith('Jython'): @@ -56,7 +56,7 @@ if cql_zip: sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver)) # the driver needs dependencies -third_parties = ('pure_sasl-', 'wcwidth-') +third_parties = ('pure_sasl-', 'wcwidth-', 'pyasyncore-') for lib in third_parties: lib_zip = find_zip(lib) diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index 7c0caf69a1..511e5d9743 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -23,6 +23,8 @@ # ################################ +[ $DEBUG ] && set -x + WORKSPACE=$1 [ "x${WORKSPACE}" != "x" ] || WORKSPACE="$(readlink -f $(dirname "$0")/..)" @@ -63,6 +65,10 @@ rm -fr ${DIST_DIR}/venv ${DIST_DIR}/test/{html,output,logs} virtualenv-clone ${BUILD_HOME}/env${python_version} ${BUILD_DIR}/venv || virtualenv --python=python3 ${BUILD_DIR}/venv source ${BUILD_DIR}/venv/bin/activate +# Force pip's legacy pkg_resources metadata backend (instead of the importlib.metadata +# backend that pip defaults to on Python 3.11+). see ubuntu-test.docker +export _PIP_USE_IMPORTLIB_METADATA=0 + pip install --exists-action w -r ${CASSANDRA_DIR}/pylib/requirements.txt pip freeze diff --git a/pylib/requirements.txt b/pylib/requirements.txt index d084f18ad5..9e64628628 100644 --- a/pylib/requirements.txt +++ b/pylib/requirements.txt @@ -23,3 +23,4 @@ coverage pytest wcwidth +pyasyncore diff --git a/pylib/setup.py b/pylib/setup.py index 1dfd8cdc89..712315f5fd 100755 --- a/pylib/setup.py +++ b/pylib/setup.py @@ -23,6 +23,8 @@ def get_extensions(): if "--no-compile" in sys.argv: return [] + assert sys.version_info < (3, 12), "Cython < 3.0 is incompatible with Python 3.12+ C-API changes. See CASSANDRA-21482." + from Cython.Build import cythonize return cythonize("cqlshlib/copyutil.py")