From 36bfe355da672ed29d64a8df81035e4b9cc95987 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 Jan 2023 22:21:24 +0100 Subject: [PATCH] Backport #31126 to v1.46.x (#31960) * fix python build on kokoro macos monterey * avoid using realpath * fix grpc_distribtests_python on macos monterey * try upgrade setuptools * remove python2.7 from grpc_build_artifacts.sh as well * python3.11 not needed yet for the backport * stop building python3.6 wheel on macos --- .../internal_ci/macos/grpc_build_artifacts.sh | 10 ++++------ .../macos/grpc_distribtests_python.sh | 10 ++++------ tools/run_tests/artifacts/artifact_targets.py | 3 +-- .../artifacts/build_artifact_python.sh | 7 ++++--- tools/run_tests/helper_scripts/build_python.sh | 18 ++---------------- tools/run_tests/helper_scripts/run_python.sh | 3 ++- 6 files changed, 17 insertions(+), 34 deletions(-) diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 1011a00d0d1..6af0493aea9 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -29,12 +29,10 @@ source tools/internal_ci/helper_scripts/prepare_build_macos_rc # TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build) # install cython for all python versions -python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.9 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.7 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.8 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.9 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.10 -m pip install -U cython setuptools==65.4.1 wheel --user gem install rubygems-update update_rubygems diff --git a/tools/internal_ci/macos/grpc_distribtests_python.sh b/tools/internal_ci/macos/grpc_distribtests_python.sh index 5068bd19013..28737582408 100644 --- a/tools/internal_ci/macos/grpc_distribtests_python.sh +++ b/tools/internal_ci/macos/grpc_distribtests_python.sh @@ -26,12 +26,10 @@ source tools/internal_ci/helper_scripts/prepare_build_macos_rc # TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build) # install cython for all python versions -python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user -python3.9 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.7 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.8 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.9 -m pip install -U cython setuptools==65.4.1 wheel --user +python3.10 -m pip install -U cython setuptools==65.4.1 wheel --user # Build all python macos artifacts (this step actually builds all the binary wheels and source archives) tools/run_tests/task_runner.py -f artifact macos python ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 048fe25c96d..28f6ed8b5fb 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -471,8 +471,7 @@ def targets(): PythonArtifact('musllinux_1_1', 'x86', 'cp37-cp37m'), PythonArtifact('musllinux_1_1', 'x86', 'cp38-cp38'), PythonArtifact('musllinux_1_1', 'x86', 'cp39-cp39'), - PythonArtifact('macos', 'x64', 'python3.6', presubmit=True), - PythonArtifact('macos', 'x64', 'python3.7'), + PythonArtifact('macos', 'x64', 'python3.7', presubmit=True), PythonArtifact('macos', 'x64', 'python3.8'), PythonArtifact('macos', 'x64', 'python3.9'), PythonArtifact('macos', 'x64', 'python3.10', presubmit=True), diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index c380ab855c6..e74aa944c36 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -134,10 +134,11 @@ ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_whee # the artifacts output dir. if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] then - # Ensure the generated artifacts are valid. - # TODO(jtattermusch): avoid the need for always re-installing virtualenv and twine + # Install virtualenv if it isn't already available. + # TODO(jtattermusch): cleanup the virtualenv version fallback logic. "${PYTHON}" -m pip install virtualenv - "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==16.7.9 && "${PYTHON}" -m virtualenv venv; } + "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==20.0.23 && "${PYTHON}" -m virtualenv venv; } + # Ensure the generated artifacts are valid using "twine check" venv/bin/python -m pip install "twine<=2.0" venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/* rm -rf venv/ diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index b45a5074a3e..86f18841dba 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -86,20 +86,6 @@ function toolchain() { fi } -# TODO(jtattermusch): this adds dependency on grealpath on mac -# (brew install coreutils) for little reason. -# Command to invoke the linux command `realpath` or equivalent. -function script_realpath() { - # Find `realpath` - if [ -x "$(command -v realpath)" ]; then - realpath "$@" - elif [ -x "$(command -v grealpath)" ]; then - grealpath "$@" - else - exit 1 - fi -} - #################### # Script Arguments # #################### @@ -137,9 +123,9 @@ if [[ "$(inside_venv)" ]]; then VENV_PYTHON="$PYTHON" else # Instantiate the virtualenv from the Python version passed in. - $PYTHON -m pip install --user virtualenv==16.7.9 + $PYTHON -m pip install --user virtualenv==20.0.23 $PYTHON -m virtualenv "$VENV" - VENV_PYTHON=$(script_realpath "$VENV/$VENV_RELATIVE_PYTHON") + VENV_PYTHON="$(pwd)/$VENV/$VENV_RELATIVE_PYTHON" fi diff --git a/tools/run_tests/helper_scripts/run_python.sh b/tools/run_tests/helper_scripts/run_python.sh index 72e659125d8..40c6939859b 100755 --- a/tools/run_tests/helper_scripts/run_python.sh +++ b/tools/run_tests/helper_scripts/run_python.sh @@ -18,7 +18,8 @@ set -ex # change to grpc repo root cd "$(dirname "$0")/../../.." -PYTHON=$(realpath "${1:-py36/bin/python}") +# TODO(jtattermusch): is the $(pwd) prefix actually useful? +PYTHON="$(pwd)/${1:-py37/bin/python}" ROOT=$(pwd)