From f23f1bb51fe9e8db2a1a11f7fe532745eb9e00a2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 25 Jan 2022 19:18:58 +0100 Subject: [PATCH] Use ccache for selected C++ and python builds (with redis server as cache) (#28661) * add cmake support for ccache * cleanup: use --env-file for docker run invocations * make python build compatible with using ccache * enable building using ccache in selected kokoro jobs * print ccache stats and the end of run_tests.py --- CMakeLists.txt | 3 ++ cmake/ccache.cmake | 26 ++++++++++++++++ templates/CMakeLists.txt.template | 3 ++ tools/docker_runners/run_in_docker.sh | 6 +++- .../helper_scripts/prepare_ccache_rc | 22 +++++++++++++ .../helper_scripts/prepare_ccache_symlinks_rc | 31 +++++++++++++++++++ .../internal_ci/linux/grpc_build_artifacts.sh | 3 ++ .../linux/grpc_distribtests_python.sh | 3 ++ .../linux/grpc_run_tests_matrix.sh | 3 ++ .../artifacts/build_artifact_python.sh | 4 +++ .../dockerize/build_and_run_docker.sh | 6 +--- .../dockerize/build_docker_and_run_tests.sh | 8 +---- .../dockerize/build_interop_image.sh | 3 +- .../dockerize/docker_propagate_env.list | 17 ++++++++++ tools/run_tests/dockerize/docker_run_tests.sh | 5 +++ .../run_tests/helper_scripts/build_python.sh | 14 ++------- 16 files changed, 131 insertions(+), 26 deletions(-) create mode 100644 cmake/ccache.cmake create mode 100644 tools/internal_ci/helper_scripts/prepare_ccache_rc create mode 100644 tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc create mode 100644 tools/run_tests/dockerize/docker_propagate_env.list diff --git a/CMakeLists.txt b/CMakeLists.txt index efd344d89a7..9e09cba2926 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,6 +305,9 @@ if(UNIX) endif() endif() +# configure ccache if requested +include(cmake/ccache.cmake) + include(cmake/abseil-cpp.cmake) include(cmake/address_sorting.cmake) include(cmake/benchmark.cmake) diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake new file mode 100644 index 00000000000..40d9c27a40b --- /dev/null +++ b/cmake/ccache.cmake @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configure ccache if requested by environment variable GRPC_BUILD_ENABLE_CCACHE + +if ($ENV{GRPC_BUILD_ENABLE_CCACHE}) + find_program(gRPC_CCACHE_BINARY ccache) + if(gRPC_CCACHE_BINARY) + message(STATUS "Will use ccache as compiler launcher: ${gRPC_CCACHE_BINARY}") + set(CMAKE_C_COMPILER_LAUNCHER ${gRPC_CCACHE_BINARY}) + set(CMAKE_CXX_COMPILER_LAUNCHER ${gRPC_CCACHE_BINARY}) + else() + message(STATUS "Build will not use ccache (ccache binary not found).") + endif() +endif() \ No newline at end of file diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 8d9374793f5..6ec36e6abfa 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -343,6 +343,9 @@ endif() endif() + # configure ccache if requested + include(cmake/ccache.cmake) + include(cmake/abseil-cpp.cmake) include(cmake/address_sorting.cmake) include(cmake/benchmark.cmake) diff --git a/tools/docker_runners/run_in_docker.sh b/tools/docker_runners/run_in_docker.sh index d891043dfa1..9c34e5e6ae6 100755 --- a/tools/docker_runners/run_in_docker.sh +++ b/tools/docker_runners/run_in_docker.sh @@ -72,6 +72,10 @@ DOCKER_CLEANUP_ARGS=( "--rm=true" ) +DOCKER_PROPAGATE_ENV_ARGS=( + "--env-file=tools/run_tests/dockerize/docker_propagate_env.list" \ +) + # Uncomment to run the docker container as current user's UID and GID. # That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace), # but it can have some other disadvantages. E.g.: @@ -88,4 +92,4 @@ set -ex # Run command inside C# docker container. # - the local clone of grpc repository will be mounted as /workspace. -exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*" +exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" "${DOCKER_PROPAGATE_ENV_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*" diff --git a/tools/internal_ci/helper_scripts/prepare_ccache_rc b/tools/internal_ci/helper_scripts/prepare_ccache_rc new file mode 100644 index 00000000000..031df5e579d --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_ccache_rc @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to configure ccache (compiler cache to speed up builds) + +# Allow use of ccache in builds that support that. +export GRPC_BUILD_ENABLE_CCACHE=true + +# Redis instance housed in grpc-testing cloud project serves as the main compiler cache +export CCACHE_SECONDARY_STORAGE=redis://10.76.145.84:6379 diff --git a/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc b/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc new file mode 100644 index 00000000000..eb666d53107 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to create symlinks to ccache + +# TODO: handle values like "0", "false" etc. +if [ "${GRPC_BUILD_ENABLE_CCACHE}" != "" ] +then + if [ -x "$(command -v ccache)" ] + then + TEMP_CCACHE_BINDIR="$(mktemp -d)" + # TODO: symlinks for crosscompilers... + for compiler in "gcc" "g++" "clang" "clang++" "cc" "c++" + do + ln -s "$(which ccache)" "${TEMP_CCACHE_BINDIR}/${compiler}" + done + export PATH="${TEMP_CCACHE_BINDIR}:$PATH" + fi +fi diff --git a/tools/internal_ci/linux/grpc_build_artifacts.sh b/tools/internal_ci/linux/grpc_build_artifacts.sh index ad08a086977..29694e70946 100755 --- a/tools/internal_ci/linux/grpc_build_artifacts.sh +++ b/tools/internal_ci/linux/grpc_build_artifacts.sh @@ -26,6 +26,9 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc # prerequisites for ruby artifact build on linux source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc +# configure ccache +source tools/internal_ci/helper_scripts/prepare_ccache_rc + tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index 874e978dfb0..2b6dae25263 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -28,6 +28,9 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc # under qemu emulator. source tools/internal_ci/helper_scripts/prepare_qemu_rc +# configure ccache +source tools/internal_ci/helper_scripts/prepare_ccache_rc + # Build all python linux artifacts (this step actually builds all the binary wheels and source archives) tools/run_tests/task_runner.py -f artifact linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts/sponge_log.xml || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 8f5d4f2f527..da208fc14f0 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -23,6 +23,9 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +# configure ccache +source tools/internal_ci/helper_scripts/prepare_ccache_rc + # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then export RUN_TESTS_FLAGS="--filter_pr_tests --base_branch origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH $RUN_TESTS_FLAGS" diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 8f18eb38d8b..fe1d066cf50 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -21,6 +21,10 @@ export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export PYTHON=${PYTHON:-python} export AUDITWHEEL=${AUDITWHEEL:-auditwheel} +# activate ccache if desired +# shellcheck disable=SC1091 +source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc + # Needed for building binary distribution wheels -- bdist_wheel "${PYTHON}" -m pip install --upgrade wheel diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index f265ad56418..3614212496b 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -60,11 +60,7 @@ docker run \ "$@" \ --cap-add SYS_PTRACE \ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \ - -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ - -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \ - -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \ - -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \ - -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \ + --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \ -v "$git_root:/var/local/jenkins/grpc:ro" \ -w /var/local/git/grpc \ --name="$CONTAINER_NAME" \ diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index a155429c7a1..7b2bb4b73ea 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -66,13 +66,7 @@ docker run \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e HOST_GIT_ROOT="$git_root" \ -e LOCAL_GIT_ROOT=$docker_instance_git_root \ - -e "BUILD_ID=$BUILD_ID" \ - -e "BUILD_URL=$BUILD_URL" \ - -e "JOB_BASE_NAME=$JOB_BASE_NAME" \ - -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \ - -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \ - -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \ - -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \ + --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \ $DOCKER_TTY_ARGS \ --sysctl net.ipv6.conf.all.disable_ipv6=0 \ -v ~/.config/gcloud:/root/.config/gcloud \ diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index 5a8abcb0bd4..f2bacae0b2e 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -113,8 +113,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" # shellcheck disable=SC2086 (docker run \ --cap-add SYS_PTRACE \ - -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ - -e THIS_IS_REALLY_NEEDED_ONCE_AGAIN='For issue 4835. See https://github.com/docker/docker/issues/14203 for why docker is awful' \ + --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \ $DOCKER_TTY_ARGS \ $MOUNT_ARGS \ $BUILD_INTEROP_DOCKER_EXTRA_ARGS \ diff --git a/tools/run_tests/dockerize/docker_propagate_env.list b/tools/run_tests/dockerize/docker_propagate_env.list new file mode 100644 index 00000000000..972feccf1f6 --- /dev/null +++ b/tools/run_tests/dockerize/docker_propagate_env.list @@ -0,0 +1,17 @@ +# Pass this file to "docker run" using --env-file argument. +# Variables listed in this file will be set or propagated +# to the docker container. + +# TODO(jtattermusch): revisit whether this very old hack is still needed. +THIS_IS_REALLY_NEEDED="see https://github.com/docker/docker/issues/14203 for why docker is awful" + +# Kokoro sets environment variables for each build and we want them to be +# available inside the test docker containers. +KOKORO_BUILD_ID +KOKORO_BUILD_NUMBER +KOKORO_BUILD_URL +KOKORO_JOB_NAME + +# Propagate ccache configuration to the docker containers. +GRPC_BUILD_ENABLE_CCACHE +CCACHE_SECONDARY_STORAGE diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh index 2a3b6f53efa..bb93aac5ecc 100755 --- a/tools/run_tests/dockerize/docker_run_tests.sh +++ b/tools/run_tests/dockerize/docker_run_tests.sh @@ -45,4 +45,9 @@ find . -name report.xml -print0 | xargs -0 -r zip reports.zip find . -name sponge_log.xml -print0 | xargs -0 -r zip reports.zip find . -name 'report_*.xml' -print0 | xargs -0 -r zip reports.zip +if [ -x "$(command -v ccache)" ] +then + ccache --show-stats || true +fi + exit $exit_code diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index bc034cec105..b45a5074a3e 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -125,17 +125,9 @@ export LANG=en_US.UTF-8 DEFAULT_PARALLEL_JOBS=$(nproc) || DEFAULT_PARALLEL_JOBS=4 export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=${GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS:-$DEFAULT_PARALLEL_JOBS} -# If ccache is available on Linux, use it. -if [ "$(is_linux)" ]; then - # We're not on Darwin (Mac OS X) - if [ -x "$(command -v ccache)" ]; then - if [ -x "$(command -v gcc)" ]; then - export CC='ccache gcc' - elif [ -x "$(command -v clang)" ]; then - export CC='ccache clang' - fi - fi -fi +# activate ccache if desired +# shellcheck disable=SC1091 +source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc ############################ # Perform build operations #