Use ccache for selected C++ and python builds (with redis server as cache) (#28661)

* add cmake support for ccache

* cleanup: use --env-file for docker run invocations

* make python build compatible with using ccache

* enable building using ccache in selected kokoro jobs

* print ccache stats and the end of run_tests.py

CMakeLists.txt

@@ -305,6 +305,9 @@ if(UNIX)
   endif()
 endif()
 
+# configure ccache if requested
+include(cmake/ccache.cmake)
+
 include(cmake/abseil-cpp.cmake)
 include(cmake/address_sorting.cmake)
 include(cmake/benchmark.cmake)

cmake/ccache.cmake

@@ -0,0 +1,26 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configure ccache if requested by environment variable GRPC_BUILD_ENABLE_CCACHE
+
+if ($ENV{GRPC_BUILD_ENABLE_CCACHE})
+  find_program(gRPC_CCACHE_BINARY ccache)
+  if(gRPC_CCACHE_BINARY)
+    message(STATUS "Will use ccache as compiler launcher: ${gRPC_CCACHE_BINARY}")
+    set(CMAKE_C_COMPILER_LAUNCHER   ${gRPC_CCACHE_BINARY})
+    set(CMAKE_CXX_COMPILER_LAUNCHER ${gRPC_CCACHE_BINARY})
+  else()
+    message(STATUS "Build will not use ccache (ccache binary not found).")
+  endif()
+endif()

templates/CMakeLists.txt.template

@@ -343,6 +343,9 @@
     endif()
   endif()
 
+  # configure ccache if requested
+  include(cmake/ccache.cmake)
+
   include(cmake/abseil-cpp.cmake)
   include(cmake/address_sorting.cmake)
   include(cmake/benchmark.cmake)

tools/docker_runners/run_in_docker.sh

@@ -72,6 +72,10 @@ DOCKER_CLEANUP_ARGS=(
   "--rm=true"
 )
 
+DOCKER_PROPAGATE_ENV_ARGS=(
+  "--env-file=tools/run_tests/dockerize/docker_propagate_env.list" \
+)
+
 # Uncomment to run the docker container as current user's UID and GID.
 # That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace),
 # but it can have some other disadvantages. E.g.:
@@ -88,4 +92,4 @@ set -ex
 
 # Run command inside C# docker container.
 # - the local clone of grpc repository will be mounted as /workspace.
-exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*"
+exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" "${DOCKER_PROPAGATE_ENV_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*"

tools/internal_ci/helper_scripts/prepare_ccache_rc

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this rc script to configure ccache (compiler cache to speed up builds)
+
+# Allow use of ccache in builds that support that.
+export GRPC_BUILD_ENABLE_CCACHE=true
+
+# Redis instance housed in grpc-testing cloud project serves as the main compiler cache
+export CCACHE_SECONDARY_STORAGE=redis://10.76.145.84:6379

tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this rc script to create symlinks to ccache
+
+# TODO: handle values like "0", "false" etc.
+if [ "${GRPC_BUILD_ENABLE_CCACHE}" != "" ]
+then
+  if [ -x "$(command -v ccache)" ]
+  then
+    TEMP_CCACHE_BINDIR="$(mktemp -d)"
+    # TODO: symlinks for crosscompilers...
+    for compiler in "gcc" "g++" "clang" "clang++" "cc" "c++"
+    do
+      ln -s "$(which ccache)" "${TEMP_CCACHE_BINDIR}/${compiler}"
+    done
+    export PATH="${TEMP_CCACHE_BINDIR}:$PATH"
+  fi
+fi

tools/internal_ci/linux/grpc_build_artifacts.sh

@@ -26,6 +26,9 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 # prerequisites for ruby artifact build on linux
 source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc
 
+# configure ccache
+source tools/internal_ci/helper_scripts/prepare_ccache_rc
+
 tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 || FAILED="true"
 
 tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -28,6 +28,9 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 # under qemu emulator.
 source tools/internal_ci/helper_scripts/prepare_qemu_rc
 
+# configure ccache
+source tools/internal_ci/helper_scripts/prepare_ccache_rc
+
 # Build all python linux artifacts (this step actually builds all the binary wheels and source archives)
 tools/run_tests/task_runner.py -f artifact linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts/sponge_log.xml || FAILED="true"
 

tools/internal_ci/linux/grpc_run_tests_matrix.sh

@@ -23,6 +23,9 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
+# configure ccache
+source tools/internal_ci/helper_scripts/prepare_ccache_rc
+
 # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests
 if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then
   export RUN_TESTS_FLAGS="--filter_pr_tests --base_branch origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH $RUN_TESTS_FLAGS"

tools/run_tests/artifacts/build_artifact_python.sh

@@ -21,6 +21,10 @@ export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 export PYTHON=${PYTHON:-python}
 export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 
+# activate ccache if desired
+# shellcheck disable=SC1091
+source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc
+
 # Needed for building binary distribution wheels -- bdist_wheel
 "${PYTHON}" -m pip install --upgrade wheel
 

tools/run_tests/dockerize/build_and_run_docker.sh

@@ -60,11 +60,7 @@ docker run \
   "$@" \
   --cap-add SYS_PTRACE \
   -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \
-  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
-  -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \
-  -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \
-  -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \
-  -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \
+  --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \
   -v "$git_root:/var/local/jenkins/grpc:ro" \
   -w /var/local/git/grpc \
   --name="$CONTAINER_NAME" \

tools/run_tests/dockerize/build_docker_and_run_tests.sh

@@ -66,13 +66,7 @@ docker run \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
   -e HOST_GIT_ROOT="$git_root" \
   -e LOCAL_GIT_ROOT=$docker_instance_git_root \
-  -e "BUILD_ID=$BUILD_ID" \
-  -e "BUILD_URL=$BUILD_URL" \
-  -e "JOB_BASE_NAME=$JOB_BASE_NAME" \
-  -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \
-  -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \
-  -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \
-  -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \
+  --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \
   $DOCKER_TTY_ARGS \
   --sysctl net.ipv6.conf.all.disable_ipv6=0 \
   -v ~/.config/gcloud:/root/.config/gcloud \

tools/run_tests/dockerize/build_interop_image.sh

@@ -113,8 +113,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)"
 # shellcheck disable=SC2086
 (docker run \
   --cap-add SYS_PTRACE \
-  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
-  -e THIS_IS_REALLY_NEEDED_ONCE_AGAIN='For issue 4835. See https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  --env-file "tools/run_tests/dockerize/docker_propagate_env.list" \
   $DOCKER_TTY_ARGS \
   $MOUNT_ARGS \
   $BUILD_INTEROP_DOCKER_EXTRA_ARGS \

tools/run_tests/dockerize/docker_propagate_env.list

@@ -0,0 +1,17 @@
+# Pass this file to "docker run" using --env-file argument.
+# Variables listed in this file will be set or propagated
+# to the docker container.
+
+# TODO(jtattermusch): revisit whether this very old hack is still needed.
+THIS_IS_REALLY_NEEDED="see https://github.com/docker/docker/issues/14203 for why docker is awful"
+
+# Kokoro sets environment variables for each build and we want them to be
+# available inside the test docker containers.
+KOKORO_BUILD_ID
+KOKORO_BUILD_NUMBER
+KOKORO_BUILD_URL
+KOKORO_JOB_NAME
+
+# Propagate ccache configuration to the docker containers.
+GRPC_BUILD_ENABLE_CCACHE
+CCACHE_SECONDARY_STORAGE

tools/run_tests/dockerize/docker_run_tests.sh

@@ -45,4 +45,9 @@ find . -name report.xml -print0 | xargs -0 -r zip reports.zip
 find . -name sponge_log.xml -print0 | xargs -0 -r zip reports.zip
 find . -name 'report_*.xml' -print0 | xargs -0 -r zip reports.zip
 
+if [ -x "$(command -v ccache)" ]
+then
+  ccache --show-stats || true
+fi
+
 exit $exit_code

tools/run_tests/helper_scripts/build_python.sh

@@ -125,17 +125,9 @@ export LANG=en_US.UTF-8
 DEFAULT_PARALLEL_JOBS=$(nproc) || DEFAULT_PARALLEL_JOBS=4
 export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=${GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS:-$DEFAULT_PARALLEL_JOBS}
 
-# If ccache is available on Linux, use it.
-if [ "$(is_linux)" ]; then
-  # We're not on Darwin (Mac OS X)
-  if [ -x "$(command -v ccache)" ]; then
-    if [ -x "$(command -v gcc)" ]; then
-      export CC='ccache gcc'
-    elif [ -x "$(command -v clang)" ]; then
-      export CC='ccache clang'
-    fi
-  fi
-fi
+# activate ccache if desired
+# shellcheck disable=SC1091
+source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc
 
 ############################
 # Perform build operations #