[bazelify tests] Add "concurrent_bazel.sh" wrapper to enable parallel invocations of bazel (#34568)

(I had to first solve a problem with "dubious file ownership" error that
was happening inside the grpc_repo_archive action when using the
concurrent wrapper).

Example:
```
tools/docker_runners/examples/concurrent_bazel.sh --bazelrc=tools/remote_build/linux.bazelrc test --genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:runtests_csharp_linux_dbg
```

templates/tools/dockerfile/git_avoid_dubious_ownership_error_allusers.include

@@ -0,0 +1,10 @@
+#=================
+# Setup git to access working directory across docker boundary.
+# This avoids the "fatal: detected dubious ownership in repository XYZ"
+# git error. Using "--system" makes the setting work
+# for all users or even current user is not properly defined
+# (which can happen e.g. inside execution environment
+# of a bazel action)
+
+RUN git config --system --add safe.directory '*'
+RUN git config --system protocol.file.allow always

templates/tools/dockerfile/test/bazel/Dockerfile.template

@@ -17,6 +17,7 @@
   <%include file="../../oss_fuzz_base.include"/>
   <%include file="../../bazel.include"/>
   <%include file="../../git_avoid_dubious_ownership_error.include"/>
+  <%include file="../../git_avoid_dubious_ownership_error_allusers.include"/>
 
   RUN apt-get install -y gdb
   

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -89,7 +89,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:7b044d6848f82234dba81b38d8eca220b608f830f93b42932df59ed6fe20b24d",
     "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52",
     "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:64ffc5d1e117172ca4dda89720087616830996181192de25fe10e03a88f0b3e5",
-    "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:d3291d3129469bd4bdd58b5dd81eb6c5668548a4d6e6c8e952fb372d1874b078",
+    "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:023fdfd072e278d4129f2c8f8564f6697e767c9847aae8c9411ea853f3faaa6f",
     "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:1b7200c85ba8bc10529e1867a27ee81ad303ec8c8efec8bbc5863b4ee8ac6d03",
     "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:38c1da5544c717ef0e3d3492833b0fd1ce5c70fbe71eea8a9bfb0ab0b1217f1b",
     "tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:4d4bc5f15e03f3d3d8fd889670ecde2c66a2e4d2dd9db80733c05c1d90c8a248",

tools/docker_runners/examples/concurrent_bazel.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This wrapper allows running concurrent instances of bazel build/test by running
+# bazel under a docker container.
+# This is especially useful for running bazel RBE builds, since inside
+# the container, bazel won't have access to local build cache.
+# Access to the local workspace is provided by mounting the workspace
+# as a volume to the docker container. That also means that any changes
+# in the workspace will be visible to the bazel instance running inside
+# the container (but also this is a similar scenario to making changes
+# to local files when bazel is running normally).
+
+# Usage:
+# tools/docker_runners/examples/concurrent_bazel.sh ANY_NORMAL_BAZEL_FLAGS_HERE
+
+set -ex
+
+# change to grpc repo root
+cd "$(dirname "$0")/../../.."
+
+# use the default docker image used for bazel builds
+export DOCKERFILE_DIR=tools/dockerfile/test/bazel
+
+# Bazel RBE uses application default credentials from localhost to authenticate with RBE servers. Use a trick to make the credentails accessible from inside the docker container."
+APPLICATION_DEFAULT_CREDENTIALS_DIR="$HOME/.config/gcloud"
+export DOCKER_EXTRA_ARGS="-v=${APPLICATION_DEFAULT_CREDENTIALS_DIR}:/application_default_credentials:ro -e=GOOGLE_APPLICATION_CREDENTIALS=/application_default_credentials/application_default_credentials.json"
+
+# Run bazel inside a docker container (local git workspace will be mounted to the container)
+tools/docker_runners/run_in_docker.sh bazel "$@"

tools/dockerfile/test/bazel.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:d64f53cefe4fe5908e00130a7a0dbe6253d9f0b8@sha256:d3291d3129469bd4bdd58b5dd81eb6c5668548a4d6e6c8e952fb372d1874b078
+us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:c8658dbcb51352d5e9c5d9f15282835e45d5ce80@sha256:023fdfd072e278d4129f2c8f8564f6697e767c9847aae8c9411ea853f3faaa6f

tools/dockerfile/test/bazel/Dockerfile

@@ -65,6 +65,17 @@ RUN BAZEL_ARCH_SUFFIX="$(uname -m | sed s/aarch64/arm64/)" \
 RUN git config --global --add safe.directory '*'
 RUN git config --global protocol.file.allow always
 
+#=================
+# Setup git to access working directory across docker boundary.
+# This avoids the "fatal: detected dubious ownership in repository XYZ"
+# git error. Using "--system" makes the setting work
+# for all users or even current user is not properly defined
+# (which can happen e.g. inside execution environment
+# of a bazel action)
+
+RUN git config --system --add safe.directory '*'
+RUN git config --system protocol.file.allow always
+
 
 RUN apt-get install -y gdb