natas_hw
/
grpc
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RBE] Updated RBE Linux image (#36685) 

A regular upgrade but it's an attempt to resolve false-positive ASAN errors found in https://github.com/grpc/grpc/pull/36626 ([log](https://btx.cloud.google.com/invocations/7e003b33-401b-48a3-963f-13d4807ba970/targets))
- Upgraded Clang 15 to 19
- Upgraded Bazel 6 to 7

Closes #36685

PiperOrigin-RevId: 637069576
This commit is contained in:
Esun Kim 2024-05-24 16:51:14 -07:00 committed by Copybara-Service
parent 621aa4e5ce
commit 6c4c3f5b51
12 changed files with 925 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
templates/tools/dockerfile/test/rbe_ubuntu2004/Dockerfile.template
View File

@ -35,7 +35,7 @@
# * for sanitizers to work, we need MSAN and TSAN enabled versions of libc++
# Check https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/latest.txt
# for the value of LLVM_LATEST.
ARG LLVM_LATEST="f2b94bd7eaa83d853dc7568fac87b1f8bf4ddec6"
ARG LLVM_LATEST="3b5e7c83a6e226d5bd7ed2e9b67449b64812074c"
RUN wget -q https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/clang_r<%text>${LLVM_LATEST}</%text>.tar.gz -O /tmp/clang.tar.gz ${'\\'}
&& tar -xzf /tmp/clang.tar.gz -C /usr/local && rm /tmp/clang.tar.gz
RUN wget -q https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/libcxx_r<%text>${LLVM_LATEST}</%text>.tar.gz -O /tmp/libcxx.tar.gz ${'\\'}
@ -50,7 +50,8 @@
ENV ASAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV CC=/usr/local/bin/clang
ENV GCOV=/dev/null
ENV LD_LIBRARY_PATH=/usr/local/lib/x86_64-unknown-linux-gnu
ENV LIBRARY_PATH=/usr/local/lib
ENV LD_LIBRARY_PATH=/usr/local/lib
ENV MSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV TSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV UBSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer

6
third_party/toolchains/generate_linux_rbe_configs.sh vendored
View File

@ -40,9 +40,9 @@ LINUX_RBE_DOCKERFILE_DIR=tools/dockerfile/test/rbe_ubuntu2004
LINUX_RBE_DOCKER_IMAGE=$(cat ${LINUX_RBE_DOCKERFILE_DIR}.current_version)
# Bazel version used for configuring
# Needs to be one of the versions from bazel/supported_versions.txt chosen so that the result is compatible
# with other supported bazel versions.
BAZEL_VERSION=6.1.2
# Needs to be the oldest of the versions from bazel/supported_versions.txt chosen
# so that the result is compatible with other supported bazel versions.
BAZEL_VERSION=6.5.0
# TODO(jtattermusch): experiment with --cpp_env_json to simplify bazel build configuration.

9
third_party/toolchains/rbe_ubuntu2004/cc/BUILD vendored
View File

@ -80,22 +80,21 @@ cc_toolchain_config(
target_libc = "glibc_2.19",
abi_version = "clang",
abi_libc_version = "glibc_2.19",
cxx_builtin_include_directories = ["/usr/local/lib/clang/15.0.0/include",
cxx_builtin_include_directories = ["/usr/local/lib/clang/19/include",
"/usr/local/include",
"/usr/include/x86_64-linux-gnu",
"/usr/include",
"/usr/local/lib/clang/15.0.0/share",
"/usr/local/lib/clang/19/share",
"/usr/include/c++/9",
"/usr/include/x86_64-linux-gnu/c++/9",
"/usr/include/c++/9/backward",
"/usr/local/include/x86_64-unknown-linux-gnu/c++/v1",
"/usr/local/include/c++/v1"],
tool_paths = {"ar": "/usr/bin/ar",
"ld": "/usr/bin/ld",
"llvm-cov": "/usr/local/bin/llvm-cov",
"llvm-profdata": "/usr/local/bin/llvm-profdata",
"cpp": "/usr/bin/cpp",
"gcc": "/usr/local/bin/clang-15",
"gcc": "/usr/local/bin/clang-19",
"dwp": "/usr/bin/dwp",
"gcov": "/dev/null",
"nm": "/usr/bin/nm",
@ -117,7 +116,7 @@ cc_toolchain_config(
"-ffunction-sections",
"-fdata-sections"],
dbg_compile_flags = ["-g"],
cxx_flags = ["-std=c++0x"],
cxx_flags = ["-std=c++14"],
link_flags = ["-fuse-ld=/usr/local/bin/ld.lld",
"-Wl,-no-as-needed",
"-Wl,-z,relro,-z,now",

7
third_party/toolchains/rbe_ubuntu2004/cc/builtin_include_directory_paths vendored
View File

@ -1,16 +1,15 @@
This file is generated by cc_configure and contains builtin include directories
that /usr/local/bin/clang-15 reported. This file is a dependency of every compilation action and
that /usr/local/bin/clang-19 reported. This file is a dependency of every compilation action and
changes to it will be reflected in the action cache key. When some of these
paths change, Bazel will make sure to rerun the action, even though none of
declared action inputs or the action commandline changes.
/usr/local/lib/clang/15.0.0/include
/usr/local/lib/clang/19/include
/usr/local/include
/usr/include/x86_64-linux-gnu
/usr/include
/usr/local/lib/clang/15.0.0/share
/usr/local/lib/clang/19/share
/usr/include/c++/9
/usr/include/x86_64-linux-gnu/c++/9
/usr/include/c++/9/backward
/usr/local/include/x86_64-unknown-linux-gnu/c++/v1
/usr/local/include/c++/v1

27
third_party/toolchains/rbe_ubuntu2004/cc/cc_toolchain_config.bzl vendored
View File

@ -811,6 +811,24 @@ def _impl(ctx):
],
)
is_linux = ctx.attr.target_libc != "macosx"
if is_linux:
versioned_library_flag_group = flag_group(
flags = ["-l:%{libraries_to_link.name}"],
expand_if_equal = variable_with_value(
name = "libraries_to_link.type",
value = "versioned_dynamic_library",
),
)
else:
versioned_library_flag_group = flag_group(
flags = ["%{libraries_to_link.path}"],
expand_if_equal = variable_with_value(
name = "libraries_to_link.type",
value = "versioned_dynamic_library",
),
)
libraries_to_link_feature = feature(
name = "libraries_to_link",
flag_sets = [
@ -868,13 +886,7 @@ def _impl(ctx):
value = "dynamic_library",
),
),
flag_group(
flags = ["-l:%{libraries_to_link.name}"],
expand_if_equal = variable_with_value(
name = "libraries_to_link.type",
value = "versioned_dynamic_library",
),
),
versioned_library_flag_group,
flag_group(
flags = ["-Wl,-no-whole-archive"],
expand_if_true = "libraries_to_link.is_whole_archive",
@ -1283,7 +1295,6 @@ def _impl(ctx):
],
)
is_linux = ctx.attr.target_libc != "macosx"
libtool_feature = feature(
name = "libtool",
enabled = not is_linux,

2
third_party/toolchains/rbe_ubuntu2004/cc/cc_wrapper.sh vendored
View File

@ -22,4 +22,4 @@ set -eu
# Call the C++ compiler
/usr/local/bin/clang-15 "$@"
/usr/local/bin/clang-19 "$@"

1168
third_party/toolchains/rbe_ubuntu2004/cc/module.modulemap vendored
View File

File diff suppressed because it is too large Load Diff

2
third_party/toolchains/rbe_ubuntu2004/config/BUILD vendored
View File

@ -42,7 +42,7 @@ platform(
"@bazel_tools//tools/cpp:clang",
],
exec_properties = {
"container-image": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8",
"container-image": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:b3eb1a17b7b091e3c5648a803076b2c40601242ff91c04d55997af6641305f68",
"OSFamily": "Linux",
},
)

2
tools/bazelify_tests/dockerimage_current_versions.bzl
View File

@ -110,7 +110,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
"tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:c541f0ce00ca542ed4ae86870b6b73c929227127c13e532d98a9afd28604b830",
"tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:fccca33a655c7aa89dd7ebd9492cbcc1f636bd2a004cd939d1982cfce3d68326",
"tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:4c539fc93d343324309939c2964204c90f4054cd9eeed093af315cb0f0ef7c26",
"tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:4393e3fd8481fb4a391143cd5604299a299d94814a49eb5b7f310523ffbb6e13",
"tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:b3eb1a17b7b091e3c5648a803076b2c40601242ff91c04d55997af6641305f68",
"tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:d2e79919b2e2d4cc36a29682ecb5170641df4fb506cfb453978ffdeb8a841bd9",
"tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:6e8b4696ba0661f11a31ed0992a94d2efcd889a018f57160f0e2fb62963f3593",
"tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:4b3bc3934f41dc1e123db3123841af84f07dd68f44b564e29ba1d85bb21df707",

2
tools/dockerfile/test/rbe_ubuntu2004.current_version
View File

@ -1 +1 @@
us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004:58594aefea3481a4839f8411e365b58ef9f34fcc@sha256:4393e3fd8481fb4a391143cd5604299a299d94814a49eb5b7f310523ffbb6e13
us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004:087453aa2851cb7b77b581c0b366ca4b53efa862@sha256:b3eb1a17b7b091e3c5648a803076b2c40601242ff91c04d55997af6641305f68

5
tools/dockerfile/test/rbe_ubuntu2004/Dockerfile
View File

@ -57,7 +57,7 @@ RUN ln -s /usr/bin/python3 /usr/bin/python
# * for sanitizers to work, we need MSAN and TSAN enabled versions of libc++
# Check https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/latest.txt
# for the value of LLVM_LATEST.
ARG LLVM_LATEST="f2b94bd7eaa83d853dc7568fac87b1f8bf4ddec6"
ARG LLVM_LATEST="3b5e7c83a6e226d5bd7ed2e9b67449b64812074c"
RUN wget -q https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/clang_r${LLVM_LATEST}.tar.gz -O /tmp/clang.tar.gz \
&& tar -xzf /tmp/clang.tar.gz -C /usr/local && rm /tmp/clang.tar.gz
RUN wget -q https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04/libcxx_r${LLVM_LATEST}.tar.gz -O /tmp/libcxx.tar.gz \
@ -72,7 +72,8 @@ RUN wget -q https://storage.googleapis.com/clang-builds-stable/clang-ubuntu20_04
ENV ASAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV CC=/usr/local/bin/clang
ENV GCOV=/dev/null
ENV LD_LIBRARY_PATH=/usr/local/lib/x86_64-unknown-linux-gnu
ENV LIBRARY_PATH=/usr/local/lib
ENV LD_LIBRARY_PATH=/usr/local/lib
ENV MSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV TSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer
ENV UBSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer

12
tools/remote_build/linux.bazelrc
View File

@ -57,7 +57,11 @@ build:msan --linkopt=--stdlib=libc++
# use MSAN-instrumented version of libc++
# TODO(jtattermusch): include the modified LD_LIBRARY_PATH in an msan-specific
# toolchain configuration, instead of needing to explicitly set it here.
build:msan --action_env=LD_LIBRARY_PATH=/usr/local/libcxx-msan/lib/x86_64-unknown-linux-gnu
build:msan --action_env=LD_LIBRARY_PATH=/usr/local/libcxx-msan/lib
# TODO(veblush): Remove the following after upgrading cloud-cpp-cloud to
# the version having https://github.com/googleapis/google-cloud-cpp/pull/14151
build:msan --cxxopt='-std=c++17'
build:msan --host_cxxopt='-std=c++17'
# thread sanitizer: most settings are already in %workspace%/.bazelrc
# we only need a few additional ones that are Foundry specific
@ -84,4 +88,8 @@ build:ubsan --action_env=MSAN_OPTIONS=halt_on_error=0
build:ubsan --test_tag_filters=-no_linux,-noubsan
# use MSAN-instrumented version of libc++
# TODO(jtattermusch): run UBSAN without enabling MSAN too.
build:ubsan --action_env=LD_LIBRARY_PATH=/usr/local/libcxx-msan/lib/x86_64-unknown-linux-gnu
build:ubsan --action_env=LD_LIBRARY_PATH=/usr/local/libcxx-msan/lib
# TODO(veblush): Remove the following after upgrading cloud-cpp-cloud to
# the version having https://github.com/googleapis/google-cloud-cpp/pull/14151
build:ubsan --cxxopt='-std=c++17'
build:ubsan --host_cxxopt='-std=c++17'