From 288fe131b435c1374698b1cce856ec755d015da0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 15 Jun 2021 18:50:13 +0200 Subject: [PATCH] Reintroduce #22964 with extra fixes (#26465) * Revert "Revert "Upgrade dotnet SDK 3 for C# build (#22964)" (#26462)" This reverts commit 736d914cfd5cf184040efd8ca2224f5caf6fc107. * Revert "Exclude csharp from performance test continuous runs. (#26458)" This reverts commit 6744e8f84ae2c012053d22b1cc2f336498678f16. * switch remaining tests to C# buster image * remove C#'s global.json (which requires dotnet SDK 3+) * dotnet SDK3 currently breaks C# package build * regenerate dockerfiles --- src/csharp/install_dotnet_sdk.ps1 | 5 ++++- templates/tools/dockerfile/csharp_deps.include | 2 +- .../dockerfile/csharp_dotnetcli_deps.include | 12 +----------- .../grpc_interop_csharp/Dockerfile.template | 2 +- .../Dockerfile.template | 2 +- .../Dockerfile.template | 2 +- .../interoptest/grpc_interop_csharp/Dockerfile | 16 +++------------- .../grpc_interop_csharpcoreclr/Dockerfile | 16 +++------------- .../Dockerfile | 16 +++------------- .../linux/grpc_e2e_performance_gke.sh | 5 +++-- .../internal_ci/linux/grpc_e2e_performance_v2.sh | 5 +++-- tools/internal_ci/linux/grpc_xds_csharp.sh | 2 +- tools/internal_ci/linux/grpc_xds_v3_csharp.sh | 2 +- tools/run_tests/artifacts/package_targets.py | 4 ++-- tools/run_tests/run_tests.py | 2 +- 15 files changed, 29 insertions(+), 64 deletions(-) rename templates/tools/dockerfile/test/{csharp_stretch_x64 => csharp_buster_x64}/Dockerfile.template (97%) rename tools/dockerfile/test/{csharp_stretch_x64 => csharp_buster_x64}/Dockerfile (81%) diff --git a/src/csharp/install_dotnet_sdk.ps1 b/src/csharp/install_dotnet_sdk.ps1 index 1685e417822..779c5673763 100644 --- a/src/csharp/install_dotnet_sdk.ps1 +++ b/src/csharp/install_dotnet_sdk.ps1 @@ -19,4 +19,7 @@ $InstallScriptPath = Join-Path "$env:TEMP" 'dotnet-install.ps1' # Download install script Write-Host "Downloading install script: $InstallScriptUrl => $InstallScriptPath" Invoke-WebRequest -Uri $InstallScriptUrl -OutFile $InstallScriptPath -&$InstallScriptPath -Version 2.1.504 + +# Installed versions should be kept in sync with +# templates/tools/dockerfile/csharp_dotnetcli_deps.include +&$InstallScriptPath -Version 2.1.816 diff --git a/templates/tools/dockerfile/csharp_deps.include b/templates/tools/dockerfile/csharp_deps.include index fdede9e840b..f959be2569a 100644 --- a/templates/tools/dockerfile/csharp_deps.include +++ b/templates/tools/dockerfile/csharp_deps.include @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean # Install mono RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list +RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list RUN apt-get update && apt-get install -y ${'\\'} mono-devel ${'\\'} ca-certificates-mono ${'\\'} diff --git a/templates/tools/dockerfile/csharp_dotnetcli_deps.include b/templates/tools/dockerfile/csharp_dotnetcli_deps.include index f3335926402..073d6b3b092 100644 --- a/templates/tools/dockerfile/csharp_dotnetcli_deps.include +++ b/templates/tools/dockerfile/csharp_dotnetcli_deps.include @@ -1,21 +1,11 @@ # Install dotnet SDK -ENV DOTNET_SDK_VERSION 2.1.500 +ENV DOTNET_SDK_VERSION 2.1.816 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz ${'\\'} && mkdir -p /usr/share/dotnet ${'\\'} && tar -zxf dotnet.tar.gz -C /usr/share/dotnet ${'\\'} && rm dotnet.tar.gz ${'\\'} && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - -# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1) -RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz ${'\\'} - && mkdir -p dotnet_old ${'\\'} - && tar zxf dotnet_old.tar.gz -C dotnet_old ${'\\'} - && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ ${'\\'} - && rm -rf dotnet_old/ dotnet_old.tar.gz -RUN apt-get update && apt-get install -y libunwind8 && apt-get clean - - # Trigger the population of the local package cache ENV NUGET_XMLDOC_MODE skip RUN mkdir warmup ${'\\'} diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template index abf1a3853d3..640c7d6dcc6 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:stretch + FROM debian:buster <%include file="../../apt_get_basic.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template index abf1a3853d3..640c7d6dcc6 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:stretch + FROM debian:buster <%include file="../../apt_get_basic.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/test/csharp_stretch_x64/Dockerfile.template b/templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template similarity index 97% rename from templates/tools/dockerfile/test/csharp_stretch_x64/Dockerfile.template rename to templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template index 355b47e18a0..8d6c11932b5 100644 --- a/templates/tools/dockerfile/test/csharp_stretch_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:stretch + FROM debian:buster <%include file="../../apt_get_basic.include"/> <%include file="../../python_deps.include"/> diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 89a4261ffde..e022ba2ab72 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:stretch +FROM debian:buster # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean # Install mono RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list +RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ @@ -82,23 +82,13 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # Install dotnet SDK -ENV DOTNET_SDK_VERSION 2.1.500 +ENV DOTNET_SDK_VERSION 2.1.816 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \ && mkdir -p /usr/share/dotnet \ && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \ && rm dotnet.tar.gz \ && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - -# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1) -RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \ - && mkdir -p dotnet_old \ - && tar zxf dotnet_old.tar.gz -C dotnet_old \ - && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \ - && rm -rf dotnet_old/ dotnet_old.tar.gz -RUN apt-get update && apt-get install -y libunwind8 && apt-get clean - - # Trigger the population of the local package cache ENV NUGET_XMLDOC_MODE skip RUN mkdir warmup \ diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index 89a4261ffde..e022ba2ab72 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:stretch +FROM debian:buster # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean # Install mono RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list +RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ @@ -82,23 +82,13 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # Install dotnet SDK -ENV DOTNET_SDK_VERSION 2.1.500 +ENV DOTNET_SDK_VERSION 2.1.816 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \ && mkdir -p /usr/share/dotnet \ && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \ && rm dotnet.tar.gz \ && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - -# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1) -RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \ - && mkdir -p dotnet_old \ - && tar zxf dotnet_old.tar.gz -C dotnet_old \ - && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \ - && rm -rf dotnet_old/ dotnet_old.tar.gz -RUN apt-get update && apt-get install -y libunwind8 && apt-get clean - - # Trigger the population of the local package cache ENV NUGET_XMLDOC_MODE skip RUN mkdir warmup \ diff --git a/tools/dockerfile/test/csharp_stretch_x64/Dockerfile b/tools/dockerfile/test/csharp_buster_x64/Dockerfile similarity index 81% rename from tools/dockerfile/test/csharp_stretch_x64/Dockerfile rename to tools/dockerfile/test/csharp_buster_x64/Dockerfile index 892be06b38f..473b4904435 100644 --- a/tools/dockerfile/test/csharp_stretch_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_buster_x64/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:stretch +FROM debian:buster # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean # Install mono RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list +RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ @@ -85,23 +85,13 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # Install dotnet SDK -ENV DOTNET_SDK_VERSION 2.1.500 +ENV DOTNET_SDK_VERSION 2.1.816 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \ && mkdir -p /usr/share/dotnet \ && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \ && rm dotnet.tar.gz \ && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - -# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1) -RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \ - && mkdir -p dotnet_old \ - && tar zxf dotnet_old.tar.gz -C dotnet_old \ - && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \ - && rm -rf dotnet_old/ dotnet_old.tar.gz -RUN apt-get update && apt-get install -y libunwind8 && apt-get clean - - # Trigger the population of the local package cache ENV NUGET_XMLDOC_MODE skip RUN mkdir warmup \ diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index 62c43456291..674e967ba8f 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -88,8 +88,8 @@ buildConfigs() { -o "./loadtest_with_prebuilt_workers_${pool}.yaml" } -buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l go -l java -l python -l ruby -buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l go -l java +buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l csharp -l go -l java -l python -l ruby +buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l csharp -l go -l java # Delete prebuilt images on exit. deleteImages() { @@ -103,6 +103,7 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ -l "cxx:${GRPC_CORE_GITREF}" \ + -l "csharp:${GRPC_CORE_GITREF}" \ -l "go:${GRPC_GO_GITREF}" \ -l "java:${GRPC_JAVA_GITREF}" \ -l "python:${GRPC_CORE_GITREF}" \ diff --git a/tools/internal_ci/linux/grpc_e2e_performance_v2.sh b/tools/internal_ci/linux/grpc_e2e_performance_v2.sh index e449abf10db..9e60c4cba82 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_v2.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_v2.sh @@ -89,8 +89,8 @@ buildConfigs() { -o "./loadtest_with_prebuilt_workers_${pool}.yaml" } -buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l go -l java -l python -l ruby -buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l go -l java +buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l csharp -l go -l java -l python -l ruby +buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l csharp -l go -l java # Delete prebuilt images on exit. deleteImages() { @@ -104,6 +104,7 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ -l "cxx:${GRPC_CORE_GITREF}" \ + -l "csharp:${GRPC_CORE_GITREF}" \ -l "go:${GRPC_GO_GITREF}" \ -l "java:${GRPC_JAVA_GITREF}" \ -l "python:${GRPC_CORE_GITREF}" \ diff --git a/tools/internal_ci/linux/grpc_xds_csharp.sh b/tools/internal_ci/linux/grpc_xds_csharp.sh index 26bb7e35444..3767a60caa4 100755 --- a/tools/internal_ci/linux/grpc_xds_csharp.sh +++ b/tools/internal_ci/linux/grpc_xds_csharp.sh @@ -20,7 +20,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -export DOCKERFILE_DIR=tools/dockerfile/test/csharp_stretch_x64 +export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64 export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh export OUTPUT_DIR=reports exec tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/internal_ci/linux/grpc_xds_v3_csharp.sh b/tools/internal_ci/linux/grpc_xds_v3_csharp.sh index 64b986c309e..0286581c119 100755 --- a/tools/internal_ci/linux/grpc_xds_v3_csharp.sh +++ b/tools/internal_ci/linux/grpc_xds_v3_csharp.sh @@ -20,7 +20,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -export DOCKERFILE_DIR=tools/dockerfile/test/csharp_stretch_x64 +export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64 export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_xds_v3_csharp_test_in_docker.sh export OUTPUT_DIR=reports exec tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 7123d180f5e..01dc7c03418 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -90,11 +90,11 @@ class CSharpPackage: def build_jobspec(self): if self.unity: return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_stretch_x64', + self.name, 'tools/dockerfile/test/csharp_buster_x64', 'src/csharp/build_unitypackage.sh') else: return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_stretch_x64', + self.name, 'tools/dockerfile/test/csharp_buster_x64', 'src/csharp/build_nuget.sh') def __str__(self): diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index afb96226dbb..f61387b5165 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -922,7 +922,7 @@ class CSharpLanguage(object): self._cmake_arch_option = 'x64' else: _check_compiler(self.args.compiler, ['default', 'coreclr']) - self._docker_distro = 'stretch' + self._docker_distro = 'buster' def test_specs(self): with open('src/csharp/tests.json') as f: