Reintroduce #22964 with extra fixes (#26465)

* Revert "Revert "Upgrade dotnet SDK 3 for C# build (#22964)" (#26462)"

This reverts commit 736d914cfd.

* Revert "Exclude csharp from performance test continuous runs. (#26458)"

This reverts commit 6744e8f84a.

* switch remaining tests to C# buster image

* remove C#'s global.json (which requires dotnet SDK 3+)

* dotnet SDK3 currently breaks C# package build

* regenerate dockerfiles

src/csharp/install_dotnet_sdk.ps1

@@ -19,4 +19,7 @@ $InstallScriptPath = Join-Path  "$env:TEMP" 'dotnet-install.ps1'
 # Download install script
 Write-Host "Downloading install script: $InstallScriptUrl => $InstallScriptPath"
 Invoke-WebRequest -Uri $InstallScriptUrl -OutFile $InstallScriptPath
-&$InstallScriptPath -Version 2.1.504
+
+# Installed versions should be kept in sync with
+# templates/tools/dockerfile/csharp_dotnetcli_deps.include
+&$InstallScriptPath -Version 2.1.816

templates/tools/dockerfile/csharp_deps.include

@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y ${'\\'}
     mono-devel ${'\\'}
     ca-certificates-mono ${'\\'}

templates/tools/dockerfile/csharp_dotnetcli_deps.include

@@ -1,21 +1,11 @@
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 2.1.816
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz ${'\\'}
     && mkdir -p /usr/share/dotnet ${'\\'}
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet ${'\\'}
     && rm dotnet.tar.gz ${'\\'}
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz ${'\\'}
-    && mkdir -p dotnet_old ${'\\'}
-    && tar zxf dotnet_old.tar.gz -C dotnet_old ${'\\'}
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ ${'\\'}
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
-
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip
 RUN mkdir warmup ${'\\'}

templates/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

templates/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

templates/tools/dockerfile/test/csharp_buster_x64/Dockerfile.template

@@ -14,7 +14,7 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:stretch
+  FROM debian:buster
   
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../python_deps.include"/>

tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -82,23 +82,13 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 2.1.816
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
-
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip
 RUN mkdir warmup \

tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -74,7 +74,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -82,23 +82,13 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 2.1.816
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
-
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip
 RUN mkdir warmup \

tools/dockerfile/test/csharp_buster_x64/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:stretch
+FROM debian:buster
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y cmake && apt-get clean
 # Install mono
 RUN apt-get update && apt-get install -y apt-transport-https dirmngr && apt-get clean
 RUN apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-RUN echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
 RUN apt-get update && apt-get install -y \
     mono-devel \
     ca-certificates-mono \
@@ -85,23 +85,13 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean
 
 # Install dotnet SDK
-ENV DOTNET_SDK_VERSION 2.1.500
+ENV DOTNET_SDK_VERSION 2.1.816
 RUN curl -sSL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \
     && mkdir -p /usr/share/dotnet \
     && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \
     && rm dotnet.tar.gz \
     && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 
-
-# Install .NET Core 1.1.10 runtime (required to run netcoreapp1.1)
-RUN curl -sSL -o dotnet_old.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Runtime/1.1.10/dotnet-debian.9-x64.1.1.10.tar.gz \
-    && mkdir -p dotnet_old \
-    && tar zxf dotnet_old.tar.gz -C dotnet_old \
-    && cp -r dotnet_old/shared/Microsoft.NETCore.App/1.1.10/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ \
-    && rm -rf dotnet_old/ dotnet_old.tar.gz
-RUN apt-get update && apt-get install -y libunwind8 && apt-get clean
-
-
 # Trigger the population of the local package cache
 ENV NUGET_XMLDOC_MODE skip
 RUN mkdir warmup \

tools/internal_ci/linux/grpc_e2e_performance_gke.sh

@@ -88,8 +88,8 @@ buildConfigs() {
         -o "./loadtest_with_prebuilt_workers_${pool}.yaml"
 }
 
-buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l go -l java -l python -l ruby
-buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l go -l java
+buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l csharp -l go -l java -l python -l ruby
+buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l csharp -l go -l java
 
 # Delete prebuilt images on exit.
 deleteImages() {
@@ -103,6 +103,7 @@ trap deleteImages EXIT
 # Build and push prebuilt images for running tests.
 time ../test-infra/bin/prepare_prebuilt_workers \
     -l "cxx:${GRPC_CORE_GITREF}" \
+    -l "csharp:${GRPC_CORE_GITREF}" \
     -l "go:${GRPC_GO_GITREF}" \
     -l "java:${GRPC_JAVA_GITREF}" \
     -l "python:${GRPC_CORE_GITREF}" \

tools/internal_ci/linux/grpc_e2e_performance_v2.sh

@@ -89,8 +89,8 @@ buildConfigs() {
         -o "./loadtest_with_prebuilt_workers_${pool}.yaml"
 }
 
-buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l go -l java -l python -l ruby
-buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l go -l java
+buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -l c++ -l csharp -l go -l java -l python -l ruby
+buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" -l c++ -l csharp -l go -l java
 
 # Delete prebuilt images on exit.
 deleteImages() {
@@ -104,6 +104,7 @@ trap deleteImages EXIT
 # Build and push prebuilt images for running tests.
 time ../test-infra/bin/prepare_prebuilt_workers \
     -l "cxx:${GRPC_CORE_GITREF}" \
+    -l "csharp:${GRPC_CORE_GITREF}" \
     -l "go:${GRPC_GO_GITREF}" \
     -l "java:${GRPC_JAVA_GITREF}" \
     -l "python:${GRPC_CORE_GITREF}" \

tools/internal_ci/linux/grpc_xds_csharp.sh

@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-export DOCKERFILE_DIR=tools/dockerfile/test/csharp_stretch_x64
+export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64
 export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh
 export OUTPUT_DIR=reports
 exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/grpc_xds_v3_csharp.sh

@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-export DOCKERFILE_DIR=tools/dockerfile/test/csharp_stretch_x64
+export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64
 export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_xds_v3_csharp_test_in_docker.sh
 export OUTPUT_DIR=reports
 exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/run_tests/artifacts/package_targets.py

@@ -90,11 +90,11 @@ class CSharpPackage:
     def build_jobspec(self):
         if self.unity:
             return create_docker_jobspec(
-                self.name, 'tools/dockerfile/test/csharp_stretch_x64',
+                self.name, 'tools/dockerfile/test/csharp_buster_x64',
                 'src/csharp/build_unitypackage.sh')
         else:
             return create_docker_jobspec(
-                self.name, 'tools/dockerfile/test/csharp_stretch_x64',
+                self.name, 'tools/dockerfile/test/csharp_buster_x64',
                 'src/csharp/build_nuget.sh')
 
     def __str__(self):

tools/run_tests/run_tests.py

@@ -922,7 +922,7 @@ class CSharpLanguage(object):
             self._cmake_arch_option = 'x64'
         else:
             _check_compiler(self.args.compiler, ['default', 'coreclr'])
-            self._docker_distro = 'stretch'
+            self._docker_distro = 'buster'
 
     def test_specs(self):
         with open('src/csharp/tests.json') as f: