Merge pull request #5369 from jtattermusch/run_tests_improvements

Run tests improvements

templates/tools/dockerfile/apt_get_basic.include

@@ -1,3 +1,4 @@
+<%page args="skip_golang=False"/>\
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y ${'\\'}
   autoconf ${'\\'}
@@ -9,7 +10,9 @@ RUN apt-get update && apt-get install -y ${'\\'}
   gcc ${'\\'}
   gcc-multilib ${'\\'}
   git ${'\\'}
+% if not skip_golang:
   golang ${'\\'}
+% endif
   gyp ${'\\'}
   lcov ${'\\'}
   libc6 ${'\\'}

templates/tools/dockerfile/run_tests_addons.include

@@ -1,7 +1,10 @@
+<%page args="skip_zookeeper=False"/>\
 <%include file="ccache_setup.include"/>
+% if not skip_zookeeper:
 #======================
 # Zookeeper dependencies
 # TODO(jtattermusch): is zookeeper still needed?
 RUN apt-get install -y libzookeeper-mt-dev
+% endif
 
 RUN mkdir /var/local/jenkins

templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template

@@ -0,0 +1,45 @@
+%YAML 1.2
+--- |
+  # Copyright 2015-2016, Google Inc.
+  # All rights reserved.
+  #
+  # Redistribution and use in source and binary forms, with or without
+  # modification, are permitted provided that the following conditions are
+  # met:
+  #
+  #     * Redistributions of source code must retain the above copyright
+  # notice, this list of conditions and the following disclaimer.
+  #     * Redistributions in binary form must reproduce the above
+  # copyright notice, this list of conditions and the following disclaimer
+  # in the documentation and/or other materials provided with the
+  # distribution.
+  #     * Neither the name of Google Inc. nor the names of its
+  # contributors may be used to endorse or promote products derived from
+  # this software without specific prior written permission.
+  #
+  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  
+  FROM debian:squeeze
+  
+  <%include file="../../apt_get_basic.include" args="skip_golang=True"/>
+  
+  # libgflags-dev is not available on squeezy
+  RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean
+  
+  RUN apt-get update && apt-get -y install python-pip && apt-get clean
+  RUN pip install argparse
+  
+  <%include file="../../run_tests_addons.include" args="skip_zookeeper=True"/>
+  # Define the default command.
+  CMD ["bash"]
+  

templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template

@@ -0,0 +1,39 @@
+%YAML 1.2
+--- |
+  # Copyright 2015-2016, Google Inc.
+  # All rights reserved.
+  #
+  # Redistribution and use in source and binary forms, with or without
+  # modification, are permitted provided that the following conditions are
+  # met:
+  #
+  #     * Redistributions of source code must retain the above copyright
+  # notice, this list of conditions and the following disclaimer.
+  #     * Redistributions in binary form must reproduce the above
+  # copyright notice, this list of conditions and the following disclaimer
+  # in the documentation and/or other materials provided with the
+  # distribution.
+  #     * Neither the name of Google Inc. nor the names of its
+  # contributors may be used to endorse or promote products derived from
+  # this software without specific prior written permission.
+  #
+  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  
+  FROM ubuntu:16.04
+  
+  <%include file="../../apt_get_basic.include"/>
+  <%include file="../../cxx_deps.include"/>
+  <%include file="../../run_tests_addons.include"/>
+  # Define the default command.
+  CMD ["bash"]
+  

tools/dockerfile/test/cxx_squeeze_x64/Dockerfile

@@ -0,0 +1,84 @@
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM debian:squeeze
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  gyp \
+  lcov \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  perl \
+  strace \
+  python-dev \
+  python-setuptools \
+  python-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+#================
+# Build profiling
+RUN apt-get update && apt-get install -y time && apt-get clean
+
+
+# libgflags-dev is not available on squeezy
+RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean
+
+RUN apt-get update && apt-get -y install python-pip && apt-get clean
+RUN pip install argparse
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile

@@ -0,0 +1,86 @@
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM ubuntu:16.04
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  golang \
+  gyp \
+  lcov \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  perl \
+  strace \
+  python-dev \
+  python-setuptools \
+  python-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+#================
+# Build profiling
+RUN apt-get update && apt-get install -y time && apt-get clean
+
+#=================
+# C++ dependencies
+RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+#======================
+# Zookeeper dependencies
+# TODO(jtattermusch): is zookeeper still needed?
+RUN apt-get install -y libzookeeper-mt-dev
+
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]

tools/run_tests/run_tests.py

@@ -55,8 +55,8 @@ import report_utils
 import watch_dirs
 
 
-ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
-os.chdir(ROOT)
+os.chdir(_ROOT)
 
 
 _FORCE_ENVIRON_FOR_WRAPPERS = {}
@@ -118,6 +118,16 @@ def get_c_tests(travis, test_lang) :
                 not (travis and tgt['flaky'])]
 
 
+def _check_compiler(compiler, supported_compilers):
+  if compiler not in supported_compilers:
+    raise Exception('Compiler %s not supported.' % compiler)
+
+
+def _is_use_docker_child():
+  """Returns True if running running as a --use_docker child."""
+  return True if os.getenv('RUN_TESTS_COMMAND') else False
+
+
 class CLanguage(object):
 
   def __init__(self, make_target, test_lang):
@@ -125,44 +135,55 @@ class CLanguage(object):
     self.platform = platform_string()
     self.test_lang = test_lang
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    if self.platform == 'windows':
+      self._make_options = [_windows_toolset_option(self.args.compiler),
+                            _windows_arch_option(self.args.arch)]
+    else:
+      self._make_options = []
+      self._docker_distro = self._get_docker_distro(self.args.use_docker,
+                                                    self.args.compiler)
+
+  def test_specs(self):
     out = []
-    binaries = get_c_tests(args.travis, self.test_lang)
+    binaries = get_c_tests(self.args.travis, self.test_lang)
     for target in binaries:
-      if config.build_config in target['exclude_configs']:
+      if self.config.build_config in target['exclude_configs']:
         continue
       if self.platform == 'windows':
         binary = 'vsprojects/%s%s/%s.exe' % (
-            'x64/' if args.arch == 'x64' else '',
+            'x64/' if self.args.arch == 'x64' else '',
-            _WINDOWS_CONFIG[config.build_config],
+            _MSBUILD_CONFIG[self.config.build_config],
             target['name'])
       else:
-        binary = 'bins/%s/%s' % (config.build_config, target['name'])
+        binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
       if os.path.isfile(binary):
         cmdline = [binary] + target['args']
-        out.append(config.job_spec(cmdline, [binary],
+        out.append(self.config.job_spec(cmdline, [binary],
-                                   shortname=' '.join(cmdline),
+                                        shortname=' '.join(cmdline),
-                                   cpu_cost=target['cpu_cost'],
+                                        cpu_cost=target['cpu_cost'],
-                                   environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
+                                        environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
-                                            os.path.abspath(os.path.dirname(
+                                                 _ROOT + '/src/core/tsi/test_creds/ca.pem'}))
-                                                sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
+      elif self.args.regex == '.*' or self.platform == 'windows':
-      elif args.regex == '.*' or platform_string() == 'windows':
         print '\nWARNING: binary not found, skipping', binary
     return sorted(out)
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
-    if platform_string() != 'windows' and test_regex != '.*':
+    test_regex = self.args.regex
+    if self.platform != 'windows' and self.args.regex != '.*':
       # use the regex to minimize the number of things to build
       return [os.path.basename(target['name'])
               for target in get_c_tests(False, self.test_lang)
               if re.search(test_regex, '/' + target['name'])]
-    if platform_string() == 'windows':
+    if self.platform == 'windows':
       # don't build tools on windows just yet
       return ['buildtests_%s' % self.make_target]
     return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
 
   def make_options(self):
-    return []
+    return self._make_options;
 
   def pre_build_steps(self):
     if self.platform == 'windows':
@@ -182,11 +203,24 @@ class CLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def _get_docker_distro(self, use_docker, compiler):
-    return True
+    if _is_use_docker_child():
+      return "already_under_docker"
+    if not use_docker:
+      _check_compiler(compiler, ['default'])
 
-  def dockerfile_dir(self, config, arch):
+    if compiler == 'gcc4.9' or compiler == 'default':
-    return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch)
+      return 'jessie'
+    elif compiler == 'gcc4.4':
+      return 'squeeze'
+    elif compiler == 'gcc5.3':
+      return 'ubuntu1604'
+    else:
+      raise Exception('Compiler %s not supported.' % compiler)
+
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/cxx_%s_%s' % (self._docker_distro,
+                                                _docker_arch_suffix(self.args.arch))
 
   def __str__(self):
     return self.make_target
@@ -198,13 +232,18 @@ class NodeLanguage(object):
     self.platform = platform_string()
     self.node_version = '0.12'
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     if self.platform == 'windows':
-      return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
+      return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
     else:
-      return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
+      return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
-                              None,
+                                   None,
-                              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+                                   environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     if self.platform == 'windows':
@@ -212,7 +251,7 @@ class NodeLanguage(object):
     else:
       return [['tools/run_tests/pre_build_node.sh', self.node_version]]
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return []
 
   def make_options(self):
@@ -230,11 +269,8 @@ class NodeLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
+    return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch)
 
   def __str__(self):
     return 'node'
@@ -242,14 +278,19 @@ class NodeLanguage(object):
 
 class PhpLanguage(object):
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
-    return [config.job_spec(['src/php/bin/run_tests.sh'], None,
+    self.config = config
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
+                                  environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['static_c', 'shared_c']
 
   def make_options(self):
@@ -264,11 +305,8 @@ class PhpLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
+    return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch)
 
   def __str__(self):
     return 'php'
@@ -280,10 +318,15 @@ class PythonLanguage(object):
     self._build_python_versions = ['2.7']
     self._has_python_versions = []
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
     environment['PYVER'] = '2.7'
-    return [config.job_spec(
+    return [self.config.job_spec(
         ['tools/run_tests/run_python.sh'],
         None,
         environ=environment,
@@ -294,7 +337,7 @@ class PythonLanguage(object):
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['static_c', 'grpc_python_plugin', 'shared_c']
 
   def make_options(self):
@@ -320,11 +363,8 @@ class PythonLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
+    return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch)
 
   def __str__(self):
     return 'python'
@@ -332,15 +372,20 @@ class PythonLanguage(object):
 
 class RubyLanguage(object):
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
-    return [config.job_spec(['tools/run_tests/run_ruby.sh'], None,
+    self.config = config
-                            timeout_seconds=10*60,
+    self.args = args
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None,
+                                 timeout_seconds=10*60,
+                                 environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return [['tools/run_tests/pre_build_ruby.sh']]
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return []
 
   def make_options(self):
@@ -355,27 +400,30 @@ class RubyLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
+    return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch)
 
   def __str__(self):
     return 'ruby'
 
 
 class CSharpLanguage(object):
+
   def __init__(self):
     self.platform = platform_string()
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     with open('src/csharp/tests.json') as f:
       tests_json = json.load(f)
     assemblies = tests_json['assemblies']
     tests = tests_json['tests']
 
-    msbuild_config = _WINDOWS_CONFIG[config.build_config]
+    msbuild_config = _MSBUILD_CONFIG[self.config.build_config]
     assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
                       for a in assemblies]
 
@@ -387,13 +435,13 @@ class CSharpLanguage(object):
     else:
       script_name = 'tools/run_tests/run_csharp.sh'
 
-    if config.build_config == 'gcov':
+    if self.config.build_config == 'gcov':
       # On Windows, we only collect C# code coverage.
       # On Linux, we only collect coverage for native extension.
       # For code coverage all tests need to run as one suite.
-      return [config.job_spec([script_name] + extra_args, None,
+      return [self.config.job_spec([script_name] + extra_args, None,
-                              shortname='csharp.coverage',
+                                    shortname='csharp.coverage',
-                              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+                                    environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
     else:
       specs = []
       for test in tests:
@@ -402,9 +450,9 @@ class CSharpLanguage(object):
           # use different output directory for each test to prevent
           # TestResult.xml clash between parallel test runs.
           cmdline += ['-work=test-result/%s' % uuid.uuid4()]
-        specs.append(config.job_spec(cmdline, None,
+        specs.append(self.config.job_spec(cmdline, None,
-                                     shortname='csharp.%s' % test,
+                                          shortname='csharp.%s' % test,
-                                     environ=_FORCE_ENVIRON_FOR_WRAPPERS))
+                                          environ=_FORCE_ENVIRON_FOR_WRAPPERS))
       return specs
 
   def pre_build_steps(self):
@@ -413,7 +461,7 @@ class CSharpLanguage(object):
     else:
       return [['tools/run_tests/pre_build_csharp.sh']]
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     # For Windows, this target doesn't really build anything,
     # everything is build by buildall script later.
     if self.platform == 'windows':
@@ -440,11 +488,8 @@ class CSharpLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
+    return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch)
 
   def __str__(self):
     return 'csharp'
@@ -452,14 +497,19 @@ class CSharpLanguage(object):
 
 class ObjCLanguage(object):
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
-    return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
+    self.config = config
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
+                                  environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['grpc_objective_c_plugin', 'interop_server']
 
   def make_options(self):
@@ -474,10 +524,7 @@ class ObjCLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
     return None
 
   def __str__(self):
@@ -486,18 +533,23 @@ class ObjCLanguage(object):
 
 class Sanity(object):
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     import yaml
     with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
-      return [config.job_spec(cmd['script'].split(), None,
+      return [self.config.job_spec(cmd['script'].split(), None,
-                              timeout_seconds=None, environ={'TEST': 'true'},
+                                   timeout_seconds=None, environ={'TEST': 'true'},
-                              cpu_cost=cmd.get('cpu_cost', 1))
+                                   cpu_cost=cmd.get('cpu_cost', 1))
               for cmd in yaml.load(f)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['run_dep_checks']
 
   def make_options(self):
@@ -512,55 +564,18 @@ class Sanity(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
+  def dockerfile_dir(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
     return 'tools/dockerfile/test/sanity'
 
   def __str__(self):
     return 'sanity'
 
 
-class Build(object):
-
-  def test_specs(self, config, args):
-    return []
-
-  def pre_build_steps(self):
-    return []
-
-  def make_targets(self, test_regex):
-    return ['static']
-
-  def make_options(self):
-    return []
-
-  def build_steps(self):
-    return []
-
-  def post_tests_steps(self):
-    return []
-
-  def makefile_name(self):
-    return 'Makefile'
-
-  def supports_multi_config(self):
-    return True
-
-  def dockerfile_dir(self, config, arch):
-    return None
-
-  def __str__(self):
-    return self.make_target
-
-
 # different configurations we can run under
 with open('tools/run_tests/configs.json') as f:
   _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))
 
 
-_DEFAULT = ['opt']
 _LANGUAGES = {
     'c++': CLanguage('cxx', 'c++'),
     'c': CLanguage('c', 'c'),
@@ -570,11 +585,11 @@ _LANGUAGES = {
     'ruby': RubyLanguage(),
     'csharp': CSharpLanguage(),
     'objc' : ObjCLanguage(),
-    'sanity': Sanity(),
+    'sanity': Sanity()
-    'build': Build(),
     }
 
-_WINDOWS_CONFIG = {
+
+_MSBUILD_CONFIG = {
     'dbg': 'Debug',
     'opt': 'Release',
     'gcov': 'Debug',
@@ -651,14 +666,6 @@ def _docker_arch_suffix(arch):
     sys.exit(1)
 
 
-def _get_dockerfile_dir(language, cfg, arch):
-  """Returns dockerfile to use"""
-  custom = language.dockerfile_dir(cfg, arch)
-  if custom:
-    return custom
-  else:
-    return 'tools/dockerfile/grpc_tests_multilang_%s' % _docker_arch_suffix(arch)
-
 def runs_per_test_type(arg_str):
     """Auxilary function to parse the "runs_per_test" flag.
 
@@ -682,9 +689,8 @@ def runs_per_test_type(arg_str):
 # parse command line
 argp = argparse.ArgumentParser(description='Run grpc tests.')
 argp.add_argument('-c', '--config',
-                  choices=['all'] + sorted(_CONFIGS.keys()),
+                  choices=sorted(_CONFIGS.keys()),
-                  nargs='+',
+                  default='opt')
-                  default=_DEFAULT)
 argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
         help='A positive integer or "inf". If "inf", all tests will run in an '
              'infinite loop. Especially useful in combination with "-f"')
@@ -728,9 +734,11 @@ argp.add_argument('--arch',
                   default='default',
                   help='Selects architecture to target. For some platforms "default" is the only supported choice.')
 argp.add_argument('--compiler',
-                  choices=['default', 'vs2010', 'vs2013', 'vs2015'],
+                  choices=['default',
+                           'gcc4.4', 'gcc4.9', 'gcc5.3',
+                           'vs2010', 'vs2013', 'vs2015'],
                   default='default',
-                  help='Selects compiler to use. For some platforms "default" is the only supported choice.')
+                  help='Selects compiler to use. Allowed values depend on the platform and language.')
 argp.add_argument('--build_only',
                   default=False,
                   action='store_const',
@@ -776,11 +784,8 @@ if need_to_regenerate_projects:
 
 
 # grab config
-run_configs = set(_CONFIGS[cfg]
+run_config = _CONFIGS[args.config]
-                  for cfg in itertools.chain.from_iterable(
+build_config = run_config.build_config
-                      _CONFIGS.iterkeys() if x == 'all' else [x]
-                      for x in args.config))
-build_configs = set(cfg.build_config for cfg in run_configs)
 
 if args.travis:
   _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
@@ -791,17 +796,13 @@ else:
   lang_list = args.language
 # We don't support code coverage on some languages
 if 'gcov' in args.config:
-  for bad in ['objc', 'sanity', 'build']:
+  for bad in ['objc', 'sanity']:
     if bad in lang_list:
       lang_list.remove(bad)
 
 languages = set(_LANGUAGES[l] for l in lang_list)
-
+for l in languages:
-if len(build_configs) > 1:
+  l.configure(run_config, args)
-  for language in languages:
-    if not language.supports_multi_config():
-      print language, 'does not support multiple build configurations'
-      sys.exit(1)
 
 language_make_options=[]
 if any(language.make_options() for language in languages):
@@ -811,10 +812,6 @@ if any(language.make_options() for language in languages):
   else:
     language_make_options = next(iter(languages)).make_options()
 
-if len(languages) != 1 or len(build_configs) != 1:
-  print 'Multi-language and multi-config testing is not supported.'
-  sys.exit(1)
-
 if args.use_docker:
   if not args.travis:
     print 'Seen --use_docker flag, will run tests under docker.'
@@ -824,14 +821,18 @@ if args.use_docker:
     print 'copied to the docker environment.'
     time.sleep(5)
 
+  dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
+  if len(dockerfile_dirs) > 1:
+    print 'Languages to be tested require running under different docker images.'
+    sys.exit(1)
+  dockerfile_dir = next(iter(dockerfile_dirs))
+
   child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ]
   run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])
 
   env = os.environ.copy()
   env['RUN_TESTS_COMMAND'] = run_tests_cmd
-  env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)),
+  env['DOCKERFILE_DIR'] = dockerfile_dir
-                                              next(iter(build_configs)),
-                                              args.arch)
   env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh'
   if args.xml_report:
     env['XML_REPORT'] = args.xml_report
@@ -843,10 +844,6 @@ if args.use_docker:
                         env=env)
   sys.exit(0)
 
-if platform_string() != 'windows' and args.compiler != 'default':
-    print 'Compiler %s not supported on current platform.' % args.compiler
-    sys.exit(1)
-
 _check_arch_option(args.arch)
 
 def make_jobspec(cfg, targets, makefile='Makefile'):
@@ -861,9 +858,7 @@ def make_jobspec(cfg, targets, makefile='Makefile'):
     return [
       jobset.JobSpec([_windows_build_bat(args.compiler),
                       'vsprojects\\%s.sln' % target,
-                      '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg],
+                      '/p:Configuration=%s' % _MSBUILD_CONFIG[cfg]] +
-                      _windows_toolset_option(args.compiler),
-                      _windows_arch_option(args.arch)] +
                       extra_args +
                       language_make_options,
                       shell=True, timeout_seconds=None)
@@ -886,32 +881,29 @@ make_targets = {}
 for l in languages:
   makefile = l.makefile_name()
   make_targets[makefile] = make_targets.get(makefile, set()).union(
-      set(l.make_targets(args.regex)))
+      set(l.make_targets()))
 
 def build_step_environ(cfg):
   environ = {'CONFIG': cfg}
-  msbuild_cfg = _WINDOWS_CONFIG.get(cfg)
+  msbuild_cfg = _MSBUILD_CONFIG.get(cfg)
   if msbuild_cfg:
     environ['MSBUILD_CONFIG'] = msbuild_cfg
   return environ
 
 build_steps = list(set(
-                   jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5)
+                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
-                   for cfg in build_configs
                    for l in languages
                    for cmdline in l.pre_build_steps()))
 if make_targets:
-  make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems())
+  make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
   build_steps.extend(set(make_commands))
 build_steps.extend(set(
-                   jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None)
+                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
-                   for cfg in build_configs
                    for l in languages
                    for cmdline in l.build_steps()))
 
 post_tests_steps = list(set(
-                        jobset.JobSpec(cmdline, environ=build_step_environ(cfg))
+                        jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
-                        for cfg in build_configs
                         for l in languages
                         for cmdline in l.post_tests_steps()))
 runs_per_test = args.runs_per_test
@@ -1024,7 +1016,7 @@ def _start_port_server(port_server_port):
           print 'last ditch attempt to contact port server succeeded'
           break
         except:
-          traceback.print_exc();
+          traceback.print_exc()
           port_log = open(logfile, 'r').read()
           print port_log
           sys.exit(1)
@@ -1044,7 +1036,7 @@ def _start_port_server(port_server_port):
         time.sleep(1)
         waits += 1
       except:
-        traceback.print_exc();
+        traceback.print_exc()
         port_server.kill()
         raise
 
@@ -1102,9 +1094,8 @@ def _build_and_run(
     infinite_runs = runs_per_test == 0
     one_run = set(
       spec
-      for config in run_configs
       for language in languages
-      for spec in language.test_specs(config, args)
+      for spec in language.test_specs()
       if re.search(args.regex, spec.shortname))
     # When running on travis, we want out test runs to be as similar as possible
     # for reproducibility purposes.