1922 lines
64 KiB
Python
Executable File
1922 lines
64 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright 2015 gRPC authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Run tests in parallel."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import ast
|
|
import collections
|
|
import glob
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import multiprocessing
|
|
import os
|
|
import os.path
|
|
import platform
|
|
import random
|
|
import re
|
|
import shlex
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import traceback
|
|
import uuid
|
|
|
|
import six
|
|
from six.moves import urllib
|
|
|
|
import python_utils.jobset as jobset
|
|
import python_utils.report_utils as report_utils
|
|
import python_utils.start_port_server as start_port_server
|
|
import python_utils.watch_dirs as watch_dirs
|
|
|
|
try:
|
|
from python_utils.upload_test_results import upload_results_to_bq
|
|
except ImportError:
|
|
pass # It's ok to not import because this is only necessary to upload results to BQ.
|
|
|
|
gcp_utils_dir = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), "../gcp/utils")
|
|
)
|
|
sys.path.append(gcp_utils_dir)
|
|
|
|
_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../.."))
|
|
os.chdir(_ROOT)
|
|
|
|
_FORCE_ENVIRON_FOR_WRAPPERS = {
|
|
"GRPC_VERBOSITY": "DEBUG",
|
|
}
|
|
|
|
_POLLING_STRATEGIES = {
|
|
"linux": ["epoll1", "poll"],
|
|
"mac": ["poll"],
|
|
}
|
|
|
|
|
|
def platform_string():
|
|
return jobset.platform_string()
|
|
|
|
|
|
_DEFAULT_TIMEOUT_SECONDS = 5 * 60
|
|
_PRE_BUILD_STEP_TIMEOUT_SECONDS = 10 * 60
|
|
|
|
|
|
def run_shell_command(cmd, env=None, cwd=None):
|
|
try:
|
|
subprocess.check_output(cmd, shell=True, env=env, cwd=cwd)
|
|
except subprocess.CalledProcessError as e:
|
|
logging.exception(
|
|
"Error while running command '%s'. Exit status %d. Output:\n%s",
|
|
e.cmd,
|
|
e.returncode,
|
|
e.output,
|
|
)
|
|
raise
|
|
|
|
|
|
def max_parallel_tests_for_current_platform():
|
|
# Too much test parallelization has only been seen to be a problem
|
|
# so far on windows.
|
|
if jobset.platform_string() == "windows":
|
|
return 64
|
|
return 1024
|
|
|
|
|
|
def _print_debug_info_epilogue(dockerfile_dir=None):
|
|
"""Use to print useful info for debug/repro just before exiting."""
|
|
print("")
|
|
print("=== run_tests.py DEBUG INFO ===")
|
|
print('command: "%s"' % " ".join(sys.argv))
|
|
if dockerfile_dir:
|
|
print("dockerfile: %s" % dockerfile_dir)
|
|
kokoro_job_name = os.getenv("KOKORO_JOB_NAME")
|
|
if kokoro_job_name:
|
|
print("kokoro job name: %s" % kokoro_job_name)
|
|
print("===============================")
|
|
|
|
|
|
# SimpleConfig: just compile with CONFIG=config, and run the binary to test
|
|
class Config(object):
|
|
def __init__(
|
|
self,
|
|
config,
|
|
environ=None,
|
|
timeout_multiplier=1,
|
|
tool_prefix=[],
|
|
iomgr_platform="native",
|
|
):
|
|
if environ is None:
|
|
environ = {}
|
|
self.build_config = config
|
|
self.environ = environ
|
|
self.environ["CONFIG"] = config
|
|
self.tool_prefix = tool_prefix
|
|
self.timeout_multiplier = timeout_multiplier
|
|
self.iomgr_platform = iomgr_platform
|
|
|
|
def job_spec(
|
|
self,
|
|
cmdline,
|
|
timeout_seconds=_DEFAULT_TIMEOUT_SECONDS,
|
|
shortname=None,
|
|
environ={},
|
|
cpu_cost=1.0,
|
|
flaky=False,
|
|
):
|
|
"""Construct a jobset.JobSpec for a test under this config
|
|
|
|
Args:
|
|
cmdline: a list of strings specifying the command line the test
|
|
would like to run
|
|
"""
|
|
actual_environ = self.environ.copy()
|
|
for k, v in environ.items():
|
|
actual_environ[k] = v
|
|
if not flaky and shortname and shortname in flaky_tests:
|
|
flaky = True
|
|
if shortname in shortname_to_cpu:
|
|
cpu_cost = shortname_to_cpu[shortname]
|
|
return jobset.JobSpec(
|
|
cmdline=self.tool_prefix + cmdline,
|
|
shortname=shortname,
|
|
environ=actual_environ,
|
|
cpu_cost=cpu_cost,
|
|
timeout_seconds=(
|
|
self.timeout_multiplier * timeout_seconds
|
|
if timeout_seconds
|
|
else None
|
|
),
|
|
flake_retries=4 if flaky or args.allow_flakes else 0,
|
|
timeout_retries=1 if flaky or args.allow_flakes else 0,
|
|
)
|
|
|
|
|
|
def get_c_tests(travis, test_lang):
|
|
out = []
|
|
platforms_str = "ci_platforms" if travis else "platforms"
|
|
with open("tools/run_tests/generated/tests.json") as f:
|
|
js = json.load(f)
|
|
return [
|
|
tgt
|
|
for tgt in js
|
|
if tgt["language"] == test_lang
|
|
and platform_string() in tgt[platforms_str]
|
|
and not (travis and tgt["flaky"])
|
|
]
|
|
|
|
|
|
def _check_compiler(compiler, supported_compilers):
|
|
if compiler not in supported_compilers:
|
|
raise Exception(
|
|
"Compiler %s not supported (on this platform)." % compiler
|
|
)
|
|
|
|
|
|
def _check_arch(arch, supported_archs):
|
|
if arch not in supported_archs:
|
|
raise Exception("Architecture %s not supported." % arch)
|
|
|
|
|
|
def _is_use_docker_child():
|
|
"""Returns True if running running as a --use_docker child."""
|
|
return True if os.getenv("DOCKER_RUN_SCRIPT_COMMAND") else False
|
|
|
|
|
|
_PythonConfigVars = collections.namedtuple(
|
|
"_ConfigVars",
|
|
[
|
|
"shell",
|
|
"builder",
|
|
"builder_prefix_arguments",
|
|
"venv_relative_python",
|
|
"toolchain",
|
|
"runner",
|
|
],
|
|
)
|
|
|
|
|
|
def _python_config_generator(name, major, minor, bits, config_vars):
|
|
build = (
|
|
config_vars.shell
|
|
+ config_vars.builder
|
|
+ config_vars.builder_prefix_arguments
|
|
+ [_python_pattern_function(major=major, minor=minor, bits=bits)]
|
|
+ [name]
|
|
+ config_vars.venv_relative_python
|
|
+ config_vars.toolchain
|
|
)
|
|
# run: [tools/run_tests/helper_scripts/run_python.sh py37/bin/python]
|
|
python_path = os.path.join(name, config_vars.venv_relative_python[0])
|
|
run = config_vars.shell + config_vars.runner + [python_path]
|
|
return PythonConfig(name, build, run, python_path)
|
|
|
|
|
|
def _pypy_config_generator(name, major, config_vars):
|
|
# Something like "py37/bin/python"
|
|
python_path = os.path.join(name, config_vars.venv_relative_python[0])
|
|
return PythonConfig(
|
|
name,
|
|
config_vars.shell
|
|
+ config_vars.builder
|
|
+ config_vars.builder_prefix_arguments
|
|
+ [_pypy_pattern_function(major=major)]
|
|
+ [name]
|
|
+ config_vars.venv_relative_python
|
|
+ config_vars.toolchain,
|
|
config_vars.shell + config_vars.runner + [python_path],
|
|
python_path,
|
|
)
|
|
|
|
|
|
def _python_pattern_function(major, minor, bits):
|
|
# Bit-ness is handled by the test machine's environment
|
|
if os.name == "nt":
|
|
if bits == "64":
|
|
return "/c/Python{major}{minor}/python.exe".format(
|
|
major=major, minor=minor, bits=bits
|
|
)
|
|
else:
|
|
return "/c/Python{major}{minor}_{bits}bits/python.exe".format(
|
|
major=major, minor=minor, bits=bits
|
|
)
|
|
else:
|
|
return "python{major}.{minor}".format(major=major, minor=minor)
|
|
|
|
|
|
def _pypy_pattern_function(major):
|
|
if major == "2":
|
|
return "pypy"
|
|
elif major == "3":
|
|
return "pypy3"
|
|
else:
|
|
raise ValueError("Unknown PyPy major version")
|
|
|
|
|
|
class CLanguage(object):
|
|
def __init__(self, lang_suffix, test_lang):
|
|
self.lang_suffix = lang_suffix
|
|
self.platform = platform_string()
|
|
self.test_lang = test_lang
|
|
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
if self.platform == "windows":
|
|
_check_compiler(
|
|
self.args.compiler,
|
|
[
|
|
"default",
|
|
"cmake",
|
|
"cmake_ninja_vs2019",
|
|
"cmake_ninja_vs2022",
|
|
"cmake_vs2019",
|
|
"cmake_vs2022",
|
|
],
|
|
)
|
|
_check_arch(self.args.arch, ["default", "x64", "x86"])
|
|
|
|
activate_vs_tools = ""
|
|
if (
|
|
self.args.compiler == "cmake_ninja_vs2019"
|
|
or self.args.compiler == "cmake"
|
|
or self.args.compiler == "default"
|
|
):
|
|
# cmake + ninja build is the default because it is faster and supports boringssl assembly optimizations
|
|
# the compiler used is exactly the same as for cmake_vs2017
|
|
cmake_generator = "Ninja"
|
|
activate_vs_tools = "2019"
|
|
elif self.args.compiler == "cmake_ninja_vs2022":
|
|
cmake_generator = "Ninja"
|
|
activate_vs_tools = "2022"
|
|
elif self.args.compiler == "cmake_vs2019":
|
|
cmake_generator = "Visual Studio 16 2019"
|
|
elif self.args.compiler == "cmake_vs2022":
|
|
cmake_generator = "Visual Studio 17 2022"
|
|
else:
|
|
print("should never reach here.")
|
|
sys.exit(1)
|
|
|
|
self._cmake_configure_extra_args = list(
|
|
self.args.cmake_configure_extra_args
|
|
)
|
|
self._cmake_generator_windows = cmake_generator
|
|
# required to pass as cmake "-A" configuration for VS builds (but not for Ninja)
|
|
self._cmake_architecture_windows = (
|
|
"x64" if self.args.arch == "x64" else "Win32"
|
|
)
|
|
# when builing with Ninja, the VS common tools need to be activated first
|
|
self._activate_vs_tools_windows = activate_vs_tools
|
|
# "x64_x86" means create 32bit binaries, but use 64bit toolkit to secure more memory for the build
|
|
self._vs_tools_architecture_windows = (
|
|
"x64" if self.args.arch == "x64" else "x64_x86"
|
|
)
|
|
|
|
else:
|
|
if self.platform == "linux":
|
|
# Allow all the known architectures. _check_arch_option has already checked that we're not doing
|
|
# something illegal when not running under docker.
|
|
_check_arch(self.args.arch, ["default", "x64", "x86", "arm64"])
|
|
else:
|
|
_check_arch(self.args.arch, ["default"])
|
|
|
|
(
|
|
self._docker_distro,
|
|
self._cmake_configure_extra_args,
|
|
) = self._compiler_options(
|
|
self.args.use_docker,
|
|
self.args.compiler,
|
|
self.args.cmake_configure_extra_args,
|
|
)
|
|
|
|
def test_specs(self):
|
|
out = []
|
|
binaries = get_c_tests(self.args.travis, self.test_lang)
|
|
for target in binaries:
|
|
if target.get("boringssl", False):
|
|
# cmake doesn't build boringssl tests
|
|
continue
|
|
auto_timeout_scaling = target.get("auto_timeout_scaling", True)
|
|
polling_strategies = (
|
|
_POLLING_STRATEGIES.get(self.platform, ["all"])
|
|
if target.get("uses_polling", True)
|
|
else ["none"]
|
|
)
|
|
for polling_strategy in polling_strategies:
|
|
env = {
|
|
"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": _ROOT
|
|
+ "/src/core/tsi/test_creds/ca.pem",
|
|
"GRPC_POLL_STRATEGY": polling_strategy,
|
|
"GRPC_VERBOSITY": "DEBUG",
|
|
}
|
|
resolver = os.environ.get("GRPC_DNS_RESOLVER", None)
|
|
if resolver:
|
|
env["GRPC_DNS_RESOLVER"] = resolver
|
|
shortname_ext = (
|
|
""
|
|
if polling_strategy == "all"
|
|
else " GRPC_POLL_STRATEGY=%s" % polling_strategy
|
|
)
|
|
if polling_strategy in target.get("excluded_poll_engines", []):
|
|
continue
|
|
|
|
timeout_scaling = 1
|
|
if auto_timeout_scaling:
|
|
config = self.args.config
|
|
if (
|
|
"asan" in config
|
|
or config == "msan"
|
|
or config == "tsan"
|
|
or config == "ubsan"
|
|
or config == "helgrind"
|
|
or config == "memcheck"
|
|
):
|
|
# Scale overall test timeout if running under various sanitizers.
|
|
# scaling value is based on historical data analysis
|
|
timeout_scaling *= 3
|
|
|
|
if self.config.build_config in target["exclude_configs"]:
|
|
continue
|
|
if self.args.iomgr_platform in target.get("exclude_iomgrs", []):
|
|
continue
|
|
|
|
if self.platform == "windows":
|
|
if self._cmake_generator_windows == "Ninja":
|
|
binary = "cmake/build/%s.exe" % target["name"]
|
|
else:
|
|
binary = "cmake/build/%s/%s.exe" % (
|
|
_MSBUILD_CONFIG[self.config.build_config],
|
|
target["name"],
|
|
)
|
|
else:
|
|
binary = "cmake/build/%s" % target["name"]
|
|
|
|
cpu_cost = target["cpu_cost"]
|
|
if cpu_cost == "capacity":
|
|
cpu_cost = multiprocessing.cpu_count()
|
|
if os.path.isfile(binary):
|
|
list_test_command = None
|
|
filter_test_command = None
|
|
|
|
# these are the flag defined by gtest and benchmark framework to list
|
|
# and filter test runs. We use them to split each individual test
|
|
# into its own JobSpec, and thus into its own process.
|
|
if "benchmark" in target and target["benchmark"]:
|
|
with open(os.devnull, "w") as fnull:
|
|
tests = subprocess.check_output(
|
|
[binary, "--benchmark_list_tests"], stderr=fnull
|
|
)
|
|
for line in tests.decode().split("\n"):
|
|
test = line.strip()
|
|
if not test:
|
|
continue
|
|
cmdline = [
|
|
binary,
|
|
"--benchmark_filter=%s$" % test,
|
|
] + target["args"]
|
|
out.append(
|
|
self.config.job_spec(
|
|
cmdline,
|
|
shortname="%s %s"
|
|
% (" ".join(cmdline), shortname_ext),
|
|
cpu_cost=cpu_cost,
|
|
timeout_seconds=target.get(
|
|
"timeout_seconds",
|
|
_DEFAULT_TIMEOUT_SECONDS,
|
|
)
|
|
* timeout_scaling,
|
|
environ=env,
|
|
)
|
|
)
|
|
elif "gtest" in target and target["gtest"]:
|
|
# here we parse the output of --gtest_list_tests to build up a complete
|
|
# list of the tests contained in a binary for each test, we then
|
|
# add a job to run, filtering for just that test.
|
|
with open(os.devnull, "w") as fnull:
|
|
tests = subprocess.check_output(
|
|
[binary, "--gtest_list_tests"], stderr=fnull
|
|
)
|
|
base = None
|
|
for line in tests.decode().split("\n"):
|
|
i = line.find("#")
|
|
if i >= 0:
|
|
line = line[:i]
|
|
if not line:
|
|
continue
|
|
if line[0] != " ":
|
|
base = line.strip()
|
|
else:
|
|
assert base is not None
|
|
assert line[1] == " "
|
|
test = base + line.strip()
|
|
cmdline = [
|
|
binary,
|
|
"--gtest_filter=%s" % test,
|
|
] + target["args"]
|
|
out.append(
|
|
self.config.job_spec(
|
|
cmdline,
|
|
shortname="%s %s"
|
|
% (" ".join(cmdline), shortname_ext),
|
|
cpu_cost=cpu_cost,
|
|
timeout_seconds=target.get(
|
|
"timeout_seconds",
|
|
_DEFAULT_TIMEOUT_SECONDS,
|
|
)
|
|
* timeout_scaling,
|
|
environ=env,
|
|
)
|
|
)
|
|
else:
|
|
cmdline = [binary] + target["args"]
|
|
shortname = target.get(
|
|
"shortname",
|
|
" ".join(shlex.quote(arg) for arg in cmdline),
|
|
)
|
|
shortname += shortname_ext
|
|
out.append(
|
|
self.config.job_spec(
|
|
cmdline,
|
|
shortname=shortname,
|
|
cpu_cost=cpu_cost,
|
|
flaky=target.get("flaky", False),
|
|
timeout_seconds=target.get(
|
|
"timeout_seconds", _DEFAULT_TIMEOUT_SECONDS
|
|
)
|
|
* timeout_scaling,
|
|
environ=env,
|
|
)
|
|
)
|
|
elif self.args.regex == ".*" or self.platform == "windows":
|
|
print("\nWARNING: binary not found, skipping", binary)
|
|
return sorted(out)
|
|
|
|
def pre_build_steps(self):
|
|
return []
|
|
|
|
def build_steps(self):
|
|
if self.platform == "windows":
|
|
return [
|
|
[
|
|
"tools\\run_tests\\helper_scripts\\build_cxx.bat",
|
|
"-DgRPC_BUILD_MSVC_MP_COUNT=%d" % self.args.jobs,
|
|
]
|
|
+ self._cmake_configure_extra_args
|
|
]
|
|
else:
|
|
return [
|
|
["tools/run_tests/helper_scripts/build_cxx.sh"]
|
|
+ self._cmake_configure_extra_args
|
|
]
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
environ = {"GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX": self.lang_suffix}
|
|
if self.platform == "windows":
|
|
environ["GRPC_CMAKE_GENERATOR"] = self._cmake_generator_windows
|
|
environ[
|
|
"GRPC_CMAKE_ARCHITECTURE"
|
|
] = self._cmake_architecture_windows
|
|
environ[
|
|
"GRPC_BUILD_ACTIVATE_VS_TOOLS"
|
|
] = self._activate_vs_tools_windows
|
|
environ[
|
|
"GRPC_BUILD_VS_TOOLS_ARCHITECTURE"
|
|
] = self._vs_tools_architecture_windows
|
|
elif self.platform == "linux":
|
|
environ["GRPC_RUNTESTS_ARCHITECTURE"] = self.args.arch
|
|
return environ
|
|
|
|
def post_tests_steps(self):
|
|
if self.platform == "windows":
|
|
return []
|
|
else:
|
|
return [["tools/run_tests/helper_scripts/post_tests_c.sh"]]
|
|
|
|
def _clang_cmake_configure_extra_args(self, version_suffix=""):
|
|
return [
|
|
"-DCMAKE_C_COMPILER=clang%s" % version_suffix,
|
|
"-DCMAKE_CXX_COMPILER=clang++%s" % version_suffix,
|
|
]
|
|
|
|
def _compiler_options(
|
|
self, use_docker, compiler, cmake_configure_extra_args
|
|
):
|
|
"""Returns docker distro and cmake configure args to use for given compiler."""
|
|
if cmake_configure_extra_args:
|
|
# only allow specifying extra cmake args for "vanilla" compiler
|
|
_check_compiler(compiler, ["default", "cmake"])
|
|
return ("nonexistent_docker_distro", cmake_configure_extra_args)
|
|
if not use_docker and not _is_use_docker_child():
|
|
# if not running under docker, we cannot ensure the right compiler version will be used,
|
|
# so we only allow the non-specific choices.
|
|
_check_compiler(compiler, ["default", "cmake"])
|
|
|
|
if compiler == "default" or compiler == "cmake":
|
|
return ("debian11", [])
|
|
elif compiler == "gcc8":
|
|
return ("gcc_8", [])
|
|
elif compiler == "gcc10.2":
|
|
return ("debian11", [])
|
|
elif compiler == "gcc10.2_openssl102":
|
|
return (
|
|
"debian11_openssl102",
|
|
[
|
|
"-DgRPC_SSL_PROVIDER=package",
|
|
],
|
|
)
|
|
elif compiler == "gcc10.2_openssl111":
|
|
return (
|
|
"debian11_openssl111",
|
|
[
|
|
"-DgRPC_SSL_PROVIDER=package",
|
|
],
|
|
)
|
|
elif compiler == "gcc12":
|
|
return ("gcc_12", ["-DCMAKE_CXX_STANDARD=20"])
|
|
elif compiler == "gcc12_openssl309":
|
|
return (
|
|
"debian12_openssl309",
|
|
[
|
|
"-DgRPC_SSL_PROVIDER=package",
|
|
],
|
|
)
|
|
elif compiler == "gcc_musl":
|
|
return ("alpine", [])
|
|
elif compiler == "clang6":
|
|
return ("clang_6", self._clang_cmake_configure_extra_args())
|
|
elif compiler == "clang17":
|
|
return ("clang_17", self._clang_cmake_configure_extra_args())
|
|
else:
|
|
raise Exception("Compiler %s not supported." % compiler)
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/cxx_%s_%s" % (
|
|
self._docker_distro,
|
|
_docker_arch_suffix(self.args.arch),
|
|
)
|
|
|
|
def __str__(self):
|
|
return self.lang_suffix
|
|
|
|
|
|
class Php7Language(object):
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
_check_compiler(self.args.compiler, ["default"])
|
|
|
|
def test_specs(self):
|
|
return [
|
|
self.config.job_spec(
|
|
["src/php/bin/run_tests.sh"],
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
)
|
|
]
|
|
|
|
def pre_build_steps(self):
|
|
return []
|
|
|
|
def build_steps(self):
|
|
return [["tools/run_tests/helper_scripts/build_php.sh"]]
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
return [["tools/run_tests/helper_scripts/post_tests_php.sh"]]
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/php7_debian11_%s" % _docker_arch_suffix(
|
|
self.args.arch
|
|
)
|
|
|
|
def __str__(self):
|
|
return "php7"
|
|
|
|
|
|
class PythonConfig(
|
|
collections.namedtuple(
|
|
"PythonConfig", ["name", "build", "run", "python_path"]
|
|
)
|
|
):
|
|
"""Tuple of commands (named s.t. 'what it says on the tin' applies)"""
|
|
|
|
|
|
class PythonLanguage(object):
|
|
_TEST_SPECS_FILE = {
|
|
"native": ["src/python/grpcio_tests/tests/tests.json"],
|
|
"asyncio": ["src/python/grpcio_tests/tests_aio/tests.json"],
|
|
}
|
|
|
|
_TEST_COMMAND = {
|
|
"native": "test_lite",
|
|
"asyncio": "test_aio",
|
|
}
|
|
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
self.pythons = self._get_pythons(self.args)
|
|
|
|
def test_specs(self):
|
|
# load list of known test suites
|
|
jobs = []
|
|
|
|
# Run tests across all supported interpreters.
|
|
for python_config in self.pythons:
|
|
# Run non-io-manager-specific tests.
|
|
if os.name != "nt":
|
|
jobs.append(
|
|
self.config.job_spec(
|
|
[
|
|
python_config.python_path,
|
|
"tools/distrib/python/xds_protos/generated_file_import_test.py",
|
|
],
|
|
timeout_seconds=60,
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
shortname=f"{python_config.name}.xds_protos",
|
|
)
|
|
)
|
|
|
|
# Run main test suite across all support IO managers.
|
|
for io_platform in self._TEST_SPECS_FILE:
|
|
test_cases = []
|
|
for tests_json_file_name in self._TEST_SPECS_FILE[io_platform]:
|
|
with open(tests_json_file_name) as tests_json_file:
|
|
test_cases.extend(json.load(tests_json_file))
|
|
|
|
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
|
|
# TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not
|
|
# designed for non-native IO manager. It has a side-effect that
|
|
# overrides threading settings in C-Core.
|
|
if io_platform != "native":
|
|
environment["GRPC_ENABLE_FORK_SUPPORT"] = "0"
|
|
jobs.extend(
|
|
[
|
|
self.config.job_spec(
|
|
python_config.run
|
|
+ [self._TEST_COMMAND[io_platform]],
|
|
timeout_seconds=8 * 60,
|
|
environ=dict(
|
|
GRPC_PYTHON_TESTRUNNER_FILTER=str(test_case),
|
|
**environment,
|
|
),
|
|
shortname=f"{python_config.name}.{io_platform}.{test_case}",
|
|
)
|
|
for test_case in test_cases
|
|
]
|
|
)
|
|
return jobs
|
|
|
|
def pre_build_steps(self):
|
|
return []
|
|
|
|
def build_steps(self):
|
|
return [config.build for config in self.pythons]
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
if self.config.build_config != "gcov":
|
|
return []
|
|
else:
|
|
return [["tools/run_tests/helper_scripts/post_tests_python.sh"]]
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/python_%s_%s" % (
|
|
self._python_docker_distro_name(),
|
|
_docker_arch_suffix(self.args.arch),
|
|
)
|
|
|
|
def _python_docker_distro_name(self):
|
|
"""Choose the docker image to use based on python version."""
|
|
if self.args.compiler == "python_alpine":
|
|
return "alpine"
|
|
else:
|
|
return "debian11_default"
|
|
|
|
def _get_pythons(self, args):
|
|
"""Get python runtimes to test with, based on current platform, architecture, compiler etc."""
|
|
if args.iomgr_platform != "native":
|
|
raise ValueError(
|
|
"Python builds no longer differentiate IO Manager platforms,"
|
|
' please use "native"'
|
|
)
|
|
|
|
if args.arch == "x86":
|
|
bits = "32"
|
|
else:
|
|
bits = "64"
|
|
|
|
if os.name == "nt":
|
|
shell = ["bash"]
|
|
builder = [
|
|
os.path.abspath(
|
|
"tools/run_tests/helper_scripts/build_python_msys2.sh"
|
|
)
|
|
]
|
|
builder_prefix_arguments = ["MINGW{}".format(bits)]
|
|
venv_relative_python = ["Scripts/python.exe"]
|
|
toolchain = ["mingw32"]
|
|
else:
|
|
shell = []
|
|
builder = [
|
|
os.path.abspath(
|
|
"tools/run_tests/helper_scripts/build_python.sh"
|
|
)
|
|
]
|
|
builder_prefix_arguments = []
|
|
venv_relative_python = ["bin/python"]
|
|
toolchain = ["unix"]
|
|
|
|
runner = [
|
|
os.path.abspath("tools/run_tests/helper_scripts/run_python.sh")
|
|
]
|
|
|
|
config_vars = _PythonConfigVars(
|
|
shell,
|
|
builder,
|
|
builder_prefix_arguments,
|
|
venv_relative_python,
|
|
toolchain,
|
|
runner,
|
|
)
|
|
|
|
# TODO: Supported version range should be defined by a single
|
|
# source of truth.
|
|
python38_config = _python_config_generator(
|
|
name="py38",
|
|
major="3",
|
|
minor="8",
|
|
bits=bits,
|
|
config_vars=config_vars,
|
|
)
|
|
python39_config = _python_config_generator(
|
|
name="py39",
|
|
major="3",
|
|
minor="9",
|
|
bits=bits,
|
|
config_vars=config_vars,
|
|
)
|
|
python310_config = _python_config_generator(
|
|
name="py310",
|
|
major="3",
|
|
minor="10",
|
|
bits=bits,
|
|
config_vars=config_vars,
|
|
)
|
|
python311_config = _python_config_generator(
|
|
name="py311",
|
|
major="3",
|
|
minor="11",
|
|
bits=bits,
|
|
config_vars=config_vars,
|
|
)
|
|
python312_config = _python_config_generator(
|
|
name="py312",
|
|
major="3",
|
|
minor="12",
|
|
bits=bits,
|
|
config_vars=config_vars,
|
|
)
|
|
pypy27_config = _pypy_config_generator(
|
|
name="pypy", major="2", config_vars=config_vars
|
|
)
|
|
pypy32_config = _pypy_config_generator(
|
|
name="pypy3", major="3", config_vars=config_vars
|
|
)
|
|
|
|
if args.compiler == "default":
|
|
if os.name == "nt":
|
|
return (python38_config,)
|
|
elif os.uname()[0] == "Darwin":
|
|
# NOTE(rbellevi): Testing takes significantly longer on
|
|
# MacOS, so we restrict the number of interpreter versions
|
|
# tested.
|
|
return (python38_config,)
|
|
elif platform.machine() == "aarch64":
|
|
# Currently the python_debian11_default_arm64 docker image
|
|
# only has python3.9 installed (and that seems sufficient
|
|
# for arm64 testing)
|
|
return (python39_config,)
|
|
else:
|
|
# Default set tested on master. Test oldest and newest.
|
|
return (
|
|
python38_config,
|
|
python312_config,
|
|
)
|
|
elif args.compiler == "python3.8":
|
|
return (python38_config,)
|
|
elif args.compiler == "python3.9":
|
|
return (python39_config,)
|
|
elif args.compiler == "python3.10":
|
|
return (python310_config,)
|
|
elif args.compiler == "python3.11":
|
|
return (python311_config,)
|
|
elif args.compiler == "python3.12":
|
|
return (python312_config,)
|
|
elif args.compiler == "pypy":
|
|
return (pypy27_config,)
|
|
elif args.compiler == "pypy3":
|
|
return (pypy32_config,)
|
|
elif args.compiler == "python_alpine":
|
|
return (python39_config,)
|
|
elif args.compiler == "all_the_cpythons":
|
|
return (
|
|
python38_config,
|
|
python39_config,
|
|
python310_config,
|
|
python311_config,
|
|
python312_config,
|
|
)
|
|
else:
|
|
raise Exception("Compiler %s not supported." % args.compiler)
|
|
|
|
def __str__(self):
|
|
return "python"
|
|
|
|
|
|
class RubyLanguage(object):
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
_check_compiler(self.args.compiler, ["default"])
|
|
|
|
def test_specs(self):
|
|
tests = [
|
|
self.config.job_spec(
|
|
["tools/run_tests/helper_scripts/run_ruby.sh"],
|
|
timeout_seconds=10 * 60,
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
)
|
|
]
|
|
# TODO(apolcyn): re-enable the following tests after
|
|
# https://bugs.ruby-lang.org/issues/15499 is fixed:
|
|
# They previously worked on ruby 2.5 but needed to be disabled
|
|
# after dropping support for ruby 2.5:
|
|
# - src/ruby/end2end/channel_state_test.rb
|
|
# - src/ruby/end2end/sig_int_during_channel_watch_test.rb
|
|
# TODO(apolcyn): the following test is skipped because it sometimes
|
|
# hits "Bus Error" crashes while requiring the grpc/ruby C-extension.
|
|
# This crashes have been unreproducible outside of CI. Also see
|
|
# b/266212253.
|
|
# - src/ruby/end2end/grpc_class_init_test.rb
|
|
for test in [
|
|
"src/ruby/end2end/fork_test.rb",
|
|
"src/ruby/end2end/simple_fork_test.rb",
|
|
"src/ruby/end2end/prefork_without_using_grpc_test.rb",
|
|
"src/ruby/end2end/prefork_postfork_loop_test.rb",
|
|
"src/ruby/end2end/secure_fork_test.rb",
|
|
"src/ruby/end2end/bad_usage_fork_test.rb",
|
|
"src/ruby/end2end/sig_handling_test.rb",
|
|
"src/ruby/end2end/channel_closing_test.rb",
|
|
"src/ruby/end2end/killed_client_thread_test.rb",
|
|
"src/ruby/end2end/forking_client_test.rb",
|
|
"src/ruby/end2end/multiple_killed_watching_threads_test.rb",
|
|
"src/ruby/end2end/load_grpc_with_gc_stress_test.rb",
|
|
"src/ruby/end2end/client_memory_usage_test.rb",
|
|
"src/ruby/end2end/package_with_underscore_test.rb",
|
|
"src/ruby/end2end/graceful_sig_handling_test.rb",
|
|
"src/ruby/end2end/graceful_sig_stop_test.rb",
|
|
"src/ruby/end2end/errors_load_before_grpc_lib_test.rb",
|
|
"src/ruby/end2end/logger_load_before_grpc_lib_test.rb",
|
|
"src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb",
|
|
"src/ruby/end2end/call_credentials_timeout_test.rb",
|
|
"src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb",
|
|
]:
|
|
if test in [
|
|
"src/ruby/end2end/fork_test.rb",
|
|
"src/ruby/end2end/simple_fork_test.rb",
|
|
"src/ruby/end2end/secure_fork_test.rb",
|
|
"src/ruby/end2end/bad_usage_fork_test.rb",
|
|
"src/ruby/end2end/prefork_without_using_grpc_test.rb",
|
|
"src/ruby/end2end/prefork_postfork_loop_test.rb",
|
|
"src/ruby/end2end/fork_test_repro_35489.rb",
|
|
]:
|
|
# Skip fork tests in general until https://github.com/grpc/grpc/issues/34442
|
|
# is fixed. Otherwise we see too many flakes.
|
|
# After that's fixed, we should continue to skip on mac
|
|
# indefinitely, and on "dbg" builds until the Event Engine
|
|
# migration completes.
|
|
continue
|
|
tests.append(
|
|
self.config.job_spec(
|
|
["ruby", test],
|
|
shortname=test,
|
|
timeout_seconds=20 * 60,
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
)
|
|
)
|
|
return tests
|
|
|
|
def pre_build_steps(self):
|
|
return [["tools/run_tests/helper_scripts/pre_build_ruby.sh"]]
|
|
|
|
def build_steps(self):
|
|
return [["tools/run_tests/helper_scripts/build_ruby.sh"]]
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
return [["tools/run_tests/helper_scripts/post_tests_ruby.sh"]]
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/ruby_debian11_%s" % _docker_arch_suffix(
|
|
self.args.arch
|
|
)
|
|
|
|
def __str__(self):
|
|
return "ruby"
|
|
|
|
|
|
class CSharpLanguage(object):
|
|
def __init__(self):
|
|
self.platform = platform_string()
|
|
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
_check_compiler(self.args.compiler, ["default", "coreclr", "mono"])
|
|
if self.args.compiler == "default":
|
|
# test both runtimes by default
|
|
self.test_runtimes = ["coreclr", "mono"]
|
|
else:
|
|
# only test the specified runtime
|
|
self.test_runtimes = [self.args.compiler]
|
|
|
|
if self.platform == "windows":
|
|
_check_arch(self.args.arch, ["default"])
|
|
self._cmake_arch_option = "x64"
|
|
else:
|
|
self._docker_distro = "debian11"
|
|
|
|
def test_specs(self):
|
|
with open("src/csharp/tests.json") as f:
|
|
tests_by_assembly = json.load(f)
|
|
|
|
msbuild_config = _MSBUILD_CONFIG[self.config.build_config]
|
|
nunit_args = ["--labels=All", "--noresult", "--workers=1"]
|
|
|
|
specs = []
|
|
for test_runtime in self.test_runtimes:
|
|
if test_runtime == "coreclr":
|
|
assembly_extension = ".dll"
|
|
assembly_subdir = "bin/%s/netcoreapp3.1" % msbuild_config
|
|
runtime_cmd = ["dotnet", "exec"]
|
|
elif test_runtime == "mono":
|
|
assembly_extension = ".exe"
|
|
assembly_subdir = "bin/%s/net45" % msbuild_config
|
|
if self.platform == "windows":
|
|
runtime_cmd = []
|
|
elif self.platform == "mac":
|
|
# mono before version 5.2 on MacOS defaults to 32bit runtime
|
|
runtime_cmd = ["mono", "--arch=64"]
|
|
else:
|
|
runtime_cmd = ["mono"]
|
|
else:
|
|
raise Exception('Illegal runtime "%s" was specified.')
|
|
|
|
for assembly in six.iterkeys(tests_by_assembly):
|
|
assembly_file = "src/csharp/%s/%s/%s%s" % (
|
|
assembly,
|
|
assembly_subdir,
|
|
assembly,
|
|
assembly_extension,
|
|
)
|
|
|
|
# normally, run each test as a separate process
|
|
for test in tests_by_assembly[assembly]:
|
|
cmdline = (
|
|
runtime_cmd
|
|
+ [assembly_file, "--test=%s" % test]
|
|
+ nunit_args
|
|
)
|
|
specs.append(
|
|
self.config.job_spec(
|
|
cmdline,
|
|
shortname="csharp.%s.%s" % (test_runtime, test),
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
)
|
|
)
|
|
return specs
|
|
|
|
def pre_build_steps(self):
|
|
if self.platform == "windows":
|
|
return [["tools\\run_tests\\helper_scripts\\pre_build_csharp.bat"]]
|
|
else:
|
|
return [["tools/run_tests/helper_scripts/pre_build_csharp.sh"]]
|
|
|
|
def build_steps(self):
|
|
if self.platform == "windows":
|
|
return [["tools\\run_tests\\helper_scripts\\build_csharp.bat"]]
|
|
else:
|
|
return [["tools/run_tests/helper_scripts/build_csharp.sh"]]
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
if self.platform == "windows":
|
|
return {"ARCHITECTURE": self._cmake_arch_option}
|
|
else:
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
if self.platform == "windows":
|
|
return [["tools\\run_tests\\helper_scripts\\post_tests_csharp.bat"]]
|
|
else:
|
|
return [["tools/run_tests/helper_scripts/post_tests_csharp.sh"]]
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/csharp_%s_%s" % (
|
|
self._docker_distro,
|
|
_docker_arch_suffix(self.args.arch),
|
|
)
|
|
|
|
def __str__(self):
|
|
return "csharp"
|
|
|
|
|
|
class ObjCLanguage(object):
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
_check_compiler(self.args.compiler, ["default"])
|
|
|
|
def test_specs(self):
|
|
out = []
|
|
out.append(
|
|
self.config.job_spec(
|
|
["src/objective-c/tests/build_one_example.sh"],
|
|
timeout_seconds=20 * 60,
|
|
shortname="ios-buildtest-example-sample",
|
|
cpu_cost=1e6,
|
|
environ={
|
|
"SCHEME": "Sample",
|
|
"EXAMPLE_PATH": "src/objective-c/examples/Sample",
|
|
},
|
|
)
|
|
)
|
|
# TODO(jtattermusch): Create bazel target for the sample and remove the test task from here.
|
|
out.append(
|
|
self.config.job_spec(
|
|
["src/objective-c/tests/build_one_example.sh"],
|
|
timeout_seconds=20 * 60,
|
|
shortname="ios-buildtest-example-switftsample",
|
|
cpu_cost=1e6,
|
|
environ={
|
|
"SCHEME": "SwiftSample",
|
|
"EXAMPLE_PATH": "src/objective-c/examples/SwiftSample",
|
|
},
|
|
)
|
|
)
|
|
out.append(
|
|
self.config.job_spec(
|
|
["src/objective-c/tests/build_one_example.sh"],
|
|
timeout_seconds=20 * 60,
|
|
shortname="ios-buildtest-example-switft-use-frameworks",
|
|
cpu_cost=1e6,
|
|
environ={
|
|
"SCHEME": "SwiftUseFrameworks",
|
|
"EXAMPLE_PATH": "src/objective-c/examples/SwiftUseFrameworks",
|
|
},
|
|
)
|
|
)
|
|
|
|
# Disabled due to #20258
|
|
# TODO (mxyan): Reenable this test when #20258 is resolved.
|
|
# out.append(
|
|
# self.config.job_spec(
|
|
# ['src/objective-c/tests/build_one_example_bazel.sh'],
|
|
# timeout_seconds=20 * 60,
|
|
# shortname='ios-buildtest-example-watchOS-sample',
|
|
# cpu_cost=1e6,
|
|
# environ={
|
|
# 'SCHEME': 'watchOS-sample-WatchKit-App',
|
|
# 'EXAMPLE_PATH': 'src/objective-c/examples/watchOS-sample',
|
|
# 'FRAMEWORKS': 'NO'
|
|
# }))
|
|
|
|
# TODO(jtattermusch): move the test out of the test/core/iomgr/CFStreamTests directory?
|
|
# How does one add the cfstream dependency in bazel?
|
|
out.append(
|
|
self.config.job_spec(
|
|
["test/core/iomgr/ios/CFStreamTests/build_and_run_tests.sh"],
|
|
timeout_seconds=60 * 60,
|
|
shortname="ios-test-cfstream-tests",
|
|
cpu_cost=1e6,
|
|
environ=_FORCE_ENVIRON_FOR_WRAPPERS,
|
|
)
|
|
)
|
|
return sorted(out)
|
|
|
|
def pre_build_steps(self):
|
|
return []
|
|
|
|
def build_steps(self):
|
|
return []
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
return []
|
|
|
|
def dockerfile_dir(self):
|
|
return None
|
|
|
|
def __str__(self):
|
|
return "objc"
|
|
|
|
|
|
class Sanity(object):
|
|
def __init__(self, config_file):
|
|
self.config_file = config_file
|
|
|
|
def configure(self, config, args):
|
|
self.config = config
|
|
self.args = args
|
|
_check_compiler(self.args.compiler, ["default"])
|
|
|
|
def test_specs(self):
|
|
import yaml
|
|
|
|
with open("tools/run_tests/sanity/%s" % self.config_file, "r") as f:
|
|
environ = {"TEST": "true"}
|
|
if _is_use_docker_child():
|
|
environ["CLANG_FORMAT_SKIP_DOCKER"] = "true"
|
|
environ["CLANG_TIDY_SKIP_DOCKER"] = "true"
|
|
environ["IWYU_SKIP_DOCKER"] = "true"
|
|
# sanity tests run tools/bazel wrapper concurrently
|
|
# and that can result in a download/run race in the wrapper.
|
|
# under docker we already have the right version of bazel
|
|
# so we can just disable the wrapper.
|
|
environ["DISABLE_BAZEL_WRAPPER"] = "true"
|
|
return [
|
|
self.config.job_spec(
|
|
cmd["script"].split(),
|
|
timeout_seconds=45 * 60,
|
|
environ=environ,
|
|
cpu_cost=cmd.get("cpu_cost", 1),
|
|
)
|
|
for cmd in yaml.safe_load(f)
|
|
]
|
|
|
|
def pre_build_steps(self):
|
|
return []
|
|
|
|
def build_steps(self):
|
|
return []
|
|
|
|
def build_steps_environ(self):
|
|
"""Extra environment variables set for pre_build_steps and build_steps jobs."""
|
|
return {}
|
|
|
|
def post_tests_steps(self):
|
|
return []
|
|
|
|
def dockerfile_dir(self):
|
|
return "tools/dockerfile/test/sanity"
|
|
|
|
def __str__(self):
|
|
return "sanity"
|
|
|
|
|
|
# different configurations we can run under
|
|
with open("tools/run_tests/generated/configs.json") as f:
|
|
_CONFIGS = dict(
|
|
(cfg["config"], Config(**cfg)) for cfg in ast.literal_eval(f.read())
|
|
)
|
|
|
|
_LANGUAGES = {
|
|
"c++": CLanguage("cxx", "c++"),
|
|
"c": CLanguage("c", "c"),
|
|
"php7": Php7Language(),
|
|
"python": PythonLanguage(),
|
|
"ruby": RubyLanguage(),
|
|
"csharp": CSharpLanguage(),
|
|
"objc": ObjCLanguage(),
|
|
"sanity": Sanity("sanity_tests.yaml"),
|
|
"clang-tidy": Sanity("clang_tidy_tests.yaml"),
|
|
}
|
|
|
|
_MSBUILD_CONFIG = {
|
|
"dbg": "Debug",
|
|
"opt": "Release",
|
|
"gcov": "Debug",
|
|
}
|
|
|
|
|
|
def _build_step_environ(cfg, extra_env={}):
|
|
"""Environment variables set for each build step."""
|
|
environ = {"CONFIG": cfg, "GRPC_RUN_TESTS_JOBS": str(args.jobs)}
|
|
msbuild_cfg = _MSBUILD_CONFIG.get(cfg)
|
|
if msbuild_cfg:
|
|
environ["MSBUILD_CONFIG"] = msbuild_cfg
|
|
environ.update(extra_env)
|
|
return environ
|
|
|
|
|
|
def _windows_arch_option(arch):
|
|
"""Returns msbuild cmdline option for selected architecture."""
|
|
if arch == "default" or arch == "x86":
|
|
return "/p:Platform=Win32"
|
|
elif arch == "x64":
|
|
return "/p:Platform=x64"
|
|
else:
|
|
print("Architecture %s not supported." % arch)
|
|
sys.exit(1)
|
|
|
|
|
|
def _check_arch_option(arch):
|
|
"""Checks that architecture option is valid."""
|
|
if platform_string() == "windows":
|
|
_windows_arch_option(arch)
|
|
elif platform_string() == "linux":
|
|
# On linux, we need to be running under docker with the right architecture.
|
|
runtime_machine = platform.machine()
|
|
runtime_arch = platform.architecture()[0]
|
|
if arch == "default":
|
|
return
|
|
elif (
|
|
runtime_machine == "x86_64"
|
|
and runtime_arch == "64bit"
|
|
and arch == "x64"
|
|
):
|
|
return
|
|
elif (
|
|
runtime_machine == "x86_64"
|
|
and runtime_arch == "32bit"
|
|
and arch == "x86"
|
|
):
|
|
return
|
|
elif (
|
|
runtime_machine == "aarch64"
|
|
and runtime_arch == "64bit"
|
|
and arch == "arm64"
|
|
):
|
|
return
|
|
else:
|
|
print(
|
|
"Architecture %s does not match current runtime architecture."
|
|
% arch
|
|
)
|
|
sys.exit(1)
|
|
else:
|
|
if args.arch != "default":
|
|
print(
|
|
"Architecture %s not supported on current platform." % args.arch
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
def _docker_arch_suffix(arch):
|
|
"""Returns suffix to dockerfile dir to use."""
|
|
if arch == "default" or arch == "x64":
|
|
return "x64"
|
|
elif arch == "x86":
|
|
return "x86"
|
|
elif arch == "arm64":
|
|
return "arm64"
|
|
else:
|
|
print("Architecture %s not supported with current settings." % arch)
|
|
sys.exit(1)
|
|
|
|
|
|
def runs_per_test_type(arg_str):
|
|
"""Auxiliary function to parse the "runs_per_test" flag.
|
|
|
|
Returns:
|
|
A positive integer or 0, the latter indicating an infinite number of
|
|
runs.
|
|
|
|
Raises:
|
|
argparse.ArgumentTypeError: Upon invalid input.
|
|
"""
|
|
if arg_str == "inf":
|
|
return 0
|
|
try:
|
|
n = int(arg_str)
|
|
if n <= 0:
|
|
raise ValueError
|
|
return n
|
|
except:
|
|
msg = "'{}' is not a positive integer or 'inf'".format(arg_str)
|
|
raise argparse.ArgumentTypeError(msg)
|
|
|
|
|
|
def percent_type(arg_str):
|
|
pct = float(arg_str)
|
|
if pct > 100 or pct < 0:
|
|
raise argparse.ArgumentTypeError(
|
|
"'%f' is not a valid percentage in the [0, 100] range" % pct
|
|
)
|
|
return pct
|
|
|
|
|
|
# This is math.isclose in python >= 3.5
|
|
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
|
|
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
|
|
|
|
|
|
def _shut_down_legacy_server(legacy_server_port):
|
|
"""Shut down legacy version of port server."""
|
|
try:
|
|
version = int(
|
|
urllib.request.urlopen(
|
|
"http://localhost:%d/version_number" % legacy_server_port,
|
|
timeout=10,
|
|
).read()
|
|
)
|
|
except:
|
|
pass
|
|
else:
|
|
urllib.request.urlopen(
|
|
"http://localhost:%d/quitquitquit" % legacy_server_port
|
|
).read()
|
|
|
|
|
|
def _calculate_num_runs_failures(list_of_results):
|
|
"""Calculate number of runs and failures for a particular test.
|
|
|
|
Args:
|
|
list_of_results: (List) of JobResult object.
|
|
Returns:
|
|
A tuple of total number of runs and failures.
|
|
"""
|
|
num_runs = len(list_of_results) # By default, there is 1 run per JobResult.
|
|
num_failures = 0
|
|
for jobresult in list_of_results:
|
|
if jobresult.retries > 0:
|
|
num_runs += jobresult.retries
|
|
if jobresult.num_failures > 0:
|
|
num_failures += jobresult.num_failures
|
|
return num_runs, num_failures
|
|
|
|
|
|
class BuildAndRunError(object):
|
|
"""Represents error type in _build_and_run."""
|
|
|
|
BUILD = object()
|
|
TEST = object()
|
|
POST_TEST = object()
|
|
|
|
|
|
# returns a list of things that failed (or an empty list on success)
|
|
def _build_and_run(
|
|
check_cancelled, newline_on_success, xml_report=None, build_only=False
|
|
):
|
|
"""Do one pass of building & running tests."""
|
|
# build latest sequentially
|
|
num_failures, resultset = jobset.run(
|
|
build_steps,
|
|
maxjobs=1,
|
|
stop_on_failure=True,
|
|
newline_on_success=newline_on_success,
|
|
travis=args.travis,
|
|
)
|
|
if num_failures:
|
|
return [BuildAndRunError.BUILD]
|
|
|
|
if build_only:
|
|
if xml_report:
|
|
report_utils.render_junit_xml_report(
|
|
resultset, xml_report, suite_name=args.report_suite_name
|
|
)
|
|
return []
|
|
|
|
# start antagonists
|
|
antagonists = [
|
|
subprocess.Popen(["tools/run_tests/python_utils/antagonist.py"])
|
|
for _ in range(0, args.antagonists)
|
|
]
|
|
start_port_server.start_port_server()
|
|
resultset = None
|
|
num_test_failures = 0
|
|
try:
|
|
infinite_runs = runs_per_test == 0
|
|
one_run = set(
|
|
spec
|
|
for language in languages
|
|
for spec in language.test_specs()
|
|
if (
|
|
re.search(args.regex, spec.shortname)
|
|
and (
|
|
args.regex_exclude == ""
|
|
or not re.search(args.regex_exclude, spec.shortname)
|
|
)
|
|
)
|
|
)
|
|
# When running on travis, we want out test runs to be as similar as possible
|
|
# for reproducibility purposes.
|
|
if args.travis and args.max_time <= 0:
|
|
massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost)
|
|
else:
|
|
# whereas otherwise, we want to shuffle things up to give all tests a
|
|
# chance to run.
|
|
massaged_one_run = list(
|
|
one_run
|
|
) # random.sample needs an indexable seq.
|
|
num_jobs = len(massaged_one_run)
|
|
# for a random sample, get as many as indicated by the 'sample_percent'
|
|
# argument. By default this arg is 100, resulting in a shuffle of all
|
|
# jobs.
|
|
sample_size = int(num_jobs * args.sample_percent / 100.0)
|
|
massaged_one_run = random.sample(massaged_one_run, sample_size)
|
|
if not isclose(args.sample_percent, 100.0):
|
|
assert (
|
|
args.runs_per_test == 1
|
|
), "Can't do sampling (-p) over multiple runs (-n)."
|
|
print(
|
|
"Running %d tests out of %d (~%d%%)"
|
|
% (sample_size, num_jobs, args.sample_percent)
|
|
)
|
|
if infinite_runs:
|
|
assert (
|
|
len(massaged_one_run) > 0
|
|
), "Must have at least one test for a -n inf run"
|
|
runs_sequence = (
|
|
itertools.repeat(massaged_one_run)
|
|
if infinite_runs
|
|
else itertools.repeat(massaged_one_run, runs_per_test)
|
|
)
|
|
all_runs = itertools.chain.from_iterable(runs_sequence)
|
|
|
|
if args.quiet_success:
|
|
jobset.message(
|
|
"START",
|
|
"Running tests quietly, only failing tests will be reported",
|
|
do_newline=True,
|
|
)
|
|
num_test_failures, resultset = jobset.run(
|
|
all_runs,
|
|
check_cancelled,
|
|
newline_on_success=newline_on_success,
|
|
travis=args.travis,
|
|
maxjobs=args.jobs,
|
|
maxjobs_cpu_agnostic=max_parallel_tests_for_current_platform(),
|
|
stop_on_failure=args.stop_on_failure,
|
|
quiet_success=args.quiet_success,
|
|
max_time=args.max_time,
|
|
)
|
|
if resultset:
|
|
for k, v in sorted(resultset.items()):
|
|
num_runs, num_failures = _calculate_num_runs_failures(v)
|
|
if num_failures > 0:
|
|
if num_failures == num_runs: # what about infinite_runs???
|
|
jobset.message("FAILED", k, do_newline=True)
|
|
else:
|
|
jobset.message(
|
|
"FLAKE",
|
|
"%s [%d/%d runs flaked]"
|
|
% (k, num_failures, num_runs),
|
|
do_newline=True,
|
|
)
|
|
finally:
|
|
for antagonist in antagonists:
|
|
antagonist.kill()
|
|
if args.bq_result_table and resultset:
|
|
upload_extra_fields = {
|
|
"compiler": args.compiler,
|
|
"config": args.config,
|
|
"iomgr_platform": args.iomgr_platform,
|
|
"language": args.language[
|
|
0
|
|
], # args.language is a list but will always have one element when uploading to BQ is enabled.
|
|
"platform": platform_string(),
|
|
}
|
|
try:
|
|
upload_results_to_bq(
|
|
resultset, args.bq_result_table, upload_extra_fields
|
|
)
|
|
except NameError as e:
|
|
logging.warning(
|
|
e
|
|
) # It's fine to ignore since this is not critical
|
|
if xml_report and resultset:
|
|
report_utils.render_junit_xml_report(
|
|
resultset,
|
|
xml_report,
|
|
suite_name=args.report_suite_name,
|
|
multi_target=args.report_multi_target,
|
|
)
|
|
|
|
number_failures, _ = jobset.run(
|
|
post_tests_steps,
|
|
maxjobs=1,
|
|
stop_on_failure=False,
|
|
newline_on_success=newline_on_success,
|
|
travis=args.travis,
|
|
)
|
|
|
|
out = []
|
|
if number_failures:
|
|
out.append(BuildAndRunError.POST_TEST)
|
|
if num_test_failures:
|
|
out.append(BuildAndRunError.TEST)
|
|
|
|
return out
|
|
|
|
|
|
# parse command line
|
|
argp = argparse.ArgumentParser(description="Run grpc tests.")
|
|
argp.add_argument(
|
|
"-c", "--config", choices=sorted(_CONFIGS.keys()), default="opt"
|
|
)
|
|
argp.add_argument(
|
|
"-n",
|
|
"--runs_per_test",
|
|
default=1,
|
|
type=runs_per_test_type,
|
|
help=(
|
|
'A positive integer or "inf". If "inf", all tests will run in an '
|
|
'infinite loop. Especially useful in combination with "-f"'
|
|
),
|
|
)
|
|
argp.add_argument("-r", "--regex", default=".*", type=str)
|
|
argp.add_argument("--regex_exclude", default="", type=str)
|
|
argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int)
|
|
argp.add_argument("-s", "--slowdown", default=1.0, type=float)
|
|
argp.add_argument(
|
|
"-p",
|
|
"--sample_percent",
|
|
default=100.0,
|
|
type=percent_type,
|
|
help="Run a random sample with that percentage of tests",
|
|
)
|
|
argp.add_argument(
|
|
"-t",
|
|
"--travis",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help=(
|
|
"When set, indicates that the script is running on CI (= not locally)."
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--newline_on_success", default=False, action="store_const", const=True
|
|
)
|
|
argp.add_argument(
|
|
"-l",
|
|
"--language",
|
|
choices=sorted(_LANGUAGES.keys()),
|
|
nargs="+",
|
|
required=True,
|
|
)
|
|
argp.add_argument(
|
|
"-S", "--stop_on_failure", default=False, action="store_const", const=True
|
|
)
|
|
argp.add_argument(
|
|
"--use_docker",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help="Run all the tests under docker. That provides "
|
|
+ "additional isolation and prevents the need to install "
|
|
+ "language specific prerequisites. Only available on Linux.",
|
|
)
|
|
argp.add_argument(
|
|
"--allow_flakes",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help=(
|
|
"Allow flaky tests to show as passing (re-runs failed tests up to five"
|
|
" times)"
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--arch",
|
|
choices=["default", "x86", "x64", "arm64"],
|
|
default="default",
|
|
help=(
|
|
'Selects architecture to target. For some platforms "default" is the'
|
|
" only supported choice."
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--compiler",
|
|
choices=[
|
|
"default",
|
|
"gcc8",
|
|
"gcc10.2",
|
|
"gcc10.2_openssl102",
|
|
"gcc10.2_openssl111",
|
|
"gcc12",
|
|
"gcc12_openssl309",
|
|
"gcc_musl",
|
|
"clang6",
|
|
"clang17",
|
|
# TODO: Automatically populate from supported version
|
|
"python3.7",
|
|
"python3.8",
|
|
"python3.9",
|
|
"python3.10",
|
|
"python3.11",
|
|
"python3.12",
|
|
"pypy",
|
|
"pypy3",
|
|
"python_alpine",
|
|
"all_the_cpythons",
|
|
"coreclr",
|
|
"cmake",
|
|
"cmake_ninja_vs2019",
|
|
"cmake_ninja_vs2022",
|
|
"cmake_vs2019",
|
|
"cmake_vs2022",
|
|
"mono",
|
|
],
|
|
default="default",
|
|
help=(
|
|
"Selects compiler to use. Allowed values depend on the platform and"
|
|
" language."
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--iomgr_platform",
|
|
choices=["native", "gevent", "asyncio"],
|
|
default="native",
|
|
help="Selects iomgr platform to build on",
|
|
)
|
|
argp.add_argument(
|
|
"--build_only",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help="Perform all the build steps but don't run any tests.",
|
|
)
|
|
argp.add_argument(
|
|
"--measure_cpu_costs",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help="Measure the cpu costs of tests",
|
|
)
|
|
argp.add_argument("-a", "--antagonists", default=0, type=int)
|
|
argp.add_argument(
|
|
"-x",
|
|
"--xml_report",
|
|
default=None,
|
|
type=str,
|
|
help="Generates a JUnit-compatible XML report",
|
|
)
|
|
argp.add_argument(
|
|
"--report_suite_name",
|
|
default="tests",
|
|
type=str,
|
|
help="Test suite name to use in generated JUnit XML report",
|
|
)
|
|
argp.add_argument(
|
|
"--report_multi_target",
|
|
default=False,
|
|
const=True,
|
|
action="store_const",
|
|
help=(
|
|
"Generate separate XML report for each test job (Looks better in UIs)."
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--quiet_success",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help=(
|
|
"Don't print anything when a test passes. Passing tests also will not"
|
|
" be reported in XML report. "
|
|
)
|
|
+ "Useful when running many iterations of each test (argument -n).",
|
|
)
|
|
argp.add_argument(
|
|
"--force_default_poller",
|
|
default=False,
|
|
action="store_const",
|
|
const=True,
|
|
help="Don't try to iterate over many polling strategies when they exist",
|
|
)
|
|
argp.add_argument(
|
|
"--force_use_pollers",
|
|
default=None,
|
|
type=str,
|
|
help=(
|
|
"Only use the specified comma-delimited list of polling engines. "
|
|
"Example: --force_use_pollers epoll1,poll "
|
|
" (This flag has no effect if --force_default_poller flag is also used)"
|
|
),
|
|
)
|
|
argp.add_argument(
|
|
"--max_time", default=-1, type=int, help="Maximum test runtime in seconds"
|
|
)
|
|
argp.add_argument(
|
|
"--bq_result_table",
|
|
default="",
|
|
type=str,
|
|
nargs="?",
|
|
help="Upload test results to a specified BQ table.",
|
|
)
|
|
argp.add_argument(
|
|
"--cmake_configure_extra_args",
|
|
default=[],
|
|
nargs="+",
|
|
help="Extra arguments that will be passed to the cmake configure command. Only works for C/C++.",
|
|
)
|
|
args = argp.parse_args()
|
|
|
|
flaky_tests = set()
|
|
shortname_to_cpu = {}
|
|
|
|
if args.force_default_poller:
|
|
_POLLING_STRATEGIES = {}
|
|
elif args.force_use_pollers:
|
|
_POLLING_STRATEGIES[platform_string()] = args.force_use_pollers.split(",")
|
|
|
|
jobset.measure_cpu_costs = args.measure_cpu_costs
|
|
|
|
# grab config
|
|
run_config = _CONFIGS[args.config]
|
|
build_config = run_config.build_config
|
|
|
|
languages = set(_LANGUAGES[l] for l in args.language)
|
|
for l in languages:
|
|
l.configure(run_config, args)
|
|
|
|
if len(languages) != 1:
|
|
print("Building multiple languages simultaneously is not supported!")
|
|
sys.exit(1)
|
|
|
|
# If --use_docker was used, respawn the run_tests.py script under a docker container
|
|
# instead of continuing.
|
|
if args.use_docker:
|
|
if not args.travis:
|
|
print("Seen --use_docker flag, will run tests under docker.")
|
|
print("")
|
|
print(
|
|
"IMPORTANT: The changes you are testing need to be locally"
|
|
" committed"
|
|
)
|
|
print(
|
|
"because only the committed changes in the current branch will be"
|
|
)
|
|
print("copied to the docker environment.")
|
|
time.sleep(5)
|
|
|
|
dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
|
|
if len(dockerfile_dirs) > 1:
|
|
print(
|
|
"Languages to be tested require running under different docker "
|
|
"images."
|
|
)
|
|
sys.exit(1)
|
|
else:
|
|
dockerfile_dir = next(iter(dockerfile_dirs))
|
|
|
|
child_argv = [arg for arg in sys.argv if not arg == "--use_docker"]
|
|
run_tests_cmd = "python3 tools/run_tests/run_tests.py %s" % " ".join(
|
|
child_argv[1:]
|
|
)
|
|
|
|
env = os.environ.copy()
|
|
env["DOCKERFILE_DIR"] = dockerfile_dir
|
|
env["DOCKER_RUN_SCRIPT"] = "tools/run_tests/dockerize/docker_run.sh"
|
|
env["DOCKER_RUN_SCRIPT_COMMAND"] = run_tests_cmd
|
|
|
|
retcode = subprocess.call(
|
|
"tools/run_tests/dockerize/build_and_run_docker.sh", shell=True, env=env
|
|
)
|
|
_print_debug_info_epilogue(dockerfile_dir=dockerfile_dir)
|
|
sys.exit(retcode)
|
|
|
|
_check_arch_option(args.arch)
|
|
|
|
# collect pre-build steps (which get retried if they fail, e.g. to avoid
|
|
# flakes on downloading dependencies etc.)
|
|
build_steps = list(
|
|
set(
|
|
jobset.JobSpec(
|
|
cmdline,
|
|
environ=_build_step_environ(
|
|
build_config, extra_env=l.build_steps_environ()
|
|
),
|
|
timeout_seconds=_PRE_BUILD_STEP_TIMEOUT_SECONDS,
|
|
flake_retries=2,
|
|
)
|
|
for l in languages
|
|
for cmdline in l.pre_build_steps()
|
|
)
|
|
)
|
|
|
|
# collect build steps
|
|
build_steps.extend(
|
|
set(
|
|
jobset.JobSpec(
|
|
cmdline,
|
|
environ=_build_step_environ(
|
|
build_config, extra_env=l.build_steps_environ()
|
|
),
|
|
timeout_seconds=None,
|
|
)
|
|
for l in languages
|
|
for cmdline in l.build_steps()
|
|
)
|
|
)
|
|
|
|
# collect post test steps
|
|
post_tests_steps = list(
|
|
set(
|
|
jobset.JobSpec(
|
|
cmdline,
|
|
environ=_build_step_environ(
|
|
build_config, extra_env=l.build_steps_environ()
|
|
),
|
|
)
|
|
for l in languages
|
|
for cmdline in l.post_tests_steps()
|
|
)
|
|
)
|
|
runs_per_test = args.runs_per_test
|
|
|
|
errors = _build_and_run(
|
|
check_cancelled=lambda: False,
|
|
newline_on_success=args.newline_on_success,
|
|
xml_report=args.xml_report,
|
|
build_only=args.build_only,
|
|
)
|
|
if not errors:
|
|
jobset.message("SUCCESS", "All tests passed", do_newline=True)
|
|
else:
|
|
jobset.message("FAILED", "Some tests failed", do_newline=True)
|
|
|
|
if not _is_use_docker_child():
|
|
# if --use_docker was used, the outer invocation of run_tests.py will
|
|
# print the debug info instead.
|
|
_print_debug_info_epilogue()
|
|
|
|
exit_code = 0
|
|
if BuildAndRunError.BUILD in errors:
|
|
exit_code |= 1
|
|
if BuildAndRunError.TEST in errors:
|
|
exit_code |= 2
|
|
if BuildAndRunError.POST_TEST in errors:
|
|
exit_code |= 4
|
|
sys.exit(exit_code)
|