natas_hw
/
grpc
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Python3-ize tools/buildgen. 

* Ran `2to3` on `tools/buildgen` and the `gen_build_yaml.py` files.
* Updated the invocations of python to be explicitly Python 3.
* Changed the loader in `mako_renderer.py` to allow pickle to find the module for pickling `external_version.Version`
* Added a few open-as-binary to placate readers/writers expecting bytes.
* Hand tweaked the templates to remove `iteritems` and fiddle with encodings.

Partially addresses #24359.
This commit is contained in:
capstan 2020-10-23 01:48:12 +02:00
parent d39e7133c5
commit 7267c8fcd9
22 changed files with 107 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/benchmark/gen_build_yaml.py
View File

@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import os
import sys
import glob

2
src/c-ares/gen_build_yaml.py
View File

@ -145,4 +145,4 @@ try:
except:
pass
print yaml.dump(out)
print(yaml.dump(out))

2
src/proto/gen_build_yaml.py
View File

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generates the appropriate build.json data for all the proto files."""
from __future__ import print_function
import yaml
import collections
import os

1
src/re2/gen_build_yaml.py
View File

@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import os
import sys
import glob

1
src/upb/gen_build_yaml.py
View File

@ -16,7 +16,6 @@
# TODO: This should ideally be in upb submodule to avoid hardcoding this here.
from __future__ import print_function
import re
import os
import sys

2
src/zlib/gen_build_yaml.py
View File

@ -58,4 +58,4 @@ try:
except:
pass
print yaml.dump(out)
print(yaml.dump(out))

6
templates/Makefile.template
View File

@ -127,7 +127,7 @@
# Configurations (as defined under "configs" section in build_handwritten.yaml)
% for name, args in configs.iteritems():
% for name, args in configs.items():
VALID_CONFIG_${name} = 1
% if args.get('compile_the_world', False):
REQUIRE_CUSTOM_LIBRARIES_${name} = 1
@ -1020,7 +1020,7 @@
LIB${lib.name.upper()}_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIB${lib.name.upper()}_SRC))))
% if lib.get('defaults', None):
% for name, value in defaults.get(lib.defaults).iteritems():
% for name, value in defaults.get(lib.defaults).items():
$(LIB${lib.name.upper()}_OBJS): ${name} += ${value}
% endfor
% endif
@ -1361,7 +1361,7 @@
% endif
% if tgt.get('defaults', None):
% for name, value in defaults.get(tgt.defaults).iteritems():
% for name, value in defaults.get(tgt.defaults).items():
$(${tgt.name.upper()}_OBJS): ${name} += ${value}
% endfor
% endif

2
templates/grpc.gyp.template
View File

@ -36,7 +36,7 @@
},
'target_defaults': {
'configurations': {
% for name, args in configs.iteritems():
% for name, args in configs.items():
% if name in ['dbg', 'opt']:
'${{'dbg':'Debug', 'opt': 'Release'}[name]}': {
% for arg, prop in [('CPPFLAGS', 'cflags'), ('DEFINES', 'defines')]:

2
templates/src/objective-c/BoringSSL-GRPC.podspec.template
View File

@ -244,7 +244,7 @@
# the correct location in BoringSSL.
base64 -D <<EOF | gunzip > src/include/openssl/boringssl_prefix_symbols.h
% for line in compress_boringssl_prefix_header():
${line}
${line.decode('utf-8')}
% endfor
EOF

2
templates/src/python/grpcio/grpc_core_dependencies.py.template
View File

@ -33,7 +33,7 @@
]
ASM_SOURCE_FILES = {
% for asm, asm_src in asm_srcs.iteritems():
% for asm, asm_src in asm_srcs.items():
'${asm}': [
% for src in asm_src:
'${src}',

2
templates/test/core/end2end/end2end_nosec_tests.cc.template
View File

@ -1,4 +1,4 @@
%YAML 1.2
--- |
<%namespace file="end2end_defs.include" import="*"/>\
${end2end_selector(k for k, v in core_end2end_tests.iteritems() if not v)}
${end2end_selector(k for k, v in core_end2end_tests.items() if not v)}

2
templates/tools/run_tests/generated/configs.json.template
View File

@ -3,7 +3,7 @@
<%
import json
out_configs = []
for name, args in configs.iteritems():
for name, args in configs.items():
config_args={}
config_args['config'] = name
if args.get('valgrind', None) is not None:

5
test/core/end2end/gen_build_yaml.py
View File

@ -33,7 +33,10 @@ def struct(**kwargs):
# generate_tests.bzl is now the source of truth for end2end tests.
# The .bzl file is basically a python file and we can "execute" it
# to get access to the variables it defines.
execfile('test/core/end2end/generate_tests.bzl')
exec(
compile(
open('test/core/end2end/generate_tests.bzl', "rb").read(),
'test/core/end2end/generate_tests.bzl', 'exec'))
def main():

3
tools/buildgen/build_cleaner.py
View File

@ -32,7 +32,8 @@ _ELEM_KEYS = [
def repr_ordered_dict(dumper, odict):
return dumper.represent_mapping(u'tag:yaml.org,2002:map', odict.items())
return dumper.represent_mapping('tag:yaml.org,2002:map',
list(odict.items()))
yaml.add_representer(collections.OrderedDict, repr_ordered_dict)

146
tools/buildgen/extract_metadata_from_bazel_xml.py
View File

@ -176,7 +176,8 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False):
# all libs that are not in the dictionary are considered external.
external_deps = list(
sorted(filter(lambda lib_name: lib_name not in lib_dict, lib_names)))
sorted([lib_name for lib_name in lib_names if lib_name not in lib_dict
]))
if verbose:
print('topo_ordering ' + str(lib_names))
print(' external_deps ' + str(external_deps))
@ -207,12 +208,12 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False):
def _populate_transitive_deps(bazel_rules):
"""Add 'transitive_deps' field for each of the rules"""
transitive_deps = {}
for rule_name in bazel_rules.iterkeys():
for rule_name in bazel_rules.keys():
transitive_deps[rule_name] = set(bazel_rules[rule_name]['deps'])
while True:
deps_added = 0
for rule_name in bazel_rules.iterkeys():
for rule_name in bazel_rules.keys():
old_deps = transitive_deps[rule_name]
new_deps = set(old_deps)
for dep_name in old_deps:
@ -223,7 +224,7 @@ def _populate_transitive_deps(bazel_rules):
if deps_added == 0:
break
for rule_name, bazel_rule in bazel_rules.iteritems():
for rule_name, bazel_rule in bazel_rules.items():
bazel_rule['transitive_deps'] = list(sorted(transitive_deps[rule_name]))
@ -337,7 +338,7 @@ def _expand_intermediate_deps(target_dict, public_dep_names, bazel_rules):
def _generate_build_metadata(build_extra_metadata, bazel_rules):
"""Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific "extra metadata"."""
lib_names = build_extra_metadata.keys()
lib_names = list(build_extra_metadata.keys())
result = {}
for lib_name in lib_names:
@ -381,12 +382,13 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
# dep names need to be updated as well
for lib_dict_to_update in result.values():
lib_dict_to_update['deps'] = list(
map(lambda dep: to_name if dep == lib_name else dep,
lib_dict_to_update['deps']))
lib_dict_to_update['deps'] = list([
to_name if dep == lib_name else dep
for dep in lib_dict_to_update['deps']
])
# make sure deps are listed in reverse topological order (e.g. "grpc gpr" and not "gpr grpc")
for lib_dict in result.itervalues():
for lib_dict in result.values():
lib_dict['deps'] = list(
reversed(_sort_by_build_order(lib_dict['deps'], result, 'deps')))
@ -394,36 +396,35 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
def _convert_to_build_yaml_like(lib_dict):
lib_names = list(
filter(
lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
'library', lib_dict.keys()))
target_names = list(
filter(
lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
'target', lib_dict.keys()))
test_names = list(
filter(
lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
'test', lib_dict.keys()))
lib_names = [
lib_name for lib_name in list(lib_dict.keys())
if lib_dict[lib_name].get('_TYPE', 'library') == 'library'
]
target_names = [
lib_name for lib_name in list(lib_dict.keys())
if lib_dict[lib_name].get('_TYPE', 'library') == 'target'
]
test_names = [
lib_name for lib_name in list(lib_dict.keys())
if lib_dict[lib_name].get('_TYPE', 'library') == 'test'
]
# list libraries and targets in predefined order
lib_list = list(map(lambda lib_name: lib_dict[lib_name], lib_names))
target_list = list(map(lambda lib_name: lib_dict[lib_name], target_names))
test_list = list(map(lambda lib_name: lib_dict[lib_name], test_names))
lib_list = [lib_dict[lib_name] for lib_name in lib_names]
target_list = [lib_dict[lib_name] for lib_name in target_names]
test_list = [lib_dict[lib_name] for lib_name in test_names]
# get rid of temporary private fields prefixed with "_" and some other useless fields
for lib in lib_list:
for field_to_remove in filter(lambda k: k.startswith('_'), lib.keys()):
for field_to_remove in [k for k in lib.keys() if k.startswith('_')]:
lib.pop(field_to_remove, None)
for target in target_list:
for field_to_remove in filter(lambda k: k.startswith('_'),
target.keys()):
for field_to_remove in [k for k in target.keys() if k.startswith('_')]:
target.pop(field_to_remove, None)
target.pop('public_headers',
None) # public headers make no sense for targets
for test in test_list:
for field_to_remove in filter(lambda k: k.startswith('_'), test.keys()):
for field_to_remove in [k for k in test.keys() if k.startswith('_')]:
test.pop(field_to_remove, None)
test.pop('public_headers',
None) # public headers make no sense for tests
@ -440,7 +441,7 @@ def _convert_to_build_yaml_like(lib_dict):
def _extract_cc_tests(bazel_rules):
"""Gets list of cc_test tests from bazel rules"""
result = []
for bazel_rule in bazel_rules.itervalues():
for bazel_rule in bazel_rules.values():
if bazel_rule['class'] == 'cc_test':
test_name = bazel_rule['name']
if test_name.startswith('//'):
@ -453,65 +454,64 @@ def _exclude_unwanted_cc_tests(tests):
"""Filters out bazel tests that we don't want to run with other build systems or we cannot build them reasonably"""
# most qps tests are autogenerated, we are fine without them
tests = list(
filter(lambda test: not test.startswith('test/cpp/qps:'), tests))
tests = [test for test in tests if not test.startswith('test/cpp/qps:')]
# we have trouble with census dependency outside of bazel
tests = list(
filter(lambda test: not test.startswith('test/cpp/ext/filters/census:'),
tests))
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/microbenchmarks:bm_opencensus_plugin'), tests))
tests = [
test for test in tests
if not test.startswith('test/cpp/ext/filters/census:')
]
tests = [
test for test in tests
if not test.startswith('test/cpp/microbenchmarks:bm_opencensus_plugin')
]
# missing opencensus/stats/stats.h
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/end2end:server_load_reporting_end2end_test'), tests))
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/server/load_reporter:lb_load_reporter_test'), tests))
tests = [
test for test in tests if not test.startswith(
'test/cpp/end2end:server_load_reporting_end2end_test')
]
tests = [
test for test in tests if not test.startswith(
'test/cpp/server/load_reporter:lb_load_reporter_test')
]
# The test uses --running_under_bazel cmdline argument
# To avoid the trouble needing to adjust it, we just skip the test
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/naming:resolver_component_tests_runner_invoker'),
tests))
tests = [
test for test in tests if not test.startswith(
'test/cpp/naming:resolver_component_tests_runner_invoker')
]
# the test requires 'client_crash_test_server' to be built
tests = list(
filter(
lambda test: not test.startswith('test/cpp/end2end:time_change_test'
), tests))
tests = [
test for test in tests
if not test.startswith('test/cpp/end2end:time_change_test')
]
# the test requires 'client_crash_test_server' to be built
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/end2end:client_crash_test'), tests))
tests = [
test for test in tests
if not test.startswith('test/cpp/end2end:client_crash_test')
]
# the test requires 'server_crash_test_client' to be built
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/end2end:server_crash_test'), tests))
tests = [
test for test in tests
if not test.startswith('test/cpp/end2end:server_crash_test')
]
# test never existed under build.yaml and it fails -> skip it
tests = list(
filter(
lambda test: not test.startswith(
'test/core/tsi:ssl_session_cache_test'), tests))
tests = [
test for test in tests
if not test.startswith('test/core/tsi:ssl_session_cache_test')
]
# the binary of this test does not get built with cmake
tests = list(
filter(
lambda test: not test.startswith(
'test/cpp/util:channelz_sampler_test'), tests))
tests = [
test for test in tests
if not test.startswith('test/cpp/util:channelz_sampler_test')
]
return tests
@ -594,14 +594,14 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules):
# detect duplicate test names
tests_by_simple_name = {}
for test_name, test_dict in test_metadata.iteritems():
for test_name, test_dict in test_metadata.items():
simple_test_name = test_dict['_RENAME']
if not simple_test_name in tests_by_simple_name:
tests_by_simple_name[simple_test_name] = []
tests_by_simple_name[simple_test_name].append(test_name)
# choose alternative names for tests with a name collision
for collision_list in tests_by_simple_name.itervalues():
for collision_list in tests_by_simple_name.values():
if len(collision_list) > 1:
for test_name in collision_list:
long_name = test_name.replace('/', '_').replace(':', '_')

2
tools/buildgen/generate_build_additions.sh
View File

@ -33,6 +33,6 @@ gen_build_files=""
for gen_build_yaml in $gen_build_yaml_dirs
do
output_file=`mktemp /tmp/genXXXXXX`
python $gen_build_yaml/gen_build_yaml.py > $output_file
python3 $gen_build_yaml/gen_build_yaml.py > $output_file
gen_build_files="$gen_build_files $output_file"
done

2
tools/buildgen/generate_projects.py
View File

@ -93,7 +93,7 @@ jobset.run(pre_jobs, maxjobs=args.jobs)
jobset.run(jobs, maxjobs=args.jobs)
if test is not None:
for s, g in test.iteritems():
for s, g in test.items():
if os.path.isfile(g):
assert 0 == os.system('diff %s %s' % (s, g)), s
os.unlink(g)

4
tools/buildgen/generate_projects.sh
View File

@ -20,7 +20,7 @@ export TEST=${TEST:-false}
echo "Generating build_autogenerated.yaml from bazel BUILD file"
rm -f build_autogenerated.yaml
python tools/buildgen/extract_metadata_from_bazel_xml.py
python3 tools/buildgen/extract_metadata_from_bazel_xml.py
cd `dirname $0`/../..
mako_renderer=tools/buildgen/mako_renderer.py
@ -35,6 +35,6 @@ TEST=true tools/buildgen/build_cleaner.py build_autogenerated.yaml
# Instead of generating from a single build.yaml, we've split it into
# - build_handwritten.yaml: manually written metadata
# - build_autogenerated.yaml: generated from bazel BUILD file
python tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $*
python3 tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $*
rm $gen_build_files

19
tools/buildgen/mako_renderer.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -21,7 +21,7 @@ Just a wrapper around the mako rendering library.
import getopt
import imp
import os
import cPickle as pickle
import pickle
import shutil
import sys
@ -36,16 +36,11 @@ import yaml
def import_plugin(name):
_, base_ex = os.path.split(name)
base, _ = os.path.splitext(base_ex)
with open(name, 'r') as plugin_file:
plugin_code = plugin_file.read()
plugin_module = imp.new_module(base)
exec plugin_code in plugin_module.__dict__
return plugin_module
return imp.load_source(base, name)
def out(msg):
print >> sys.stderr, msg
print(msg, file=sys.stderr)
def showhelp():
@ -103,7 +98,7 @@ def main(argv):
0,
os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), 'plugins')))
with open(arg, 'r') as dict_file:
with open(arg, 'rb') as dict_file:
dictionary = pickle.load(dict_file)
got_preprocessed_input = True
elif opt == '-d':
@ -125,7 +120,7 @@ def main(argv):
dictionary[k] = bunch.to_bunch(v)
if preprocessed_output:
with open(preprocessed_output, 'w') as dict_file:
with open(preprocessed_output, 'wb') as dict_file:
pickle.dump(dictionary, dict_file)
cleared_dir = False
@ -134,7 +129,7 @@ def main(argv):
with open(arg) as f:
srcs = list(yaml.load_all(f.read()))
for src in srcs:
if isinstance(src, basestring):
if isinstance(src, str):
assert len(srcs) == 1
template = Template(src,
filename=arg,

6
tools/buildgen/plugins/expand_filegroups.py
View File

@ -62,7 +62,7 @@ def mako_plugin(dictionary):
for lst in FILEGROUP_LISTS:
fg[lst] = fg.get(lst, [])
fg['own_%s' % lst] = list(fg[lst])
for attr, val in FILEGROUP_DEFAULTS.iteritems():
for attr, val in FILEGROUP_DEFAULTS.items():
if attr not in fg:
fg[attr] = val
@ -113,7 +113,7 @@ def mako_plugin(dictionary):
thing['used_by'] = []
thing_deps = lambda t: t.get('uses', []) + t.get('filegroups', []) + t.get(
'deps', [])
for thing in things.itervalues():
for thing in things.values():
done = set()
todo = thing_deps(thing)
while todo:
@ -125,7 +125,7 @@ def mako_plugin(dictionary):
done.add(cur)
# the above expansion can introduce duplicate filenames: contract them here
for fg in filegroups.itervalues():
for fg in filegroups.values():
for lst in FILEGROUP_LISTS:
fg[lst] = uniquify(fg.get(lst, []))

2
tools/buildgen/plugins/generate_vsprojects.py
View File

@ -61,7 +61,7 @@ def mako_plugin(dictionary):
name = target['name']
guid = re.sub('(........)(....)(....)(....)(.*)',
r'{\1-\2-\3-\4-\5}',
hashlib.md5(name).hexdigest())
hashlib.md5(name.encode('utf-8')).hexdigest())
target['vs_project_guid'] = guid.upper()
# Exclude projects without a visual project guid, such as the tests.
projects = [

3
tools/buildgen/plugins/list_api.py
View File

@ -64,4 +64,5 @@ def mako_plugin(dictionary):
if __name__ == '__main__':
print yaml.dump([api for api in list_c_apis(headers_under('include/grpc'))])
print(yaml.dump([api for api in list_c_apis(headers_under('include/grpc'))
]))