diff --git a/src/benchmark/gen_build_yaml.py b/src/benchmark/gen_build_yaml.py index 6d9079bb8b6..45c39ffb970 100755 --- a/src/benchmark/gen_build_yaml.py +++ b/src/benchmark/gen_build_yaml.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function import os import sys import glob diff --git a/src/c-ares/gen_build_yaml.py b/src/c-ares/gen_build_yaml.py index 27903d35716..d12203d7031 100755 --- a/src/c-ares/gen_build_yaml.py +++ b/src/c-ares/gen_build_yaml.py @@ -145,4 +145,4 @@ try: except: pass -print yaml.dump(out) +print(yaml.dump(out)) diff --git a/src/proto/gen_build_yaml.py b/src/proto/gen_build_yaml.py index 3900ac25690..e310d4c8937 100755 --- a/src/proto/gen_build_yaml.py +++ b/src/proto/gen_build_yaml.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Generates the appropriate build.json data for all the proto files.""" -from __future__ import print_function + import yaml import collections import os diff --git a/src/re2/gen_build_yaml.py b/src/re2/gen_build_yaml.py index bdb1f7b544e..cd66a13e46e 100755 --- a/src/re2/gen_build_yaml.py +++ b/src/re2/gen_build_yaml.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function import os import sys import glob diff --git a/src/upb/gen_build_yaml.py b/src/upb/gen_build_yaml.py index cd585095ef6..3bbc44cdded 100755 --- a/src/upb/gen_build_yaml.py +++ b/src/upb/gen_build_yaml.py @@ -16,7 +16,6 @@ # TODO: This should ideally be in upb submodule to avoid hardcoding this here. -from __future__ import print_function import re import os import sys diff --git a/src/zlib/gen_build_yaml.py b/src/zlib/gen_build_yaml.py index a27d8c0a359..c58ff8ab3a9 100755 --- a/src/zlib/gen_build_yaml.py +++ b/src/zlib/gen_build_yaml.py @@ -58,4 +58,4 @@ try: except: pass -print yaml.dump(out) +print(yaml.dump(out)) diff --git a/templates/Makefile.template b/templates/Makefile.template index b9999a60f43..50f5d42aa5b 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -127,7 +127,7 @@ # Configurations (as defined under "configs" section in build_handwritten.yaml) - % for name, args in configs.iteritems(): + % for name, args in configs.items(): VALID_CONFIG_${name} = 1 % if args.get('compile_the_world', False): REQUIRE_CUSTOM_LIBRARIES_${name} = 1 @@ -1020,7 +1020,7 @@ LIB${lib.name.upper()}_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIB${lib.name.upper()}_SRC)))) % if lib.get('defaults', None): - % for name, value in defaults.get(lib.defaults).iteritems(): + % for name, value in defaults.get(lib.defaults).items(): $(LIB${lib.name.upper()}_OBJS): ${name} += ${value} % endfor % endif @@ -1361,7 +1361,7 @@ % endif % if tgt.get('defaults', None): - % for name, value in defaults.get(tgt.defaults).iteritems(): + % for name, value in defaults.get(tgt.defaults).items(): $(${tgt.name.upper()}_OBJS): ${name} += ${value} % endfor % endif diff --git a/templates/grpc.gyp.template b/templates/grpc.gyp.template index a722dfc9bb9..49ce883ef8f 100644 --- a/templates/grpc.gyp.template +++ b/templates/grpc.gyp.template @@ -36,7 +36,7 @@ }, 'target_defaults': { 'configurations': { - % for name, args in configs.iteritems(): + % for name, args in configs.items(): % if name in ['dbg', 'opt']: '${{'dbg':'Debug', 'opt': 'Release'}[name]}': { % for arg, prop in [('CPPFLAGS', 'cflags'), ('DEFINES', 'defines')]: diff --git a/templates/src/objective-c/BoringSSL-GRPC.podspec.template b/templates/src/objective-c/BoringSSL-GRPC.podspec.template index 11c93ac8761..4855d9d59b7 100644 --- a/templates/src/objective-c/BoringSSL-GRPC.podspec.template +++ b/templates/src/objective-c/BoringSSL-GRPC.podspec.template @@ -244,7 +244,7 @@ # the correct location in BoringSSL. base64 -D < src/include/openssl/boringssl_prefix_symbols.h % for line in compress_boringssl_prefix_header(): - ${line} + ${line.decode('utf-8')} % endfor EOF diff --git a/templates/src/python/grpcio/grpc_core_dependencies.py.template b/templates/src/python/grpcio/grpc_core_dependencies.py.template index 153e7f8aef8..619dea97975 100644 --- a/templates/src/python/grpcio/grpc_core_dependencies.py.template +++ b/templates/src/python/grpcio/grpc_core_dependencies.py.template @@ -33,7 +33,7 @@ ] ASM_SOURCE_FILES = { - % for asm, asm_src in asm_srcs.iteritems(): + % for asm, asm_src in asm_srcs.items(): '${asm}': [ % for src in asm_src: '${src}', diff --git a/templates/test/core/end2end/end2end_nosec_tests.cc.template b/templates/test/core/end2end/end2end_nosec_tests.cc.template index 3719ded75d1..e0023dd2d91 100644 --- a/templates/test/core/end2end/end2end_nosec_tests.cc.template +++ b/templates/test/core/end2end/end2end_nosec_tests.cc.template @@ -1,4 +1,4 @@ %YAML 1.2 --- | <%namespace file="end2end_defs.include" import="*"/>\ - ${end2end_selector(k for k, v in core_end2end_tests.iteritems() if not v)} + ${end2end_selector(k for k, v in core_end2end_tests.items() if not v)} diff --git a/templates/tools/run_tests/generated/configs.json.template b/templates/tools/run_tests/generated/configs.json.template index 5c82dfb3479..e26522ef59e 100644 --- a/templates/tools/run_tests/generated/configs.json.template +++ b/templates/tools/run_tests/generated/configs.json.template @@ -3,7 +3,7 @@ <% import json out_configs = [] - for name, args in configs.iteritems(): + for name, args in configs.items(): config_args={} config_args['config'] = name if args.get('valgrind', None) is not None: diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 3cb31686619..fb21f65be22 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -33,7 +33,10 @@ def struct(**kwargs): # generate_tests.bzl is now the source of truth for end2end tests. # The .bzl file is basically a python file and we can "execute" it # to get access to the variables it defines. -execfile('test/core/end2end/generate_tests.bzl') +exec( + compile( + open('test/core/end2end/generate_tests.bzl', "rb").read(), + 'test/core/end2end/generate_tests.bzl', 'exec')) def main(): diff --git a/tools/buildgen/build_cleaner.py b/tools/buildgen/build_cleaner.py index 4be078f1a04..cda6f715123 100755 --- a/tools/buildgen/build_cleaner.py +++ b/tools/buildgen/build_cleaner.py @@ -32,7 +32,8 @@ _ELEM_KEYS = [ def repr_ordered_dict(dumper, odict): - return dumper.represent_mapping(u'tag:yaml.org,2002:map', odict.items()) + return dumper.represent_mapping('tag:yaml.org,2002:map', + list(odict.items())) yaml.add_representer(collections.OrderedDict, repr_ordered_dict) diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index c5fc9305643..2bf63cdc5d1 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -176,7 +176,8 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False): # all libs that are not in the dictionary are considered external. external_deps = list( - sorted(filter(lambda lib_name: lib_name not in lib_dict, lib_names))) + sorted([lib_name for lib_name in lib_names if lib_name not in lib_dict + ])) if verbose: print('topo_ordering ' + str(lib_names)) print(' external_deps ' + str(external_deps)) @@ -207,12 +208,12 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False): def _populate_transitive_deps(bazel_rules): """Add 'transitive_deps' field for each of the rules""" transitive_deps = {} - for rule_name in bazel_rules.iterkeys(): + for rule_name in bazel_rules.keys(): transitive_deps[rule_name] = set(bazel_rules[rule_name]['deps']) while True: deps_added = 0 - for rule_name in bazel_rules.iterkeys(): + for rule_name in bazel_rules.keys(): old_deps = transitive_deps[rule_name] new_deps = set(old_deps) for dep_name in old_deps: @@ -223,7 +224,7 @@ def _populate_transitive_deps(bazel_rules): if deps_added == 0: break - for rule_name, bazel_rule in bazel_rules.iteritems(): + for rule_name, bazel_rule in bazel_rules.items(): bazel_rule['transitive_deps'] = list(sorted(transitive_deps[rule_name])) @@ -337,7 +338,7 @@ def _expand_intermediate_deps(target_dict, public_dep_names, bazel_rules): def _generate_build_metadata(build_extra_metadata, bazel_rules): """Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific "extra metadata".""" - lib_names = build_extra_metadata.keys() + lib_names = list(build_extra_metadata.keys()) result = {} for lib_name in lib_names: @@ -381,12 +382,13 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules): # dep names need to be updated as well for lib_dict_to_update in result.values(): - lib_dict_to_update['deps'] = list( - map(lambda dep: to_name if dep == lib_name else dep, - lib_dict_to_update['deps'])) + lib_dict_to_update['deps'] = list([ + to_name if dep == lib_name else dep + for dep in lib_dict_to_update['deps'] + ]) # make sure deps are listed in reverse topological order (e.g. "grpc gpr" and not "gpr grpc") - for lib_dict in result.itervalues(): + for lib_dict in result.values(): lib_dict['deps'] = list( reversed(_sort_by_build_order(lib_dict['deps'], result, 'deps'))) @@ -394,36 +396,35 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules): def _convert_to_build_yaml_like(lib_dict): - lib_names = list( - filter( - lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') == - 'library', lib_dict.keys())) - target_names = list( - filter( - lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') == - 'target', lib_dict.keys())) - test_names = list( - filter( - lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') == - 'test', lib_dict.keys())) + lib_names = [ + lib_name for lib_name in list(lib_dict.keys()) + if lib_dict[lib_name].get('_TYPE', 'library') == 'library' + ] + target_names = [ + lib_name for lib_name in list(lib_dict.keys()) + if lib_dict[lib_name].get('_TYPE', 'library') == 'target' + ] + test_names = [ + lib_name for lib_name in list(lib_dict.keys()) + if lib_dict[lib_name].get('_TYPE', 'library') == 'test' + ] # list libraries and targets in predefined order - lib_list = list(map(lambda lib_name: lib_dict[lib_name], lib_names)) - target_list = list(map(lambda lib_name: lib_dict[lib_name], target_names)) - test_list = list(map(lambda lib_name: lib_dict[lib_name], test_names)) + lib_list = [lib_dict[lib_name] for lib_name in lib_names] + target_list = [lib_dict[lib_name] for lib_name in target_names] + test_list = [lib_dict[lib_name] for lib_name in test_names] # get rid of temporary private fields prefixed with "_" and some other useless fields for lib in lib_list: - for field_to_remove in filter(lambda k: k.startswith('_'), lib.keys()): + for field_to_remove in [k for k in lib.keys() if k.startswith('_')]: lib.pop(field_to_remove, None) for target in target_list: - for field_to_remove in filter(lambda k: k.startswith('_'), - target.keys()): + for field_to_remove in [k for k in target.keys() if k.startswith('_')]: target.pop(field_to_remove, None) target.pop('public_headers', None) # public headers make no sense for targets for test in test_list: - for field_to_remove in filter(lambda k: k.startswith('_'), test.keys()): + for field_to_remove in [k for k in test.keys() if k.startswith('_')]: test.pop(field_to_remove, None) test.pop('public_headers', None) # public headers make no sense for tests @@ -440,7 +441,7 @@ def _convert_to_build_yaml_like(lib_dict): def _extract_cc_tests(bazel_rules): """Gets list of cc_test tests from bazel rules""" result = [] - for bazel_rule in bazel_rules.itervalues(): + for bazel_rule in bazel_rules.values(): if bazel_rule['class'] == 'cc_test': test_name = bazel_rule['name'] if test_name.startswith('//'): @@ -453,65 +454,64 @@ def _exclude_unwanted_cc_tests(tests): """Filters out bazel tests that we don't want to run with other build systems or we cannot build them reasonably""" # most qps tests are autogenerated, we are fine without them - tests = list( - filter(lambda test: not test.startswith('test/cpp/qps:'), tests)) + tests = [test for test in tests if not test.startswith('test/cpp/qps:')] # we have trouble with census dependency outside of bazel - tests = list( - filter(lambda test: not test.startswith('test/cpp/ext/filters/census:'), - tests)) - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/microbenchmarks:bm_opencensus_plugin'), tests)) + tests = [ + test for test in tests + if not test.startswith('test/cpp/ext/filters/census:') + ] + tests = [ + test for test in tests + if not test.startswith('test/cpp/microbenchmarks:bm_opencensus_plugin') + ] # missing opencensus/stats/stats.h - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/end2end:server_load_reporting_end2end_test'), tests)) - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/server/load_reporter:lb_load_reporter_test'), tests)) + tests = [ + test for test in tests if not test.startswith( + 'test/cpp/end2end:server_load_reporting_end2end_test') + ] + tests = [ + test for test in tests if not test.startswith( + 'test/cpp/server/load_reporter:lb_load_reporter_test') + ] # The test uses --running_under_bazel cmdline argument # To avoid the trouble needing to adjust it, we just skip the test - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/naming:resolver_component_tests_runner_invoker'), - tests)) + tests = [ + test for test in tests if not test.startswith( + 'test/cpp/naming:resolver_component_tests_runner_invoker') + ] # the test requires 'client_crash_test_server' to be built - tests = list( - filter( - lambda test: not test.startswith('test/cpp/end2end:time_change_test' - ), tests)) + tests = [ + test for test in tests + if not test.startswith('test/cpp/end2end:time_change_test') + ] # the test requires 'client_crash_test_server' to be built - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/end2end:client_crash_test'), tests)) + tests = [ + test for test in tests + if not test.startswith('test/cpp/end2end:client_crash_test') + ] # the test requires 'server_crash_test_client' to be built - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/end2end:server_crash_test'), tests)) + tests = [ + test for test in tests + if not test.startswith('test/cpp/end2end:server_crash_test') + ] # test never existed under build.yaml and it fails -> skip it - tests = list( - filter( - lambda test: not test.startswith( - 'test/core/tsi:ssl_session_cache_test'), tests)) + tests = [ + test for test in tests + if not test.startswith('test/core/tsi:ssl_session_cache_test') + ] # the binary of this test does not get built with cmake - tests = list( - filter( - lambda test: not test.startswith( - 'test/cpp/util:channelz_sampler_test'), tests)) + tests = [ + test for test in tests + if not test.startswith('test/cpp/util:channelz_sampler_test') + ] return tests @@ -594,14 +594,14 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules): # detect duplicate test names tests_by_simple_name = {} - for test_name, test_dict in test_metadata.iteritems(): + for test_name, test_dict in test_metadata.items(): simple_test_name = test_dict['_RENAME'] if not simple_test_name in tests_by_simple_name: tests_by_simple_name[simple_test_name] = [] tests_by_simple_name[simple_test_name].append(test_name) # choose alternative names for tests with a name collision - for collision_list in tests_by_simple_name.itervalues(): + for collision_list in tests_by_simple_name.values(): if len(collision_list) > 1: for test_name in collision_list: long_name = test_name.replace('/', '_').replace(':', '_') diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 84252ddde73..9ad98786fda 100755 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -33,6 +33,6 @@ gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs do output_file=`mktemp /tmp/genXXXXXX` - python $gen_build_yaml/gen_build_yaml.py > $output_file + python3 $gen_build_yaml/gen_build_yaml.py > $output_file gen_build_files="$gen_build_files $output_file" done diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 88183f8024d..d86cfcdf74f 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -93,7 +93,7 @@ jobset.run(pre_jobs, maxjobs=args.jobs) jobset.run(jobs, maxjobs=args.jobs) if test is not None: - for s, g in test.iteritems(): + for s, g in test.items(): if os.path.isfile(g): assert 0 == os.system('diff %s %s' % (s, g)), s os.unlink(g) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 872af701aea..81c16153c71 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -20,7 +20,7 @@ export TEST=${TEST:-false} echo "Generating build_autogenerated.yaml from bazel BUILD file" rm -f build_autogenerated.yaml -python tools/buildgen/extract_metadata_from_bazel_xml.py +python3 tools/buildgen/extract_metadata_from_bazel_xml.py cd `dirname $0`/../.. mako_renderer=tools/buildgen/mako_renderer.py @@ -35,6 +35,6 @@ TEST=true tools/buildgen/build_cleaner.py build_autogenerated.yaml # Instead of generating from a single build.yaml, we've split it into # - build_handwritten.yaml: manually written metadata # - build_autogenerated.yaml: generated from bazel BUILD file -python tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $* +python3 tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $* rm $gen_build_files diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index 3c0cef429e9..f0202cd5ff9 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ Just a wrapper around the mako rendering library. import getopt import imp import os -import cPickle as pickle +import pickle import shutil import sys @@ -36,16 +36,11 @@ import yaml def import_plugin(name): _, base_ex = os.path.split(name) base, _ = os.path.splitext(base_ex) - - with open(name, 'r') as plugin_file: - plugin_code = plugin_file.read() - plugin_module = imp.new_module(base) - exec plugin_code in plugin_module.__dict__ - return plugin_module + return imp.load_source(base, name) def out(msg): - print >> sys.stderr, msg + print(msg, file=sys.stderr) def showhelp(): @@ -103,7 +98,7 @@ def main(argv): 0, os.path.abspath( os.path.join(os.path.dirname(sys.argv[0]), 'plugins'))) - with open(arg, 'r') as dict_file: + with open(arg, 'rb') as dict_file: dictionary = pickle.load(dict_file) got_preprocessed_input = True elif opt == '-d': @@ -125,7 +120,7 @@ def main(argv): dictionary[k] = bunch.to_bunch(v) if preprocessed_output: - with open(preprocessed_output, 'w') as dict_file: + with open(preprocessed_output, 'wb') as dict_file: pickle.dump(dictionary, dict_file) cleared_dir = False @@ -134,7 +129,7 @@ def main(argv): with open(arg) as f: srcs = list(yaml.load_all(f.read())) for src in srcs: - if isinstance(src, basestring): + if isinstance(src, str): assert len(srcs) == 1 template = Template(src, filename=arg, diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py index d6eea6c7139..b45e68243f2 100755 --- a/tools/buildgen/plugins/expand_filegroups.py +++ b/tools/buildgen/plugins/expand_filegroups.py @@ -62,7 +62,7 @@ def mako_plugin(dictionary): for lst in FILEGROUP_LISTS: fg[lst] = fg.get(lst, []) fg['own_%s' % lst] = list(fg[lst]) - for attr, val in FILEGROUP_DEFAULTS.iteritems(): + for attr, val in FILEGROUP_DEFAULTS.items(): if attr not in fg: fg[attr] = val @@ -113,7 +113,7 @@ def mako_plugin(dictionary): thing['used_by'] = [] thing_deps = lambda t: t.get('uses', []) + t.get('filegroups', []) + t.get( 'deps', []) - for thing in things.itervalues(): + for thing in things.values(): done = set() todo = thing_deps(thing) while todo: @@ -125,7 +125,7 @@ def mako_plugin(dictionary): done.add(cur) # the above expansion can introduce duplicate filenames: contract them here - for fg in filegroups.itervalues(): + for fg in filegroups.values(): for lst in FILEGROUP_LISTS: fg[lst] = uniquify(fg.get(lst, [])) diff --git a/tools/buildgen/plugins/generate_vsprojects.py b/tools/buildgen/plugins/generate_vsprojects.py index f7ef49288f2..6fbad5e1652 100755 --- a/tools/buildgen/plugins/generate_vsprojects.py +++ b/tools/buildgen/plugins/generate_vsprojects.py @@ -61,7 +61,7 @@ def mako_plugin(dictionary): name = target['name'] guid = re.sub('(........)(....)(....)(....)(.*)', r'{\1-\2-\3-\4-\5}', - hashlib.md5(name).hexdigest()) + hashlib.md5(name.encode('utf-8')).hexdigest()) target['vs_project_guid'] = guid.upper() # Exclude projects without a visual project guid, such as the tests. projects = [ diff --git a/tools/buildgen/plugins/list_api.py b/tools/buildgen/plugins/list_api.py index f7ecb97666b..87a592d8464 100755 --- a/tools/buildgen/plugins/list_api.py +++ b/tools/buildgen/plugins/list_api.py @@ -64,4 +64,5 @@ def mako_plugin(dictionary): if __name__ == '__main__': - print yaml.dump([api for api in list_c_apis(headers_under('include/grpc'))]) + print(yaml.dump([api for api in list_c_apis(headers_under('include/grpc')) + ]))