From d9892bdd8032485919067748c5eedcf52f56240f Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Mon, 4 Jul 2016 00:52:39 -0700 Subject: [PATCH 01/97] Moved sending of initial metadata from server into server handler methods --- src/ruby/lib/grpc/generic/active_call.rb | 47 +++- src/ruby/lib/grpc/generic/bidi_call.rb | 16 +- src/ruby/lib/grpc/generic/rpc_desc.rb | 9 +- src/ruby/lib/grpc/generic/rpc_server.rb | 23 +- src/ruby/spec/generic/active_call_spec.rb | 329 +++++++++++++++++++++- src/ruby/spec/generic/rpc_desc_spec.rb | 14 +- 6 files changed, 412 insertions(+), 26 deletions(-) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index 4260d854376..d43a9e7a4b7 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -58,7 +58,7 @@ module GRPC include Core::TimeConsts include Core::CallOps extend Forwardable - attr_reader(:deadline) + attr_reader :deadline, :metadata_sent, :metadata_to_send def_delegators :@call, :cancel, :metadata, :write_flag, :write_flag=, :peer, :peer_cert, :trailing_metadata @@ -101,7 +101,7 @@ module GRPC # @param metadata_received [true|false] indicates if metadata has already # been received. Should always be true for server calls def initialize(call, marshal, unmarshal, deadline, started: true, - metadata_received: false) + metadata_received: false, metadata_to_send: nil) fail(TypeError, '!Core::Call') unless call.is_a? Core::Call @call = call @deadline = deadline @@ -110,6 +110,14 @@ module GRPC @metadata_received = metadata_received @metadata_sent = started @op_notifier = nil + + fail(ArgumentError, 'Already sent md') if started && metadata_to_send + @metadata_to_send = metadata_to_send || {} unless started + end + + def send_initial_metadata + fail 'Already sent metadata' if @metadata_sent + start_call(@metadata_to_send) end # output_metadata are provides access to hash that can be used to @@ -187,7 +195,7 @@ module GRPC # @param marshalled [false, true] indicates if the object is already # marshalled. def remote_send(req, marshalled = false) - # TODO(murgatroid99): ensure metadata was sent + start_call(@metadata_to_send) unless @metadata_sent GRPC.logger.debug("sending #{req}, marshalled? #{marshalled}") payload = marshalled ? req : @marshal.call(req) @call.run_batch(SEND_MESSAGE => payload) @@ -203,6 +211,7 @@ module GRPC # list, mulitple metadata for its key are sent def send_status(code = OK, details = '', assert_finished = false, metadata: {}) + start_call unless @metadata_sent ops = { SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata) } @@ -392,9 +401,12 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Enumerator, nil] a response Enumerator def bidi_streamer(requests, metadata: {}, &blk) - start_call(metadata) - bd = BidiCall.new(@call, @marshal, @unmarshal, + start_call(metadata) unless @metadata_sent + bd = BidiCall.new(@call, + @marshal, + @unmarshal, metadata_received: @metadata_received) + bd.run_on_client(requests, @op_notifier, &blk) end @@ -410,8 +422,12 @@ module GRPC # # @param gen_each_reply [Proc] generates the BiDi stream replies def run_server_bidi(gen_each_reply) - bd = BidiCall.new(@call, @marshal, @unmarshal, - metadata_received: @metadata_received) + bd = BidiCall.new(@call, + @marshal, + @unmarshal, + metadata_received: @metadata_received, + req_view: MultiReqView.new(self)) + bd.run_on_server(gen_each_reply) end @@ -428,6 +444,11 @@ module GRPC @op_notifier.notify(self) end + def merge_metadata_to_send(new_metadata = {}) + fail('cant change metadata after already sent') if @metadata_sent + @metadata_to_send.merge!(new_metadata) + end + private # Starts the call if not already started @@ -454,12 +475,20 @@ module GRPC # SingleReqView limits access to an ActiveCall's methods for use in server # handlers that receive just one request. SingleReqView = view_class(:cancelled?, :deadline, :metadata, - :output_metadata, :peer, :peer_cert) + :output_metadata, :peer, :peer_cert, + :send_initial_metadata, + :metadata_to_send, + :merge_metadata_to_send, + :metadata_sent) # MultiReqView limits access to an ActiveCall's methods for use in # server client_streamer handlers. MultiReqView = view_class(:cancelled?, :deadline, :each_queued_msg, - :each_remote_read, :metadata, :output_metadata) + :each_remote_read, :metadata, :output_metadata, + :send_initial_metadata, + :metadata_to_send, + :merge_metadata_to_send, + :metadata_sent) # Operation limits access to an ActiveCall's methods for use as # a Operation on the client. diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index 425dc3e5198..0b6ef4918c6 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -56,7 +56,8 @@ module GRPC # @param unmarshal [Function] f(string)->obj that unmarshals responses # @param metadata_received [true|false] indicates if metadata has already # been received. Should always be true for server calls - def initialize(call, marshal, unmarshal, metadata_received: false) + def initialize(call, marshal, unmarshal, metadata_received: false, + req_view: nil) fail(ArgumentError, 'not a call') unless call.is_a? Core::Call @call = call @marshal = marshal @@ -68,6 +69,7 @@ module GRPC @writes_complete = false @complete = false @done_mutex = Mutex.new + @req_view = req_view end # Begins orchestration of the Bidi stream for a client sending requests. @@ -97,7 +99,15 @@ module GRPC # # @param gen_each_reply [Proc] generates the BiDi stream replies. def run_on_server(gen_each_reply) - replys = gen_each_reply.call(each_queued_msg) + # Pass in the optional call object parameter if possible + if gen_each_reply.arity == 1 + replys = gen_each_reply.call(each_queued_msg) + elsif gen_each_reply.arity == 2 + replys = gen_each_reply.call(each_queued_msg, @req_view) + else + fail 'Illegal arity of reply generator' + end + @loop_th = start_read_loop(is_client: false) write_loop(replys, is_client: false) end @@ -162,6 +172,8 @@ module GRPC payload = @marshal.call(req) # Fails if status already received begin + @req_view.send_initial_metadata unless + @req_view.nil? || @req_view.metadata_sent @call.run_batch(SEND_MESSAGE => payload) rescue GRPC::Core::CallError => e # This is almost definitely caused by a status arriving while still diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb index 913f55d0d3b..584fe781698 100644 --- a/src/ruby/lib/grpc/generic/rpc_desc.rb +++ b/src/ruby/lib/grpc/generic/rpc_desc.rb @@ -104,7 +104,14 @@ module GRPC end def assert_arity_matches(mth) - if request_response? || server_streamer? + # A bidi handler function can optionally be passed a second + # call object parameter for access to metadata, cancelling, etc. + if bidi_streamer? + if mth.arity != 2 && mth.arity != 1 + fail arity_error(mth, 2, "should be #{mth.name}(req, call) or " \ + "#{mth.name}(req)") + end + elsif request_response? || server_streamer? if mth.arity != 2 fail arity_error(mth, 2, "should be #{mth.name}(req, call)") end diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index c92a532a500..b1d30b8e38c 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -339,8 +339,11 @@ module GRPC return an_rpc if @pool.jobs_waiting <= @max_waiting_requests GRPC.logger.warn("NOT AVAILABLE: too many jobs_waiting: #{an_rpc}") noop = proc { |x| x } + + # Create a new active call that knows that metadata hasn't been + # sent yet c = ActiveCall.new(an_rpc.call, noop, noop, an_rpc.deadline, - metadata_received: true) + metadata_received: true, started: false) c.send_status(GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, '') nil end @@ -351,8 +354,11 @@ module GRPC return an_rpc if rpc_descs.key?(mth) GRPC.logger.warn("UNIMPLEMENTED: #{an_rpc}") noop = proc { |x| x } + + # Create a new active call that knows that + # metadata hasn't been sent yet c = ActiveCall.new(an_rpc.call, noop, noop, an_rpc.deadline, - metadata_received: true) + metadata_received: true, started: false) c.send_status(GRPC::Core::StatusCodes::UNIMPLEMENTED, '') nil end @@ -400,17 +406,20 @@ module GRPC unless @connect_md_proc.nil? connect_md = @connect_md_proc.call(an_rpc.method, an_rpc.metadata) end - an_rpc.call.run_batch(SEND_INITIAL_METADATA => connect_md) return nil unless available?(an_rpc) return nil unless implemented?(an_rpc) - # Create the ActiveCall + # Create the ActiveCall. Indicate that metadata hasnt been sent yet. GRPC.logger.info("deadline is #{an_rpc.deadline}; (now=#{Time.now})") rpc_desc = rpc_descs[an_rpc.method.to_sym] - c = ActiveCall.new(an_rpc.call, rpc_desc.marshal_proc, - rpc_desc.unmarshal_proc(:input), an_rpc.deadline, - metadata_received: true) + c = ActiveCall.new(an_rpc.call, + rpc_desc.marshal_proc, + rpc_desc.unmarshal_proc(:input), + an_rpc.deadline, + metadata_received: true, + started: false, + metadata_to_send: connect_md) mth = an_rpc.method.to_sym [c, mth] end diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index 018580e0dfa..0c72be9a98a 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -60,8 +60,10 @@ describe GRPC::ActiveCall do end describe '#multi_req_view' do - it 'exposes a fixed subset of the ActiveCall methods' do - want = %w(cancelled?, deadline, each_remote_read, metadata, shutdown) + it 'exposes a fixed subset of the ActiveCall.methods' do + want = %w(cancelled?, deadline, each_remote_read, metadata, \ + shutdown, peer, peer_cert, send_initial_metadata, \ + initial_metadata_sent) v = @client_call.multi_req_view want.each do |w| expect(v.methods.include?(w)) @@ -70,8 +72,10 @@ describe GRPC::ActiveCall do end describe '#single_req_view' do - it 'exposes a fixed subset of the ActiveCall methods' do - want = %w(cancelled?, deadline, metadata, shutdown) + it 'exposes a fixed subset of the ActiveCall.methods' do + want = %w(cancelled?, deadline, metadata, shutdown, \ + send_initial_metadata, metadata_to_send, \ + merge_metadata_to_send, initial_metadata_sent) v = @client_call.single_req_view want.each do |w| expect(v.methods.include?(w)) @@ -149,6 +153,158 @@ describe GRPC::ActiveCall do end end + describe 'sending initial metadata', send_initial_metadata: true do + it 'sends metadata before sending a message if it hasnt been sent yet' do + call = make_test_call + @client_call = ActiveCall.new( + call, + @pass_through, + @pass_through, + deadline, + started: false) + + metadata = { key: 'dummy_val', other: 'other_val' } + expect(@client_call.metadata_sent).to eq(false) + @client_call.merge_metadata_to_send(metadata) + + message = 'dummy message' + + expect(call).to( + receive(:run_batch) + .with( + hash_including( + CallOps::SEND_INITIAL_METADATA => metadata)).once) + + expect(call).to( + receive(:run_batch).with(hash_including( + CallOps::SEND_MESSAGE => message)).once) + @client_call.remote_send(message) + + expect(@client_call.metadata_sent).to eq(true) + end + + it 'doesnt send metadata if it thinks its already been sent' do + call = make_test_call + + @client_call = ActiveCall.new(call, + @pass_through, + @pass_through, + deadline) + + expect(@client_call.metadata_sent).to eql(true) + expect(call).to( + receive(:run_batch).with(hash_including( + CallOps::SEND_INITIAL_METADATA)).never) + + @client_call.remote_send('test message') + end + + it 'sends metadata if it is explicitly sent and ok to do so' do + call = make_test_call + + @client_call = ActiveCall.new(call, + @pass_through, + @pass_through, + deadline, + started: false) + + expect(@client_call.metadata_sent).to eql(false) + + metadata = { test_key: 'val' } + @client_call.merge_metadata_to_send(metadata) + expect(@client_call.metadata_to_send).to eq(metadata) + + expect(call).to( + receive(:run_batch).with(hash_including( + CallOps::SEND_INITIAL_METADATA => + metadata)).once) + @client_call.send_initial_metadata + end + + it 'explicit sending fails if metadata has already been sent' do + call = make_test_call + + @client_call = ActiveCall.new(call, + @pass_through, + @pass_through, + deadline) + + expect(@client_call.metadata_sent).to eql(true) + + blk = proc do + @client_call.send_initial_metadata + end + + expect { blk.call }.to raise_error + end + end + + describe '#merge_metadata_to_send', merge_metadata_to_send: true do + it 'adds to existing metadata when there is existing metadata to send' do + call = make_test_call + starting_metadata = { k1: 'key1_val', k2: 'key2_val' } + @client_call = ActiveCall.new( + call, + @pass_through, @pass_through, + deadline, + started: false, + metadata_to_send: starting_metadata) + + expect(@client_call.metadata_to_send).to eq(starting_metadata) + + @client_call.merge_metadata_to_send( + k3: 'key3_val', + k4: 'key4_val') + + expected_md_to_send = { + k1: 'key1_val', + k2: 'key2_val', + k3: 'key3_val', + k4: 'key4_val' } + + expect(@client_call.metadata_to_send).to eq(expected_md_to_send) + + @client_call.merge_metadata_to_send(k5: 'key5_val') + expected_md_to_send.merge!(k5: 'key5_val') + expect(@client_call.metadata_to_send).to eq(expected_md_to_send) + end + + it 'overrides existing metadata if adding metadata with an existing key' do + call = make_test_call + starting_metadata = { k1: 'key1_val', k2: 'key2_val' } + @client_call = ActiveCall.new( + call, + @pass_through, + @pass_through, + deadline, + started: false, + metadata_to_send: starting_metadata) + + expect(@client_call.metadata_to_send).to eq(starting_metadata) + @client_call.merge_metadata_to_send(k1: 'key1_new_val') + expect(@client_call.metadata_to_send).to eq(k1: 'key1_new_val', + k2: 'key2_val') + end + + it 'fails when initial metadata has already been sent' do + call = make_test_call + @client_call = ActiveCall.new( + call, + @pass_through, + @pass_through, + deadline, + started: true) + + expect(@client_call.metadata_sent).to eq(true) + + blk = proc do + @client_call.merge_metadata_to_send(k1: 'key1_val') + end + + expect { blk.call }.to raise_error + end + end + describe '#client_invoke' do it 'sends metadata to the server when present' do call = make_test_call @@ -163,7 +319,26 @@ describe GRPC::ActiveCall do end end - describe '#remote_read' do + describe '#send_status', send_status: true do + it 'works when no metadata or messages have been sent yet' do + call = make_test_call + ActiveCall.client_invoke(call) + + recvd_rpc = @server.request_call + server_call = ActiveCall.new( + recvd_rpc.call, + @pass_through, + @pass_through, + deadline, + started: false) + + expect(server_call.metadata_sent).to eq(false) + blk = proc { server_call.send_status(OK) } + expect { blk.call }.to_not raise_error + end + end + + describe '#remote_read', remote_read: true do it 'reads the response sent by a server' do call = make_test_call ActiveCall.client_invoke(call) @@ -205,6 +380,31 @@ describe GRPC::ActiveCall do expect(client_call.metadata).to eq(expected) end + it 'get a status from server when nothing else sent from server' do + client_call = make_test_call + ActiveCall.client_invoke(client_call) + + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + + server_call = ActiveCall.new( + recvd_call, + @pass_through, + @pass_through, + deadline, + started: false) + + server_call.send_status(OK, 'OK') + + # Check that we can receive initial metadata and a status + client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil) + batch_result = client_call.run_batch( + CallOps::RECV_STATUS_ON_CLIENT => nil) + + expect(batch_result.status.code).to eq(OK) + end + it 'get a nil msg before a status when an OK status is sent' do call = make_test_call ActiveCall.client_invoke(call) @@ -329,6 +529,125 @@ describe GRPC::ActiveCall do end end + # Test sending of the initial metadata in #run_server_bidi + # from the server handler both implicitly and explicitly, + # when the server handler function has one argument and two arguments + describe '#run_server_bidi sanity tests', run_server_bidi: true do + it 'sends the initial metadata implicitly if not already sent' do + requests = ['first message', 'second message'] + server_to_client_metadata = { 'test_key' => 'test_val' } + server_status = OK + + client_call = make_test_call + client_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) + + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = ActiveCall.new(recvd_call, + @pass_through, + @pass_through, + deadline, + metadata_received: true, + started: false, + metadata_to_send: server_to_client_metadata) + + # Server handler that doesn't have access to a "call" + # It echoes the requests + fake_gen_each_reply_with_no_call_param = proc do |msgs| + msgs + end + + server_thread = Thread.new do + server_call.run_server_bidi( + fake_gen_each_reply_with_no_call_param) + server_call.send_status(server_status) + end + + # Send the requests and send a close so the server can send a status + requests.each do |message| + client_call.run_batch(CallOps::SEND_MESSAGE => message) + end + client_call.run_batch(CallOps::SEND_CLOSE_FROM_CLIENT => nil) + + server_thread.join + + # Expect that initial metadata was sent, + # the requests were echoed, and a status was sent + batch_result = client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil) + expect(batch_result.metadata).to eq(server_to_client_metadata) + + requests.each do |message| + batch_result = client_call.run_batch( + CallOps::RECV_MESSAGE => nil) + expect(batch_result.message).to eq(message) + end + + batch_result = client_call.run_batch( + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(batch_result.status.code).to eq(server_status) + end + + it 'sends the metadata when sent explicitly and not already sent' do + requests = ['first message', 'second message'] + server_to_client_metadata = { 'test_key' => 'test_val' } + server_status = OK + + client_call = make_test_call + client_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) + + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = ActiveCall.new(recvd_call, + @pass_through, + @pass_through, + deadline, + metadata_received: true, + started: false) + + # Fake server handler that has access to a "call" object and + # uses it to explicitly update and sent the initial metadata + fake_gen_each_reply_with_call_param = proc do |msgs, call_param| + call_param.merge_metadata_to_send(server_to_client_metadata) + call_param.send_initial_metadata + msgs + end + + server_thread = Thread.new do + server_call.run_server_bidi( + fake_gen_each_reply_with_call_param) + server_call.send_status(server_status) + end + + # Send requests and a close from the client so the server + # can send a status + requests.each do |message| + client_call.run_batch( + CallOps::SEND_MESSAGE => message) + end + client_call.run_batch( + CallOps::SEND_CLOSE_FROM_CLIENT => nil) + + server_thread.join + + # Verify that the correct metadata was sent, the requests + # were echoed, and the correct status was sent + batch_result = client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil) + expect(batch_result.metadata).to eq(server_to_client_metadata) + + requests.each do |message| + batch_result = client_call.run_batch( + CallOps::RECV_MESSAGE => nil) + expect(batch_result.message).to eq(message) + end + + batch_result = client_call.run_batch( + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(batch_result.status.code).to eq(server_status) + end + end + def expect_server_to_receive(sent_text, **kw) c = expect_server_to_be_invoked(**kw) expect(c.remote_read).to eq(sent_text) diff --git a/src/ruby/spec/generic/rpc_desc_spec.rb b/src/ruby/spec/generic/rpc_desc_spec.rb index d2080b7ca2d..1a895005bc3 100644 --- a/src/ruby/spec/generic/rpc_desc_spec.rb +++ b/src/ruby/spec/generic/rpc_desc_spec.rb @@ -196,6 +196,9 @@ describe GRPC::RpcDesc do def fake_svstream(_arg1, _arg2) end + def fake_three_args(_arg1, _arg2, _arg3) + end + it 'raises when a request_response does not have 2 args' do [:fake_clstream, :no_arg].each do |mth| blk = proc do @@ -244,8 +247,8 @@ describe GRPC::RpcDesc do expect(&blk).to_not raise_error end - it 'raises when a bidi streamer does not have 1 arg' do - [:fake_svstream, :no_arg].each do |mth| + it 'raises when a bidi streamer does not have 1 or 2 args' do + [:fake_three_args, :no_arg].each do |mth| blk = proc do @bidi_streamer.assert_arity_matches(method(mth)) end @@ -259,6 +262,13 @@ describe GRPC::RpcDesc do end expect(&blk).to_not raise_error end + + it 'passes when a bidi streamer has 2 args' do + blk = proc do + @bidi_streamer.assert_arity_matches(method(:fake_svstream)) + end + expect(&blk).to_not raise_error + end end describe '#request_response?' do From 29ca79be8939c0386f1c7b17ba66cc3b105c7fc1 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 12:00:08 -0700 Subject: [PATCH 02/97] Command processing, validation --- Makefile | 67 +++- build.yaml | 26 +- test/cpp/util/grpc_cli.cc | 202 ++-------- test/cpp/util/grpc_tool.cc | 369 ++++++++++++++++++ test/cpp/util/grpc_tool.h | 47 +++ test/cpp/util/grpc_tool_test.cc | 219 +++++++++++ tools/run_tests/sources_and_headers.json | 32 ++ tools/run_tests/tests.json | 21 + .../grpc_cli_libs/grpc_cli_libs.vcxproj | 3 + .../grpc_cli_libs.vcxproj.filters | 6 + .../vcxproj/test/grpc_cli/grpc_cli.vcxproj | 6 +- .../grpc_tool_test/grpc_tool_test.vcxproj | 286 ++++++++++++++ .../grpc_tool_test.vcxproj.filters | 232 +++++++++++ 13 files changed, 1332 insertions(+), 184 deletions(-) create mode 100644 test/cpp/util/grpc_tool.cc create mode 100644 test/cpp/util/grpc_tool.h create mode 100644 test/cpp/util/grpc_tool_test.cc create mode 100644 vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj create mode 100644 vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters diff --git a/Makefile b/Makefile index 37999d3b1e2..a1165955a9a 100644 --- a/Makefile +++ b/Makefile @@ -1047,6 +1047,7 @@ grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_python_plugin: $(BINDIR)/$(CONFIG)/grpc_python_plugin grpc_ruby_plugin: $(BINDIR)/$(CONFIG)/grpc_ruby_plugin +grpc_tool_test: $(BINDIR)/$(CONFIG)/grpc_tool_test grpclb_api_test: $(BINDIR)/$(CONFIG)/grpclb_api_test hybrid_end2end_test: $(BINDIR)/$(CONFIG)/hybrid_end2end_test interop_client: $(BINDIR)/$(CONFIG)/interop_client @@ -1402,6 +1403,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/generic_end2end_test \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/hybrid_end2end_test \ $(BINDIR)/$(CONFIG)/interop_client \ @@ -1486,6 +1488,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/generic_end2end_test \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/hybrid_end2end_test \ $(BINDIR)/$(CONFIG)/interop_client \ @@ -1772,6 +1775,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/generic_end2end_test || ( echo test generic_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing golden_file_test" $(Q) $(BINDIR)/$(CONFIG)/golden_file_test || ( echo test golden_file_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_tool_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_api_test" $(Q) $(BINDIR)/$(CONFIG)/grpclb_api_test || ( echo test grpclb_api_test failed ; exit 1 ) $(E) "[RUN] Testing hybrid_end2end_test" @@ -4089,6 +4094,7 @@ endif LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ + test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ @@ -10998,16 +11004,16 @@ $(BINDIR)/$(CONFIG)/grpc_cli: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a deps_grpc_cli: $(GRPC_CLI_OBJS:.o=.dep) @@ -11204,6 +11210,60 @@ ifneq ($(NO_DEPS),true) endif +GRPC_TOOL_TEST_SRC = \ + $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc \ + test/cpp/util/grpc_tool_test.cc \ + test/cpp/util/string_ref_helper.cc \ + +GRPC_TOOL_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_TOOL_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_tool_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/grpc_tool_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_tool_test: $(PROTOBUF_DEP) $(GRPC_TOOL_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_TOOL_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_tool_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo_messages.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_tool_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/test/cpp/util/string_ref_helper.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_tool_test: $(GRPC_TOOL_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_TOOL_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_tool_test.o: $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/util/string_ref_helper.o: $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc + + GRPCLB_API_TEST_SRC = \ $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc \ test/cpp/grpclb/grpclb_api_test.cc \ @@ -14920,6 +14980,7 @@ test/cpp/util/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/util/byte_buffer_proto_helper.cc: $(OPENSSL_DEP) test/cpp/util/cli_call.cc: $(OPENSSL_DEP) test/cpp/util/create_test_channel.cc: $(OPENSSL_DEP) +test/cpp/util/grpc_tool.cc: $(OPENSSL_DEP) test/cpp/util/proto_file_parser.cc: $(OPENSSL_DEP) test/cpp/util/proto_reflection_descriptor_database.cc: $(OPENSSL_DEP) test/cpp/util/string_ref_helper.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 237394c2056..5b2c5818501 100644 --- a/build.yaml +++ b/build.yaml @@ -1029,10 +1029,12 @@ libs: language: c++ headers: - test/cpp/util/cli_call.h + - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h src: - test/cpp/util/cli_call.cc + - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc deps: @@ -2658,9 +2660,9 @@ targets: - test/cpp/util/grpc_cli.cc deps: - grpc_cli_libs + - grpc++_reflection - grpc++_test_util - grpc_test_util - - grpc++_reflection - grpc++ - grpc - gpr_test_util @@ -2725,6 +2727,28 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' +- name: grpc_tool_test + gtest: true + build: test + language: c++ + headers: + - test/cpp/util/string_ref_helper.h + src: + - src/proto/grpc/testing/echo.proto + - src/proto/grpc/testing/echo_messages.proto + - test/cpp/util/grpc_tool_test.cc + - test/cpp/util/string_ref_helper.cc + deps: + - grpc_cli_libs + - grpc++_reflection + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + filegroups: + - grpc++_codegen_proto + - grpc++_config_proto - name: grpclb_api_test gtest: true build: test diff --git a/test/cpp/util/grpc_cli.cc b/test/cpp/util/grpc_cli.cc index 53529da782c..28c87956d67 100644 --- a/test/cpp/util/grpc_cli.cc +++ b/test/cpp/util/grpc_cli.cc @@ -34,21 +34,18 @@ /* A command line tool to talk to a grpc server. Example of talking to grpc interop server: - grpc_cli call localhost:50051 UnaryCall "response_size:10" \ - --protofiles=src/proto/grpc/testing/test.proto --enable_ssl=false + grpc_cli call localhost:50051 UnaryCall src/proto/grpc/testing/test.proto \ + "response_size:10" --enable_ssl=false Options: - 1. --protofiles, use this flag to provide a proto file if the server does - does not have the reflection service. - 2. --proto_path, if your proto file is not under current working directory, + 1. --proto_path, if your proto file is not under current working directory, use this flag to provide a search root. It should work similar to the - counterpart in protoc. This option is valid only when protofiles is - provided. - 3. --metadata specifies metadata to be sent to the server, such as: + counterpart in protoc. + 2. --metadata specifies metadata to be sent to the server, such as: --metadata="MyHeaderKey1:Value1:MyHeaderKey2:Value2" - 4. --enable_ssl, whether to use tls. - 5. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call - 6. --input_binary_file, a file containing the serialized request. The file + 3. --enable_ssl, whether to use tls. + 4. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call + 3. --input_binary_file, a file containing the serialized request. The file can be generated by calling something like: protoc --proto_path=src/proto/grpc/testing/ \ --encode=grpc.testing.SimpleRequest \ @@ -56,7 +53,7 @@ < input.txt > input.bin If this is used and no proto file is provided in the argument list, the method string has to be exact in the form of /package.service/method. - 7. --output_binary_file, a file to write binary format response into, it can + 4. --output_binary_file, a file to write binary format response into, it can be later decoded using protoc: protoc --proto_path=src/proto/grpc/testing/ \ --decode=grpc.testing.SimpleResponse \ @@ -64,182 +61,33 @@ < output.bin > output.txt */ -#include #include +#include #include -#include #include -#include -#include -#include -#include -#include - -#include "test/cpp/util/cli_call.h" -#include "test/cpp/util/proto_file_parser.h" -#include "test/cpp/util/string_ref_helper.h" +#include +#include "test/cpp/util/grpc_tool.h" #include "test/cpp/util/test_config.h" -DEFINE_bool(enable_ssl, true, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); -DEFINE_string(input_binary_file, "", - "Path to input file containing serialized request."); -DEFINE_string(output_binary_file, "", - "Path to output file to write serialized response."); -DEFINE_string(metadata, "", - "Metadata to send to server, in the form of key1:val1:key2:val2"); -DEFINE_string(proto_path, ".", "Path to look for the proto file."); -// TODO(zyc): support a list of input proto files -DEFINE_string(protofiles, "", "Name of the proto file."); +DEFINE_string(outfile, "", "Output file (default is stdout)"); -void ParseMetadataFlag( - std::multimap* client_metadata) { - if (FLAGS_metadata.empty()) { - return; - } - std::vector fields; - const char* delim = ":"; - size_t cur, next = -1; - do { - cur = next + 1; - next = FLAGS_metadata.find_first_of(delim, cur); - fields.push_back(FLAGS_metadata.substr(cur, next - cur)); - } while (next != grpc::string::npos); - if (fields.size() % 2) { - std::cout << "Failed to parse metadata flag" << std::endl; - exit(1); - } - for (size_t i = 0; i < fields.size(); i += 2) { - client_metadata->insert( - std::pair(fields[i], fields[i + 1])); - } -} - -template -void PrintMetadata(const T& m, const grpc::string& message) { - if (m.empty()) { - return; - } - std::cout << message << std::endl; - grpc::string pair; - for (typename T::const_iterator iter = m.begin(); iter != m.end(); ++iter) { - pair.clear(); - pair.append(iter->first.data(), iter->first.size()); - pair.append(" : "); - pair.append(iter->second.data(), iter->second.size()); - std::cout << pair << std::endl; +static bool SimplePrint(const grpc::string& outfile, + const grpc::string& output) { + if (outfile.empty()) { + std::cout << output; + } else { + std::ofstream output_file(outfile, std::ios::trunc | std::ios::binary); + output_file << output; + output_file.close(); } + return true; } int main(int argc, char** argv) { grpc::testing::InitTest(&argc, &argv, true); - if (argc < 4 || grpc::string(argv[1]) != "call") { - std::cout << "Usage: grpc_cli call server_host:port method_name " - << "[proto file] [text format request] []" << std::endl; - return 1; - } - - grpc::string request_text; - grpc::string server_address(argv[2]); - grpc::string method_name(argv[3]); - std::unique_ptr parser; - grpc::string serialized_request_proto; - - if (argc == 5) { - request_text = argv[4]; - } - - std::shared_ptr creds; - if (!FLAGS_enable_ssl) { - creds = grpc::InsecureChannelCredentials(); - } else { - if (FLAGS_use_auth) { - creds = grpc::GoogleDefaultCredentials(); - } else { - creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); - } - } - std::shared_ptr channel = - grpc::CreateChannel(server_address, creds); - - if (request_text.empty() && FLAGS_input_binary_file.empty()) { - if (isatty(STDIN_FILENO)) { - std::cout << "reading request message from stdin..." << std::endl; - } - std::stringstream input_stream; - input_stream << std::cin.rdbuf(); - request_text = input_stream.str(); - } - - if (!request_text.empty()) { - if (!FLAGS_protofiles.empty()) { - parser.reset(new grpc::testing::ProtoFileParser( - FLAGS_proto_path, FLAGS_protofiles, method_name)); - } else { - parser.reset(new grpc::testing::ProtoFileParser(channel, method_name)); - } - method_name = parser->GetFullMethodName(); - if (parser->HasError()) { - return 1; - } - - if (!FLAGS_input_binary_file.empty()) { - std::cout - << "warning: request given in argv, ignoring --input_binary_file" - << std::endl; - } - } - - if (parser) { - serialized_request_proto = - parser->GetSerializedProto(request_text, true /* is_request */); - if (parser->HasError()) { - return 1; - } - } else if (!FLAGS_input_binary_file.empty()) { - std::ifstream input_file(FLAGS_input_binary_file, - std::ios::in | std::ios::binary); - std::stringstream input_stream; - input_stream << input_file.rdbuf(); - serialized_request_proto = input_stream.str(); - } - std::cout << "connecting to " << server_address << std::endl; - - grpc::string serialized_response_proto; - std::multimap client_metadata; - std::multimap server_initial_metadata, - server_trailing_metadata; - ParseMetadataFlag(&client_metadata); - PrintMetadata(client_metadata, "Sending client initial metadata:"); - grpc::Status s = grpc::testing::CliCall::Call( - channel, method_name, serialized_request_proto, - &serialized_response_proto, client_metadata, &server_initial_metadata, - &server_trailing_metadata); - PrintMetadata(server_initial_metadata, - "Received initial metadata from server:"); - PrintMetadata(server_trailing_metadata, - "Received trailing metadata from server:"); - if (s.ok()) { - std::cout << "Rpc succeeded with OK status" << std::endl; - if (parser) { - grpc::string response_text = parser->GetTextFormat( - serialized_response_proto, false /* is_request */); - if (parser->HasError()) { - return 1; - } - std::cout << "Response: \n " << response_text << std::endl; - } - if (!FLAGS_output_binary_file.empty()) { - std::ofstream output_file(FLAGS_output_binary_file, - std::ios::trunc | std::ios::binary); - output_file << serialized_response_proto; - } - } else { - std::cout << "Rpc failed with status code " << s.error_code() - << ", error message: " << s.error_message() << std::endl; - } - - return 0; + return grpc::testing::GrpcToolMainLib( + argc, (const char**)argv, + std::bind(SimplePrint, FLAGS_outfile, std::placeholders::_1)); } diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc new file mode 100644 index 00000000000..38ac9e55a03 --- /dev/null +++ b/test/cpp/util/grpc_tool.cc @@ -0,0 +1,369 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "grpc_tool.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "test/cpp/util/cli_call.h" + +#include "test/cpp/util/proto_file_parser.h" +#include "test/cpp/util/proto_reflection_descriptor_database.h" +#include "test/cpp/util/string_ref_helper.h" +#include "test/cpp/util/test_config.h" + +DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); +DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_string(input_binary_file, "", + "Path to input file containing serialized request."); +DEFINE_string(output_binary_file, "", + "Path to output file to write serialized response."); +DEFINE_string(metadata, "", + "Metadata to send to server, in the form of key1:val1:key2:val2"); +DEFINE_string(proto_path, ".", "Path to look for the proto file."); +// TODO(zyc): support a list of input proto files +DEFINE_string(protofiles, "", "Name of the proto file."); + +namespace grpc { +namespace testing { +namespace { + +class GrpcTool { + public: + explicit GrpcTool(); + virtual ~GrpcTool() {} + bool Help(int argc, const char** argv, OutputCallback callback); + bool CallMethod(int argc, const char** argv, OutputCallback callback); + void SetPrintCommandMode(int exit_status) { + print_command_usage_ = true; + usage_exit_status_ = exit_status; + } + + private: + void CommandUsage(const grpc::string& usage) const; + bool print_command_usage_; + int usage_exit_status_; +}; + +template +std::function BindWith4Args( + T&& func) { + return std::bind(std::forward(func), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, + std::placeholders::_4); +} + +template +size_t ArraySize(T& a) { + return ((sizeof(a) / sizeof(*(a))) / + static_cast(!(sizeof(a) % sizeof(*(a))))); +} + +void ParseMetadataFlag( + std::multimap* client_metadata) { + if (FLAGS_metadata.empty()) { + return; + } + std::vector fields; + const char* delim = ":"; + size_t cur, next = -1; + do { + cur = next + 1; + next = FLAGS_metadata.find_first_of(delim, cur); + fields.push_back(FLAGS_metadata.substr(cur, next - cur)); + } while (next != grpc::string::npos); + if (fields.size() % 2) { + fprintf(stderr, "Failed to parse metadata flag.\n"); + exit(1); + } + for (size_t i = 0; i < fields.size(); i += 2) { + client_metadata->insert( + std::pair(fields[i], fields[i + 1])); + } +} + +template +void PrintMetadata(const T& m, const grpc::string& message) { + if (m.empty()) { + return; + } + fprintf(stderr, "%s\n", message.c_str()); + grpc::string pair; + for (typename T::const_iterator iter = m.begin(); iter != m.end(); ++iter) { + pair.clear(); + pair.append(iter->first.data(), iter->first.size()); + pair.append(" : "); + pair.append(iter->second.data(), iter->second.size()); + fprintf(stderr, "%s\n", pair.c_str()); + } +} + +struct Command { + const char* command; + std::function function; + int min_args; + int max_args; +}; + +const Command ops[] = { + {"help", BindWith4Args(&GrpcTool::Help), 0, INT_MAX}, + // {"ls", BindWith4Args(&GrpcTool::ListServices), 1, 3}, + // {"list", BindWith4Args(&GrpcTool::ListServices), 1, 3}, + {"call", BindWith4Args(&GrpcTool::CallMethod), 2, 3}, + // {"type", BindWith4Args(&GrpcTool::PrintType), 2, 2}, + // {"parse", BindWith4Args(&GrpcTool::ParseMessage), 2, 3}, + // {"totext", BindWith4Args(&GrpcTool::ToText), 2, 3}, + // {"tobinary", BindWith4Args(&GrpcTool::ToBinary), 2, 3}, +}; + +void Usage(const grpc::string& msg) { + fprintf( + stderr, + "%s\n" + // " grpc_cli ls ... ; List services\n" + " grpc_cli call ... ; Call method\n" + // " grpc_cli type ... ; Print type\n" + // " grpc_cli parse ... ; Parse message\n" + // " grpc_cli totext ... ; Convert binary message to text\n" + // " grpc_cli tobinary ... ; Convert text message to binary\n" + " grpc_cli help ... ; Print this message, or per-command usage\n" + "\n", + msg.c_str()); + + exit(1); +} + +const Command* FindCommand(const grpc::string& name) { + for (int i = 0; i < (int)ArraySize(ops); i++) { + if (name == ops[i].command) { + return &ops[i]; + } + } + return NULL; +} +} // namespace + +int GrpcToolMainLib(int argc, const char** argv, OutputCallback callback) { + if (argc < 2) { + Usage("No command specified"); + } + + grpc::string command = argv[1]; + argc -= 2; + argv += 2; + + const Command* cmd = FindCommand(command); + if (cmd != NULL) { + GrpcTool grpc_tool; + if (argc < cmd->min_args || argc > cmd->max_args) { + // Force the command to print its usage message + fprintf(stderr, "\nWrong number of arguments for %s\n", command.c_str()); + grpc_tool.SetPrintCommandMode(1); + return cmd->function(&grpc_tool, -1, NULL, callback); + } + const bool ok = cmd->function(&grpc_tool, argc, argv, callback); + return ok ? 0 : 1; + } else { + Usage("Invalid command '" + grpc::string(command.c_str()) + "'"); + } + return 1; +} + +GrpcTool::GrpcTool() : print_command_usage_(false), usage_exit_status_(0) {} + +void GrpcTool::CommandUsage(const grpc::string& usage) const { + if (print_command_usage_) { + fprintf(stderr, "\n%s%s\n", usage.c_str(), + (usage.empty() || usage[usage.size() - 1] != '\n') ? "\n" : ""); + exit(usage_exit_status_); + } +} + +bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { + CommandUsage( + "Print help\n" + " grpc_cli help [subcommand]\n"); + + if (argc == 0) { + Usage(""); + } else { + const Command* cmd = FindCommand(argv[0]); + if (cmd == NULL) { + Usage("Unknown command '" + grpc::string(argv[0]) + "'"); + } + SetPrintCommandMode(0); + cmd->function(this, -1, NULL, callback); + } + return true; +} + +bool GrpcTool::CallMethod(int argc, const char** argv, + OutputCallback callback) { + CommandUsage( + "Call method\n" + " grpc_cli call
[.] \n" + "
; host:port\n" + " ; Exported service name\n" + " ; Method name\n" + " ; Text protobuffer (overrides infile)\n" + " --protofiles ; Comma separated proto files used as a" + " fallback when parsing request/response\n" + " --proto_path ; The search path of proto files, valid" + " only when --protofiles is given\n" + " --metadata ; The metadata to be sent to the server\n" + " --enable_ssl ; Set whether to use tls\n" + " --use_auth ; Set whether to create default google" + " credentials\n" + " --outfile ; Output filename (defaults to stdout)\n" + " --input_binary_file ; Path to input file in binary format\n" + " --binary_output ; Path to output file in binary format\n"); + + std::stringstream output_ss; + grpc::string request_text; + grpc::string server_address(argv[0]); + grpc::string method_name(argv[1]); + std::unique_ptr parser; + grpc::string serialized_request_proto; + + if (argc == 3) { + request_text = argv[2]; + } + + std::shared_ptr creds; + if (!FLAGS_enable_ssl) { + creds = grpc::InsecureChannelCredentials(); + } else { + if (FLAGS_use_auth) { + creds = grpc::GoogleDefaultCredentials(); + } else { + creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); + } + } + std::shared_ptr channel = + grpc::CreateChannel(server_address, creds); + + if (request_text.empty() && FLAGS_input_binary_file.empty()) { + if (isatty(STDIN_FILENO)) { + std::cout << "reading request message from stdin..." << std::endl; + } + std::stringstream input_stream; + input_stream << std::cin.rdbuf(); + request_text = input_stream.str(); + } + + if (!request_text.empty()) { + if (!FLAGS_protofiles.empty()) { + parser.reset(new grpc::testing::ProtoFileParser( + FLAGS_proto_path, FLAGS_protofiles, method_name)); + } else { + parser.reset(new grpc::testing::ProtoFileParser(channel, method_name)); + } + method_name = parser->GetFullMethodName(); + if (parser->HasError()) { + return 1; + } + + if (!FLAGS_input_binary_file.empty()) { + std::cout + << "warning: request given in argv, ignoring --input_binary_file" + << std::endl; + } + } + + if (parser) { + serialized_request_proto = + parser->GetSerializedProto(request_text, true /* is_request */); + if (parser->HasError()) { + return 1; + } + } else if (!FLAGS_input_binary_file.empty()) { + std::ifstream input_file(FLAGS_input_binary_file, + std::ios::in | std::ios::binary); + std::stringstream input_stream; + input_stream << input_file.rdbuf(); + serialized_request_proto = input_stream.str(); + } + std::cout << "connecting to " << server_address << std::endl; + + grpc::string serialized_response_proto; + std::multimap client_metadata; + std::multimap server_initial_metadata, + server_trailing_metadata; + ParseMetadataFlag(&client_metadata); + PrintMetadata(client_metadata, "Sending client initial metadata:"); + grpc::Status s = grpc::testing::CliCall::Call( + channel, method_name, serialized_request_proto, + &serialized_response_proto, client_metadata, &server_initial_metadata, + &server_trailing_metadata); + PrintMetadata(server_initial_metadata, + "Received initial metadata from server:"); + PrintMetadata(server_trailing_metadata, + "Received trailing metadata from server:"); + if (s.ok()) { + std::cout << "Rpc succeeded with OK status" << std::endl; + if (parser) { + grpc::string response_text = parser->GetTextFormat( + serialized_response_proto, false /* is_request */); + if (parser->HasError()) { + return false; + } + output_ss << "Response: \n " << response_text << std::endl; + } + if (!FLAGS_output_binary_file.empty()) { + std::ofstream output_file(FLAGS_output_binary_file, + std::ios::trunc | std::ios::binary); + output_file << serialized_response_proto; + } + } else { + std::cout << "Rpc failed with status code " << s.error_code() + << ", error message: " << s.error_message() << std::endl; + } + + return callback(output_ss.str()); +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h new file mode 100644 index 00000000000..d779737cd5c --- /dev/null +++ b/test/cpp/util/grpc_tool.h @@ -0,0 +1,47 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include + +namespace grpc { +namespace testing { + +typedef std::function OutputCallback; + +int GrpcToolMainLib(int argc, const char **argv, OutputCallback callback); + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc new file mode 100644 index 00000000000..77c3f3fc24d --- /dev/null +++ b/test/cpp/util/grpc_tool_test.cc @@ -0,0 +1,219 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/cpp/util/grpc_tool.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "src/proto/grpc/testing/echo.pb.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +#include "test/cpp/util/string_ref_helper.h" + +using grpc::testing::EchoRequest; +using grpc::testing::EchoResponse; + +namespace grpc { +namespace testing { + +class TestServiceImpl : public ::grpc::testing::EchoTestService::Service { + public: + Status Echo(ServerContext* context, const EchoRequest* request, + EchoResponse* response) GRPC_OVERRIDE { + if (!context->client_metadata().empty()) { + for (std::multimap::const_iterator + iter = context->client_metadata().begin(); + iter != context->client_metadata().end(); ++iter) { + context->AddInitialMetadata(ToString(iter->first), + ToString(iter->second)); + } + } + context->AddTrailingMetadata("trailing_key", "trailing_value"); + response->set_message(request->message()); + return Status::OK; + } +}; + +class GrpcToolTest : public ::testing::Test { + protected: + GrpcToolTest() {} + + void SetUp() GRPC_OVERRIDE { + int port = grpc_pick_unused_port_or_die(); + server_address_ << "localhost:" << port; + // Setup server + ServerBuilder builder; + builder.AddListeningPort(server_address_.str(), + InsecureServerCredentials()); + builder.RegisterService(&service_); + server_ = builder.BuildAndStart(); + } + + void TearDown() GRPC_OVERRIDE { server_->Shutdown(); } + + void ResetStub() { + channel_ = + CreateChannel(server_address_.str(), InsecureChannelCredentials()); + stub_ = grpc::testing::EchoTestService::NewStub(channel_); + } + + std::shared_ptr channel_; + std::unique_ptr stub_; + std::unique_ptr server_; + std::ostringstream server_address_; + TestServiceImpl service_; + reflection::ProtoServerReflectionPlugin plugin_; +}; + +static bool PrintStream(std::stringstream* ss, const grpc::string& output) { + (*ss) << output << std::endl; + return true; +} + +template +static size_t ArraySize(T& a) { + return ((sizeof(a) / sizeof(*(a))) / + static_cast(!(sizeof(a) % sizeof(*(a))))); +} + +#define USAGE_REGEX "( grpc_cli .+\n){2,10}" + +TEST_F(GrpcToolTest, NoCommand) { + // Test input "grpc_cli" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), "No command specified\n" USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, InvalidCommand) { + // Test input "grpc_cli" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "abc"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), "Invalid command 'abc'\n" USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, HelpCommand) { + // Test input "grpc_cli help" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "help"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT(GrpcToolMainLib(ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, + std::placeholders::_1)), + ::testing::ExitedWithCode(1), USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, CallCommand) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "message: 'Hello'"}; + + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + // Expected output: "message: \"Hello\"" + EXPECT_TRUE(NULL != + strstr(output_stream.str().c_str(), "message: \"Hello\"")); +} + +TEST_F(GrpcToolTest, TooFewArguments) { + // Test input "grpc_cli call localhost: Echo "message: 'Hello'" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", "Echo"}; + + // Exit with 1 + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, TooManyArguments) { + // Test input "grpc_cli call localhost: Echo Echo "message: 'Hello'" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + "Echo", "Echo", "message: 'Hello'"}; + + // Exit with 1 + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 4a27a1d8752..f7232a24f32 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2226,6 +2226,35 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_codegen_proto", + "grpc++_config_proto", + "grpc++_reflection", + "grpc_cli_libs", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/testing/echo.grpc.pb.h", + "src/proto/grpc/testing/echo.pb.h", + "src/proto/grpc/testing/echo_messages.grpc.pb.h", + "src/proto/grpc/testing/echo_messages.pb.h", + "test/cpp/util/string_ref_helper.h" + ], + "language": "c++", + "name": "grpc_tool_test", + "src": [ + "test/cpp/util/grpc_tool_test.cc", + "test/cpp/util/string_ref_helper.cc", + "test/cpp/util/string_ref_helper.h" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "grpc", @@ -4466,6 +4495,7 @@ ], "headers": [ "test/cpp/util/cli_call.h", + "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.h" ], @@ -4474,6 +4504,8 @@ "src": [ "test/cpp/util/cli_call.cc", "test/cpp/util/cli_call.h", + "test/cpp/util/grpc_tool.cc", + "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index d94301b946b..379833bde5e 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2269,6 +2269,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_tool_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index d25c692e3e8..3269bab56fa 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -148,12 +148,15 @@ + + + diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index 4add8ed5e13..cbce2f23120 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -4,6 +4,9 @@ test\cpp\util + + test\cpp\util + test\cpp\util @@ -15,6 +18,9 @@ test\cpp\util + + test\cpp\util + test\cpp\util diff --git a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj index 9c8cdc54c25..f9dbe1dcb69 100644 --- a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj +++ b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj @@ -167,15 +167,15 @@ {86E35862-43E8-F59E-F906-AFE0348AD3D2} + + {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} + {0BE77741-552A-929B-A497-4EF7ECE17A64} {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - - {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} diff --git a/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj new file mode 100644 index 00000000000..c6f65aa30b8 --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj @@ -0,0 +1,286 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {F00D82D4-E988-6D2F-F0B9-9E82BCC2A2B2} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + grpc_tool_test + static + Debug + static + Debug + + + grpc_tool_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {86E35862-43E8-F59E-F906-AFE0348AD3D2} + + + {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters new file mode 100644 index 00000000000..731eb2e6ffb --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters @@ -0,0 +1,232 @@ + + + + + src\proto\grpc\testing + + + src\proto\grpc\testing + + + test\cpp\util + + + test\cpp\util + + + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen\security + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc++\impl\codegen + + + + + test\cpp\util + + + + + + {89fed779-17c5-23da-c8a2-9e868ff34480} + + + {96e4a1a8-0b91-1a6d-ae4d-ddf33abb93c0} + + + {1d9dcc6f-7c1b-cdc3-4c35-73d5968dfd92} + + + {5eca7690-973a-c8ed-84d6-5325f8de43ac} + + + {5789073e-5b84-0ec9-af06-47866647874d} + + + {d3f3293f-204f-7771-fcdf-de673f6b06b6} + + + {7e90f37b-f9cc-0725-b2c1-12aa7d4809ba} + + + {7e4b71ef-8125-6446-bfc1-9bc90beed59c} + + + {169774bd-5c6c-6827-66a4-326b4aef44d6} + + + {1b609b37-ef2a-e5eb-e1ba-ad9e79c77438} + + + {cd1e35d8-8a61-62fe-6ce1-c8936872d1ef} + + + {f7ee4df5-1f47-1e7f-c91e-350382c1b729} + + + {f2166b83-6b0b-d53b-b58b-627bd9efcad2} + + + {bbe36cbc-7fbe-2817-0bd0-d03726f323e6} + + + {e106cd7b-cfa0-0645-f1a9-2acedc23afe7} + + + + From 28263275f0ab22d5a9d01f7e5afddbcc24a9d22c Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 14:38:07 -0700 Subject: [PATCH 03/97] Remove unnecessary dependencies --- Makefile | 7 +++--- build.yaml | 6 ++--- test/cpp/util/grpc_cli.cc | 23 +++++++++++-------- test/cpp/util/grpc_tool.cc | 22 ++++++++++-------- test/cpp/util/grpc_tool.h | 9 ++++---- tools/run_tests/sources_and_headers.json | 13 ++++++----- .../grpc_cli_libs/grpc_cli_libs.vcxproj | 6 +++++ .../grpc_cli_libs.vcxproj.filters | 6 +++++ .../vcxproj/test/grpc_cli/grpc_cli.vcxproj | 9 -------- 9 files changed, 56 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index a1165955a9a..f3a9b2bc7a8 100644 --- a/Makefile +++ b/Makefile @@ -4097,6 +4097,7 @@ LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ + test/cpp/util/string_ref_helper.cc \ PUBLIC_HEADERS_CXX += \ @@ -11004,16 +11005,16 @@ $(BINDIR)/$(CONFIG)/grpc_cli: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a deps_grpc_cli: $(GRPC_CLI_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index 5b2c5818501..afad1e113c5 100644 --- a/build.yaml +++ b/build.yaml @@ -1032,15 +1032,18 @@ libs: - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h + - test/cpp/util/string_ref_helper.h src: - test/cpp/util/cli_call.cc - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc + - test/cpp/util/string_ref_helper.cc deps: - grpc++_reflection - grpc++ - grpc_plugin_support + - grpc++_test_config - name: grpc_plugin_support build: protoc language: c++ @@ -2661,11 +2664,8 @@ targets: deps: - grpc_cli_libs - grpc++_reflection - - grpc++_test_util - - grpc_test_util - grpc++ - grpc - - gpr_test_util - gpr - grpc++_test_config - name: grpc_cpp_plugin diff --git a/test/cpp/util/grpc_cli.cc b/test/cpp/util/grpc_cli.cc index 28c87956d67..897ac5ce4a0 100644 --- a/test/cpp/util/grpc_cli.cc +++ b/test/cpp/util/grpc_cli.cc @@ -33,19 +33,24 @@ /* A command line tool to talk to a grpc server. + Run `grpc_cli help` command to see its usage information. + Example of talking to grpc interop server: - grpc_cli call localhost:50051 UnaryCall src/proto/grpc/testing/test.proto \ - "response_size:10" --enable_ssl=false + grpc_cli call localhost:50051 UnaryCall "response_size:10" \ + --protofiles=src/proto/grpc/testing/test.proto --enable_ssl=false Options: - 1. --proto_path, if your proto file is not under current working directory, + 1. --protofiles, use this flag to provide a proto file if the server does + does not have the reflection service. + 2. --proto_path, if your proto file is not under current working directory, use this flag to provide a search root. It should work similar to the - counterpart in protoc. - 2. --metadata specifies metadata to be sent to the server, such as: + counterpart in protoc. This option is valid only when protofiles is + provided. + 3. --metadata specifies metadata to be sent to the server, such as: --metadata="MyHeaderKey1:Value1:MyHeaderKey2:Value2" - 3. --enable_ssl, whether to use tls. - 4. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call - 3. --input_binary_file, a file containing the serialized request. The file + 4. --enable_ssl, whether to use tls. + 5. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call + 6. --input_binary_file, a file containing the serialized request. The file can be generated by calling something like: protoc --proto_path=src/proto/grpc/testing/ \ --encode=grpc.testing.SimpleRequest \ @@ -53,7 +58,7 @@ < input.txt > input.bin If this is used and no proto file is provided in the argument list, the method string has to be exact in the form of /package.service/method. - 4. --output_binary_file, a file to write binary format response into, it can + 7. --output_binary_file, a file to write binary format response into, it can be later decoded using protoc: protoc --proto_path=src/proto/grpc/testing/ \ --decode=grpc.testing.SimpleResponse \ diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 38ac9e55a03..5b3f565ff50 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -31,7 +31,7 @@ * */ -#include "grpc_tool.h" +#include "test/cpp/util/grpc_tool.h" #include #include @@ -47,7 +47,6 @@ #include #include #include -#include #include "test/cpp/util/cli_call.h" #include "test/cpp/util/proto_file_parser.h" @@ -75,8 +74,8 @@ class GrpcTool { public: explicit GrpcTool(); virtual ~GrpcTool() {} - bool Help(int argc, const char** argv, OutputCallback callback); - bool CallMethod(int argc, const char** argv, OutputCallback callback); + bool Help(int argc, const char** argv, GrpcToolOutputCallback callback); + bool CallMethod(int argc, const char** argv, GrpcToolOutputCallback callback); void SetPrintCommandMode(int exit_status) { print_command_usage_ = true; usage_exit_status_ = exit_status; @@ -89,8 +88,8 @@ class GrpcTool { }; template -std::function BindWith4Args( - T&& func) { +std::function +BindWith4Args(T&& func) { return std::bind(std::forward(func), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4); @@ -143,7 +142,8 @@ void PrintMetadata(const T& m, const grpc::string& message) { struct Command { const char* command; - std::function function; + std::function + function; int min_args; int max_args; }; @@ -186,7 +186,8 @@ const Command* FindCommand(const grpc::string& name) { } } // namespace -int GrpcToolMainLib(int argc, const char** argv, OutputCallback callback) { +int GrpcToolMainLib(int argc, const char** argv, + GrpcToolOutputCallback callback) { if (argc < 2) { Usage("No command specified"); } @@ -222,7 +223,8 @@ void GrpcTool::CommandUsage(const grpc::string& usage) const { } } -bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { +bool GrpcTool::Help(int argc, const char** argv, + GrpcToolOutputCallback callback) { CommandUsage( "Print help\n" " grpc_cli help [subcommand]\n"); @@ -241,7 +243,7 @@ bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { } bool GrpcTool::CallMethod(int argc, const char** argv, - OutputCallback callback) { + GrpcToolOutputCallback callback) { CommandUsage( "Call method\n" " grpc_cli call
[.] \n" diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h index d779737cd5c..9e61abf6418 100644 --- a/test/cpp/util/grpc_tool.h +++ b/test/cpp/util/grpc_tool.h @@ -31,17 +31,16 @@ * */ +#include #include -#include - -#include namespace grpc { namespace testing { -typedef std::function OutputCallback; +typedef std::function GrpcToolOutputCallback; -int GrpcToolMainLib(int argc, const char **argv, OutputCallback callback); +int GrpcToolMainLib(int argc, const char **argv, + GrpcToolOutputCallback callback); } // namespace testing } // namespace grpc diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index f7232a24f32..e2d2796184f 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2130,14 +2130,11 @@ { "deps": [ "gpr", - "gpr_test_util", "grpc", "grpc++", "grpc++_reflection", "grpc++_test_config", - "grpc++_test_util", - "grpc_cli_libs", - "grpc_test_util" + "grpc_cli_libs" ], "headers": [], "language": "c++", @@ -4491,13 +4488,15 @@ "deps": [ "grpc++", "grpc++_reflection", + "grpc++_test_config", "grpc_plugin_support" ], "headers": [ "test/cpp/util/cli_call.h", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", - "test/cpp/util/proto_reflection_descriptor_database.h" + "test/cpp/util/proto_reflection_descriptor_database.h", + "test/cpp/util/string_ref_helper.h" ], "language": "c++", "name": "grpc_cli_libs", @@ -4509,7 +4508,9 @@ "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", - "test/cpp/util/proto_reflection_descriptor_database.h" + "test/cpp/util/proto_reflection_descriptor_database.h", + "test/cpp/util/string_ref_helper.cc", + "test/cpp/util/string_ref_helper.h" ], "third_party": false, "type": "lib" diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index 3269bab56fa..59b2923b037 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -151,6 +151,7 @@ + @@ -161,6 +162,8 @@ + + @@ -172,6 +175,9 @@ {B6E81D84-2ACB-41B8-8781-493A944C7817} + + {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} + diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index cbce2f23120..d400f8eccf2 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -13,6 +13,9 @@ test\cpp\util + + test\cpp\util + @@ -27,6 +30,9 @@ test\cpp\util + + test\cpp\util + diff --git a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj index f9dbe1dcb69..78a0a63b5d7 100644 --- a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj +++ b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj @@ -170,21 +170,12 @@ {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} - - {0BE77741-552A-929B-A497-4EF7ECE17A64} - - - {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} {29D16885-7228-4C31-81ED-5F9187C7F2A9} - - {EAB0A629-17A9-44DB-B5FF-E91A721FE037} - {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} From 724a4e253a579f44d6b3d4ab9599f771837d7580 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 14:56:06 -0700 Subject: [PATCH 04/97] Add header guard --- test/cpp/util/grpc_tool.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h index 9e61abf6418..0c7217d962a 100644 --- a/test/cpp/util/grpc_tool.h +++ b/test/cpp/util/grpc_tool.h @@ -31,6 +31,9 @@ * */ +#ifndef GRPC_TEST_CPP_UTIL_GRPC_TOOL_H +#define GRPC_TEST_CPP_UTIL_GRPC_TOOL_H + #include #include @@ -44,3 +47,5 @@ int GrpcToolMainLib(int argc, const char **argv, } // namespace testing } // namespace grpc + +#endif // GRPC_TEST_CPP_UTIL_GRPC_TOOL_H From 6b86080912d347efd336d9ebba61439cd7950ed6 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 25 Jul 2016 16:10:35 -0700 Subject: [PATCH 05/97] fixing issues from cronet end to end testing changed core logic for executing multiple ops. added call cancellation logic simplified the code. --- .../cronet/transport/cronet_transport.c | 377 ++++++++++++------ 1 file changed, 266 insertions(+), 111 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 25d8aca2508..d589d750be3 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -65,10 +65,11 @@ typedef struct grpc_cronet_transport grpc_cronet_transport; enum send_state { CRONET_SEND_IDLE = 0, - CRONET_REQ_STARTED, CRONET_SEND_HEADER, - CRONET_WRITE, + CRONET_WRITE_PENDING, CRONET_WRITE_COMPLETED, + CRONET_WAIT_FOR_CANCEL, + CRONET_STREAM_CLOSED, }; enum recv_state { @@ -91,13 +92,14 @@ enum e_caller { }; enum callback_id { - CB_SEND_INITIAL_METADATA = 0, - CB_SEND_MESSAGE, - CB_SEND_TRAILING_METADATA, - CB_RECV_MESSAGE, - CB_RECV_INITIAL_METADATA, - CB_RECV_TRAILING_METADATA, - CB_NUM_CALLBACKS + OP_SEND_INITIAL_METADATA = 0, + OP_SEND_MESSAGE, + OP_SEND_TRAILING_METADATA, + OP_RECV_MESSAGE, + OP_RECV_INITIAL_METADATA, + OP_RECV_TRAILING_METADATA, + OP_CANCEL_ERROR, + OP_NUM_CALLBACKS }; struct stream_obj { @@ -117,23 +119,29 @@ struct stream_obj { // Hold the URL char *url; - bool response_headers_received; - bool read_requested; - bool response_trailers_received; + // One bit per operation + bool op_requested[OP_NUM_CALLBACKS]; + bool op_done[OP_NUM_CALLBACKS]; + // Set to true when server indicates no more data will be sent bool read_closed; // Recv message stuff grpc_byte_buffer **recv_message; // Initial metadata stuff grpc_metadata_batch *recv_initial_metadata; + grpc_chttp2_incoming_metadata_buffer initial_metadata; // Trailing metadata stuff grpc_metadata_batch *recv_trailing_metadata; grpc_chttp2_incoming_metadata_buffer imb; + bool imb_valid; // true if there are any valid entries in imb. // This mutex protects receive state machine execution gpr_mu recv_mu; - // we can queue up up to 2 callbacks for each OP - grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; + + // Callbacks to be called when operations complete + grpc_closure *cb_recv_initial_metadata_ready; + grpc_closure *cb_recv_message_ready; + grpc_closure *on_complete; // storage for header cronet_bidirectional_stream_header *headers; @@ -156,34 +164,63 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_pollset_set *pollset_set) {} -static void enqueue_callbacks(grpc_closure *callback_list[]) { +// Client creates a bunch of operations and invokes "call_start_batch" +// call_start_batch creates a stream_op structure. this structure has info +// needed for executing all the ops. It has on_complete callback that needs +// to be called when all ops are executed. This function keeps track of all +// outstanding operations. It returns true if all operations that were part of +// the stream_op have been completed. +static bool is_op_complete(stream_obj *s) { + int i; + // Check if any requested op is pending + for (i = 0; i < OP_NUM_CALLBACKS; i++) { + if (s->op_requested[i] && !s->op_done[i]) { + gpr_log(GPR_DEBUG, "is_op_complete is FALSE because of %d", i); + return false; + } + } + // Clear the requested/done bits and return true + for (i = 0; i < OP_NUM_CALLBACKS; i++) { + s->op_requested[i] = s->op_done[i] = false; + } + return true; +} + +static void enqueue_callback(grpc_closure *callback) { + GPR_ASSERT(callback); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - if (callback_list[0]) { - grpc_exec_ctx_sched(&exec_ctx, callback_list[0], GRPC_ERROR_NONE, NULL); - callback_list[0] = NULL; - } - if (callback_list[1]) { - grpc_exec_ctx_sched(&exec_ctx, callback_list[1], GRPC_ERROR_NONE, NULL); - callback_list[1] = NULL; - } + grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); grpc_exec_ctx_finish(&exec_ctx); } static void on_canceled(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled %p", stream); + gpr_log(GPR_DEBUG, "on_canceled(%p)", stream); } + stream_obj *s = (stream_obj *)stream->annotation; + s->op_done[OP_CANCEL_ERROR] = true; + + // Terminate any read callback + if (s->cb_recv_message_ready) { + enqueue_callback(s->cb_recv_message_ready); + s->cb_recv_message_ready = 0; + s->op_done[OP_RECV_MESSAGE] = true; + } + // Don't wait to get any trailing metadata + s->op_done[OP_RECV_TRAILING_METADATA] = true; + + next_send_step(s); } static void on_failed(cronet_bidirectional_stream *stream, int net_error) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); + gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); } } static void on_succeeded(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded %p", stream); + gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); } } @@ -191,31 +228,38 @@ static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received"); + gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + trailers); } stream_obj *s = (stream_obj *)stream->annotation; memset(&s->imb, 0, sizeof(s->imb)); + s->imb_valid = false; grpc_chttp2_incoming_metadata_buffer_init(&s->imb); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + trailers->headers[i].value); + } + grpc_chttp2_incoming_metadata_buffer_add( &s->imb, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(trailers->headers[i].key), grpc_mdstr_from_string(trailers->headers[i].value))); + s->imb_valid = true; } - s->response_trailers_received = true; + s->op_done[OP_RECV_TRAILING_METADATA] = true; next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); } static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed"); + gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); - s->cronet_send_state = CRONET_WRITE_COMPLETED; + s->op_done[OP_SEND_MESSAGE] = true; next_send_step(s); } @@ -245,14 +289,14 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", - count, s->total_read_bytes, s->remaining_read_bytes); + gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); } if (count > 0) { GPR_ASSERT(s->recv_message); s->remaining_read_bytes -= count; next_recv_step(s, ON_READ_COMPLETE); } else { + gpr_log(GPR_DEBUG, "read_closed = true"); s->read_closed = true; next_recv_step(s, ON_READ_COMPLETE); } @@ -263,21 +307,39 @@ static void on_response_headers_received( const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received"); + gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + headers, negotiated_protocol); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); - s->response_headers_received = true; + + memset(&s->initial_metadata, 0, sizeof(s->initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->initial_metadata); + unsigned int i = 0; + for (i = 0; i < headers->count; i++) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "header key=%s, value=%s", headers->headers[i].key, + headers->headers[i].value); + } + grpc_chttp2_incoming_metadata_buffer_add( + &s->initial_metadata, + grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(headers->headers[i].key), + grpc_mdstr_from_string(headers->headers[i].value))); + } + + grpc_chttp2_incoming_metadata_buffer_publish(&s->initial_metadata, + s->recv_initial_metadata); + enqueue_callback(s->cb_recv_initial_metadata_ready); + s->op_done[OP_RECV_INITIAL_METADATA] = true; next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); } static void on_request_headers_sent(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent"); + gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); - s->cronet_send_state = CRONET_SEND_HEADER; + s->op_done[OP_SEND_INITIAL_METADATA] = true; next_send_step(s); } @@ -293,11 +355,13 @@ static cronet_bidirectional_stream_callback callbacks = { on_canceled}; static void invoke_closing_callback(stream_obj *s) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); - if (s->callback_list[CB_RECV_TRAILING_METADATA]) { - enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]); + if (!is_op_complete(s)) return; + + if (s->imb_valid) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, + s->recv_trailing_metadata); } + enqueue_callback(s->on_complete); } static void set_recv_state(stream_obj *s, enum recv_state state) { @@ -309,29 +373,35 @@ static void set_recv_state(stream_obj *s, enum recv_state state) { // This is invoked from perform_stream_op, and all on_xxxx callbacks. static void next_recv_step(stream_obj *s, enum e_caller caller) { + // gpr_log(GPR_DEBUG, "locking mutex %p", &s->recv_mu); gpr_mu_lock(&s->recv_mu); switch (s->cronet_recv_state) { case CRONET_RECV_IDLE: if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); + gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE, caller=%d", + caller); } if (caller == PERFORM_STREAM_OP || caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->response_trailers_received) { - invoke_closing_callback(s); + if (s->read_closed && s->op_done[OP_RECV_TRAILING_METADATA]) { set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->response_headers_received == true && - s->read_requested == true) { + } else if (s->op_done[OP_RECV_INITIAL_METADATA] == true && + s->op_requested[OP_RECV_MESSAGE]) { set_recv_state(s, CRONET_RECV_READ_LENGTH); s->total_read_bytes = s->remaining_read_bytes = GRPC_HEADER_SIZE_IN_BYTES; GPR_ASSERT(s->read_buffer); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", + s->cbs, s->read_buffer, s->remaining_read_bytes); } cronet_bidirectional_stream_read(s->cbs, s->read_buffer, s->remaining_read_bytes); } + } else if (caller == ON_RESPONSE_TRAILERS_RECEIVED) { + // We get here when we receive trailers directly, i.e. without + // going through a data read operation. + set_recv_state(s, CRONET_RECV_CLOSED); } break; case CRONET_RECV_READ_LENGTH: @@ -340,8 +410,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { } if (caller == ON_READ_COMPLETE) { if (s->read_closed) { - invoke_closing_callback(s); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; set_recv_state(s, CRONET_RECV_CLOSED); } else { GPR_ASSERT(s->remaining_read_bytes == 0); @@ -352,7 +422,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); GPR_ASSERT(s->read_buffer); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", + s->cbs, s->read_buffer, s->remaining_read_bytes); } if (s->remaining_read_bytes > 0) { cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, @@ -361,8 +432,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { // Calling the closing callback directly since this is a 0 byte read // for an empty message. process_recv_message(s, NULL); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - invoke_closing_callback(s); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; set_recv_state(s, CRONET_RECV_CLOSED); } } @@ -386,7 +457,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { uint8_t *p = (uint8_t *)s->read_buffer; process_recv_message(s, p); set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; } } break; @@ -396,7 +468,9 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { GPR_ASSERT(0); // Should not reach here break; } + invoke_closing_callback(s); gpr_mu_unlock(&s->recv_mu); + // gpr_log(GPR_DEBUG, "unlocking mutex %p", &s->recv_mu); } // This function takes the data from s->write_slice_buffer and assembles into @@ -417,27 +491,69 @@ static void create_grpc_frame(stream_obj *s) { // append actual data memcpy(p, raw_data, length); } - -static void do_write(stream_obj *s) { +// Return false if there is no data to write +static bool do_write(stream_obj *s) { gpr_slice_buffer *sb = &s->write_slice_buffer; GPR_ASSERT(sb->count <= 1); if (sb->count > 0) { create_grpc_frame(s); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write(%p,%p,%d,%d)", + s->cbs, s->write_buffer, (int)s->write_buffer_size, false); } cronet_bidirectional_stream_write(s->cbs, s->write_buffer, (int)s->write_buffer_size, false); + return true; + } else { + return false; } } +static bool init_cronet_stream(stream_obj *s, grpc_transport *gt) { + GPR_ASSERT(s->cbs == NULL); + grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; + GPR_ASSERT(ct->engine); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + } + s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); + GPR_ASSERT(s->cbs); + s->read_closed = false; + + for (int i = 0; i < OP_NUM_CALLBACKS; i++) { + s->op_requested[i] = s->op_done[i] = false; + } + s->cronet_send_state = CRONET_SEND_IDLE; + s->cronet_recv_state = CRONET_RECV_IDLE; +} + +static bool do_close_connection(stream_obj *s) { + s->op_done[OP_SEND_TRAILING_METADATA] = true; + if (s->cbs) { + // Send an "empty" write to the far end to signal that we're done. + // This will induce the server to send down trailers. + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write length 0"); + } + cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); + return true; + } else { + // We never created a stream. This was probably an empty request. + invoke_closing_callback(s); + return true; + } + return false; +} + // static void next_send_step(stream_obj *s) { + gpr_log(GPR_DEBUG, "next_send_step cronet_send_state=%d", + s->cronet_send_state); switch (s->cronet_send_state) { case CRONET_SEND_IDLE: GPR_ASSERT( s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_REQ_STARTED; + s->cronet_send_state = CRONET_SEND_HEADER; if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); } @@ -447,11 +563,51 @@ static void next_send_step(stream_obj *s) { gpr_free(s->header_array.headers); break; case CRONET_SEND_HEADER: - do_write(s); - s->cronet_send_state = CRONET_WRITE; + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } else { + s->cronet_send_state = CRONET_WRITE_PENDING; + } + break; + case CRONET_WRITE_PENDING: + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } else { + s->cronet_send_state = CRONET_WRITE_COMPLETED; + } break; case CRONET_WRITE_COMPLETED: - do_write(s); + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } + break; + case CRONET_STREAM_CLOSED: + s->cronet_send_state = CRONET_SEND_IDLE; + break; + case CRONET_WAIT_FOR_CANCEL: + invoke_closing_callback(s); + s->cronet_send_state = CRONET_SEND_IDLE; break; default: GPR_ASSERT(0); @@ -493,7 +649,7 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, // Create URL by appending :path value to the hostname gpr_asprintf(&s->url, "https://%s%s", host, value); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); + // gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); } continue; } @@ -511,6 +667,13 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; GPR_ASSERT(ct->engine); stream_obj *s = (stream_obj *)gs; + // Initialize a cronet bidirectional stream if it doesn't exist. + if (s->cbs == NULL) { + init_cronet_stream(s, gt); + } + + s->on_complete = op->on_complete; + if (op->recv_trailing_metadata) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, @@ -518,8 +681,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, op->on_complete); } s->recv_trailing_metadata = op->recv_trailing_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); - s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; + s->op_requested[OP_RECV_TRAILING_METADATA] = true; } if (op->recv_message) { if (grpc_cronet_trace) { @@ -527,24 +689,19 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, op->on_complete); } s->recv_message = (grpc_byte_buffer **)op->recv_message; - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); - s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; - s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete; - s->read_requested = true; - next_recv_step(s, PERFORM_STREAM_OP); + s->cb_recv_message_ready = op->recv_message_ready; + s->op_requested[OP_RECV_MESSAGE] = true; } if (op->recv_initial_metadata) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); + gpr_log(GPR_DEBUG, + "perform_stream_op - recv_initial_metadata on_complete=%p, " + "on_ready=%p", + op->on_complete, op->recv_initial_metadata_ready); } s->recv_initial_metadata = op->recv_initial_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); - s->callback_list[CB_RECV_INITIAL_METADATA][0] = - op->recv_initial_metadata_ready; - s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; + s->cb_recv_initial_metadata_ready = op->recv_initial_metadata_ready; + s->op_requested[OP_RECV_INITIAL_METADATA] = true; } if (op->send_initial_metadata) { if (grpc_cronet_trace) { @@ -558,8 +715,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->header_array.count = s->num_headers; s->header_array.capacity = s->num_headers; s->header_array.headers = s->headers; - GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]); - s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; + s->op_requested[OP_SEND_INITIAL_METADATA] = true; } if (op->send_message) { if (grpc_cronet_trace) { @@ -572,21 +728,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, // TODO (makdharma): add compression support GPR_ASSERT(op->send_message->flags == 0); gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - if (s->cbs == NULL) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - s->response_trailers_received = false; - s->response_headers_received = false; - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; - } - GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); - s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; - next_send_step(s); + s->op_requested[OP_SEND_MESSAGE] = true; } if (op->send_trailing_metadata) { if (grpc_cronet_trace) { @@ -594,27 +736,24 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, "perform_stream_op - send_trailing_metadata: on_complete=%p", op->on_complete); } - GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); - s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); - } + s->op_requested[OP_SEND_TRAILING_METADATA] = true; } + if (op->cancel_error) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - cancel_error: on_complete=%p", + op->on_complete); + } + s->op_requested[OP_CANCEL_ERROR] = true; + } + next_send_step(s); + next_recv_step(s, PERFORM_STREAM_OP); } static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, const void *server_data) { stream_obj *s = (stream_obj *)gs; - memset(s->callback_list, 0, sizeof(s->callback_list)); + memset(s, 0, sizeof(stream_obj)); s->cbs = NULL; gpr_mu_init(&s->recv_mu); s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); @@ -636,6 +775,7 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_free(s->read_buffer); gpr_free(s->write_buffer); gpr_free(s->url); + gpr_log(GPR_DEBUG, "destroying %p", &s->recv_mu); gpr_mu_destroy(&s->recv_mu); if (and_free_memory) { gpr_free(and_free_memory); @@ -650,13 +790,28 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { } } +static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Unimplemented method"); + } + return NULL; +} + +static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_transport_op *op) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Unimplemented method"); + } + return NULL; +} + const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), "cronet_http", init_stream, set_pollset_do_nothing, set_pollset_set_do_nothing, perform_stream_op, - NULL, + perform_op, destroy_stream, destroy_transport, - NULL}; + get_peer}; From e7068c836ee7077c9e61d8e6a354420477b610c5 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 17:30:45 -0700 Subject: [PATCH 06/97] Remove references to string_ref_helper --- Makefile | 1 - build.yaml | 2 -- test/cpp/util/grpc_tool.cc | 1 - tools/run_tests/sources_and_headers.json | 7 ++----- vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj | 3 --- .../vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters | 6 ------ 6 files changed, 2 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 3f01664b41b..d5b2413ac37 100644 --- a/Makefile +++ b/Makefile @@ -4164,7 +4164,6 @@ LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ - test/cpp/util/string_ref_helper.cc \ PUBLIC_HEADERS_CXX += \ diff --git a/build.yaml b/build.yaml index 5eff6f197ca..0937d1f6cdf 100644 --- a/build.yaml +++ b/build.yaml @@ -1045,13 +1045,11 @@ libs: - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h - - test/cpp/util/string_ref_helper.h src: - test/cpp/util/cli_call.cc - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc - - test/cpp/util/string_ref_helper.cc deps: - grpc++_reflection - grpc++ diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 5b3f565ff50..e227e6027dc 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -51,7 +51,6 @@ #include "test/cpp/util/proto_file_parser.h" #include "test/cpp/util/proto_reflection_descriptor_database.h" -#include "test/cpp/util/string_ref_helper.h" #include "test/cpp/util/test_config.h" DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 21a2da9051a..4b5557dc682 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4501,8 +4501,7 @@ "test/cpp/util/cli_call.h", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", - "test/cpp/util/proto_reflection_descriptor_database.h", - "test/cpp/util/string_ref_helper.h" + "test/cpp/util/proto_reflection_descriptor_database.h" ], "language": "c++", "name": "grpc_cli_libs", @@ -4514,9 +4513,7 @@ "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", - "test/cpp/util/proto_reflection_descriptor_database.h", - "test/cpp/util/string_ref_helper.cc", - "test/cpp/util/string_ref_helper.h" + "test/cpp/util/proto_reflection_descriptor_database.h" ], "third_party": false, "type": "lib" diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index 59b2923b037..09034dc33ef 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -151,7 +151,6 @@ - @@ -162,8 +161,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index d400f8eccf2..cbce2f23120 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -13,9 +13,6 @@ test\cpp\util - - test\cpp\util - @@ -30,9 +27,6 @@ test\cpp\util - - test\cpp\util - From 636ef6fb63fb6873a302797376674abbf6d2d4d7 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 26 Jul 2016 10:15:26 -0700 Subject: [PATCH 07/97] minor compiler warning fix --- src/core/ext/transport/cronet/transport/cronet_transport.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index d589d750be3..0a079927ea8 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -509,7 +509,7 @@ static bool do_write(stream_obj *s) { } } -static bool init_cronet_stream(stream_obj *s, grpc_transport *gt) { +static void init_cronet_stream(stream_obj *s, grpc_transport *gt) { GPR_ASSERT(s->cbs == NULL); grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; GPR_ASSERT(ct->engine); @@ -802,7 +802,6 @@ static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "Unimplemented method"); } - return NULL; } const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), From f4046cdcedbe6be6de173139f4217b34dc876fdd Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 26 Jul 2016 12:22:42 -0700 Subject: [PATCH 08/97] Fix thread leak --- test/cpp/util/grpc_tool_test.cc | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 77c3f3fc24d..57eeed7234a 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -79,29 +79,25 @@ class GrpcToolTest : public ::testing::Test { protected: GrpcToolTest() {} - void SetUp() GRPC_OVERRIDE { + // SetUpServer cannot be used with EXPECT_EXIT. grpc_pick_unused_port_or_die() + // uses atexit() to free chosen ports, and it will spawn a new thread in + // resolve_address_posix.c:192 at exit time. + const grpc::string SetUpServer() { + std::ostringstream server_address; int port = grpc_pick_unused_port_or_die(); - server_address_ << "localhost:" << port; + server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address_.str(), + builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); + return server_address.str(); } - void TearDown() GRPC_OVERRIDE { server_->Shutdown(); } + void ShutdownServer() { server_->Shutdown(); } - void ResetStub() { - channel_ = - CreateChannel(server_address_.str(), InsecureChannelCredentials()); - stub_ = grpc::testing::EchoTestService::NewStub(channel_); - } - - std::shared_ptr channel_; - std::unique_ptr stub_; std::unique_ptr server_; - std::ostringstream server_address_; TestServiceImpl service_; reflection::ProtoServerReflectionPlugin plugin_; }; @@ -163,7 +159,8 @@ TEST_F(GrpcToolTest, HelpCommand) { TEST_F(GrpcToolTest, CallCommand) { // Test input "grpc_cli call Echo" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); + + const grpc::string server_address = SetUpServer(); const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", "message: 'Hello'"}; @@ -173,12 +170,12 @@ TEST_F(GrpcToolTest, CallCommand) { // Expected output: "message: \"Hello\"" EXPECT_TRUE(NULL != strstr(output_stream.str().c_str(), "message: \"Hello\"")); + ShutdownServer(); } TEST_F(GrpcToolTest, TooFewArguments) { // Test input "grpc_cli call localhost: Echo "message: 'Hello'" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); const char* argv[] = {"grpc_cli", "call", "Echo"}; // Exit with 1 @@ -194,8 +191,7 @@ TEST_F(GrpcToolTest, TooFewArguments) { TEST_F(GrpcToolTest, TooManyArguments) { // Test input "grpc_cli call localhost: Echo Echo "message: 'Hello'" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + const char* argv[] = {"grpc_cli", "call", "localhost:10000", "Echo", "Echo", "message: 'Hello'"}; // Exit with 1 From 2f7060bad91c4e3f89b5e3c6818edb810b5058ca Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 26 Jul 2016 16:03:28 -0700 Subject: [PATCH 09/97] test harness changes for compiling --- .../CoreCronetEnd2EndTests.m | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index 58abb492cea..d2181120e93 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -77,7 +77,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_secure_fullstack( gpr_malloc(sizeof(fullstack_secure_fixture_data)); memset(&f, 0, sizeof(f)); - gpr_join_host_port(&ffd->localaddr, "localhost", port); + gpr_join_host_port(&ffd->localaddr, "127.0.0.1", port); f.fixture_data = ffd; f.cq = grpc_completion_queue_create(NULL); @@ -124,15 +124,24 @@ static void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture *f) { } static void cronet_init_client_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { + grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { grpc_arg ssl_name_override = {GRPC_ARG_STRING, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, {"foo.test.google.fr"}}; grpc_channel_args *new_client_args = grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); - [Cronet setHttp2Enabled:YES]; - [Cronet start]; + static bool done = false; + // TODO (makdharma): DO NOT CHECK IN THIS HACK!!! + if (!done) { + done = true; + [Cronet setHttp2Enabled:YES]; + NSURL *url = [[[NSFileManager defaultManager] + URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] lastObject]; + NSLog(@"Documents directory: %@", url); + [Cronet start]; + [Cronet startNetLogToFile: @"Documents/cronet_netlog.json" logBytes:YES]; + } cronet_engine *cronetEngine = [Cronet getGlobalEngine]; cronet_init_client_secure_fullstack(f, new_client_args, cronetEngine); @@ -236,7 +245,7 @@ static char *roots_filename; } - (void)testBinaryMetadata { - [self testIndividualCase:"binary_metadata"]; + //[self testIndividualCase:"binary_metadata"]; } - (void)testCallCreds { From 68ca3511264c8af56d70f7576ae84dc296e29989 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 26 Jul 2016 16:48:46 -0700 Subject: [PATCH 10/97] Fix sanity issues --- test/cpp/util/grpc_tool_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 57eeed7234a..0f3086b4428 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -88,8 +88,7 @@ class GrpcToolTest : public ::testing::Test { server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address.str(), - InsecureServerCredentials()); + builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); return server_address.str(); From fcb5271105ec2c439f83ec3cce6456b73824b6be Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 29 Jul 2016 10:44:01 -0700 Subject: [PATCH 11/97] Update node protobuf dependency to 3.0.0 where applicable. Also update example dependency to grpc 1.0.0 --- examples/node/package.json | 4 ++-- package.json | 2 +- src/node/health_check/package.json | 4 ++-- templates/package.json.template | 2 +- templates/src/node/health_check/package.json.template | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/node/package.json b/examples/node/package.json index 2cae031175e..6317838295a 100644 --- a/examples/node/package.json +++ b/examples/node/package.json @@ -3,8 +3,8 @@ "version": "0.1.0", "dependencies": { "async": "^1.5.2", - "google-protobuf": "^3.0.0-alpha.5", - "grpc": "^0.14.0", + "google-protobuf": "^3.0.0", + "grpc": "^1.0.0", "lodash": "^4.6.1", "minimist": "^1.2.0" } diff --git a/package.json b/package.json index 35410642c78..ac936fea84c 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "devDependencies": { "async": "^1.5.0", "google-auth-library": "^0.9.2", - "google-protobuf": "^3.0.0-alpha.5", + "google-protobuf": "^3.0.0", "istanbul": "^0.3.21", "jsdoc": "^3.3.2", "jshint": "^2.5.0", diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 582d5601016..bdacb66825d 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -15,9 +15,9 @@ } ], "dependencies": { - "grpc": "^0.15.0", + "grpc": "^1.0.0-pre1", "lodash": "^3.9.3", - "google-protobuf": "^3.0.0-alpha.5" + "google-protobuf": "^3.0.0" }, "files": [ "LICENSE", diff --git a/templates/package.json.template b/templates/package.json.template index f68f64d0475..e9596d4d4cb 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -37,7 +37,7 @@ "devDependencies": { "async": "^1.5.0", "google-auth-library": "^0.9.2", - "google-protobuf": "^3.0.0-alpha.5", + "google-protobuf": "^3.0.0", "istanbul": "^0.3.21", "jsdoc": "^3.3.2", "jshint": "^2.5.0", diff --git a/templates/src/node/health_check/package.json.template b/templates/src/node/health_check/package.json.template index 96b9748aa74..c2bb232245d 100644 --- a/templates/src/node/health_check/package.json.template +++ b/templates/src/node/health_check/package.json.template @@ -17,9 +17,9 @@ } ], "dependencies": { - "grpc": "^0.15.0", + "grpc": "^${settings.node_version}", "lodash": "^3.9.3", - "google-protobuf": "^3.0.0-alpha.5" + "google-protobuf": "^3.0.0" }, "files": [ "LICENSE", From 3ed66976f6a430d402670f7fa5886d9f164c4d12 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 29 Jul 2016 11:17:52 -0700 Subject: [PATCH 12/97] Update ruby google-protobuf dependency to 3.0 --- grpc.gemspec | 2 +- templates/grpc.gemspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc.gemspec b/grpc.gemspec index 369851b0f24..00113cba924 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -27,7 +27,7 @@ Gem::Specification.new do |s| s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' + s.add_dependency 'google-protobuf', '~> 3.0' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index ce775ffb90b..f95adaf30fa 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -29,7 +29,7 @@ s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' + s.add_dependency 'google-protobuf', '~> 3.0' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' From ff47bc0daf9c6081506e4ec462208cc990c7cde1 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 29 Jul 2016 16:55:35 -0700 Subject: [PATCH 13/97] rewrite the core logic to work with end to end tests. --- .../cronet/transport/cronet_transport.c | 1112 +++++++---------- 1 file changed, 440 insertions(+), 672 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 0a079927ea8..694d346fc33 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -51,47 +51,7 @@ #define GRPC_HEADER_SIZE_IN_BYTES 5 -// Global flag that gets set with GRPC_TRACE env variable -int grpc_cronet_trace = 1; - -// Cronet transport object -struct grpc_cronet_transport { - grpc_transport base; /* must be first element in this structure */ - cronet_engine *engine; - char *host; -}; - -typedef struct grpc_cronet_transport grpc_cronet_transport; - -enum send_state { - CRONET_SEND_IDLE = 0, - CRONET_SEND_HEADER, - CRONET_WRITE_PENDING, - CRONET_WRITE_COMPLETED, - CRONET_WAIT_FOR_CANCEL, - CRONET_STREAM_CLOSED, -}; - -enum recv_state { - CRONET_RECV_IDLE = 0, - CRONET_RECV_READ_LENGTH, - CRONET_RECV_READ_DATA, - CRONET_RECV_CLOSED, -}; - -static const char *recv_state_name[] = { - "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", - "CRONET_RECV_CLOSED"}; - -// Enum that identifies calling function. -enum e_caller { - PERFORM_STREAM_OP, - ON_READ_COMPLETE, - ON_RESPONSE_HEADERS_RECEIVED, - ON_RESPONSE_TRAILERS_RECEIVED -}; - -enum callback_id { +enum OP_ID { OP_SEND_INITIAL_METADATA = 0, OP_SEND_MESSAGE, OP_SEND_TRAILING_METADATA, @@ -99,91 +59,240 @@ enum callback_id { OP_RECV_INITIAL_METADATA, OP_RECV_TRAILING_METADATA, OP_CANCEL_ERROR, - OP_NUM_CALLBACKS + OP_ON_COMPLETE, + OP_NUM_OPS +}; + +/* Cronet callbacks */ + +static void on_request_headers_sent(cronet_bidirectional_stream *); +static void on_response_headers_received(cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *, + const char *); +static void on_write_completed(cronet_bidirectional_stream *, const char *); +static void on_read_completed(cronet_bidirectional_stream *, char *, int); +static void on_response_trailers_received(cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *); +static void on_succeeded(cronet_bidirectional_stream *); +static void on_failed(cronet_bidirectional_stream *, int); +//static void on_canceled(cronet_bidirectional_stream *); +static cronet_bidirectional_stream_callback cronet_callbacks = { + on_request_headers_sent, + on_response_headers_received, + on_read_completed, + on_write_completed, + on_response_trailers_received, + on_succeeded, + on_failed, + NULL //on_canceled +}; + +// Cronet transport object +struct grpc_cronet_transport { + grpc_transport base; /* must be first element in this structure */ + cronet_engine *engine; + char *host; +}; +typedef struct grpc_cronet_transport grpc_cronet_transport; + +struct read_state { + // vars to store data coming from cronet + char *read_buffer; + bool length_field_received; + int received_bytes; + int remaining_bytes; + int length_field; + char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; + char *payload_field; + + // vars for holding data destined for the application + struct grpc_slice_buffer_stream sbs; + gpr_slice_buffer read_slice_buffer; + + // vars for trailing metadata + grpc_chttp2_incoming_metadata_buffer trailing_metadata; + bool trailing_metadata_valid; + + // vars for initial metadata + grpc_chttp2_incoming_metadata_buffer initial_metadata; +}; + +struct write_state { + char *write_buffer; +}; + +#define MAX_PENDING_OPS 10 +struct op_storage { + grpc_transport_stream_op pending_ops[MAX_PENDING_OPS]; + int wrptr; + int rdptr; + int num_pending_ops; }; struct stream_obj { - // we store received bytes here as they trickle in. - gpr_slice_buffer write_slice_buffer; + grpc_transport_stream_op *curr_op; + grpc_cronet_transport curr_ct; + grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; - gpr_slice slice; - gpr_slice_buffer read_slice_buffer; - struct grpc_slice_buffer_stream sbs; - char *read_buffer; - int remaining_read_bytes; - int total_read_bytes; - char *write_buffer; - size_t write_buffer_size; + // TODO (makdharma) : make a sub structure for tracking state + bool state_op_done[OP_NUM_OPS]; + bool state_callback_received[OP_NUM_OPS]; - // Hold the URL - char *url; - - // One bit per operation - bool op_requested[OP_NUM_CALLBACKS]; - bool op_done[OP_NUM_CALLBACKS]; - // Set to true when server indicates no more data will be sent - bool read_closed; - - // Recv message stuff - grpc_byte_buffer **recv_message; - // Initial metadata stuff - grpc_metadata_batch *recv_initial_metadata; - grpc_chttp2_incoming_metadata_buffer initial_metadata; - // Trailing metadata stuff - grpc_metadata_batch *recv_trailing_metadata; - grpc_chttp2_incoming_metadata_buffer imb; - bool imb_valid; // true if there are any valid entries in imb. - - // This mutex protects receive state machine execution - gpr_mu recv_mu; - - // Callbacks to be called when operations complete - grpc_closure *cb_recv_initial_metadata_ready; - grpc_closure *cb_recv_message_ready; - grpc_closure *on_complete; - - // storage for header - cronet_bidirectional_stream_header *headers; - uint32_t num_headers; - cronet_bidirectional_stream_header_array header_array; - // state tracking - enum recv_state cronet_recv_state; - enum send_state cronet_send_state; + // Read state + struct read_state rs; + // Write state + struct write_state ws; + // OP storage + struct op_storage storage; }; - typedef struct stream_obj stream_obj; -static void next_send_step(stream_obj *s); -static void next_recv_step(stream_obj *s, enum e_caller caller); +/* Globals */ +cronet_bidirectional_stream_header_array header_array; -static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) {} +// +static void execute_curr_stream_op(stream_obj *s); -static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, - grpc_transport *gt, grpc_stream *gs, - grpc_pollset_set *pollset_set) {} +/************************************************************* + Op Storage +*/ -// Client creates a bunch of operations and invokes "call_start_batch" -// call_start_batch creates a stream_op structure. this structure has info -// needed for executing all the ops. It has on_complete callback that needs -// to be called when all ops are executed. This function keeps track of all -// outstanding operations. It returns true if all operations that were part of -// the stream_op have been completed. -static bool is_op_complete(stream_obj *s) { - int i; - // Check if any requested op is pending - for (i = 0; i < OP_NUM_CALLBACKS; i++) { - if (s->op_requested[i] && !s->op_done[i]) { - gpr_log(GPR_DEBUG, "is_op_complete is FALSE because of %d", i); - return false; +static void add_pending_op(struct op_storage *storage, grpc_transport_stream_op *op) { + GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); + storage->num_pending_ops++; + gpr_log(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", + storage->wrptr, storage->num_pending_ops); + memcpy(&storage->pending_ops[storage->wrptr], op, sizeof(grpc_transport_stream_op)); + storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; +} + +static grpc_transport_stream_op *pop_pending_op(struct op_storage *storage) { + if (storage->num_pending_ops == 0) return NULL; + grpc_transport_stream_op *result = &storage->pending_ops[storage->rdptr]; + storage->rdptr = (storage->rdptr + 1) % MAX_PENDING_OPS; + storage->num_pending_ops--; + gpr_log(GPR_DEBUG, "popping op @rdptr=%d. %d more left in queue", + storage->rdptr, storage->num_pending_ops); + return result; +} + +/************************************************************* +Cronet Callback Ipmlementation +*/ +static void on_failed(cronet_bidirectional_stream *stream, int net_error) { + gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); +} + +static void on_succeeded(cronet_bidirectional_stream *stream) { + gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); +} + + +static void on_request_headers_sent(cronet_bidirectional_stream *stream) { + gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); + stream_obj *s = (stream_obj *)stream->annotation; + s->state_op_done[OP_SEND_INITIAL_METADATA] = true; + s->state_callback_received[OP_SEND_INITIAL_METADATA] = true; + execute_curr_stream_op(s); +} + +static void on_response_headers_received( + cronet_bidirectional_stream *stream, + const cronet_bidirectional_stream_header_array *headers, + const char *negotiated_protocol) { + gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + headers, negotiated_protocol); + + stream_obj *s = (stream_obj *)stream->annotation; + memset(&s->rs.initial_metadata, 0, sizeof(s->rs.initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->rs.initial_metadata); + unsigned int i = 0; + for (i = 0; i < headers->count; i++) { + grpc_chttp2_incoming_metadata_buffer_add( + &s->rs.initial_metadata, + grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(headers->headers[i].key), + grpc_mdstr_from_string(headers->headers[i].value))); + } + s->state_callback_received[OP_RECV_INITIAL_METADATA] = true; + execute_curr_stream_op(s); +} + +static void on_write_completed(cronet_bidirectional_stream *stream, + const char *data) { + gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); + stream_obj *s = (stream_obj *)stream->annotation; + if (s->ws.write_buffer) { + gpr_free(s->ws.write_buffer); + s->ws.write_buffer = NULL; + } + s->state_callback_received[OP_SEND_MESSAGE] = true; + execute_curr_stream_op(s); +} + +static void on_read_completed(cronet_bidirectional_stream *stream, char *data, + int count) { + stream_obj *s = (stream_obj *)stream->annotation; + gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + if (count > 0) { + s->rs.received_bytes += count; + s->rs.remaining_bytes -= count; + if (s->rs.remaining_bytes > 0) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer + s->rs.received_bytes, s->rs.remaining_bytes); + } else { + execute_curr_stream_op(s); } + s->state_callback_received[OP_RECV_MESSAGE] = true; } - // Clear the requested/done bits and return true - for (i = 0; i < OP_NUM_CALLBACKS; i++) { - s->op_requested[i] = s->op_done[i] = false; +} + +static void on_response_trailers_received( + cronet_bidirectional_stream *stream, + const cronet_bidirectional_stream_header_array *trailers) { + gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + trailers); + stream_obj *s = (stream_obj *)stream->annotation; + memset(&s->rs.trailing_metadata, 0, sizeof(s->rs.trailing_metadata)); + s->rs.trailing_metadata_valid = false; + grpc_chttp2_incoming_metadata_buffer_init(&s->rs.trailing_metadata); + unsigned int i = 0; + for (i = 0; i < trailers->count; i++) { + gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + trailers->headers[i].value); + grpc_chttp2_incoming_metadata_buffer_add( + &s->rs.trailing_metadata, grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(trailers->headers[i].key), + grpc_mdstr_from_string(trailers->headers[i].value))); + s->rs.trailing_metadata_valid = true; } - return true; + s->state_callback_received[OP_RECV_TRAILING_METADATA] = true; + execute_curr_stream_op(s); +} + +/************************************************************* +Utility functions. Can be in their own file +*/ +// This function takes the data from s->write_slice_buffer and assembles into +// a contiguous byte stream with 5 byte gRPC header prepended. +static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, + char **pp_write_buffer, int *p_write_buffer_size) { + gpr_slice slice = gpr_slice_buffer_take_first(write_slice_buffer); + size_t length = GPR_SLICE_LENGTH(slice); + // TODO (makdharma): FREE THIS!! HACK! + *p_write_buffer_size = (int)length + GRPC_HEADER_SIZE_IN_BYTES; + char *write_buffer = gpr_malloc(length + GRPC_HEADER_SIZE_IN_BYTES); + *pp_write_buffer = write_buffer; + uint8_t *p = (uint8_t *)write_buffer; + // Append 5 byte header + *p++ = 0; + *p++ = (uint8_t)(length >> 24); + *p++ = (uint8_t)(length >> 16); + *p++ = (uint8_t)(length >> 8); + *p++ = (uint8_t)(length); + // append actual data + memcpy(p, GPR_SLICE_START_PTR(slice), length); } static void enqueue_callback(grpc_closure *callback) { @@ -193,86 +302,54 @@ static void enqueue_callback(grpc_closure *callback) { grpc_exec_ctx_finish(&exec_ctx); } -static void on_canceled(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled(%p)", stream); +static void convert_metadata_to_cronet_headers( + grpc_linked_mdelem *head, + const char *host, + char **pp_url, + cronet_bidirectional_stream_header **pp_headers, + size_t *p_num_headers) { + grpc_linked_mdelem *curr = head; + // Walk the linked list and get number of header fields + uint32_t num_headers_available = 0; + while (curr != NULL) { + curr = curr->next; + num_headers_available++; } - stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_CANCEL_ERROR] = true; + // Allocate enough memory. TODO (makdharma): FREE MEMORY! HACK HACK + cronet_bidirectional_stream_header *headers = + (cronet_bidirectional_stream_header *)gpr_malloc( + sizeof(cronet_bidirectional_stream_header) * num_headers_available); + *pp_headers = headers; - // Terminate any read callback - if (s->cb_recv_message_ready) { - enqueue_callback(s->cb_recv_message_ready); - s->cb_recv_message_ready = 0; - s->op_done[OP_RECV_MESSAGE] = true; - } - // Don't wait to get any trailing metadata - s->op_done[OP_RECV_TRAILING_METADATA] = true; - - next_send_step(s); -} - -static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); - } -} - -static void on_succeeded(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); - } -} - -static void on_response_trailers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *trailers) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, - trailers); - } - stream_obj *s = (stream_obj *)stream->annotation; - - memset(&s->imb, 0, sizeof(s->imb)); - s->imb_valid = false; - grpc_chttp2_incoming_metadata_buffer_init(&s->imb); - unsigned int i = 0; - for (i = 0; i < trailers->count; i++) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, - trailers->headers[i].value); + // Walk the linked list again, this time copying the header fields. + // s->num_headers + // can be less than num_headers_available, as some headers are not used for + // cronet + curr = head; + int num_headers = 0; + while (num_headers < num_headers_available) { + grpc_mdelem *mdelem = curr->md; + curr = curr->next; + const char *key = grpc_mdstr_as_c_string(mdelem->key); + const char *value = grpc_mdstr_as_c_string(mdelem->value); + if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || + strcmp(key, ":authority") == 0) { + // Cronet populates these fields on its own. + continue; + } + if (strcmp(key, ":path") == 0) { + // Create URL by appending :path value to the hostname + gpr_asprintf(pp_url, "https://%s%s", host, value); + continue; + } + headers[num_headers].key = key; + headers[num_headers].value = value; + num_headers++; + if (curr == NULL) { + break; } - - grpc_chttp2_incoming_metadata_buffer_add( - &s->imb, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); - s->imb_valid = true; } - s->op_done[OP_RECV_TRAILING_METADATA] = true; - next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); -} - -static void on_write_completed(cronet_bidirectional_stream *stream, - const char *data) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); - } - stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_SEND_MESSAGE] = true; - next_send_step(s); -} - -static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); - uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - if (s->total_read_bytes > 0) { - // Only copy if there is non-zero number of bytes - memcpy(dst_p, recv_data, (size_t)s->total_read_bytes); - gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); - } - grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); - *s->recv_message = (grpc_byte_buffer *)&s->sbs; + *p_num_headers = num_headers; } static int parse_grpc_header(const uint8_t *data) { @@ -285,523 +362,214 @@ static int parse_grpc_header(const uint8_t *data) { return length; } -static void on_read_completed(cronet_bidirectional_stream *stream, char *data, - int count) { - stream_obj *s = (stream_obj *)stream->annotation; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); +/* +Op Execution +*/ + +static bool op_can_be_run(stream_obj *s, enum OP_ID op_id) { + if (op_id == OP_SEND_INITIAL_METADATA) { + // already executed + if (s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; } - if (count > 0) { - GPR_ASSERT(s->recv_message); - s->remaining_read_bytes -= count; - next_recv_step(s, ON_READ_COMPLETE); - } else { - gpr_log(GPR_DEBUG, "read_closed = true"); - s->read_closed = true; - next_recv_step(s, ON_READ_COMPLETE); + if (op_id == OP_RECV_INITIAL_METADATA) { + // already executed + if (s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + // we haven't sent headers yet. + if (!s->state_callback_received[OP_SEND_INITIAL_METADATA]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; } + if (op_id == OP_SEND_MESSAGE) { + // already executed + if (s->state_op_done[OP_SEND_MESSAGE]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; + } + if (op_id == OP_RECV_MESSAGE) { + // already executed + if (s->state_op_done[OP_RECV_MESSAGE]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; + } + if (op_id == OP_RECV_TRAILING_METADATA) { + // already executed + if (s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + // we haven't received trailers yet. + if (!s->state_callback_received[OP_RECV_TRAILING_METADATA]) return false; + } + if (op_id == OP_SEND_TRAILING_METADATA) { + // already executed + if (s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + // we haven't sent message yet + if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; + } + + if (op_id == OP_ON_COMPLETE) { + // already executed + if (s->state_op_done[OP_ON_COMPLETE]) return false; + // Check if every op that was asked for is done. + if (s->curr_op->send_initial_metadata && !s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; + if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; + if (s->curr_op->send_trailing_metadata && !s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + if (s->curr_op->recv_initial_metadata && !s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + if (s->curr_op->recv_message && !s->state_op_done[OP_RECV_MESSAGE]) return false; + if (s->curr_op->recv_trailing_metadata && !s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + } + return true; } -static void on_response_headers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *headers, - const char *negotiated_protocol) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, - headers, negotiated_protocol); - } - stream_obj *s = (stream_obj *)stream->annotation; - - memset(&s->initial_metadata, 0, sizeof(s->initial_metadata)); - grpc_chttp2_incoming_metadata_buffer_init(&s->initial_metadata); - unsigned int i = 0; - for (i = 0; i < headers->count; i++) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "header key=%s, value=%s", headers->headers[i].key, - headers->headers[i].value); - } - grpc_chttp2_incoming_metadata_buffer_add( - &s->initial_metadata, - grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(headers->headers[i].key), - grpc_mdstr_from_string(headers->headers[i].value))); - } - - grpc_chttp2_incoming_metadata_buffer_publish(&s->initial_metadata, - s->recv_initial_metadata); - enqueue_callback(s->cb_recv_initial_metadata_ready); - s->op_done[OP_RECV_INITIAL_METADATA] = true; - next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); -} - -static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); - } - stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_SEND_INITIAL_METADATA] = true; - next_send_step(s); -} - -// Callback function pointers (invoked by cronet in response to events) -static cronet_bidirectional_stream_callback callbacks = { - on_request_headers_sent, - on_response_headers_received, - on_read_completed, - on_write_completed, - on_response_trailers_received, - on_succeeded, - on_failed, - on_canceled}; - -static void invoke_closing_callback(stream_obj *s) { - if (!is_op_complete(s)) return; - - if (s->imb_valid) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); - } - enqueue_callback(s->on_complete); -} - -static void set_recv_state(stream_obj *s, enum recv_state state) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]); - } - s->cronet_recv_state = state; -} - -// This is invoked from perform_stream_op, and all on_xxxx callbacks. -static void next_recv_step(stream_obj *s, enum e_caller caller) { - // gpr_log(GPR_DEBUG, "locking mutex %p", &s->recv_mu); - gpr_mu_lock(&s->recv_mu); - switch (s->cronet_recv_state) { - case CRONET_RECV_IDLE: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE, caller=%d", - caller); - } - if (caller == PERFORM_STREAM_OP || - caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->op_done[OP_RECV_TRAILING_METADATA]) { - set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->op_done[OP_RECV_INITIAL_METADATA] == true && - s->op_requested[OP_RECV_MESSAGE]) { - set_recv_state(s, CRONET_RECV_READ_LENGTH); - s->total_read_bytes = s->remaining_read_bytes = - GRPC_HEADER_SIZE_IN_BYTES; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", - s->cbs, s->read_buffer, s->remaining_read_bytes); - } - cronet_bidirectional_stream_read(s->cbs, s->read_buffer, - s->remaining_read_bytes); - } - } else if (caller == ON_RESPONSE_TRAILERS_RECEIVED) { - // We get here when we receive trailers directly, i.e. without - // going through a data read operation. - set_recv_state(s, CRONET_RECV_CLOSED); - } - break; - case CRONET_RECV_READ_LENGTH: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH"); - } - if (caller == ON_READ_COMPLETE) { - if (s->read_closed) { - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - set_recv_state(s, CRONET_RECV_CLOSED); - } else { - GPR_ASSERT(s->remaining_read_bytes == 0); - set_recv_state(s, CRONET_RECV_READ_DATA); - s->total_read_bytes = s->remaining_read_bytes = - parse_grpc_header((const uint8_t *)s->read_buffer); - s->read_buffer = - gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", - s->cbs, s->read_buffer, s->remaining_read_bytes); - } - if (s->remaining_read_bytes > 0) { - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, - s->remaining_read_bytes); - } else { - // Calling the closing callback directly since this is a 0 byte read - // for an empty message. - process_recv_message(s, NULL); - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - set_recv_state(s, CRONET_RECV_CLOSED); - } - } - } - break; - case CRONET_RECV_READ_DATA: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); - } - if (caller == ON_READ_COMPLETE) { - if (s->remaining_read_bytes > 0) { - int offset = s->total_read_bytes - s->remaining_read_bytes; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read( - s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); - } else { - gpr_slice_buffer_init(&s->read_slice_buffer); - uint8_t *p = (uint8_t *)s->read_buffer; - process_recv_message(s, p); - set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - } - } - break; - case CRONET_RECV_CLOSED: - break; - default: - GPR_ASSERT(0); // Should not reach here - break; - } - invoke_closing_callback(s); - gpr_mu_unlock(&s->recv_mu); - // gpr_log(GPR_DEBUG, "unlocking mutex %p", &s->recv_mu); -} - -// This function takes the data from s->write_slice_buffer and assembles into -// a contiguous byte stream with 5 byte gRPC header prepended. -static void create_grpc_frame(stream_obj *s) { - gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer); - uint8_t *raw_data = GPR_SLICE_START_PTR(slice); - size_t length = GPR_SLICE_LENGTH(slice); - s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; - s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); - uint8_t *p = (uint8_t *)s->write_buffer; - // Append 5 byte header - *p++ = 0; - *p++ = (uint8_t)(length >> 24); - *p++ = (uint8_t)(length >> 16); - *p++ = (uint8_t)(length >> 8); - *p++ = (uint8_t)(length); - // append actual data - memcpy(p, raw_data, length); -} -// Return false if there is no data to write -static bool do_write(stream_obj *s) { - gpr_slice_buffer *sb = &s->write_slice_buffer; - GPR_ASSERT(sb->count <= 1); - if (sb->count > 0) { - create_grpc_frame(s); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write(%p,%p,%d,%d)", - s->cbs, s->write_buffer, (int)s->write_buffer_size, false); - } - cronet_bidirectional_stream_write(s->cbs, s->write_buffer, - (int)s->write_buffer_size, false); - return true; - } else { - return false; - } -} - -static void init_cronet_stream(stream_obj *s, grpc_transport *gt) { - GPR_ASSERT(s->cbs == NULL); - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - if (grpc_cronet_trace) { +static void execute_curr_stream_op(stream_obj *s) { + if (s->curr_op->send_initial_metadata && op_can_be_run(s, OP_SEND_INITIAL_METADATA)) { + // This OP is the beginning. Reset various states + memset(&s->rs, 0, sizeof(s->rs)); + memset(&s->ws, 0, sizeof(s->ws)); + memset(s->state_op_done, 0, sizeof(s->state_op_done)); + memset(s->state_callback_received, 0, sizeof(s->state_callback_received)); + // Start new cronet stream + GPR_ASSERT(s->cbs == NULL); gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - - for (int i = 0; i < OP_NUM_CALLBACKS; i++) { - s->op_requested[i] = s->op_done[i] = false; - } - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; -} - -static bool do_close_connection(stream_obj *s) { - s->op_done[OP_SEND_TRAILING_METADATA] = true; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write length 0"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - return true; - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); - return true; - } - return false; -} - -// -static void next_send_step(stream_obj *s) { - gpr_log(GPR_DEBUG, "next_send_step cronet_send_state=%d", - s->cronet_send_state); - switch (s->cronet_send_state) { - case CRONET_SEND_IDLE: - GPR_ASSERT( - s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_SEND_HEADER; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); - } - cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", - &s->header_array, false); - // we no longer need the memory that was allocated earlier. - gpr_free(s->header_array.headers); - break; - case CRONET_SEND_HEADER: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } else { - s->cronet_send_state = CRONET_WRITE_PENDING; - } - break; - case CRONET_WRITE_PENDING: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } else { - s->cronet_send_state = CRONET_WRITE_COMPLETED; - } - break; - case CRONET_WRITE_COMPLETED: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } - break; - case CRONET_STREAM_CLOSED: - s->cronet_send_state = CRONET_SEND_IDLE; - break; - case CRONET_WAIT_FOR_CANCEL: - invoke_closing_callback(s); - s->cronet_send_state = CRONET_SEND_IDLE; - break; - default: - GPR_ASSERT(0); - break; - } -} - -static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, - const char *host, - stream_obj *s) { - grpc_linked_mdelem *curr = head; - // Walk the linked list and get number of header fields - uint32_t num_headers_available = 0; - while (curr != NULL) { - curr = curr->next; - num_headers_available++; - } - // Allocate enough memory - s->headers = (cronet_bidirectional_stream_header *)gpr_malloc( - sizeof(cronet_bidirectional_stream_header) * num_headers_available); - - // Walk the linked list again, this time copying the header fields. - // s->num_headers - // can be less than num_headers_available, as some headers are not used for - // cronet - curr = head; - s->num_headers = 0; - while (s->num_headers < num_headers_available) { - grpc_mdelem *mdelem = curr->md; - curr = curr->next; - const char *key = grpc_mdstr_as_c_string(mdelem->key); - const char *value = grpc_mdstr_as_c_string(mdelem->value); - if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || - strcmp(key, ":authority") == 0) { - // Cronet populates these fields on its own. - continue; - } - if (strcmp(key, ":path") == 0) { - // Create URL by appending :path value to the hostname - gpr_asprintf(&s->url, "https://%s%s", host, value); - if (grpc_cronet_trace) { - // gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); - } - continue; - } - s->headers[s->num_headers].key = key; - s->headers[s->num_headers].value = value; - s->num_headers++; - if (curr == NULL) { - break; - } - } -} - -static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_transport_stream_op *op) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - stream_obj *s = (stream_obj *)gs; - // Initialize a cronet bidirectional stream if it doesn't exist. - if (s->cbs == NULL) { - init_cronet_stream(s, gt); - } - - s->on_complete = op->on_complete; - - if (op->recv_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_trailing_metadata: on_complete=%p", - op->on_complete); - } - s->recv_trailing_metadata = op->recv_trailing_metadata; - s->op_requested[OP_RECV_TRAILING_METADATA] = true; - } - if (op->recv_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); - } - s->recv_message = (grpc_byte_buffer **)op->recv_message; - s->cb_recv_message_ready = op->recv_message_ready; - s->op_requested[OP_RECV_MESSAGE] = true; - } - if (op->recv_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_initial_metadata on_complete=%p, " - "on_ready=%p", - op->on_complete, op->recv_initial_metadata_ready); - } - s->recv_initial_metadata = op->recv_initial_metadata; - s->cb_recv_initial_metadata_ready = op->recv_initial_metadata_ready; - s->op_requested[OP_RECV_INITIAL_METADATA] = true; - } - if (op->send_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_initial_metadata: on_complete=%p", - op->on_complete); - } - s->num_headers = 0; - convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, - ct->host, s); - s->header_array.count = s->num_headers; - s->header_array.capacity = s->num_headers; - s->header_array.headers = s->headers; - s->op_requested[OP_SEND_INITIAL_METADATA] = true; - } - if (op->send_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", - op->on_complete); - } - grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, - op->send_message->length, NULL); + s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); + char *url; + convert_metadata_to_cronet_headers(s->curr_op->send_initial_metadata->list.head, + s->curr_ct.host, &url, &header_array.headers, &header_array.count); + header_array.capacity = header_array.count; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start"); + cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); + s->state_op_done[OP_SEND_INITIAL_METADATA] = true; + } else if (s->curr_op->recv_initial_metadata && + op_can_be_run(s, OP_RECV_INITIAL_METADATA)) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->rs.initial_metadata, + s->curr_op->recv_initial_metadata); + enqueue_callback(s->curr_op->recv_initial_metadata_ready); + s->state_op_done[OP_RECV_INITIAL_METADATA] = true; + // We are ready to execute send_message. + execute_curr_stream_op(s); + } else if (s->curr_op->send_message && op_can_be_run(s, OP_SEND_MESSAGE)) { + // TODO (makdharma): Make into a standalone function + gpr_slice_buffer write_slice_buffer; + gpr_slice slice; + gpr_slice_buffer_init(&write_slice_buffer); + grpc_byte_stream_next(NULL, s->curr_op->send_message, &slice, + s->curr_op->send_message->length, NULL); // Check that compression flag is not ON. We don't support compression yet. // TODO (makdharma): add compression support - GPR_ASSERT(op->send_message->flags == 0); - gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - s->op_requested[OP_SEND_MESSAGE] = true; - } - if (op->send_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_trailing_metadata: on_complete=%p", - op->on_complete); + GPR_ASSERT(s->curr_op->send_message->flags == 0); + gpr_slice_buffer_add(&write_slice_buffer, slice); + GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet + if (write_slice_buffer.count > 0) { + int write_buffer_size; + create_grpc_frame(&write_slice_buffer, &s->ws.write_buffer, &write_buffer_size); + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", s->ws.write_buffer); + cronet_bidirectional_stream_write(s->cbs, s->ws.write_buffer, + write_buffer_size, false); // TODO: What if this is not the last write? } - s->op_requested[OP_SEND_TRAILING_METADATA] = true; - } - if (op->cancel_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - cancel_error: on_complete=%p", - op->on_complete); + s->state_op_done[OP_SEND_MESSAGE] = true; + } else if (s->curr_op->recv_message && op_can_be_run(s, OP_RECV_MESSAGE)) { + if (s->rs.length_field_received == false) { + if (s->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && s->rs.remaining_bytes == 0) { + // Start a read operation for data + s->rs.length_field_received = true; + s->rs.length_field = s->rs.remaining_bytes = + parse_grpc_header((const uint8_t *)s->rs.read_buffer); + GPR_ASSERT(s->rs.length_field > 0); // Empty message? + gpr_log(GPR_DEBUG, "length field = %d", s->rs.length_field); + s->rs.read_buffer = gpr_malloc(s->rs.length_field); + GPR_ASSERT(s->rs.read_buffer); + s->rs.remaining_bytes = s->rs.length_field; + s->rs.received_bytes = 0; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, + s->rs.remaining_bytes); + } else if (s->rs.remaining_bytes == 0) { + // Start a read operation for first 5 bytes (GRPC header) + s->rs.read_buffer = s->rs.grpc_header_bytes; + s->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + s->rs.received_bytes = 0; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, + s->rs.remaining_bytes); + } + } else if (s->rs.remaining_bytes == 0) { + gpr_log(GPR_DEBUG, "read operation complete"); + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->rs.length_field); + uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); + memcpy(dst_p, s->rs.read_buffer, (size_t)s->rs.length_field); + gpr_slice_buffer_init(&s->rs.read_slice_buffer); + gpr_slice_buffer_add(&s->rs.read_slice_buffer, read_data_slice); + grpc_slice_buffer_stream_init(&s->rs.sbs, &s->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)s->curr_op->recv_message) = (grpc_byte_buffer *)&s->rs.sbs; + enqueue_callback(s->curr_op->recv_message_ready); + s->state_op_done[OP_RECV_MESSAGE] = true; + execute_curr_stream_op(s); } - s->op_requested[OP_CANCEL_ERROR] = true; + } else if (s->curr_op->recv_trailing_metadata && + op_can_be_run(s, OP_RECV_TRAILING_METADATA)) { + if (s->rs.trailing_metadata_valid) { + grpc_chttp2_incoming_metadata_buffer_publish( + &s->rs.trailing_metadata, s->curr_op->recv_trailing_metadata); + s->rs.trailing_metadata_valid = false; + } + s->state_op_done[OP_RECV_TRAILING_METADATA] = true; + execute_curr_stream_op(s); + } else if (s->curr_op->send_trailing_metadata && + op_can_be_run(s, OP_SEND_TRAILING_METADATA)) { + + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + cronet_bidirectional_stream_write(s->cbs, "", 0, true); + s->state_op_done[OP_SEND_TRAILING_METADATA] = true; + } else if (op_can_be_run(s, OP_ON_COMPLETE)) { + // All ops are complete. Call the on_complete callback + enqueue_callback(s->curr_op->on_complete); + s->state_op_done[OP_ON_COMPLETE] = true; + cronet_bidirectional_stream_destroy(s->cbs); + s->cbs = NULL; } - next_send_step(s); - next_recv_step(s, PERFORM_STREAM_OP); } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, const void *server_data) { stream_obj *s = (stream_obj *)gs; - memset(s, 0, sizeof(stream_obj)); - s->cbs = NULL; - gpr_mu_init(&s->recv_mu); - s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - gpr_slice_buffer_init(&s->write_slice_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_transport - init_stream"); - } + memset(&s->storage, 0, sizeof(s->storage)); + s->curr_op = NULL; return 0; } +static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) {} + +static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, + grpc_transport *gt, grpc_stream *gs, + grpc_pollset_set *pollset_set) {} + +static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_transport_stream_op *op) { + gpr_log(GPR_DEBUG, "perform_stream_op"); + stream_obj *s = (stream_obj *)gs; + memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); + add_pending_op(&s->storage, op); + if (s->curr_op == NULL) { + s->curr_op = pop_pending_op(&s->storage); + } + s->curr_gs = gs; + execute_curr_stream_op(s); +} + static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, void *and_free_memory) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy stream"); - } - stream_obj *s = (stream_obj *)gs; - s->cbs = NULL; - gpr_free(s->read_buffer); - gpr_free(s->write_buffer); - gpr_free(s->url); - gpr_log(GPR_DEBUG, "destroying %p", &s->recv_mu); - gpr_mu_destroy(&s->recv_mu); - if (and_free_memory) { - gpr_free(and_free_memory); - } } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - gpr_free(ct->host); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy transport"); - } } static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Unimplemented method"); - } return NULL; } static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_transport_op *op) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Unimplemented method"); - } } const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), From 4d8283f93bed3b9d35362cad2c4fb38474b547d6 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:11:49 -0700 Subject: [PATCH 14/97] Add thrift module --- .gitmodules | 3 +++ third_party/thrift | 1 + 2 files changed, 4 insertions(+) create mode 160000 third_party/thrift diff --git a/.gitmodules b/.gitmodules index ce647f3c455..3bfd3c9ce10 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,3 +17,6 @@ [submodule "third_party/nanopb"] path = third_party/nanopb url = https://github.com/nanopb/nanopb.git +[submodule "third_party/thrift"] + path = third_party/thrift + url = https://github.com/apache/thrift.git diff --git a/third_party/thrift b/third_party/thrift new file mode 160000 index 00000000000..bcad91771b7 --- /dev/null +++ b/third_party/thrift @@ -0,0 +1 @@ +Subproject commit bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c From bc618eed71f1c2a2b69351a6aa60e8bb460773b9 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:35:51 -0700 Subject: [PATCH 15/97] thrift serializer --- Makefile | 3 + build.yaml | 9 + .../grpc++/impl/codegen/thrift_serializer.h | 148 +++++++++++++++ .../impl/codegen/thrift_serializer_inl.h | 169 ++++++++++++++++++ include/grpc++/impl/codegen/thrift_utils.h | 90 ++++++++++ tools/run_tests/sources_and_headers.json | 22 ++- .../grpc++_test_util/grpc++_test_util.vcxproj | 3 + .../grpc++_test_util.vcxproj.filters | 9 + 8 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 include/grpc++/impl/codegen/thrift_serializer.h create mode 100644 include/grpc++/impl/codegen/thrift_serializer_inl.h create mode 100644 include/grpc++/impl/codegen/thrift_utils.h diff --git a/Makefile b/Makefile index 8b6114bd7f9..986c25780f2 100644 --- a/Makefile +++ b/Makefile @@ -3923,6 +3923,9 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/time.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ + include/grpc++/impl/codegen/thrift_serializer.h \ + include/grpc++/impl/codegen/thrift_serializer_inl.h \ + include/grpc++/impl/codegen/thrift_utils.h \ LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC)))) diff --git a/build.yaml b/build.yaml index 37c95ccdc0a..35af7a6c7a5 100644 --- a/build.yaml +++ b/build.yaml @@ -780,6 +780,14 @@ filegroups: - src/cpp/ext/reflection.pb.cc uses: - grpc++_codegen_proto +- name: thrift_util + language: c++ + public_headers: + - include/grpc++/impl/codegen/thrift_serializer.h + - include/grpc++/impl/codegen/thrift_serializer_inl.h + - include/grpc++/impl/codegen/thrift_utils.h + uses: + - grpc++_codegen_base libs: - name: gpr build: all @@ -1021,6 +1029,7 @@ libs: - grpc++_codegen_base_src - grpc++_codegen_proto - grpc++_config_proto + - thrift_util - name: grpc++_unsecure build: all language: c++ diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h new file mode 100644 index 00000000000..315d76cf2a5 --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -0,0 +1,148 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + +#include +#include + +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TBinaryProtocolT; +using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TNetworkBigEndian; +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::transport::TBufferBase; +using apache::thrift::transport::TTransport; +using std::shared_ptr; + + +template +class ThriftSerializer { +public: + ThriftSerializer() + : prepared_ (false) + , lastDeserialized_ (false) + , serializeVersion_ (false) {} + + /** + * Serialize the passed type into the internal buffer + * and returns a pointer to internal buffer and its size + * + */ + template + void serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen); + + /** + * Serialize the passed type into the byte buffer + */ + template + void serialize(const T& fields, grpc_byte_buffer** bp); + + /** + * Deserialize the passed char array into the passed type, returns the number + * of bytes that have been consumed from the passed string. + */ + template + uint32_t deserialize(const uint8_t* serializedBuffer, size_t length, + T* fields); + + /** + * Deserialize the passed byte buffer to passed type, returns the number + * of bytes consumed from byte buffer + */ + template + uint32_t deserialize(grpc_byte_buffer* buffer, T* msg); + + void setSerializeVersion(bool value); + + virtual ~ThriftSerializer() {} + + + /** + * Set the container size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setContainerSizeLimit(int32_t container_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setContainerSizeLimit(container_limit); + } + + /** + * Set the string size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setStringSizeLimit(int32_t string_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setStringSizeLimit(string_limit); + } + + + private: + void prepare(); + + private: + typedef P Protocol; + bool prepared_; + bool lastDeserialized_; + boost::shared_ptr buffer_; + shared_ptr protocol_; + bool serializeVersion_; +}; // ThriftSerializer + +template +struct ThriftSerializerBinary : public ThriftSerializer > {}; + + +template +struct ThriftSerializerCompact : public ThriftSerializer >{ }; + +}}} // namespace apache::thrift::util + +#include + +#endif diff --git a/include/grpc++/impl/codegen/thrift_serializer_inl.h b/include/grpc++/impl/codegen/thrift_serializer_inl.h new file mode 100644 index 00000000000..866ecf6312d --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer_inl.h @@ -0,0 +1,169 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + +#include +#include +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TMessageType; + +template +template +void ThriftSerializer::serialize(const T& fields, + const uint8_t** serializedBuffer, size_t* serializedLen) { + + // prepare or reset buffer + if (!prepared_ || lastDeserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + lastDeserialized_ = false; + + // if required serialize protocol version + if (serializeVersion_) { + protocol_->writeMessageBegin("", TMessageType(0), 0); + } + + // serilaize fields into buffer + fields.write(protocol_.get()); + + // write the end of message + if (serializeVersion_) { + protocol_->writeMessageEnd(); + } + + // assign buffer to string + uint8_t* byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; +} + +template +template +void ThriftSerializer::serialize(const T& fields, grpc_byte_buffer** bp) { + + const uint8_t* byteBuffer; + size_t byteBufferSize; + serialize(fields, &byteBuffer, &byteBufferSize); + + gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + + *bp = grpc_raw_byte_buffer_create(&slice, 1); + + gpr_slice_unref(slice); +} + +template +template +uint32_t ThriftSerializer::deserialize(const uint8_t* serializedBuffer, + size_t length, T* fields) { + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + lastDeserialized_ = true; + + //reset buffer transport + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // read the protocol version if necessary + if (serializeVersion_) { + std::string name = ""; + TMessageType mt = (TMessageType) 0; + int32_t seq_id = 0; + protocol_->readMessageBegin(name, mt, seq_id); + } + + // deserialize buffer into fields + uint32_t len = fields->read(protocol_.get()); + + // read the end of message + if (serializeVersion_) { + protocol_->readMessageEnd(); + } + + return len; +} + +template +template +uint32_t ThriftSerializer::deserialize(grpc_byte_buffer* bp, T* fields) { + grpc_byte_buffer_reader reader; + grpc_byte_buffer_reader_init(&reader, bp); + + gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); + + uint32_t len = deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), fields); + + gpr_slice_unref(slice); + + grpc_byte_buffer_reader_destroy(&reader); + + return len; +} + +template +void ThriftSerializer::setSerializeVersion(bool value) { + serializeVersion_ = value; +} + +template +void +ThriftSerializer::prepare() +{ + + buffer_.reset(new TMemoryBuffer()); + + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + + prepared_ = true; +} + +}}} // namespace apache::thrift::util + +#endif diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h new file mode 100644 index 00000000000..629441149fd --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -0,0 +1,90 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H +#define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace grpc { + +using apache::thrift::util::ThriftSerializerCompact; + +template +class SerializationTraits::value>::type> { + public: + + static Status Serialize(const T& msg, + grpc_byte_buffer** bp, bool* own_buffer) { + + *own_buffer = true; + + ThriftSerializerCompact serializer; + + serializer.serialize(msg, bp); + + return Status(StatusCode::OK, "ok"); + } + + static Status Deserialize(grpc_byte_buffer* buffer, + T* msg, + int max_message_size) { + if (!buffer) { + return Status(StatusCode::INTERNAL, "No payload"); + } + + ThriftSerializerCompact deserializer; + deserializer.deserialize(buffer, msg); + + grpc_byte_buffer_destroy(buffer); + + return Status(StatusCode::OK, "ok"); + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e85dd2c8200..06ac86fc9c4 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4411,7 +4411,8 @@ "grpc++_codegen_base_src", "grpc++_codegen_proto", "grpc++_config_proto", - "grpc_test_util" + "grpc_test_util", + "thrift_util" ], "headers": [ "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", @@ -6784,5 +6785,24 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [ + "grpc++_codegen_base" + ], + "headers": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "language": "c++", + "name": "thrift_util", + "src": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "third_party": false, + "type": "filegroup" } ] diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj index d0fca9ba65a..96ce3b89164 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj @@ -200,6 +200,9 @@ + + + diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters index cc98c604080..2105e672df9 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters @@ -192,6 +192,15 @@ include\grpc++\impl\codegen + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + From dd7a2a35e23042413c232ea9f00a4b7dbbce5648 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 21:57:47 -0700 Subject: [PATCH 16/97] gthrift codegen and Dockerfile --- tools/grift/Dockerfile | 63 + tools/grift/grpc_plugins_generator.patch | 2417 ++++++++++++++++++++++ 2 files changed, 2480 insertions(+) create mode 100644 tools/grift/Dockerfile create mode 100644 tools/grift/grpc_plugins_generator.patch diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile new file mode 100644 index 00000000000..5238010ea92 --- /dev/null +++ b/tools/grift/Dockerfile @@ -0,0 +1,63 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:14.04 + +RUN apt-get update && \ + apt-get install -y \ + git build-essential \ + pkg-config flex \ + bison \ + libkrb5-dev \ + libsasl2-dev \ + libnuma-dev \ + pkg-config \ + libssl-dev \ + autoconf libtool \ + cmake \ + libiberty-dev \ + g++ unzip \ + curl make automake libtool + +# Configure git +RUN git config --global user.name " " && \ + git config --global user.email " " + +RUN git clone https://github.com/grpc/grpc + +RUN cd grpc && git submodule update --init + +RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch + +RUN cd grpc/third_party/protobuf && ./autogen.sh && ./configure && \ + make -j && make check -j && make install && ldconfig + +RUN cd grpc && make -j && make install + +RUN cd grpc/third_party/thrift && ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch new file mode 100644 index 00000000000..2779dfb59eb --- /dev/null +++ b/tools/grift/grpc_plugins_generator.patch @@ -0,0 +1,2417 @@ +From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 15:47:47 -0700 +Subject: [PATCH 1/3] don't build tests + +--- + Makefile.am | 7 ++----- + lib/cpp/Makefile.am | 7 ++----- + 2 files changed, 4 insertions(+), 10 deletions(-) + +diff --git a/Makefile.am b/Makefile.am +index 10fe49a..d49caac 100755 +--- a/Makefile.am ++++ b/Makefile.am +@@ -21,10 +21,6 @@ ACLOCAL_AMFLAGS = -I ./aclocal + + SUBDIRS = compiler/cpp lib + +-if WITH_TESTS +-SUBDIRS += test +-endif +- + if WITH_TUTORIAL + SUBDIRS += tutorial + endif +@@ -117,4 +113,5 @@ EXTRA_DIST = \ + CHANGES \ + NOTICE \ + README.md \ +- Thrift.podspec ++ Thrift.podspec \ ++ test +diff --git a/lib/cpp/Makefile.am b/lib/cpp/Makefile.am +index 6fd15d2..7de1fad 100755 +--- a/lib/cpp/Makefile.am ++++ b/lib/cpp/Makefile.am +@@ -27,10 +27,6 @@ moc__%.cpp: %.h + + SUBDIRS = . + +-if WITH_TESTS +-SUBDIRS += test +-endif +- + pkgconfigdir = $(libdir)/pkgconfig + + lib_LTLIBRARIES = libthrift.la +@@ -277,7 +273,8 @@ EXTRA_DIST = \ + thrift-qt.pc.in \ + thrift-qt5.pc.in \ + src/thrift/qt/CMakeLists.txt \ +- $(WINDOWS_DIST) ++ $(WINDOWS_DIST) \ ++ test + + style-local: + $(CPPSTYLE_CMD) +-- +2.8.0.rc3.226.g39d4020 + + +From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 16:16:40 -0700 +Subject: [PATCH 2/3] grpc cpp plugins generator with example + +--- + compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- + tutorial/cpp/CMakeLists.txt | 53 --- + tutorial/cpp/CppClient.cpp | 134 ++++---- + tutorial/cpp/CppServer.cpp | 226 ++++--------- + tutorial/cpp/Makefile.am | 58 ++-- + tutorial/cpp/test.thrift | 13 + + 6 files changed, 590 insertions(+), 370 deletions(-) + delete mode 100644 tutorial/cpp/CMakeLists.txt + create mode 100644 tutorial/cpp/test.thrift + +diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc +index 6c04899..9c3399b 100644 +--- a/compiler/cpp/src/generate/t_cpp_generator.cc ++++ b/compiler/cpp/src/generate/t_cpp_generator.cc +@@ -162,6 +162,8 @@ public: + bool specialized = false); + void generate_function_helpers(t_service* tservice, t_function* tfunction); + void generate_service_async_skeleton(t_service* tservice); ++ void generate_service_stub_interface(t_service* tservice); ++ void generate_service_stub(t_service* tservice); + + /** + * Serialization constructs +@@ -883,10 +885,10 @@ void t_cpp_generator::generate_struct_declaration(ofstream& out, + bool is_user_struct) { + string extends = ""; + if (is_exception) { +- extends = " : public ::apache::thrift::TException"; ++ extends = " : public apache::thrift::TException"; + } else { +- if (is_user_struct && !gen_templates_) { +- extends = " : public virtual ::apache::thrift::TBase"; ++ if (!gen_templates_) { ++ extends = " : public virtual apache::thrift::TBase"; + } + } + +@@ -1130,9 +1132,15 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + vector::const_iterator m_iter; + const vector& members = tstruct->get_members(); + ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } ++ + // Destructor + if (tstruct->annotations_.find("final") == tstruct->annotations_.end()) { +- force_cpp_out << endl << indent() << tstruct->get_name() << "::~" << tstruct->get_name() ++ force_cpp_out << endl << indent() << method_prefix << ++ tstruct->get_name() << "::~" << tstruct->get_name() + << "() throw() {" << endl; + indent_up(); + +@@ -1145,12 +1153,14 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (is_reference((*m_iter))) { + std::string type = type_name((*m_iter)->get_type()); +- out << endl << indent() << "void " << tstruct->get_name() << "::__set_" ++ out << endl << indent() << "void " << method_prefix ++ << tstruct->get_name() << "::__set_" + << (*m_iter)->get_name() << "(boost::shared_ptr<" + << type_name((*m_iter)->get_type(), false, false) << ">"; + out << " val) {" << endl; + } else { +- out << endl << indent() << "void " << tstruct->get_name() << "::__set_" ++ out << endl << indent() << "void " << method_prefix ++ << tstruct->get_name() << "::__set_" + << (*m_iter)->get_name() << "(" << type_name((*m_iter)->get_type(), false, true); + out << " val) {" << endl; + } +@@ -1177,11 +1187,16 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + * @param tstruct The struct + */ + void t_cpp_generator::generate_struct_reader(ofstream& out, t_struct* tstruct, bool pointers) { ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } ++ + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::read(Protocol_* iprot) {" << endl; ++ << method_prefix << tstruct->get_name() << "::read(Protocol_* iprot) {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::read(::apache::thrift::protocol::TProtocol* iprot) {" << endl; + } + indent_up(); +@@ -1301,14 +1316,18 @@ void t_cpp_generator::generate_struct_reader(ofstream& out, t_struct* tstruct, b + */ + void t_cpp_generator::generate_struct_writer(ofstream& out, t_struct* tstruct, bool pointers) { + string name = tstruct->get_name(); ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; ++ << method_prefix << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::write(::apache::thrift::protocol::TProtocol* oprot) const {" << endl; + } + indent_up(); +@@ -1369,14 +1388,18 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + t_struct* tstruct, + bool pointers) { + string name = tstruct->get_name(); ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; ++ << method_prefix << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::write(::apache::thrift::protocol::TProtocol* oprot) const {" << endl; + } + indent_up(); +@@ -1385,18 +1408,7 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + + indent(out) << "xfer += oprot->writeStructBegin(\"" << name << "\");" << endl; + +- bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { +- if (first) { +- first = false; +- out << endl << indent() << "if "; +- } else { +- out << " else if "; +- } +- +- out << "(this->__isset." << (*f_iter)->get_name() << ") {" << endl; +- +- indent_up(); + + // Write field header + out << indent() << "xfer += oprot->writeFieldBegin(" +@@ -1410,9 +1422,6 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + } + // Write field closer + indent(out) << "xfer += oprot->writeFieldEnd();" << endl; +- +- indent_down(); +- indent(out) << "}"; + } + + // Write the struct map +@@ -1478,9 +1487,13 @@ void t_cpp_generator::generate_struct_ostream_operator(std::ofstream& out, t_str + } + + void t_cpp_generator::generate_struct_print_method_decl(std::ofstream& out, t_struct* tstruct) { ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + out << "void "; + if (tstruct) { +- out << tstruct->get_name() << "::"; ++ out << method_prefix << tstruct->get_name() << "::"; + } + out << "printTo(std::ostream& out) const"; + } +@@ -1601,11 +1614,13 @@ void t_cpp_generator::generate_exception_what_method(std::ofstream& out, t_struc + */ + void t_cpp_generator::generate_service(t_service* tservice) { + string svcname = tservice->get_name(); ++ string ns = tservice->get_program()->get_namespace("cpp"); + + // Make output files +- string f_header_name = get_out_dir() + svcname + ".h"; ++ string f_header_name = get_out_dir() + svcname + ".grpc.thrift.h"; + f_header_.open(f_header_name.c_str()); + ++ + // Print header file includes + f_header_ << autogen_comment(); + f_header_ << "#ifndef " << svcname << "_H" << endl << "#define " << svcname << "_H" << endl +@@ -1621,15 +1636,38 @@ void t_cpp_generator::generate_service(t_service* tservice) { + f_header_ << "#include " << endl; + } + f_header_ << "#include " << endl; ++ + f_header_ << "#include \"" << get_include_prefix(*get_program()) << program_name_ << "_types.h\"" + << endl; + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != NULL) { ++ if (extends_service != nullptr) { + f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) +- << extends_service->get_name() << ".h\"" << endl; ++ << extends_service->get_name() << ".grpc.thrift.h\"" << endl; + } + ++ ++ f_header_ << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl; ++ ++ ++ f_header_ << ++ endl << ++ "namespace grpc {" << endl << ++ "class CompletionQueue;" << endl << ++ "class Channel;" << endl << ++ "class RpcService;" << endl << ++ "class ServerCompletionQueue;" << endl << ++ "class ServerContext;" << endl << ++ "}" << endl; ++ + f_header_ << endl << ns_open_ << endl << endl; + + f_header_ << "#ifdef _WIN32\n" +@@ -1638,10 +1676,13 @@ void t_cpp_generator::generate_service(t_service* tservice) { + "#endif\n\n"; + + // Service implementation file includes +- string f_service_name = get_out_dir() + svcname + ".cpp"; ++ string f_service_name = get_out_dir() + svcname + ".grpc.thrift.cpp"; + f_service_.open(f_service_name.c_str()); + f_service_ << autogen_comment(); +- f_service_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" << endl; ++ ++ f_service_ << "#include \"" << ++ get_include_prefix(*get_program()) << svcname << ".grpc.thrift.h\"" << endl; ++ + if (gen_cob_style_) { + f_service_ << "#include \"thrift/async/TAsyncChannel.h\"" << endl; + } +@@ -1652,7 +1693,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + string f_service_tcc_name = get_out_dir() + svcname + ".tcc"; + f_service_tcc_.open(f_service_tcc_name.c_str()); + f_service_tcc_ << autogen_comment(); +- f_service_tcc_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" ++ f_service_tcc_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".grpc.thrift.h\"" + << endl; + + f_service_tcc_ << "#ifndef " << svcname << "_TCC" << endl << "#define " << svcname << "_TCC" +@@ -1663,19 +1704,66 @@ void t_cpp_generator::generate_service(t_service* tservice) { + } + } + ++ f_service_ << ++ endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ endl; ++ + f_service_ << endl << ns_open_ << endl << endl; + f_service_tcc_ << endl << ns_open_ << endl << endl; + ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ ++ f_service_ << ++ "static const char* " << service_name_ << "_method_names[] = {" << endl; ++ ++ ++ indent_up(); ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ ++ for ( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "\"/" << extends_service->get_program()->get_namespace("cpp") << ++ "." << extends_service->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; ++ } ++ } ++ ++ indent_down(); ++ f_service_ << ++ "};" << endl; ++ ++ // Generate service class ++ if ( extends_service != nullptr ) { ++ f_header_ << "class " << service_name_ << " : public " << ++ extends_service->get_name() << " {" << endl << ++ "public:" << endl; ++ } ++ else { ++ f_header_ << "class " << service_name_ << "{" << endl << ++ "public:" << endl; ++ } ++ + // Generate all the components +- generate_service_interface(tservice, ""); +- generate_service_interface_factory(tservice, ""); +- generate_service_null(tservice, ""); + generate_service_helpers(tservice); +- generate_service_client(tservice, ""); +- generate_service_processor(tservice, ""); +- generate_service_multiface(tservice); +- generate_service_skeleton(tservice); +- generate_service_client(tservice, "Concurrent"); ++ generate_service_interface(tservice, ""); ++ generate_service_stub_interface(tservice); ++ generate_service_stub(tservice); + + // Generate all the cob components + if (gen_cob_style_) { +@@ -1688,10 +1776,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { + generate_service_async_skeleton(tservice); + } + ++ // Close service class ++ f_header_ << "};" << endl; ++ + f_header_ << "#ifdef _WIN32\n" + " #pragma warning( pop )\n" + "#endif\n\n"; + ++ + // Close the namespace + f_service_ << ns_close_ << endl << endl; + f_service_tcc_ << ns_close_ << endl << endl; +@@ -1729,15 +1821,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + string name_orig = ts->get_name(); + + // TODO(dreiss): Why is this stuff not in generate_function_helpers? +- ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_args"); ++ ts->set_name((*f_iter)->get_name() + "Req"); + generate_struct_declaration(f_header_, ts, false); +- generate_struct_definition(out, f_service_, ts, false); ++ generate_struct_definition(out, f_service_, ts, true); + generate_struct_reader(out, ts); + generate_struct_writer(out, ts); +- ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_pargs"); +- generate_struct_declaration(f_header_, ts, false, true, false, true); +- generate_struct_definition(out, f_service_, ts, false); +- generate_struct_writer(out, ts, true); + ts->set_name(name_orig); + + generate_function_helpers(tservice, *f_iter); +@@ -1745,13 +1833,210 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + } + + /** ++ * Generates a service Stub Interface ++ * ++ * @param tservice The service to generate a stub for. ++ * ++ */ ++void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { ++ ++ string extends = ""; ++ if (tservice->get_extends() != nullptr) { ++ extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; ++ } ++ ++ f_header_ << ++ endl << ++ "class StubInterface " << extends << " {" << endl; ++ indent_up(); ++ f_header_ << ++ " public:" << endl << ++ indent() << "virtual ~StubInterface() {}" << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "virtual ::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) = 0;" << endl; ++ } ++ indent_down(); ++ f_header_ << ++ "};" << endl << endl; ++ ++} ++void t_cpp_generator::generate_service_stub(t_service* tservice) { ++ f_header_ << ++ endl << ++ "class Stub : public StubInterface {" << ++ endl; ++ ++ indent_up(); ++ f_header_ << ++ " public:" << endl << ++ indent() << "Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel);" << ++ endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) override;" << endl; ++ } ++ ++ t_service* extends_service = tservice->get_extends(); ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) override;" << endl; ++ } ++ } ++ ++ f_header_ << ++ endl << ++ " private:" << endl << ++ indent() << "std::shared_ptr< ::grpc::ChannelInterface> channel_;" << ++ endl; ++ ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_header_ << ++ indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_header_ << ++ indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; ++ } ++ } ++ ++ indent_down(); ++ f_header_ << ++ "};" << endl << endl; ++ ++ // generate the implementaion of Stub ++ f_service_ << ++ endl << ++ service_name_ << "::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << ": channel_(channel)" << endl; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter , ++i) { ++ f_service_ << ++ indent() << ++ ", rpcmethod_" << (*f_iter)->get_name() << "_(" << ++ service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << ++ ", rpcmethod_" << (*f_iter)->get_name() << "_(" << ++ service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; ++ } ++ } ++ f_service_ << ++ indent() << "{}" << endl; ++ indent_down(); ++ ++ // generate NewStub ++ f_header_ << ++ endl << ++ "static std::unique_ptr NewStub(const std::shared_ptr\ ++ < ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions());" << ++ endl; ++ ++ // generate NewStub Implementation ++ f_service_ << ++ endl << ++ "std::unique_ptr< " << service_name_ << "::Stub> " << service_name_ << "::NewStub(const std::shared_ptr\ ++ < ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "std::unique_ptr< " << service_name_ << "::Stub> stub(new " << service_name_ << ++ "::Stub(channel));" << endl << ++ indent() << "return stub;" << endl; ++ indent_down(); ++ f_service_ << ++ "}" << endl; ++ ++ // generate stub methods ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Stub::" << function_name << ++ "(::grpc::ClientContext* context, const " << service_name_ << "::" << ++ function_name << "Req& request, " << service_name_ << "::" << ++ function_name << "Resp* response) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_" << ++ function_name << "_, context, request, response);" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ ++ } ++ ++ if (extends_service != nullptr) { ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Stub::" << function_name << ++ "(::grpc::ClientContext* context, const " << service_name_ << "::" << ++ function_name << "Req& request, " << service_name_ << "::" << ++ function_name << "Resp* response) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_" << ++ function_name << "_, context, request, response);" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ ++ } ++ } ++ ++} ++ ++ ++/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ + void t_cpp_generator::generate_service_interface(t_service* tservice, string style) { + +- string service_if_name = service_name_ + style + "If"; ++ string service_if_name = "Service"; + if (style == "CobCl") { + // Forward declare the client. + string client_name = service_name_ + "CobClient"; +@@ -1765,12 +2050,14 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + + string extends = ""; + if (tservice->get_extends() != NULL) { +- extends = " : virtual public " + type_name(tservice->get_extends()) + style + "If"; ++ extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; + if (style == "CobCl" && gen_templates_) { + // TODO(simpkins): If gen_templates_ is enabled, we currently assume all + // parent services were also generated with templates enabled. + extends += "T"; + } ++ } else { ++ extends = " : public ::grpc::Service"; + } + + if (style == "CobCl" && gen_templates_) { +@@ -1778,7 +2065,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } + f_header_ << "class " << service_if_name << extends << " {" << endl << " public:" << endl; + indent_up(); +- f_header_ << indent() << "virtual ~" << service_if_name << "() {}" << endl; ++ ++ f_header_ << indent() << "Service();" << endl; ++ f_header_ << indent() << "virtual ~Service();" << endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; +@@ -1786,7 +2075,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + if ((*f_iter)->has_doc()) + f_header_ << endl; + generate_java_doc(f_header_, *f_iter); +- f_header_ << indent() << "virtual " << function_signature(*f_iter, style) << " = 0;" << endl; ++ ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "virtual ::grpc::Status " << function_name << ++ "(::grpc::ServerContext* context, const "<< function_name << ++ "Req* request, "<< function_name << "Resp* response);" << endl; + } + indent_down(); + f_header_ << "};" << endl << endl; +@@ -1797,6 +2091,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + f_header_ << "typedef " << service_if_name << "< ::apache::thrift::protocol::TProtocol> " + << service_name_ << style << "If;" << endl << endl; + } ++ ++ // generate the service interface implementations ++ ++ f_service_ << ++ endl << ++ service_name_ << "::Service::Service() {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "(void)" << service_name_ << "_method_names;" << endl; ++ uint32_t i=0; ++ for(i=0;iget_name(); ++ f_service_ << ++ endl << ++ indent() << "AddMethod(new ::grpc::RpcServiceMethod(" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << service_name_ << "_method_names[" << i << "]," << endl << ++ indent() << "::grpc::RpcMethod::NORMAL_RPC," << endl << ++ indent() << "new ::grpc::RpcMethodHandler< " << service_name_ << "::Service, " << ++ service_name_ << "::" << function_name << "Req, " << service_name_ << "::" << ++ function_name << "Resp>(" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "std::mem_fn(&" << service_name_ << "::Service::" << function_name << "), this)));" << endl; ++ ++ indent_down(); ++ indent_down(); ++ } ++ ++ indent_down(); ++ f_service_ << ++ "}" << endl; ++ ++ f_service_ << ++ endl << ++ service_name_ << "::Service::~Service() {" << endl << ++ "}" << endl; ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Service::" << function_name << ++ "(::grpc::ServerContext* context, const " << service_name_ << "::" << function_name << ++ "Req* request, " << service_name_ << "::" << function_name << "Resp* response) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "(void) context;" << endl << ++ indent() << "(void) request;" << endl << ++ indent() << "(void) response;" << endl << ++ indent() << "return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED,\"\");" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ } ++ + } + + /** +@@ -3095,7 +3449,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* + + std::ofstream& out = (gen_templates_ ? f_service_tcc_ : f_service_); + +- t_struct result(program_, tservice->get_name() + "_" + tfunction->get_name() + "_result"); ++ t_struct result(program_, tfunction->get_name() + "Resp"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); +@@ -3109,17 +3463,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* + } + + generate_struct_declaration(f_header_, &result, false); +- generate_struct_definition(out, f_service_, &result, false); ++ generate_struct_definition(out, f_service_, &result, true); + generate_struct_reader(out, &result); + generate_struct_result_writer(out, &result); +- +- result.set_name(tservice->get_name() + "_" + tfunction->get_name() + "_presult"); +- generate_struct_declaration(f_header_, &result, false, true, true, gen_cob_style_); +- generate_struct_definition(out, f_service_, &result, false); +- generate_struct_reader(out, &result, true); +- if (gen_cob_style_) { +- generate_struct_writer(out, &result, true); +- } + } + + /** +@@ -3162,8 +3508,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + << endl; + scope_up(out); + +- string argsname = tservice->get_name() + "_" + tfunction->get_name() + "_args"; +- string resultname = tservice->get_name() + "_" + tfunction->get_name() + "_result"; ++ string argsname = tfunction->get_name() + "Req"; ++ string resultname = tfunction->get_name() + "Resp"; + + if (tfunction->is_oneway() && !unnamed_oprot_seqid) { + out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; +@@ -3320,7 +3666,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; + } + +- out << indent() << tservice->get_name() + "_" + tfunction->get_name() << "_args args;" << endl ++ out << indent() << tfunction->get_name() << "Req args;" << endl + << indent() << "void* ctx = NULL;" << endl << indent() + << "if (this->eventHandler_.get() != NULL) {" << endl << indent() + << " ctx = this->eventHandler_->getContext(" << service_func_name << ", NULL);" << endl +@@ -3487,7 +3833,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + << "this->eventHandler_.get(), ctx, " << service_func_name << ");" << endl << endl; + + // Throw the TDelayedException, and catch the result +- out << indent() << tservice->get_name() << "_" << tfunction->get_name() << "_result result;" ++ out << indent() << tfunction->get_name() << "Resp result;" + << endl << endl << indent() << "try {" << endl; + indent_up(); + out << indent() << "_throw->throw_it();" << endl << indent() << "return cob(false);" +diff --git a/tutorial/cpp/CMakeLists.txt b/tutorial/cpp/CMakeLists.txt +deleted file mode 100644 +index 8a3d085..0000000 +--- a/tutorial/cpp/CMakeLists.txt ++++ /dev/null +@@ -1,53 +0,0 @@ +-# +-# Licensed to the Apache Software Foundation (ASF) under one +-# or more contributor license agreements. See the NOTICE file +-# distributed with this work for additional information +-# regarding copyright ownership. The ASF licenses this file +-# to you under the Apache License, Version 2.0 (the +-# "License"); you may not use this file except in compliance +-# with the License. You may obtain a copy of the License at +-# +-# http://www.apache.org/licenses/LICENSE-2.0 +-# +-# Unless required by applicable law or agreed to in writing, +-# software distributed under the License is distributed on an +-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-# KIND, either express or implied. See the License for the +-# specific language governing permissions and limitations +-# under the License. +-# +- +-find_package(Boost 1.53.0 REQUIRED) +-include_directories(SYSTEM "${Boost_INCLUDE_DIRS}") +- +-#Make sure gen-cpp files can be included +-include_directories("${CMAKE_CURRENT_BINARY_DIR}") +-include_directories("${CMAKE_CURRENT_BINARY_DIR}/gen-cpp") +-include_directories("${PROJECT_SOURCE_DIR}/lib/cpp/src") +- +-include(ThriftMacros) +- +-set(tutorialgencpp_SOURCES +- gen-cpp/Calculator.cpp +- gen-cpp/SharedService.cpp +- gen-cpp/shared_constants.cpp +- gen-cpp/shared_types.cpp +- gen-cpp/tutorial_constants.cpp +- gen-cpp/tutorial_types.cpp +-) +-add_library(tutorialgencpp STATIC ${tutorialgencpp_SOURCES}) +-LINK_AGAINST_THRIFT_LIBRARY(tutorialgencpp thrift) +- +-add_custom_command(OUTPUT gen-cpp/Calculator.cpp gen-cpp/SharedService.cpp gen-cpp/shared_constants.cpp gen-cpp/shared_types.cpp gen-cpp/tutorial_constants.cpp gen-cpp/tutorial_types.cpp +- COMMAND ${THRIFT_COMPILER} --gen cpp -r ${PROJECT_SOURCE_DIR}/tutorial/tutorial.thrift +-) +- +-add_executable(TutorialServer CppServer.cpp) +-target_link_libraries(TutorialServer tutorialgencpp) +-LINK_AGAINST_THRIFT_LIBRARY(TutorialServer thrift) +-target_link_libraries(TutorialServer ${ZLIB_LIBRARIES}) +- +-add_executable(TutorialClient CppClient.cpp) +-target_link_libraries(TutorialClient tutorialgencpp) +-LINK_AGAINST_THRIFT_LIBRARY(TutorialClient thrift) +-target_link_libraries(TutorialClient ${ZLIB_LIBRARIES}) +diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/CppClient.cpp +index 2763fee..c41604e 100644 +--- a/tutorial/cpp/CppClient.cpp ++++ b/tutorial/cpp/CppClient.cpp +@@ -1,80 +1,94 @@ + /* +- * Licensed to the Apache Software Foundation (ASF) under one +- * or more contributor license agreements. See the NOTICE file +- * distributed with this work for additional information +- * regarding copyright ownership. The ASF licenses this file +- * to you under the Apache License, Version 2.0 (the +- * "License"); you may not use this file except in compliance +- * with the License. You may obtain a copy of the License at + * +- * http://www.apache.org/licenses/LICENSE-2.0 ++ * Copyright 2015, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. + */ + + #include ++#include ++#include + +-#include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" ++#include + +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; ++#include "gen-cpp/Greeter.grpc.thrift.h" + +-using namespace tutorial; +-using namespace shared; ++using grpc::Channel; ++using grpc::ClientContext; ++using grpc::Status; ++using test::Greeter; ++using namespace test; + +-int main() { +- boost::shared_ptr socket(new TSocket("localhost", 9090)); +- boost::shared_ptr transport(new TBufferedTransport(socket)); +- boost::shared_ptr protocol(new TBinaryProtocol(transport)); +- CalculatorClient client(protocol); ++class GreeterClient { ++ public: ++ GreeterClient(std::shared_ptr channel) ++ : stub_(Greeter::NewStub(channel)) {} + +- try { +- transport->open(); ++ // Assambles the client's payload, sends it and presents the response back ++ // from the server. ++ std::string SayHello(const std::string& user) { ++ // Data we are sending to the server. ++ Greeter::SayHelloReq req; ++ req.request.name = user; + +- client.ping(); +- cout << "ping()" << endl; ++ // Container for the data we expect from the server. ++ Greeter::SayHelloResp reply; + +- cout << "1 + 1 = " << client.add(1, 1) << endl; ++ // Context for the client. It could be used to convey extra information to ++ // the server and/or tweak certain RPC behaviors. ++ ClientContext context; + +- Work work; +- work.op = Operation::DIVIDE; +- work.num1 = 1; +- work.num2 = 0; ++ // The actual RPC. ++ Status status = stub_->SayHello(&context, req, &reply); + +- try { +- client.calculate(1, work); +- cout << "Whoa? We can divide by zero!" << endl; +- } catch (InvalidOperation& io) { +- cout << "InvalidOperation: " << io.why << endl; +- // or using generated operator<<: cout << io << endl; +- // or by using std::exception native method what(): cout << io.what() << endl; ++ // Act upon its status. ++ if (status.ok()) { ++ return reply.success.message; ++ } else { ++ return "RPC failed"; + } ++ } + +- work.op = Operation::SUBTRACT; +- work.num1 = 15; +- work.num2 = 10; +- int32_t diff = client.calculate(1, work); +- cout << "15 - 10 = " << diff << endl; ++ private: ++ std::unique_ptr stub_; ++}; + +- // Note that C++ uses return by reference for complex types to avoid +- // costly copy construction +- SharedStruct ss; +- client.getStruct(ss, 1); +- cout << "Received log: " << ss << endl; ++int main() { ++ // Instantiate the client. It requires a channel, out of which the actual RPCs ++ // are created. This channel models a connection to an endpoint (in this case, ++ // localhost at port 50051). We indicate that the channel isn't authenticated ++ // (use of InsecureChannelCredentials()). ++ GreeterClient greeter(grpc::CreateChannel( ++ "localhost:50051", grpc::InsecureChannelCredentials())); ++ std::string user("world"); ++ std::string reply = greeter.SayHello(user); ++ std::cout << "Greeter received: " << reply << std::endl; + +- transport->close(); +- } catch (TException& tx) { +- cout << "ERROR: " << tx.what() << endl; +- } ++ return 0; + } +diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp +index eafffa9..c838b61 100644 +--- a/tutorial/cpp/CppServer.cpp ++++ b/tutorial/cpp/CppServer.cpp +@@ -1,181 +1,87 @@ + /* +- * Licensed to the Apache Software Foundation (ASF) under one +- * or more contributor license agreements. See the NOTICE file +- * distributed with this work for additional information +- * regarding copyright ownership. The ASF licenses this file +- * to you under the Apache License, Version 2.0 (the +- * "License"); you may not use this file except in compliance +- * with the License. You may obtain a copy of the License at + * +- * http://www.apache.org/licenses/LICENSE-2.0 ++ * Copyright 2015, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. + */ + +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- + #include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" ++#include ++#include + +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::concurrency; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; +-using namespace apache::thrift::server; ++#include + +-using namespace tutorial; +-using namespace shared; ++#include "gen-cpp/Greeter.grpc.thrift.h" ++#include + +-class CalculatorHandler : public CalculatorIf { +-public: +- CalculatorHandler() {} ++using grpc::Server; ++using grpc::ServerBuilder; ++using grpc::ServerContext; ++using grpc::Status; ++using test::Greeter; + +- void ping() { cout << "ping()" << endl; } ++using namespace grpc; ++using namespace test; + +- int32_t add(const int32_t n1, const int32_t n2) { +- cout << "add(" << n1 << ", " << n2 << ")" << endl; +- return n1 + n2; +- } ++// Logic and data behind the server's behavior. ++class GreeterServiceImpl final : public Greeter::Service { ++ Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, ++ Greeter::SayHelloResp* reply) override { ++ std::string prefix("Hello "); + +- int32_t calculate(const int32_t logid, const Work& work) { +- cout << "calculate(" << logid << ", " << work << ")" << endl; +- int32_t val; ++ reply->success.message = prefix + request->request.name; + +- switch (work.op) { +- case Operation::ADD: +- val = work.num1 + work.num2; +- break; +- case Operation::SUBTRACT: +- val = work.num1 - work.num2; +- break; +- case Operation::MULTIPLY: +- val = work.num1 * work.num2; +- break; +- case Operation::DIVIDE: +- if (work.num2 == 0) { +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Cannot divide by 0"; +- throw io; +- } +- val = work.num1 / work.num2; +- break; +- default: +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Invalid Operation"; +- throw io; +- } +- +- SharedStruct ss; +- ss.key = logid; +- ss.value = to_string(val); +- +- log[logid] = ss; +- +- return val; ++ return Status::OK; + } +- +- void getStruct(SharedStruct& ret, const int32_t logid) { +- cout << "getStruct(" << logid << ")" << endl; +- ret = log[logid]; +- } +- +- void zip() { cout << "zip()" << endl; } +- +-protected: +- map log; + }; + +-/* +- CalculatorIfFactory is code generated. +- CalculatorCloneFactory is useful for getting access to the server side of the +- transport. It is also useful for making per-connection state. Without this +- CloneFactory, all connections will end up sharing the same handler instance. +-*/ +-class CalculatorCloneFactory : virtual public CalculatorIfFactory { +- public: +- virtual ~CalculatorCloneFactory() {} +- virtual CalculatorIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) +- { +- boost::shared_ptr sock = boost::dynamic_pointer_cast(connInfo.transport); +- cout << "Incoming connection\n"; +- cout << "\tSocketInfo: " << sock->getSocketInfo() << "\n"; +- cout << "\tPeerHost: " << sock->getPeerHost() << "\n"; +- cout << "\tPeerAddress: " << sock->getPeerAddress() << "\n"; +- cout << "\tPeerPort: " << sock->getPeerPort() << "\n"; +- return new CalculatorHandler; +- } +- virtual void releaseHandler( ::shared::SharedServiceIf* handler) { +- delete handler; +- } +-}; ++void RunServer() { ++ std::string server_address("0.0.0.0:50051"); ++ GreeterServiceImpl service; ++ ++ ServerBuilder builder; ++ // Listen on the given address without any authentication mechanism. ++ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); ++ // Register "service" as the instance through which we'll communicate with ++ // clients. In this case it corresponds to an *synchronous* service. ++ builder.RegisterService(&service); ++ // Finally assemble the server. ++ std::unique_ptr server(builder.BuildAndStart()); ++ std::cout << "Server listening on " << server_address << std::endl; ++ ++ // Wait for the server to shutdown. Note that some other thread must be ++ // responsible for shutting down the server for this call to ever return. ++ server->Wait(); ++} + + int main() { +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- +- /* +- // if you don't need per-connection state, do the following instead +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- */ +- +- /** +- * Here are some alternate server types... +- +- // This server only allows one connection at a time, but spawns no threads +- TSimpleServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared()); +- +- const int workerCount = 4; +- +- boost::shared_ptr threadManager = +- ThreadManager::newSimpleThreadManager(workerCount); +- threadManager->threadFactory( +- boost::make_shared()); +- threadManager->start(); +- +- // This server allows "workerCount" connection at a time, and reuses threads +- TThreadPoolServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared(), +- threadManager); +- */ ++ RunServer(); + +- cout << "Starting the server..." << endl; +- server.serve(); +- cout << "Done." << endl; + return 0; + } +diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am +index 184a69d..581f75e 100755 +--- a/tutorial/cpp/Makefile.am ++++ b/tutorial/cpp/Makefile.am +@@ -18,44 +18,38 @@ + # + AUTOMAKE_OPTIONS = subdir-objects serial-tests + +-BUILT_SOURCES = gen-cpp/shared_types.cpp \ +- gen-cpp/tutorial_types.cpp ++BUILT_SOURCES = gen-cpp/test_types.cpp + +-noinst_LTLIBRARIES = libtutorialgencpp.la +-nodist_libtutorialgencpp_la_SOURCES = \ +- gen-cpp/Calculator.cpp \ +- gen-cpp/Calculator.h \ +- gen-cpp/SharedService.cpp \ +- gen-cpp/SharedService.h \ +- gen-cpp/shared_constants.cpp \ +- gen-cpp/shared_constants.h \ +- gen-cpp/shared_types.cpp \ +- gen-cpp/shared_types.h \ +- gen-cpp/tutorial_constants.cpp \ +- gen-cpp/tutorial_constants.h \ +- gen-cpp/tutorial_types.cpp \ +- gen-cpp/tutorial_types.h ++#noinst_LTLIBRARIES = libtutorialgencpp.la ++noinst_LTLIBRARIES = libtestgencpp.la ++nodist_libtestgencpp_la_SOURCES = \ ++ gen-cpp/Greeter.grpc.thrift.cpp \ ++ gen-cpp/Greeter.grpc.thrift.h \ ++ gen-cpp/test_constants.cpp \ ++ gen-cpp/test_constants.h \ ++ gen-cpp/test_types.cpp \ ++ gen-cpp/test_types.h + + + +-libtutorialgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la ++libtestgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la + + noinst_PROGRAMS = \ +- TutorialServer \ +- TutorialClient ++ TestServer \ ++ TestClient + +-TutorialServer_SOURCES = \ ++TestServer_SOURCES = \ + CppServer.cpp + +-TutorialServer_LDADD = \ +- libtutorialgencpp.la \ ++TestServer_LDADD = \ ++ libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + +-TutorialClient_SOURCES = \ ++TestClient_SOURCES = \ + CppClient.cpp + +-TutorialClient_LDADD = \ +- libtutorialgencpp.la \ ++TestClient_LDADD = \ ++ libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + + # +@@ -63,21 +57,21 @@ TutorialClient_LDADD = \ + # + THRIFT = $(top_builddir)/compiler/cpp/thrift + +-gen-cpp/Calculator.cpp gen-cpp/SharedService.cpp gen-cpp/shared_constants.cpp gen-cpp/shared_types.cpp gen-cpp/tutorial_constants.cpp gen-cpp/tutorial_types.cpp: $(top_srcdir)/tutorial/tutorial.thrift ++gen-cpp/Greeter.grpc.thrift.cpp gen-cpp/test_constants.cpp gen-cpp/test_types.cpp: $(top_srcdir)/tutorial/cpp/test.thrift + $(THRIFT) --gen cpp -r $< + + AM_CPPFLAGS = $(BOOST_CPPFLAGS) $(LIBEVENT_CPPFLAGS) -I$(top_srcdir)/lib/cpp/src -Igen-cpp + AM_CXXFLAGS = -Wall -Wextra -pedantic +-AM_LDFLAGS = $(BOOST_LDFLAGS) $(LIBEVENT_LDFLAGS) ++AM_LDFLAGS = $(BOOST_LDFLAGS) $(LIBEVENT_LDFLAGS) `pkg-config --libs grpc++ grpc` -lpthread -ldl -lgrpc + + clean-local: +- $(RM) gen-cpp/* ++ $(RM) -r gen-cpp + +-tutorialserver: all +- ./TutorialServer ++testserver: all ++ ./TestServer + +-tutorialclient: all +- ./TutorialClient ++testclient: all ++ ./TestClient + + style-local: + $(CPPSTYLE_CMD) +diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift +new file mode 100644 +index 0000000..de3c9a4 +--- /dev/null ++++ b/tutorial/cpp/test.thrift +@@ -0,0 +1,13 @@ ++namespace cpp test ++ ++struct HelloRequest { ++ 1:string name ++} ++ ++struct HelloResponse { ++ 1:string message ++} ++ ++service Greeter { ++ HelloResponse SayHello(1:HelloRequest request); ++} +\ No newline at end of file +-- +2.8.0.rc3.226.g39d4020 + + +From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 16:23:53 -0700 +Subject: [PATCH 3/3] grpc java plugins generator + +for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift +--- + compiler/cpp/src/generate/t_java_generator.cc | 906 +++++++++++++++++++++++++- + tutorial/Makefile.am | 8 +- + 2 files changed, 887 insertions(+), 27 deletions(-) + +diff --git a/compiler/cpp/src/generate/t_java_generator.cc b/compiler/cpp/src/generate/t_java_generator.cc +index 2db8cb8..95e1ca8 100644 +--- a/compiler/cpp/src/generate/t_java_generator.cc ++++ b/compiler/cpp/src/generate/t_java_generator.cc +@@ -97,10 +97,10 @@ public: + } else if(iter->second.compare("suppress") == 0) { + suppress_generated_annotations_ = true; + } else { +- throw "unknown option java:" + iter->first + "=" + iter->second; ++ throw "unknown option java:" + iter->first + "=" + iter->second; + } + } else { +- throw "unknown option java:" + iter->first; ++ throw "unknown option java:" + iter->first; + } + } + +@@ -195,6 +195,17 @@ public: + void generate_service_async_server(t_service* tservice); + void generate_process_function(t_service* tservice, t_function* tfunction); + void generate_process_async_function(t_service* tservice, t_function* tfunction); ++ void generate_service_impl_base(t_service* tservice); ++ void generate_method_descriptors(t_service* tservice); ++ void generate_stub(t_service* tservice); ++ void generate_blocking_stub(t_service* tservice); ++ void generate_future_stub(t_service* tservice); ++ void generate_method_ids(t_service* tservice); ++ void generate_method_handlers(t_service* tservice); ++ void generate_service_descriptors(t_service* tservice); ++ void generate_service_builder(t_service* tservice); ++ void generate_arg_ids(t_service* tservice); ++ void generate_message_factory(t_service* tservice); + + void generate_java_union(t_struct* tstruct); + void generate_union_constructor(ofstream& out, t_struct* tstruct); +@@ -307,6 +318,8 @@ public: + std::string java_package(); + std::string java_type_imports(); + std::string java_suppressions(); ++ std::string grpc_imports(); ++ std::string import_extended_service(t_service* tservice); + std::string type_name(t_type* ttype, + bool in_container = false, + bool in_init = false, +@@ -368,7 +381,7 @@ private: + bool use_option_type_; + bool undated_generated_annotations_; + bool suppress_generated_annotations_; +- ++ + }; + + /** +@@ -456,6 +469,35 @@ string t_java_generator::java_suppressions() { + return "@SuppressWarnings({\"cast\", \"rawtypes\", \"serial\", \"unchecked\", \"unused\"})\n"; + } + ++string t_java_generator::grpc_imports() { ++ return ++ string() + ++ "import static io.grpc.stub.ClientCalls.asyncUnaryCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncServerStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncClientStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.blockingUnaryCall;\n" + ++ "import static io.grpc.stub.ClientCalls.blockingServerStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.futureUnaryCall;\n" + ++ "import static io.grpc.MethodDescriptor.generateFullMethodName;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnaryCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncServerStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncClientStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncBidiStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall;\n" + ++ "import io.grpc.thrift.ThriftUtils;\n\n"; ++} ++ ++string t_java_generator::import_extended_service(t_service* tservice) { ++ if (tservice == nullptr) { ++ return string() + "\n"; ++ } ++ string ns = tservice->get_program()->get_namespace("java"); ++ string extend_service_name = tservice->get_name() + "Grpc"; ++ return string() + "import " + ns + "." + extend_service_name + ";\n\n"; ++} ++ + /** + * Nothing in Java + */ +@@ -2772,25 +2814,51 @@ void t_java_generator::generate_field_value_meta_data(std::ofstream& out, t_type + */ + void t_java_generator::generate_service(t_service* tservice) { + // Make output file +- string f_service_name = package_dir_ + "/" + make_valid_java_filename(service_name_) + ".java"; ++ string f_service_name = package_dir_ + "/" + make_valid_java_filename(service_name_) + "Grpc.java"; + f_service_.open(f_service_name.c_str()); + +- f_service_ << autogen_comment() << java_package() << java_type_imports() << java_suppressions(); ++ f_service_ << ++ autogen_comment() << ++ java_package() << ++ java_type_imports() << ++ grpc_imports() << ++ import_extended_service(tservice->get_extends()); ++ java_suppressions(); ++ ++ f_service_ << ++ "public class " << service_name_ << "Grpc {" << endl << ++ endl; + +- if (!suppress_generated_annotations_) { +- generate_javax_generated_annotation(f_service_); +- } +- f_service_ << "public class " << service_name_ << " {" << endl << endl; + indent_up(); + ++ // generate constructor ++ f_service_ << ++ indent() << "private " << service_name_ << ++ "Grpc() {}" << endl << endl; ++ ++ f_service_ << ++ indent() << "public static final String SERVICE_NAME = " << ++ "\"" << package_name_ << "." << service_name_ << "\";" << endl << endl; ++ + // Generate the three main parts of the service + generate_service_interface(tservice); +- generate_service_async_interface(tservice); +- generate_service_client(tservice); +- generate_service_async_client(tservice); +- generate_service_server(tservice); +- generate_service_async_server(tservice); ++ generate_arg_ids(tservice); ++ generate_message_factory(tservice); ++ generate_service_impl_base(tservice); ++ //generate_service_async_interface(tservice); ++ //generate_service_client(tservice); ++ //generate_service_async_client(tservice); ++ //generate_service_server(tservice); ++ //generate_service_async_server(tservice); + generate_service_helpers(tservice); ++ generate_method_descriptors(tservice); ++ generate_stub(tservice); ++ generate_blocking_stub(tservice); ++ generate_future_stub(tservice); ++ generate_method_ids(tservice); ++ generate_method_handlers(tservice); ++ generate_service_descriptors(tservice); ++ generate_service_builder(tservice); + + indent_down(); + f_service_ << "}" << endl; +@@ -2805,24 +2873,820 @@ void t_java_generator::generate_service(t_service* tservice) { + void t_java_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; +- if (tservice->get_extends() != NULL) { +- extends = type_name(tservice->get_extends()); +- extends_iface = " extends " + extends + ".Iface"; +- } + + generate_java_doc(f_service_, tservice); +- f_service_ << indent() << "public interface Iface" << extends_iface << " {" << endl << endl; ++ f_service_ << indent() << ++ "@java.lang.Deprecated public static interface " << service_name_; ++ ++ if (tservice->get_extends() != nullptr) { ++ f_service_ << " extends " << tservice->get_extends()->get_name() + "Grpc." << ++ tservice->get_extends()->get_name() << endl; ++ } ++ f_service_ << " {" << endl; ++ ++ indent_up(); ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ //generate_java_doc(f_service_, *f_iter); ++ f_service_ << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << ++ "_args request," << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver);" << endl << endl; ++ } ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_arg_ids(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "private static final int ARG_IN_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ f_service_ << indent() << ++ "private static final int ARG_OUT_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ } ++ f_service_ << endl; ++ ++ if (tservice->get_extends() != nullptr) { ++ f_service_ << indent() << "// ARG IDs for extended service" << endl; ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "private static final int ARG_IN_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ f_service_ << indent() << ++ "private static final int ARG_OUT_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ } ++ f_service_ << endl; ++ } ++} ++ ++void t_java_generator::generate_message_factory(t_service* tservice) { ++ f_service_ << indent() << ++ "private static final class ThriftMessageFactory>" << endl << indent() << ++ " implements io.grpc.thrift.MessageFactory {" << endl; ++ indent_up(); ++ f_service_ << indent() << ++ "private final int id;" << endl << endl; ++ f_service_ << endl; ++ ++ f_service_ << indent() << ++ "ThriftMessageFactory(int id) {" << endl << ++ indent() << " this.id = id;" << endl << ++ indent() << "}" << endl; ++ ++ f_service_ << indent() << ++ "@java.lang.Override" << endl << ++ indent() << "public T newInstance() {" << endl; ++ indent_up(); ++ ++ f_service_ << indent() << ++ "Object o;" << endl << ++ indent() << "switch (id) {" << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "case ARG_IN_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << (*f_iter)->get_name() << "_args();" << ++ endl << indent() << " break;" << endl; ++ f_service_ << indent() << ++ "case ARG_OUT_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << (*f_iter)->get_name() << "_result();" << ++ endl << indent() << " break;" << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for (f_iter = functions.begin(); f_iter!= functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "case ARG_IN_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << extend_service_name << "." << (*f_iter)->get_name() << "_args();" << ++ endl << indent() << " break;" << endl; ++ f_service_ << indent() << ++ "case ARG_OUT_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << extend_service_name << "." << (*f_iter)->get_name() << "_result();" << ++ endl << indent() << " break;" << endl; ++ } ++ } ++ ++ f_service_ << indent() << ++ "default:" << endl << indent() << ++ " throw new AssertionError();" << endl << indent() << ++ "}" << endl; ++ ++ f_service_ << indent() << ++ "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "T t = (T) o;" << endl << indent() << ++ "return t;" << endl; ++ ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl; ++ ++ indent_down(); ++ f_service_ << indent() << "}" << endl; ++} ++ ++void t_java_generator::generate_service_impl_base(t_service* tservice) { ++ f_service_ << ++ indent() << "public static abstract class " << service_name_ << ++ "ImplBase implements " << service_name_ << ", io.grpc.BindableService {" << endl; ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << ++ "_args request, " << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnimplementedUnaryCall(METHOD_" << (*f_iter)->get_name() << ++ ", responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc" ; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ extend_service_name << "." << (*f_iter)->get_name() << ++ "_args request, " << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << extend_service_name ++ << "." << (*f_iter)->get_name() << "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnimplementedUnaryCall(METHOD_" << (*f_iter)->get_name() << ++ ", responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << ++ " public io.grpc.ServerServiceDefinition bindService() {" << endl; + indent_up(); ++ f_service_ << ++ indent() << "return " << service_name_ << "Grpc.bindService(this);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Abstract Service ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static abstract class Abstract" << service_name_ << ++ " extends " << service_name_ << "ImplBase {}" << endl << endl; ++} ++ ++void t_java_generator::generate_method_descriptors(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public static final io.grpc.MethodDescriptor<" << ++ (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << (*f_iter)->get_name() << "_result> METHOD_" << (*f_iter)->get_name() << ++ " = " << endl << indent() << " io.grpc.MethodDescriptor.create(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " io.grpc.MethodDescriptor.MethodType.UNARY," << endl << ++ indent() << " generateFullMethodName(" << "\"" << package_name_ << "." << ++ service_name_ << "\" , \"" << (*f_iter)->get_name() << "\")," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << (*f_iter)->get_name() << ++ "_args>( ARG_IN_METHOD_" << (*f_iter)->get_name() << "))," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << (*f_iter)->get_name() << ++ "_result>( ARG_OUT_METHOD_" << (*f_iter)->get_name() << ")));" << endl << endl; ++ indent_down(); ++ } ++ ++ if(tservice->get_extends() != nullptr) { ++ t_service* extends_service = tservice->get_extends(); ++ functions = extends_service->get_functions(); ++ string extend_service_name = extends_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public static final io.grpc.MethodDescriptor<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_result> METHOD_" ++ << (*f_iter)->get_name() << " = " << endl << indent() << ++ " io.grpc.MethodDescriptor.create(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " io.grpc.MethodDescriptor.MethodType.UNARY," << endl << ++ indent() << " generateFullMethodName(" << "\"" << package_name_ << "." << ++ service_name_ << "\" , \"" << (*f_iter)->get_name() << "\")," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args>( ARG_IN_METHOD_" << (*f_iter)->get_name() << "))," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << extend_service_name << "." << (*f_iter)->get_name() << ++ "_result>( ARG_OUT_METHOD_" << (*f_iter)->get_name() << ")));" << endl << endl; ++ indent_down(); ++ } ++ } ++} ++ ++void t_java_generator::generate_stub(t_service* tservice) { ++ f_service_ << ++ indent() << ++ "public static " << service_name_ << ++ "Stub newStub(io.grpc.Channel channel) {" << ++ endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << ++ "return new " << service_name_ << "Stub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "Stub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "Stub>" << endl << ++ indent() << " implements " << service_name_ << "{" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "Stub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "Stub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "Stub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "Stub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ (*f_iter)->get_name() << "_args request," << endl << indent() << ++ " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request, responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_args request," ++ << endl << indent() << " io.grpc.stub.StreamObserver<" << ++ extend_service_name << "." << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request, responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_blocking_stub(t_service* tservice) { ++ f_service_ << ++ indent() << "public static " << service_name_ << ++ "BlockingStub newBlockingStub(" << endl << ++ indent() << " io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "BlockingStub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Blocking Client ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static interface " << service_name_ << ++ "BlockingClient " ; ++ ++ if (tservice->get_extends() != nullptr) { ++ string extend_service_name = tservice->get_extends()->get_name(); ++ f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << ++ extend_service_name << "BlockingClient " ; ++ } ++ ++ f_service_ << "{" << endl; ++ ++ indent_up(); ++ + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { +- generate_java_doc(f_service_, *f_iter); +- indent(f_service_) << "public " << function_signature(*f_iter) << ";" << endl << endl; ++ f_service_ << ++ indent() << "public " << (*f_iter)->get_name() << "_result " << ++ (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << "_args request);" << endl << endl; ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Blocking Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "BlockingStub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "BlockingStub>" << endl << ++ indent() << " implements " << service_name_ << "BlockingClient {"; ++ ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "BlockingStub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "BlockingStub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "BlockingStub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "BlockingStub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public " << (*f_iter)->get_name() << "_result " << (*f_iter)->get_name() << "(" << ++ (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return blockingUnaryCall(" << endl << ++ indent() << " getChannel(), METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions(), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public " << extend_service_name << "." << (*f_iter)->get_name() << ++ "_result " << (*f_iter)->get_name() << "(" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return blockingUnaryCall(" << endl << ++ indent() << " getChannel(), METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions(), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_future_stub(t_service* tservice) { ++ f_service_ << ++ indent() << "public static " << service_name_ << ++ "FutureStub newFutureStub(" << endl << ++ indent() << " io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "FutureStub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Future Client ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static interface " << service_name_ << ++ "FutureClient " ; ++ ++ if (tservice->get_extends() != nullptr) { ++ string extend_service_name = tservice->get_extends()->get_name(); ++ f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << ++ extend_service_name << "FutureClient " ; ++ } ++ f_service_ << "{" << endl; ++ ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ (*f_iter)->get_name() << "_result> " << (*f_iter)->get_name() << "(" << endl << ++ indent() << " " << (*f_iter)->get_name() << "_args request);" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_result> " << ++ (*f_iter)->get_name() << "(" << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << ++ "_args request);" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "FutureStub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "FutureStub>" << endl << ++ indent() << " implements " << service_name_ << "FutureClient {" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "FutureStub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "FutureStub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "FutureStub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "FutureStub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ functions = tservice->get_functions(); ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ (*f_iter)->get_name() << "_result> " << (*f_iter)->get_name() << "(" << ++ endl << indent() << " " << (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return futureUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_result> " << ++ (*f_iter)->get_name() << "(" << endl << indent() << " " << ++ extend_service_name << "." << (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return futureUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_method_ids(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << "private static final int METHODID_" << ++ (*f_iter)->get_name() << " = " << i << ";" << endl; ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << "private static final int METHODID_" << ++ (*f_iter)->get_name() << " = " << i << ";" << endl; ++ } ++ } ++ f_service_ << endl; ++} ++ ++void t_java_generator::generate_method_handlers(t_service* tservice) { ++ f_service_ << ++ indent() << "private static class MethodHandlers implements" << ++ endl << indent() << " io.grpc.stub.ServerCalls.UnaryMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.ServerStreamingMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.ClientStreamingMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.BidiStreamingMethod {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "private final " << service_name_ << " serviceImpl;" << endl << ++ indent() << "private final int methodId;" << endl << endl; ++ ++ f_service_ << ++ indent() << "public MethodHandlers(" << service_name_ << " serviceImpl, int " << ++ "methodId) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "this.serviceImpl = serviceImpl;" << endl << ++ indent() << "this.methodId = methodId;" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // invoke ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "switch (methodId) {" << endl; ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "case METHODID_" << (*f_iter)->get_name() << ":" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "serviceImpl." << (*f_iter)->get_name() << "((" << (*f_iter)->get_name() << ++ "_args) request," << endl << ++ indent() << " (io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << "_result>)" << ++ " responseObserver);" << endl << ++ indent() << "break;" << endl << endl; ++ indent_down(); ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "case METHODID_" << (*f_iter)->get_name() << ":" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "serviceImpl." << (*f_iter)->get_name() << "((" << extend_service_name << ++ "." << (*f_iter)->get_name() << "_args) request," << endl << ++ indent() << " (io.grpc.stub.StreamObserver<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_result>)" << " responseObserver);" << endl << ++ indent() << "break;" << endl << endl; ++ indent_down(); ++ } + } ++ f_service_ << ++ indent() << "default:" << endl << ++ indent() << " throw new AssertionError();" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // invoke ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "public io.grpc.stub.StreamObserver invoke(" << endl << ++ indent() << " io.grpc.stub.StreamObserver responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "switch (methodId) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "default:" << endl; ++ f_service_ << ++ indent() << " throw new AssertionError();" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl; + indent_down(); + f_service_ << indent() << "}" << endl << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++ + } + ++void t_java_generator::generate_service_descriptors(t_service* tservice) { ++ // generate service descriptor ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ f_service_ << ++ indent() << "public static io.grpc.ServiceDescriptor getServiceDescriptor() {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new io.grpc.ServiceDescriptor(SERVICE_NAME"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "," << endl << ++ indent() << " METHOD_" << (*f_iter)->get_name(); ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "," << endl << ++ indent() << " METHOD_" << (*f_iter)->get_name(); ++ } ++ } ++ f_service_ << ");" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_service_builder(t_service* tservice) { ++ // bind Service ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static io.grpc.ServerServiceDefinition" << ++ " bindService(" << endl << ++ indent() << " final " << service_name_ << " serviceImpl) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor())" << ++ endl; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << " .addMethod(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " METHOD_" << (*f_iter)->get_name() << "," << endl << ++ indent() << " asyncUnaryCall(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " new MethodHandlers<" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " " << (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << (*f_iter)->get_name() << "_result>(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " serviceImpl, METHODID_" << (*f_iter)->get_name() << ")))" << endl; ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << " .addMethod(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " METHOD_" << (*f_iter)->get_name() << "," << endl << ++ indent() << " asyncUnaryCall(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " new MethodHandlers<" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_result>(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " serviceImpl, METHODID_" << (*f_iter)->get_name() << ")))" << endl; ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ } ++ } ++ f_service_ << ++ indent() << " .build();" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++ + void t_java_generator::generate_service_async_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; +diff --git a/tutorial/Makefile.am b/tutorial/Makefile.am +index 5865c54..1cffbe6 100755 +--- a/tutorial/Makefile.am ++++ b/tutorial/Makefile.am +@@ -35,11 +35,6 @@ if WITH_D + SUBDIRS += d + endif + +-if WITH_JAVA +-SUBDIRS += java +-SUBDIRS += js +-endif +- + if WITH_PYTHON + SUBDIRS += py + SUBDIRS += py.twisted +@@ -95,4 +90,5 @@ EXTRA_DIST = \ + php \ + shared.thrift \ + tutorial.thrift \ +- README.md ++ README.md \ ++ java +-- +2.8.0.rc3.226.g39d4020 + From b4280fa084ab57878fae069f59f1d834cceb1546 Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Thu, 28 Jul 2016 23:48:50 -0700 Subject: [PATCH 17/97] compare test config as objects instead of strings --- src/csharp/Grpc.Core.Tests/SanityTest.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs index 501992c5695..9d069fa432a 100644 --- a/src/csharp/Grpc.Core.Tests/SanityTest.cs +++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs @@ -58,10 +58,11 @@ namespace Grpc.Core.Tests [Test] public void TestsJsonUpToDate() { - var discoveredTests = DiscoverAllTestClasses(); - string discoveredTestsJson = JsonConvert.SerializeObject(discoveredTests, Formatting.Indented); + Dictionary> discoveredTests = DiscoverAllTestClasses(); + Dictionary> testsFromFile + = JsonConvert.DeserializeObject>>(ReadTestsJson()); - Assert.AreEqual(discoveredTestsJson, ReadTestsJson()); + Assert.AreEqual(discoveredTests, testsFromFile); } /// From a17039aaa443ac8b4f8d5bad51fd6e8c2777071f Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:32:06 -0700 Subject: [PATCH 18/97] refine code and add README --- Makefile | 1 - build.yaml | 1 - .../grpc++/impl/codegen/thrift_serializer.h | 209 ++++++++++++------ .../impl/codegen/thrift_serializer_inl.h | 169 -------------- include/grpc++/impl/codegen/thrift_utils.h | 10 +- tools/grift/README.md | 15 ++ tools/grift/grpc_plugins_generator.patch | 187 +++++++++++++++- tools/run_tests/sources_and_headers.json | 2 - .../grpc++_test_util/grpc++_test_util.vcxproj | 1 - .../grpc++_test_util.vcxproj.filters | 3 - 10 files changed, 342 insertions(+), 256 deletions(-) delete mode 100644 include/grpc++/impl/codegen/thrift_serializer_inl.h create mode 100644 tools/grift/README.md diff --git a/Makefile b/Makefile index 986c25780f2..31a8448fff6 100644 --- a/Makefile +++ b/Makefile @@ -3924,7 +3924,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ include/grpc++/impl/codegen/thrift_serializer.h \ - include/grpc++/impl/codegen/thrift_serializer_inl.h \ include/grpc++/impl/codegen/thrift_utils.h \ LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC)))) diff --git a/build.yaml b/build.yaml index 35af7a6c7a5..02ad6a8842b 100644 --- a/build.yaml +++ b/build.yaml @@ -784,7 +784,6 @@ filegroups: language: c++ public_headers: - include/grpc++/impl/codegen/thrift_serializer.h - - include/grpc++/impl/codegen/thrift_serializer_inl.h - include/grpc++/impl/codegen/thrift_utils.h uses: - grpc++_codegen_base diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 315d76cf2a5..46112ee5b23 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -36,12 +36,16 @@ #include #include - +#include +#include +#include +#include +#include #include #include +#include #include #include -#include namespace apache { namespace thrift { @@ -49,100 +53,165 @@ namespace util { using apache::thrift::protocol::TBinaryProtocolT; using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TMessageType; using apache::thrift::protocol::TNetworkBigEndian; using apache::thrift::transport::TMemoryBuffer; using apache::thrift::transport::TBufferBase; using apache::thrift::transport::TTransport; -using std::shared_ptr; - -template -class ThriftSerializer { +template class ThriftSerializer { public: ThriftSerializer() - : prepared_ (false) - , lastDeserialized_ (false) - , serializeVersion_ (false) {} - - /** - * Serialize the passed type into the internal buffer - * and returns a pointer to internal buffer and its size - * - */ - template - void serialize(const T& fields, const uint8_t** serializedBuffer, - size_t* serializedLen); - - /** - * Serialize the passed type into the byte buffer - */ - template - void serialize(const T& fields, grpc_byte_buffer** bp); - - /** - * Deserialize the passed char array into the passed type, returns the number - * of bytes that have been consumed from the passed string. - */ - template - uint32_t deserialize(const uint8_t* serializedBuffer, size_t length, - T* fields); - - /** - * Deserialize the passed byte buffer to passed type, returns the number - * of bytes consumed from byte buffer - */ - template - uint32_t deserialize(grpc_byte_buffer* buffer, T* msg); - - void setSerializeVersion(bool value); + : prepared_ (false) + , last_deserialized_ (false) + , serialize_version_ (false) {} virtual ~ThriftSerializer() {} + // Serialize the passed type into the internal buffer + // and returns a pointer to internal buffer and its size + template void Serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen) { + // prepare or reset buffer + if (!prepared_ || last_deserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + last_deserialized_ = false; - /** - * Set the container size limit to deserialize - * This function should be called after buffer_ is initialized - */ - void setContainerSizeLimit(int32_t container_limit) { + // if required serialize protocol version + if (serialize_version_) { + protocol_->writeMessageBegin("", TMessageType(0), 0); + } + + // serilaize fields into buffer + fields.write(protocol_.get()); + + // write the end of message + if (serialize_version_) { + protocol_->writeMessageEnd(); + } + + uint8_t* byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; + } + + // Serialize the passed type into the byte buffer + template void Serialize(const T& fields, grpc_byte_buffer** bp) { + + const uint8_t* byteBuffer; + size_t byteBufferSize; + + Serialize(fields, &byteBuffer, &byteBufferSize); + + gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + + *bp = grpc_raw_byte_buffer_create(&slice, 1); + + gpr_slice_unref(slice); + } + + // Deserialize the passed char array into the passed type, returns the number + // of bytes that have been consumed from the passed string. + template uint32_t Deserialize(const uint8_t* serializedBuffer, size_t length, + T* fields) { + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + last_deserialized_ = true; + + //reset buffer transport + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // read the protocol version if necessary + if (serialize_version_) { + std::string name = ""; + TMessageType mt = (TMessageType) 0; + int32_t seq_id = 0; + protocol_->readMessageBegin(name, mt, seq_id); + } + + // deserialize buffer into fields + uint32_t len = fields->read(protocol_.get()); + + // read the end of message + if (serialize_version_) { + protocol_->readMessageEnd(); + } + + return len; + } + + + // Deserialize the passed byte buffer to passed type, returns the number + // of bytes consumed from byte buffer + template uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { + + grpc_byte_buffer_reader reader; + grpc_byte_buffer_reader_init(&reader, buffer); + + gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); + + uint32_t len = Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); + + gpr_slice_unref(slice); + + grpc_byte_buffer_reader_destroy(&reader); + + return len; + } + + // set serialization version flag + void SetSerializeVersion(bool value) { + serialize_version_ = value; + } + + // Set the container size limit to deserialize + // This function should be called after buffer_ is initialized + void SetContainerSizeLimit(int32_t container_limit) { if (!prepared_) { prepare(); } protocol_->setContainerSizeLimit(container_limit); } - /** - * Set the string size limit to deserialize - * This function should be called after buffer_ is initialized - */ - void setStringSizeLimit(int32_t string_limit) { + // Set the string size limit to deserialize + // This function should be called after buffer_ is initialized + void SetStringSizeLimit(int32_t string_limit) { if (!prepared_) { prepare(); } protocol_->setStringSizeLimit(string_limit); } +private: + bool prepared_; + bool last_deserialized_; + boost::shared_ptr buffer_; + std::shared_ptr protocol_; + bool serialize_version_; - private: - void prepare(); + void prepare() { + buffer_.reset(new TMemoryBuffer()); + + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + + prepared_ = true; + } - private: - typedef P Protocol; - bool prepared_; - bool lastDeserialized_; - boost::shared_ptr buffer_; - shared_ptr protocol_; - bool serializeVersion_; }; // ThriftSerializer -template -struct ThriftSerializerBinary : public ThriftSerializer > {}; +typedef ThriftSerializer> ThriftSerializerBinary; +typedef ThriftSerializer> ThriftSerializerCompact; +} // namespace util +} // namespace thrift +} // namespace apache -template -struct ThriftSerializerCompact : public ThriftSerializer >{ }; - -}}} // namespace apache::thrift::util - -#include - -#endif +#endif \ No newline at end of file diff --git a/include/grpc++/impl/codegen/thrift_serializer_inl.h b/include/grpc++/impl/codegen/thrift_serializer_inl.h deleted file mode 100644 index 866ecf6312d..00000000000 --- a/include/grpc++/impl/codegen/thrift_serializer_inl.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - - #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H - #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H - -#include -#include -#include -#include -#include -#include -#include - -namespace apache { -namespace thrift { -namespace util { - -using apache::thrift::protocol::TMessageType; - -template -template -void ThriftSerializer::serialize(const T& fields, - const uint8_t** serializedBuffer, size_t* serializedLen) { - - // prepare or reset buffer - if (!prepared_ || lastDeserialized_) { - prepare(); - } else { - buffer_->resetBuffer(); - } - lastDeserialized_ = false; - - // if required serialize protocol version - if (serializeVersion_) { - protocol_->writeMessageBegin("", TMessageType(0), 0); - } - - // serilaize fields into buffer - fields.write(protocol_.get()); - - // write the end of message - if (serializeVersion_) { - protocol_->writeMessageEnd(); - } - - // assign buffer to string - uint8_t* byteBuffer; - uint32_t byteBufferSize; - buffer_->getBuffer(&byteBuffer, &byteBufferSize); - *serializedBuffer = byteBuffer; - *serializedLen = byteBufferSize; -} - -template -template -void ThriftSerializer::serialize(const T& fields, grpc_byte_buffer** bp) { - - const uint8_t* byteBuffer; - size_t byteBufferSize; - serialize(fields, &byteBuffer, &byteBufferSize); - - gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); - - *bp = grpc_raw_byte_buffer_create(&slice, 1); - - gpr_slice_unref(slice); -} - -template -template -uint32_t ThriftSerializer::deserialize(const uint8_t* serializedBuffer, - size_t length, T* fields) { - // prepare buffer if necessary - if (!prepared_) { - prepare(); - } - lastDeserialized_ = true; - - //reset buffer transport - buffer_->resetBuffer((uint8_t*)serializedBuffer, length); - - // read the protocol version if necessary - if (serializeVersion_) { - std::string name = ""; - TMessageType mt = (TMessageType) 0; - int32_t seq_id = 0; - protocol_->readMessageBegin(name, mt, seq_id); - } - - // deserialize buffer into fields - uint32_t len = fields->read(protocol_.get()); - - // read the end of message - if (serializeVersion_) { - protocol_->readMessageEnd(); - } - - return len; -} - -template -template -uint32_t ThriftSerializer::deserialize(grpc_byte_buffer* bp, T* fields) { - grpc_byte_buffer_reader reader; - grpc_byte_buffer_reader_init(&reader, bp); - - gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); - - uint32_t len = deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), fields); - - gpr_slice_unref(slice); - - grpc_byte_buffer_reader_destroy(&reader); - - return len; -} - -template -void ThriftSerializer::setSerializeVersion(bool value) { - serializeVersion_ = value; -} - -template -void -ThriftSerializer::prepare() -{ - - buffer_.reset(new TMemoryBuffer()); - - // create a protocol for the memory buffer transport - protocol_.reset(new Protocol(buffer_)); - - prepared_ = true; -} - -}}} // namespace apache::thrift::util - -#endif diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h index 629441149fd..14332c05219 100644 --- a/include/grpc++/impl/codegen/thrift_utils.h +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -44,9 +44,7 @@ #include #include #include -#include #include -#include #include namespace grpc { @@ -62,9 +60,9 @@ class SerializationTraits serializer; + ThriftSerializerCompact serializer; - serializer.serialize(msg, bp); + serializer.Serialize(msg, bp); return Status(StatusCode::OK, "ok"); } @@ -76,8 +74,8 @@ class SerializationTraits deserializer; - deserializer.deserialize(buffer, msg); + ThriftSerializerCompact deserializer; + deserializer.Deserialize(buffer, msg); grpc_byte_buffer_destroy(buffer); diff --git a/tools/grift/README.md b/tools/grift/README.md new file mode 100644 index 00000000000..a4cb87bff1a --- /dev/null +++ b/tools/grift/README.md @@ -0,0 +1,15 @@ +Copyright 2016 Google Inc. + +#Documentation + +grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with GRPC. + +This integration allows you to use grpc to send thrift messages in C++ and java. + +By default grift uses Compact Protocol to serialize thrift messages. + +#Installation + +Before Installing thrift make sure to apply this [patch](grpc_plugins_generate.patch) to third_party/thrift. +Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to +install thrift. \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 2779dfb59eb..9c18db35995 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -1,7 +1,7 @@ From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:47:47 -0700 -Subject: [PATCH 1/3] don't build tests +Subject: [PATCH 1/5] don't build tests --- Makefile.am | 7 ++----- @@ -62,7 +62,7 @@ index 6fd15d2..7de1fad 100755 From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 -Subject: [PATCH 2/3] grpc cpp plugins generator with example +Subject: [PATCH 2/5] grpc cpp plugins generator with example --- compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- @@ -1401,7 +1401,7 @@ index 0000000..de3c9a4 From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 -Subject: [PATCH 3/3] grpc java plugins generator +Subject: [PATCH 3/5] grpc java plugins generator for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift --- @@ -2415,3 +2415,184 @@ index 5865c54..1cffbe6 100755 -- 2.8.0.rc3.226.g39d4020 + +From a9769a0fa08f553da76215ca59a8fd797b92a853 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Mon, 1 Aug 2016 16:56:36 -0700 +Subject: [PATCH 4/5] maintain consistency with grpc + +--- + compiler/cpp/src/generate/t_cpp_generator.cc | 20 ++++++++++---------- + tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} | 0 + tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} | 0 + tutorial/cpp/Makefile.am | 8 ++++---- + 4 files changed, 14 insertions(+), 14 deletions(-) + rename tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} (100%) + rename tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} (100%) + +diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc +index 9c3399b..4e00129 100644 +--- a/compiler/cpp/src/generate/t_cpp_generator.cc ++++ b/compiler/cpp/src/generate/t_cpp_generator.cc +@@ -1641,7 +1641,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + << endl; + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != nullptr) { ++ if (extends_service) { + f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) + << extends_service->get_name() << ".grpc.thrift.h\"" << endl; + } +@@ -1733,7 +1733,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + +@@ -1749,9 +1749,9 @@ void t_cpp_generator::generate_service(t_service* tservice) { + "};" << endl; + + // Generate service class +- if ( extends_service != nullptr ) { ++ if ( extends_service) { + f_header_ << "class " << service_name_ << " : public " << +- extends_service->get_name() << " {" << endl << ++ type_name(extends_service) << " {" << endl << + "public:" << endl; + } + else { +@@ -1841,7 +1841,7 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { + + string extends = ""; +- if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; + } + +@@ -1890,7 +1890,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + } + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -1914,7 +1914,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -1944,7 +1944,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -2002,7 +2002,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { +@@ -2049,7 +2049,7 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } + + string extends = ""; +- if (tservice->get_extends() != NULL) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; + if (style == "CobCl" && gen_templates_) { + // TODO(simpkins): If gen_templates_ is enabled, we currently assume all +diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/GrpcClient.cpp +similarity index 100% +rename from tutorial/cpp/CppClient.cpp +rename to tutorial/cpp/GrpcClient.cpp +diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/GrpcServer.cpp +similarity index 100% +rename from tutorial/cpp/CppServer.cpp +rename to tutorial/cpp/GrpcServer.cpp +diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am +index 581f75e..39d85e1 100755 +--- a/tutorial/cpp/Makefile.am ++++ b/tutorial/cpp/Makefile.am +@@ -39,14 +39,14 @@ noinst_PROGRAMS = \ + TestClient + + TestServer_SOURCES = \ +- CppServer.cpp ++ GrpcServer.cpp + + TestServer_LDADD = \ + libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + + TestClient_SOURCES = \ +- CppClient.cpp ++ GrpcClient.cpp + + TestClient_LDADD = \ + libtestgencpp.la \ +@@ -78,5 +78,5 @@ style-local: + + EXTRA_DIST = \ + CMakeLists.txt \ +- CppClient.cpp \ +- CppServer.cpp ++ GrpcClient.cpp \ ++ GrpcServer.cpp +-- +2.8.0.rc3.226.g39d4020 + + +From b4bc0c810f00a1b86516774306ff4017e3d2d252 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Mon, 1 Aug 2016 17:00:17 -0700 +Subject: [PATCH 5/5] fix typo + +--- + tutorial/cpp/GrpcClient.cpp | 2 +- + tutorial/cpp/GrpcServer.cpp | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +index c41604e..41a7acf 100644 +--- a/tutorial/cpp/GrpcClient.cpp ++++ b/tutorial/cpp/GrpcClient.cpp +@@ -1,6 +1,6 @@ + /* + * +- * Copyright 2015, Google Inc. ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +index c838b61..f63db57 100644 +--- a/tutorial/cpp/GrpcServer.cpp ++++ b/tutorial/cpp/GrpcServer.cpp +@@ -1,6 +1,6 @@ + /* + * +- * Copyright 2015, Google Inc. ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +-- +2.8.0.rc3.226.g39d4020 + diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 06ac86fc9c4..82b443a4742 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -6792,14 +6792,12 @@ ], "headers": [ "include/grpc++/impl/codegen/thrift_serializer.h", - "include/grpc++/impl/codegen/thrift_serializer_inl.h", "include/grpc++/impl/codegen/thrift_utils.h" ], "language": "c++", "name": "thrift_util", "src": [ "include/grpc++/impl/codegen/thrift_serializer.h", - "include/grpc++/impl/codegen/thrift_serializer_inl.h", "include/grpc++/impl/codegen/thrift_utils.h" ], "third_party": false, diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj index 96ce3b89164..c2c7d00a6d9 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj @@ -201,7 +201,6 @@ - diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters index 2105e672df9..9b8c8ddfada 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters @@ -195,9 +195,6 @@ include\grpc++\impl\codegen - - include\grpc++\impl\codegen - include\grpc++\impl\codegen From f5dcc9b5b5d1ade84eb7bac07abbac56f5f86675 Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:57:24 -0700 Subject: [PATCH 19/97] fix typos --- tools/grift/README.md | 16 ++++++++++++++-- tools/grift/grpc_plugins_generator.patch | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tools/grift/README.md b/tools/grift/README.md index a4cb87bff1a..25c67451326 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -6,10 +6,22 @@ grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Se This integration allows you to use grpc to send thrift messages in C++ and java. -By default grift uses Compact Protocol to serialize thrift messages. +grift uses Compact Protocol to serialize thrift messages. + +##generating grpc plugins for thrift services + +###CPP +```sh + $ thrift --gen cpp +``` + +###JAVA +```sh + $ thrift --gen java +``` #Installation -Before Installing thrift make sure to apply this [patch](grpc_plugins_generate.patch) to third_party/thrift. +Before Installing thrift make sure to apply this [patch](grpc_plugins_generator.patch) to third_party/thrift. Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to install thrift. \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 9c18db35995..eeee1612519 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -2559,18 +2559,18 @@ index 581f75e..39d85e1 100755 2.8.0.rc3.226.g39d4020 -From b4bc0c810f00a1b86516774306ff4017e3d2d252 Mon Sep 17 00:00:00 2001 +From bc74fca1ee73333819724f51d5aaff3546443ed0 Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:00:17 -0700 Subject: [PATCH 5/5] fix typo --- - tutorial/cpp/GrpcClient.cpp | 2 +- + tutorial/cpp/GrpcClient.cpp | 4 ++-- tutorial/cpp/GrpcServer.cpp | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp -index c41604e..41a7acf 100644 +index c41604e..ab1fe77 100644 --- a/tutorial/cpp/GrpcClient.cpp +++ b/tutorial/cpp/GrpcClient.cpp @@ -1,6 +1,6 @@ @@ -2581,6 +2581,15 @@ index c41604e..41a7acf 100644 * All rights reserved. * * Redistribution and use in source and binary forms, with or without +@@ -50,7 +50,7 @@ class GreeterClient { + GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} + +- // Assambles the client's payload, sends it and presents the response back ++ // Assembles the client's payload, sends it and presents the response back + // from the server. + std::string SayHello(const std::string& user) { + // Data we are sending to the server. diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp index c838b61..f63db57 100644 --- a/tutorial/cpp/GrpcServer.cpp From 00be9de95cf040e2705c137ea213140562f7571d Mon Sep 17 00:00:00 2001 From: chedeti Date: Tue, 2 Aug 2016 10:44:41 -0700 Subject: [PATCH 20/97] fix codegen patch --- tools/grift/grpc_plugins_generator.patch | 722 ++++++++++------------- 1 file changed, 302 insertions(+), 420 deletions(-) diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index eeee1612519..3a6710c224e 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -1,7 +1,7 @@ From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:47:47 -0700 -Subject: [PATCH 1/5] don't build tests +Subject: [PATCH 1/3] don't build tests --- Makefile.am | 7 ++----- @@ -59,24 +59,30 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 +From c8577ad5513543c57a81ad1bf4927cc8a78baa03 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 -Subject: [PATCH 2/5] grpc cpp plugins generator with example +Subject: [PATCH 2/3] grpc cpp plugins generator with example --- - compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- + compiler/cpp/src/generate/t_cpp_generator.cc | 478 +++++++++++++++++++++++---- tutorial/cpp/CMakeLists.txt | 53 --- - tutorial/cpp/CppClient.cpp | 134 ++++---- - tutorial/cpp/CppServer.cpp | 226 ++++--------- - tutorial/cpp/Makefile.am | 58 ++-- + tutorial/cpp/CppClient.cpp | 80 ----- + tutorial/cpp/CppServer.cpp | 181 ---------- + tutorial/cpp/GrpcClient.cpp | 94 ++++++ + tutorial/cpp/GrpcServer.cpp | 87 +++++ + tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 6 files changed, 590 insertions(+), 370 deletions(-) + 8 files changed, 636 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt + delete mode 100644 tutorial/cpp/CppClient.cpp + delete mode 100644 tutorial/cpp/CppServer.cpp + create mode 100644 tutorial/cpp/GrpcClient.cpp + create mode 100644 tutorial/cpp/GrpcServer.cpp create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 6c04899..9c3399b 100644 +index 6c04899..4e00129 100644 --- a/compiler/cpp/src/generate/t_cpp_generator.cc +++ b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -162,6 +162,8 @@ public: @@ -266,7 +272,7 @@ index 6c04899..9c3399b 100644 t_service* extends_service = tservice->get_extends(); - if (extends_service != NULL) { -+ if (extends_service != nullptr) { ++ if (extends_service) { f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) - << extends_service->get_name() << ".h\"" << endl; + << extends_service->get_name() << ".grpc.thrift.h\"" << endl; @@ -355,7 +361,7 @@ index 6c04899..9c3399b 100644 + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + @@ -371,9 +377,9 @@ index 6c04899..9c3399b 100644 + "};" << endl; + + // Generate service class -+ if ( extends_service != nullptr ) { ++ if ( extends_service) { + f_header_ << "class " << service_name_ << " : public " << -+ extends_service->get_name() << " {" << endl << ++ type_name(extends_service) << " {" << endl << + "public:" << endl; + } + else { @@ -442,7 +448,7 @@ index 6c04899..9c3399b 100644 +void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { + + string extends = ""; -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; + } + @@ -491,7 +497,7 @@ index 6c04899..9c3399b 100644 + } + + t_service* extends_service = tservice->get_extends(); -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -515,7 +521,7 @@ index 6c04899..9c3399b 100644 + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -545,7 +551,7 @@ index 6c04899..9c3399b 100644 + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -603,7 +609,7 @@ index 6c04899..9c3399b 100644 + + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { @@ -642,11 +648,13 @@ index 6c04899..9c3399b 100644 if (style == "CobCl") { // Forward declare the client. string client_name = service_name_ + "CobClient"; -@@ -1765,12 +2050,14 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1764,13 +2049,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } string extends = ""; - if (tservice->get_extends() != NULL) { +- if (tservice->get_extends() != NULL) { - extends = " : virtual public " + type_name(tservice->get_extends()) + style + "If"; ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; if (style == "CobCl" && gen_templates_) { // TODO(simpkins): If gen_templates_ is enabled, we currently assume all @@ -867,11 +875,12 @@ index 8a3d085..0000000 -LINK_AGAINST_THRIFT_LIBRARY(TutorialClient thrift) -target_link_libraries(TutorialClient ${ZLIB_LIBRARIES}) diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/CppClient.cpp -index 2763fee..c41604e 100644 +deleted file mode 100644 +index 2763fee..0000000 --- a/tutorial/cpp/CppClient.cpp -+++ b/tutorial/cpp/CppClient.cpp -@@ -1,80 +1,94 @@ - /* ++++ /dev/null +@@ -1,80 +0,0 @@ +-/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information @@ -879,106 +888,52 @@ index 2763fee..c41604e 100644 - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * +- * - * http://www.apache.org/licenses/LICENSE-2.0 -+ * Copyright 2015, Google Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * * Neither the name of Google Inc. nor the names of its -+ * contributors may be used to endorse or promote products derived from -+ * this software without specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * +- * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - #include -+#include -+#include - +- */ +- +-#include +- -#include -#include -#include - -#include "../gen-cpp/Calculator.h" -+#include - +- -using namespace std; -using namespace apache::thrift; -using namespace apache::thrift::protocol; -using namespace apache::thrift::transport; -+#include "gen-cpp/Greeter.grpc.thrift.h" - +- -using namespace tutorial; -using namespace shared; -+using grpc::Channel; -+using grpc::ClientContext; -+using grpc::Status; -+using test::Greeter; -+using namespace test; - +- -int main() { - boost::shared_ptr socket(new TSocket("localhost", 9090)); - boost::shared_ptr transport(new TBufferedTransport(socket)); - boost::shared_ptr protocol(new TBinaryProtocol(transport)); - CalculatorClient client(protocol); -+class GreeterClient { -+ public: -+ GreeterClient(std::shared_ptr channel) -+ : stub_(Greeter::NewStub(channel)) {} - +- - try { - transport->open(); -+ // Assambles the client's payload, sends it and presents the response back -+ // from the server. -+ std::string SayHello(const std::string& user) { -+ // Data we are sending to the server. -+ Greeter::SayHelloReq req; -+ req.request.name = user; - +- - client.ping(); - cout << "ping()" << endl; -+ // Container for the data we expect from the server. -+ Greeter::SayHelloResp reply; - +- - cout << "1 + 1 = " << client.add(1, 1) << endl; -+ // Context for the client. It could be used to convey extra information to -+ // the server and/or tweak certain RPC behaviors. -+ ClientContext context; - +- - Work work; - work.op = Operation::DIVIDE; - work.num1 = 1; - work.num2 = 0; -+ // The actual RPC. -+ Status status = stub_->SayHello(&context, req, &reply); - +- - try { - client.calculate(1, work); - cout << "Whoa? We can divide by zero!" << endl; @@ -986,51 +941,32 @@ index 2763fee..c41604e 100644 - cout << "InvalidOperation: " << io.why << endl; - // or using generated operator<<: cout << io << endl; - // or by using std::exception native method what(): cout << io.what() << endl; -+ // Act upon its status. -+ if (status.ok()) { -+ return reply.success.message; -+ } else { -+ return "RPC failed"; - } -+ } - +- } +- - work.op = Operation::SUBTRACT; - work.num1 = 15; - work.num2 = 10; - int32_t diff = client.calculate(1, work); - cout << "15 - 10 = " << diff << endl; -+ private: -+ std::unique_ptr stub_; -+}; - +- - // Note that C++ uses return by reference for complex types to avoid - // costly copy construction - SharedStruct ss; - client.getStruct(ss, 1); - cout << "Received log: " << ss << endl; -+int main() { -+ // Instantiate the client. It requires a channel, out of which the actual RPCs -+ // are created. This channel models a connection to an endpoint (in this case, -+ // localhost at port 50051). We indicate that the channel isn't authenticated -+ // (use of InsecureChannelCredentials()). -+ GreeterClient greeter(grpc::CreateChannel( -+ "localhost:50051", grpc::InsecureChannelCredentials())); -+ std::string user("world"); -+ std::string reply = greeter.SayHello(user); -+ std::cout << "Greeter received: " << reply << std::endl; - +- - transport->close(); - } catch (TException& tx) { - cout << "ERROR: " << tx.what() << endl; - } -+ return 0; - } +-} diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp -index eafffa9..c838b61 100644 +deleted file mode 100644 +index eafffa9..0000000 --- a/tutorial/cpp/CppServer.cpp -+++ b/tutorial/cpp/CppServer.cpp -@@ -1,181 +1,87 @@ - /* ++++ /dev/null +@@ -1,181 +0,0 @@ +-/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information @@ -1038,45 +974,17 @@ index eafffa9..c838b61 100644 - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * +- * - * http://www.apache.org/licenses/LICENSE-2.0 -+ * Copyright 2015, Google Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * * Neither the name of Google Inc. nor the names of its -+ * contributors may be used to endorse or promote products derived from -+ * this software without specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * +- * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - +- */ +- -#include -#include -#include @@ -1090,55 +998,37 @@ index eafffa9..c838b61 100644 - -#include - - #include +-#include -#include -#include - -#include "../gen-cpp/Calculator.h" -+#include -+#include - +- -using namespace std; -using namespace apache::thrift; -using namespace apache::thrift::concurrency; -using namespace apache::thrift::protocol; -using namespace apache::thrift::transport; -using namespace apache::thrift::server; -+#include - +- -using namespace tutorial; -using namespace shared; -+#include "gen-cpp/Greeter.grpc.thrift.h" -+#include - +- -class CalculatorHandler : public CalculatorIf { -public: - CalculatorHandler() {} -+using grpc::Server; -+using grpc::ServerBuilder; -+using grpc::ServerContext; -+using grpc::Status; -+using test::Greeter; - +- - void ping() { cout << "ping()" << endl; } -+using namespace grpc; -+using namespace test; - +- - int32_t add(const int32_t n1, const int32_t n2) { - cout << "add(" << n1 << ", " << n2 << ")" << endl; - return n1 + n2; - } -+// Logic and data behind the server's behavior. -+class GreeterServiceImpl final : public Greeter::Service { -+ Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, -+ Greeter::SayHelloResp* reply) override { -+ std::string prefix("Hello "); - +- - int32_t calculate(const int32_t logid, const Work& work) { - cout << "calculate(" << logid << ", " << work << ")" << endl; - int32_t val; -+ reply->success.message = prefix + request->request.name; - +- - switch (work.op) { - case Operation::ADD: - val = work.num1 + work.num2; @@ -1172,8 +1062,7 @@ index eafffa9..c838b61 100644 - log[logid] = ss; - - return val; -+ return Status::OK; - } +- } - - void getStruct(SharedStruct& ret, const int32_t logid) { - cout << "getStruct(" << logid << ")" << endl; @@ -1184,8 +1073,8 @@ index eafffa9..c838b61 100644 - -protected: - map log; - }; - +-}; +- -/* - CalculatorIfFactory is code generated. - CalculatorCloneFactory is useful for getting access to the server side of the @@ -1209,26 +1098,8 @@ index eafffa9..c838b61 100644 - delete handler; - } -}; -+void RunServer() { -+ std::string server_address("0.0.0.0:50051"); -+ GreeterServiceImpl service; -+ -+ ServerBuilder builder; -+ // Listen on the given address without any authentication mechanism. -+ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); -+ // Register "service" as the instance through which we'll communicate with -+ // clients. In this case it corresponds to an *synchronous* service. -+ builder.RegisterService(&service); -+ // Finally assemble the server. -+ std::unique_ptr server(builder.BuildAndStart()); -+ std::cout << "Server listening on " << server_address << std::endl; -+ -+ // Wait for the server to shutdown. Note that some other thread must be -+ // responsible for shutting down the server for this call to ever return. -+ server->Wait(); -+} - - int main() { +- +-int main() { - TThreadedServer server( - boost::make_shared(boost::make_shared()), - boost::make_shared(9090), //port @@ -1270,15 +1141,207 @@ index eafffa9..c838b61 100644 - boost::make_shared(), - threadManager); - */ -+ RunServer(); - +- - cout << "Starting the server..." << endl; - server.serve(); - cout << "Done." << endl; - return 0; - } +- return 0; +-} +diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +new file mode 100644 +index 0000000..ab1fe77 +--- /dev/null ++++ b/tutorial/cpp/GrpcClient.cpp +@@ -0,0 +1,94 @@ ++/* ++ * ++ * Copyright 2016, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "gen-cpp/Greeter.grpc.thrift.h" ++ ++using grpc::Channel; ++using grpc::ClientContext; ++using grpc::Status; ++using test::Greeter; ++using namespace test; ++ ++class GreeterClient { ++ public: ++ GreeterClient(std::shared_ptr channel) ++ : stub_(Greeter::NewStub(channel)) {} ++ ++ // Assembles the client's payload, sends it and presents the response back ++ // from the server. ++ std::string SayHello(const std::string& user) { ++ // Data we are sending to the server. ++ Greeter::SayHelloReq req; ++ req.request.name = user; ++ ++ // Container for the data we expect from the server. ++ Greeter::SayHelloResp reply; ++ ++ // Context for the client. It could be used to convey extra information to ++ // the server and/or tweak certain RPC behaviors. ++ ClientContext context; ++ ++ // The actual RPC. ++ Status status = stub_->SayHello(&context, req, &reply); ++ ++ // Act upon its status. ++ if (status.ok()) { ++ return reply.success.message; ++ } else { ++ return "RPC failed"; ++ } ++ } ++ ++ private: ++ std::unique_ptr stub_; ++}; ++ ++int main() { ++ // Instantiate the client. It requires a channel, out of which the actual RPCs ++ // are created. This channel models a connection to an endpoint (in this case, ++ // localhost at port 50051). We indicate that the channel isn't authenticated ++ // (use of InsecureChannelCredentials()). ++ GreeterClient greeter(grpc::CreateChannel( ++ "localhost:50051", grpc::InsecureChannelCredentials())); ++ std::string user("world"); ++ std::string reply = greeter.SayHello(user); ++ std::cout << "Greeter received: " << reply << std::endl; ++ ++ return 0; ++} +diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +new file mode 100644 +index 0000000..f63db57 +--- /dev/null ++++ b/tutorial/cpp/GrpcServer.cpp +@@ -0,0 +1,87 @@ ++/* ++ * ++ * Copyright 2016, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "gen-cpp/Greeter.grpc.thrift.h" ++#include ++ ++using grpc::Server; ++using grpc::ServerBuilder; ++using grpc::ServerContext; ++using grpc::Status; ++using test::Greeter; ++ ++using namespace grpc; ++using namespace test; ++ ++// Logic and data behind the server's behavior. ++class GreeterServiceImpl final : public Greeter::Service { ++ Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, ++ Greeter::SayHelloResp* reply) override { ++ std::string prefix("Hello "); ++ ++ reply->success.message = prefix + request->request.name; ++ ++ return Status::OK; ++ } ++}; ++ ++void RunServer() { ++ std::string server_address("0.0.0.0:50051"); ++ GreeterServiceImpl service; ++ ++ ServerBuilder builder; ++ // Listen on the given address without any authentication mechanism. ++ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); ++ // Register "service" as the instance through which we'll communicate with ++ // clients. In this case it corresponds to an *synchronous* service. ++ builder.RegisterService(&service); ++ // Finally assemble the server. ++ std::unique_ptr server(builder.BuildAndStart()); ++ std::cout << "Server listening on " << server_address << std::endl; ++ ++ // Wait for the server to shutdown. Note that some other thread must be ++ // responsible for shutting down the server for this call to ever return. ++ server->Wait(); ++} ++ ++int main() { ++ RunServer(); ++ ++ return 0; ++} diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 184a69d..581f75e 100755 +index 184a69d..39d85e1 100755 --- a/tutorial/cpp/Makefile.am +++ b/tutorial/cpp/Makefile.am @@ -18,44 +18,38 @@ @@ -1325,8 +1388,9 @@ index 184a69d..581f75e 100755 + TestClient -TutorialServer_SOURCES = \ +- CppServer.cpp +TestServer_SOURCES = \ - CppServer.cpp ++ GrpcServer.cpp -TutorialServer_LDADD = \ - libtutorialgencpp.la \ @@ -1335,8 +1399,9 @@ index 184a69d..581f75e 100755 $(top_builddir)/lib/cpp/libthrift.la -TutorialClient_SOURCES = \ +- CppClient.cpp +TestClient_SOURCES = \ - CppClient.cpp ++ GrpcClient.cpp -TutorialClient_LDADD = \ - libtutorialgencpp.la \ @@ -1345,7 +1410,7 @@ index 184a69d..581f75e 100755 $(top_builddir)/lib/cpp/libthrift.la # -@@ -63,21 +57,21 @@ TutorialClient_LDADD = \ +@@ -63,26 +57,26 @@ TutorialClient_LDADD = \ # THRIFT = $(top_builddir)/compiler/cpp/thrift @@ -1374,6 +1439,13 @@ index 184a69d..581f75e 100755 style-local: $(CPPSTYLE_CMD) + + EXTRA_DIST = \ + CMakeLists.txt \ +- CppClient.cpp \ +- CppServer.cpp ++ GrpcClient.cpp \ ++ GrpcServer.cpp diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift new file mode 100644 index 0000000..de3c9a4 @@ -1398,10 +1470,10 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 +From 096042c132126536870eea118127cf1e608969bc Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 -Subject: [PATCH 3/5] grpc java plugins generator +Subject: [PATCH 3/3] grpc java plugins generator for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift --- @@ -1410,7 +1482,7 @@ for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thr 2 files changed, 887 insertions(+), 27 deletions(-) diff --git a/compiler/cpp/src/generate/t_java_generator.cc b/compiler/cpp/src/generate/t_java_generator.cc -index 2db8cb8..95e1ca8 100644 +index 2db8cb8..8b28fe2 100644 --- a/compiler/cpp/src/generate/t_java_generator.cc +++ b/compiler/cpp/src/generate/t_java_generator.cc @@ -97,10 +97,10 @@ public: @@ -1487,7 +1559,7 @@ index 2db8cb8..95e1ca8 100644 +} + +string t_java_generator::import_extended_service(t_service* tservice) { -+ if (tservice == nullptr) { ++ if (!tservice) { + return string() + "\n"; + } + string ns = tservice->get_program()->get_namespace("java"); @@ -1575,7 +1647,7 @@ index 2db8cb8..95e1ca8 100644 + f_service_ << indent() << + "@java.lang.Deprecated public static interface " << service_name_; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + f_service_ << " extends " << tservice->get_extends()->get_name() + "Grpc." << + tservice->get_extends()->get_name() << endl; + } @@ -1610,7 +1682,7 @@ index 2db8cb8..95e1ca8 100644 + } + f_service_ << endl; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + f_service_ << indent() << "// ARG IDs for extended service" << endl; + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); @@ -1663,7 +1735,7 @@ index 2db8cb8..95e1ca8 100644 + endl << indent() << " break;" << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -1721,7 +1793,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc" ; @@ -1786,7 +1858,7 @@ index 2db8cb8..95e1ca8 100644 + indent_down(); + } + -+ if(tservice->get_extends() != nullptr) { ++ if(tservice->get_extends()) { + t_service* extends_service = tservice->get_extends(); + functions = extends_service->get_functions(); + string extend_service_name = extends_service->get_name() + "Grpc"; @@ -1886,7 +1958,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -1930,7 +2002,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "@java.lang.Deprecated public static interface " << service_name_ << + "BlockingClient " ; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + string extend_service_name = tservice->get_extends()->get_name(); + f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << + extend_service_name << "BlockingClient " ; @@ -2008,7 +2080,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2050,7 +2122,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "@java.lang.Deprecated public static interface " << service_name_ << + "FutureClient " ; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + string extend_service_name = tservice->get_extends()->get_name(); + f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << + extend_service_name << "FutureClient " ; @@ -2068,7 +2140,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << " " << (*f_iter)->get_name() << "_args request);" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2141,7 +2213,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2176,7 +2248,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "private static final int METHODID_" << + (*f_iter)->get_name() << " = " << i << ";" << endl; + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2238,7 +2310,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "break;" << endl << endl; + indent_down(); + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2303,7 +2375,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "," << endl << + indent() << " METHOD_" << (*f_iter)->get_name(); + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { @@ -2352,7 +2424,7 @@ index 2db8cb8..95e1ca8 100644 + indent_down(); + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2415,193 +2487,3 @@ index 5865c54..1cffbe6 100755 -- 2.8.0.rc3.226.g39d4020 - -From a9769a0fa08f553da76215ca59a8fd797b92a853 Mon Sep 17 00:00:00 2001 -From: chedeti -Date: Mon, 1 Aug 2016 16:56:36 -0700 -Subject: [PATCH 4/5] maintain consistency with grpc - ---- - compiler/cpp/src/generate/t_cpp_generator.cc | 20 ++++++++++---------- - tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} | 0 - tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} | 0 - tutorial/cpp/Makefile.am | 8 ++++---- - 4 files changed, 14 insertions(+), 14 deletions(-) - rename tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} (100%) - rename tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} (100%) - -diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 9c3399b..4e00129 100644 ---- a/compiler/cpp/src/generate/t_cpp_generator.cc -+++ b/compiler/cpp/src/generate/t_cpp_generator.cc -@@ -1641,7 +1641,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { - << endl; - - t_service* extends_service = tservice->get_extends(); -- if (extends_service != nullptr) { -+ if (extends_service) { - f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) - << extends_service->get_name() << ".grpc.thrift.h\"" << endl; - } -@@ -1733,7 +1733,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { - indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - vector functions = extends_service->get_functions(); - vector::iterator f_iter; - -@@ -1749,9 +1749,9 @@ void t_cpp_generator::generate_service(t_service* tservice) { - "};" << endl; - - // Generate service class -- if ( extends_service != nullptr ) { -+ if ( extends_service) { - f_header_ << "class " << service_name_ << " : public " << -- extends_service->get_name() << " {" << endl << -+ type_name(extends_service) << " {" << endl << - "public:" << endl; - } - else { -@@ -1841,7 +1841,7 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { - void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { - - string extends = ""; -- if (tservice->get_extends() != nullptr) { -+ if (tservice->get_extends()) { - extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; - } - -@@ -1890,7 +1890,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - } - - t_service* extends_service = tservice->get_extends(); -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -1914,7 +1914,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -1944,7 +1944,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -2002,7 +2002,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - vector functions = extends_service->get_functions(); - vector::iterator f_iter; - for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { -@@ -2049,7 +2049,7 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty - } - - string extends = ""; -- if (tservice->get_extends() != NULL) { -+ if (tservice->get_extends()) { - extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; - if (style == "CobCl" && gen_templates_) { - // TODO(simpkins): If gen_templates_ is enabled, we currently assume all -diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/GrpcClient.cpp -similarity index 100% -rename from tutorial/cpp/CppClient.cpp -rename to tutorial/cpp/GrpcClient.cpp -diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/GrpcServer.cpp -similarity index 100% -rename from tutorial/cpp/CppServer.cpp -rename to tutorial/cpp/GrpcServer.cpp -diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 581f75e..39d85e1 100755 ---- a/tutorial/cpp/Makefile.am -+++ b/tutorial/cpp/Makefile.am -@@ -39,14 +39,14 @@ noinst_PROGRAMS = \ - TestClient - - TestServer_SOURCES = \ -- CppServer.cpp -+ GrpcServer.cpp - - TestServer_LDADD = \ - libtestgencpp.la \ - $(top_builddir)/lib/cpp/libthrift.la - - TestClient_SOURCES = \ -- CppClient.cpp -+ GrpcClient.cpp - - TestClient_LDADD = \ - libtestgencpp.la \ -@@ -78,5 +78,5 @@ style-local: - - EXTRA_DIST = \ - CMakeLists.txt \ -- CppClient.cpp \ -- CppServer.cpp -+ GrpcClient.cpp \ -+ GrpcServer.cpp --- -2.8.0.rc3.226.g39d4020 - - -From bc74fca1ee73333819724f51d5aaff3546443ed0 Mon Sep 17 00:00:00 2001 -From: chedeti -Date: Mon, 1 Aug 2016 17:00:17 -0700 -Subject: [PATCH 5/5] fix typo - ---- - tutorial/cpp/GrpcClient.cpp | 4 ++-- - tutorial/cpp/GrpcServer.cpp | 2 +- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp -index c41604e..ab1fe77 100644 ---- a/tutorial/cpp/GrpcClient.cpp -+++ b/tutorial/cpp/GrpcClient.cpp -@@ -1,6 +1,6 @@ - /* - * -- * Copyright 2015, Google Inc. -+ * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without -@@ -50,7 +50,7 @@ class GreeterClient { - GreeterClient(std::shared_ptr channel) - : stub_(Greeter::NewStub(channel)) {} - -- // Assambles the client's payload, sends it and presents the response back -+ // Assembles the client's payload, sends it and presents the response back - // from the server. - std::string SayHello(const std::string& user) { - // Data we are sending to the server. -diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp -index c838b61..f63db57 100644 ---- a/tutorial/cpp/GrpcServer.cpp -+++ b/tutorial/cpp/GrpcServer.cpp -@@ -1,6 +1,6 @@ - /* - * -- * Copyright 2015, Google Inc. -+ * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without --- -2.8.0.rc3.226.g39d4020 - From 7df3597eb0953f826487ed670bf0613e59542a0e Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 2 Aug 2016 14:38:00 -0700 Subject: [PATCH 21/97] Regenerate packages --- src/node/health_check/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 472a80b6e7f..224e4ad6ca5 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.0.0-pre1", + "grpc": "^1.0.0-pre2", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, From 82afcaa009e85cba16422b94a23387a3b7a0bd06 Mon Sep 17 00:00:00 2001 From: chedeti Date: Wed, 3 Aug 2016 16:38:05 -0700 Subject: [PATCH 22/97] rename class variables to snake_case --- .../grpc++/impl/codegen/thrift_serializer.h | 94 ++++++++++--------- include/grpc++/impl/codegen/thrift_utils.h | 21 ++--- tools/grift/Dockerfile | 4 +- tools/grift/README.md | 2 +- tools/grift/grpc_plugins_generator.patch | 93 +++++++++--------- 5 files changed, 111 insertions(+), 103 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 46112ee5b23..04dcc699deb 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -31,12 +31,9 @@ * */ - #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H - #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H +#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H +#define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H -#include -#include -#include #include #include #include @@ -46,6 +43,10 @@ #include #include #include +#include +#include +#include +#include namespace apache { namespace thrift { @@ -59,20 +60,22 @@ using apache::thrift::transport::TMemoryBuffer; using apache::thrift::transport::TBufferBase; using apache::thrift::transport::TTransport; -template class ThriftSerializer { -public: +template +class ThriftSerializer { + public: ThriftSerializer() - : prepared_ (false) - , last_deserialized_ (false) - , serialize_version_ (false) {} + : prepared_(false), + last_deserialized_(false), + serialize_version_(false) {} virtual ~ThriftSerializer() {} // Serialize the passed type into the internal buffer // and returns a pointer to internal buffer and its size - template void Serialize(const T& fields, const uint8_t** serializedBuffer, - size_t* serializedLen) { - // prepare or reset buffer + template + void Serialize(const T& fields, const uint8_t** serialized_buffer, + size_t* serialized_len) { + // prepare or reset buffer if (!prepared_ || last_deserialized_) { prepare(); } else { @@ -85,7 +88,7 @@ public: protocol_->writeMessageBegin("", TMessageType(0), 0); } - // serilaize fields into buffer + // serialize fields into buffer fields.write(protocol_.get()); // write the end of message @@ -93,22 +96,23 @@ public: protocol_->writeMessageEnd(); } - uint8_t* byteBuffer; - uint32_t byteBufferSize; - buffer_->getBuffer(&byteBuffer, &byteBufferSize); - *serializedBuffer = byteBuffer; - *serializedLen = byteBufferSize; + uint8_t* byte_buffer; + uint32_t byte_buffer_size; + buffer_->getBuffer(&byte_buffer, &byte_buffer_size); + *serialized_buffer = byte_buffer; + *serialized_len = byte_buffer_size; } // Serialize the passed type into the byte buffer - template void Serialize(const T& fields, grpc_byte_buffer** bp) { + template + void Serialize(const T& fields, grpc_byte_buffer** bp) { + const uint8_t* byte_buffer; + size_t byte_buffer_size; - const uint8_t* byteBuffer; - size_t byteBufferSize; + Serialize(fields, &byte_buffer, &byte_buffer_size); - Serialize(fields, &byteBuffer, &byteBufferSize); - - gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + gpr_slice slice = + gpr_slice_from_copied_buffer((char*)byte_buffer, byte_buffer_size); *bp = grpc_raw_byte_buffer_create(&slice, 1); @@ -117,21 +121,22 @@ public: // Deserialize the passed char array into the passed type, returns the number // of bytes that have been consumed from the passed string. - template uint32_t Deserialize(const uint8_t* serializedBuffer, size_t length, - T* fields) { + template + uint32_t Deserialize(const uint8_t* serialized_buffer, size_t length, + T* fields) { // prepare buffer if necessary if (!prepared_) { prepare(); } last_deserialized_ = true; - //reset buffer transport - buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + // reset buffer transport + buffer_->resetBuffer((uint8_t*)serialized_buffer, length); // read the protocol version if necessary if (serialize_version_) { std::string name = ""; - TMessageType mt = (TMessageType) 0; + TMessageType mt = (TMessageType)0; int32_t seq_id = 0; protocol_->readMessageBegin(name, mt, seq_id); } @@ -147,17 +152,17 @@ public: return len; } - // Deserialize the passed byte buffer to passed type, returns the number // of bytes consumed from byte buffer - template uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { - + template + uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { grpc_byte_buffer_reader reader; grpc_byte_buffer_reader_init(&reader, buffer); gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); - uint32_t len = Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); + uint32_t len = + Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); gpr_slice_unref(slice); @@ -167,9 +172,7 @@ public: } // set serialization version flag - void SetSerializeVersion(bool value) { - serialize_version_ = value; - } + void SetSerializeVersion(bool value) { serialize_version_ = value; } // Set the container size limit to deserialize // This function should be called after buffer_ is initialized @@ -189,7 +192,7 @@ public: protocol_->setStringSizeLimit(string_limit); } -private: + private: bool prepared_; bool last_deserialized_; boost::shared_ptr buffer_; @@ -197,6 +200,7 @@ private: bool serialize_version_; void prepare() { + buffer_.reset(new TMemoryBuffer()); // create a protocol for the memory buffer transport @@ -205,13 +209,15 @@ private: prepared_ = true; } -}; // ThriftSerializer +}; // ThriftSerializer -typedef ThriftSerializer> ThriftSerializerBinary; -typedef ThriftSerializer> ThriftSerializerCompact; +typedef ThriftSerializer> + ThriftSerializerBinary; +typedef ThriftSerializer> + ThriftSerializerCompact; -} // namespace util -} // namespace thrift -} // namespace apache +} // namespace util +} // namespace thrift +} // namespace apache #endif \ No newline at end of file diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h index 14332c05219..7d19b247f4c 100644 --- a/include/grpc++/impl/codegen/thrift_utils.h +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -34,16 +34,16 @@ #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H #define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H -#include -#include -#include -#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include @@ -52,23 +52,20 @@ namespace grpc { using apache::thrift::util::ThriftSerializerCompact; template -class SerializationTraits::value>::type> { +class SerializationTraits::value>::type> { public: - - static Status Serialize(const T& msg, - grpc_byte_buffer** bp, bool* own_buffer) { - + static Status Serialize(const T& msg, grpc_byte_buffer** bp, + bool* own_buffer) { *own_buffer = true; ThriftSerializerCompact serializer; - serializer.Serialize(msg, bp); return Status(StatusCode::OK, "ok"); } - static Status Deserialize(grpc_byte_buffer* buffer, - T* msg, + static Status Deserialize(grpc_byte_buffer* buffer, T* msg, int max_message_size) { if (!buffer) { return Status(StatusCode::INTERNAL, "No payload"); diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile index 5238010ea92..954640f0df0 100644 --- a/tools/grift/Dockerfile +++ b/tools/grift/Dockerfile @@ -46,8 +46,8 @@ RUN apt-get update && \ curl make automake libtool # Configure git -RUN git config --global user.name " " && \ - git config --global user.email " " +RUN git config --global user.name "Jenkins" && \ + git config --global user.email "jenkins@grpc" RUN git clone https://github.com/grpc/grpc diff --git a/tools/grift/README.md b/tools/grift/README.md index 25c67451326..2525f9b83dd 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -2,7 +2,7 @@ Copyright 2016 Google Inc. #Documentation -grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with GRPC. +grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with gRPC. This integration allows you to use grpc to send thrift messages in C++ and java. diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 3a6710c224e..a1d4cecde04 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -59,7 +59,7 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From c8577ad5513543c57a81ad1bf4927cc8a78baa03 Mon Sep 17 00:00:00 2001 +From e724d3abf096278615085bd58217321e32b43fd8 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 Subject: [PATCH 2/3] grpc cpp plugins generator with example @@ -69,16 +69,16 @@ Subject: [PATCH 2/3] grpc cpp plugins generator with example tutorial/cpp/CMakeLists.txt | 53 --- tutorial/cpp/CppClient.cpp | 80 ----- tutorial/cpp/CppServer.cpp | 181 ---------- - tutorial/cpp/GrpcClient.cpp | 94 ++++++ - tutorial/cpp/GrpcServer.cpp | 87 +++++ + tutorial/cpp/GriftClient.cpp | 93 ++++++ + tutorial/cpp/GriftServer.cpp | 93 ++++++ tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 8 files changed, 636 insertions(+), 416 deletions(-) + 8 files changed, 641 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt delete mode 100644 tutorial/cpp/CppClient.cpp delete mode 100644 tutorial/cpp/CppServer.cpp - create mode 100644 tutorial/cpp/GrpcClient.cpp - create mode 100644 tutorial/cpp/GrpcServer.cpp + create mode 100644 tutorial/cpp/GriftClient.cpp + create mode 100644 tutorial/cpp/GriftServer.cpp create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -1147,12 +1147,12 @@ index eafffa9..0000000 - cout << "Done." << endl; - return 0; -} -diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +diff --git a/tutorial/cpp/GriftClient.cpp b/tutorial/cpp/GriftClient.cpp new file mode 100644 -index 0000000..ab1fe77 +index 0000000..647a683 --- /dev/null -+++ b/tutorial/cpp/GrpcClient.cpp -@@ -0,0 +1,94 @@ ++++ b/tutorial/cpp/GriftClient.cpp +@@ -0,0 +1,93 @@ +/* + * + * Copyright 2016, Google Inc. @@ -1198,7 +1198,6 @@ index 0000000..ab1fe77 +using grpc::ClientContext; +using grpc::Status; +using test::Greeter; -+using namespace test; + +class GreeterClient { + public: @@ -1247,12 +1246,12 @@ index 0000000..ab1fe77 + + return 0; +} -diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +diff --git a/tutorial/cpp/GriftServer.cpp b/tutorial/cpp/GriftServer.cpp new file mode 100644 -index 0000000..f63db57 +index 0000000..7c01606 --- /dev/null -+++ b/tutorial/cpp/GrpcServer.cpp -@@ -0,0 +1,87 @@ ++++ b/tutorial/cpp/GriftServer.cpp +@@ -0,0 +1,93 @@ +/* + * + * Copyright 2016, Google Inc. @@ -1301,11 +1300,14 @@ index 0000000..f63db57 +using grpc::Status; +using test::Greeter; + -+using namespace grpc; -+using namespace test; -+ +// Logic and data behind the server's behavior. +class GreeterServiceImpl final : public Greeter::Service { ++ public: ++ ~GreeterServiceImpl() { ++ // shutdown server ++ server->Shutdown(); ++ } ++ + Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, + Greeter::SayHelloResp* reply) override { + std::string prefix("Hello "); @@ -1314,34 +1316,37 @@ index 0000000..f63db57 + + return Status::OK; + } ++ ++ void RunServer() { ++ std::string server_address("0.0.0.0:50051"); ++ ++ ServerBuilder builder; ++ // Listen on the given address without any authentication mechanism. ++ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); ++ // Register "service" as the instance through which we'll communicate with ++ // clients. In this case it corresponds to an *synchronous* service. ++ builder.RegisterService(this); ++ // Finally assemble the server. ++ server = builder.BuildAndStart(); ++ std::cout << "Server listening on " << server_address << std::endl; ++ ++ // Wait for the server to shutdown. Note that some other thread must be ++ // responsible for shutting down the server for this call to ever return. ++ server->Wait(); ++ } ++ ++ private: ++ std::unique_ptr server; +}; + -+void RunServer() { -+ std::string server_address("0.0.0.0:50051"); -+ GreeterServiceImpl service; -+ -+ ServerBuilder builder; -+ // Listen on the given address without any authentication mechanism. -+ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); -+ // Register "service" as the instance through which we'll communicate with -+ // clients. In this case it corresponds to an *synchronous* service. -+ builder.RegisterService(&service); -+ // Finally assemble the server. -+ std::unique_ptr server(builder.BuildAndStart()); -+ std::cout << "Server listening on " << server_address << std::endl; -+ -+ // Wait for the server to shutdown. Note that some other thread must be -+ // responsible for shutting down the server for this call to ever return. -+ server->Wait(); -+} -+ +int main() { -+ RunServer(); ++ GreeterServiceImpl service; ++ service.RunServer(); + + return 0; +} diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 184a69d..39d85e1 100755 +index 184a69d..6f91e28 100755 --- a/tutorial/cpp/Makefile.am +++ b/tutorial/cpp/Makefile.am @@ -18,44 +18,38 @@ @@ -1390,7 +1395,7 @@ index 184a69d..39d85e1 100755 -TutorialServer_SOURCES = \ - CppServer.cpp +TestServer_SOURCES = \ -+ GrpcServer.cpp ++ GriftServer.cpp -TutorialServer_LDADD = \ - libtutorialgencpp.la \ @@ -1401,7 +1406,7 @@ index 184a69d..39d85e1 100755 -TutorialClient_SOURCES = \ - CppClient.cpp +TestClient_SOURCES = \ -+ GrpcClient.cpp ++ GriftClient.cpp -TutorialClient_LDADD = \ - libtutorialgencpp.la \ @@ -1444,8 +1449,8 @@ index 184a69d..39d85e1 100755 CMakeLists.txt \ - CppClient.cpp \ - CppServer.cpp -+ GrpcClient.cpp \ -+ GrpcServer.cpp ++ GriftClient.cpp \ ++ GriftServer.cpp diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift new file mode 100644 index 0000000..de3c9a4 @@ -1470,7 +1475,7 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From 096042c132126536870eea118127cf1e608969bc Mon Sep 17 00:00:00 2001 +From f991f33dd6461eae197b6ad0e7088b571f2a7b22 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 Subject: [PATCH 3/3] grpc java plugins generator From 2e698457f370091111d0fe620bad306c743a3548 Mon Sep 17 00:00:00 2001 From: chedeti Date: Wed, 3 Aug 2016 16:45:18 -0700 Subject: [PATCH 23/97] sanity check for thrift --- tools/run_tests/sanity/check_submodules.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index b602d695649..e2bbd8cabfc 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -47,6 +47,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463) bdeb215cab2985195325fcd5e70c3fa751f46e0f third_party/protobuf (v3.0.0-beta-3.3) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) + bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift EOF diff -u $submodules $want_submodules From c6446bf3d8a2492cbe8bae213deb6b7cf547d95d Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Wed, 3 Aug 2016 15:17:01 -0700 Subject: [PATCH 24/97] add csharp hello world that uses dotnet cli --- .../helloworld-from-cli/Greeter/Helloworld.cs | 259 ++++++++++++++++++ .../Greeter/HelloworldGrpc.cs | 143 ++++++++++ .../helloworld-from-cli/Greeter/project.json | 19 ++ .../GreeterClient/Program.cs | 53 ++++ .../GreeterClient/project.json | 23 ++ .../GreeterServer/Program.cs | 66 +++++ .../GreeterServer/project.json | 23 ++ examples/csharp/helloworld-from-cli/README.md | 59 ++++ 8 files changed, 645 insertions(+) create mode 100644 examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs create mode 100644 examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs create mode 100644 examples/csharp/helloworld-from-cli/Greeter/project.json create mode 100644 examples/csharp/helloworld-from-cli/GreeterClient/Program.cs create mode 100644 examples/csharp/helloworld-from-cli/GreeterClient/project.json create mode 100644 examples/csharp/helloworld-from-cli/GreeterServer/Program.cs create mode 100644 examples/csharp/helloworld-from-cli/GreeterServer/project.json create mode 100644 examples/csharp/helloworld-from-cli/README.md diff --git a/examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs b/examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs new file mode 100644 index 00000000000..6477b4f35be --- /dev/null +++ b/examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs @@ -0,0 +1,259 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: helloworld.proto +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace Helloworld { + + /// Holder for reflection information generated from helloworld.proto + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public static partial class HelloworldReflection { + + #region Descriptor + /// File descriptor for helloworld.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static HelloworldReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "ChBoZWxsb3dvcmxkLnByb3RvEgpoZWxsb3dvcmxkIhwKDEhlbGxvUmVxdWVz", + "dBIMCgRuYW1lGAEgASgJIh0KCkhlbGxvUmVwbHkSDwoHbWVzc2FnZRgBIAEo", + "CTJJCgdHcmVldGVyEj4KCFNheUhlbGxvEhguaGVsbG93b3JsZC5IZWxsb1Jl", + "cXVlc3QaFi5oZWxsb3dvcmxkLkhlbGxvUmVwbHkiAEI2Chtpby5ncnBjLmV4", + "YW1wbGVzLmhlbGxvd29ybGRCD0hlbGxvV29ybGRQcm90b1ABogIDSExXYgZw", + "cm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { }, + new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloRequest), global::Helloworld.HelloRequest.Parser, new[]{ "Name" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloReply), global::Helloworld.HelloReply.Parser, new[]{ "Message" }, null, null, null) + })); + } + #endregion + + } + #region Messages + /// + /// The request message containing the user's name. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class HelloRequest : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new HelloRequest()); + public static pb::MessageParser Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[0]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public HelloRequest() { + OnConstruction(); + } + + partial void OnConstruction(); + + public HelloRequest(HelloRequest other) : this() { + name_ = other.name_; + } + + public HelloRequest Clone() { + return new HelloRequest(this); + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 1; + private string name_ = ""; + public string Name { + get { return name_; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + public override bool Equals(object other) { + return Equals(other as HelloRequest); + } + + public bool Equals(HelloRequest other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + return true; + } + + public override int GetHashCode() { + int hash = 1; + if (Name.Length != 0) hash ^= Name.GetHashCode(); + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + } + + public int CalculateSize() { + int size = 0; + if (Name.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + return size; + } + + public void MergeFrom(HelloRequest other) { + if (other == null) { + return; + } + if (other.Name.Length != 0) { + Name = other.Name; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 10: { + Name = input.ReadString(); + break; + } + } + } + } + + } + + /// + /// The response message containing the greetings + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class HelloReply : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new HelloReply()); + public static pb::MessageParser Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[1]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public HelloReply() { + OnConstruction(); + } + + partial void OnConstruction(); + + public HelloReply(HelloReply other) : this() { + message_ = other.message_; + } + + public HelloReply Clone() { + return new HelloReply(this); + } + + /// Field number for the "message" field. + public const int MessageFieldNumber = 1; + private string message_ = ""; + public string Message { + get { return message_; } + set { + message_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + public override bool Equals(object other) { + return Equals(other as HelloReply); + } + + public bool Equals(HelloReply other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Message != other.Message) return false; + return true; + } + + public override int GetHashCode() { + int hash = 1; + if (Message.Length != 0) hash ^= Message.GetHashCode(); + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + if (Message.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Message); + } + } + + public int CalculateSize() { + int size = 0; + if (Message.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Message); + } + return size; + } + + public void MergeFrom(HelloReply other) { + if (other == null) { + return; + } + if (other.Message.Length != 0) { + Message = other.Message; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 10: { + Message = input.ReadString(); + break; + } + } + } + } + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs b/examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs new file mode 100644 index 00000000000..041f5a78d75 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs @@ -0,0 +1,143 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: helloworld.proto +// Original file comments: +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#region Designer generated code + +using System; +using System.Threading; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Helloworld { + /// + /// The greeting service definition. + /// + public static class Greeter + { + static readonly string __ServiceName = "helloworld.Greeter"; + + static readonly Marshaller __Marshaller_HelloRequest = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloRequest.Parser.ParseFrom); + static readonly Marshaller __Marshaller_HelloReply = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloReply.Parser.ParseFrom); + + static readonly Method __Method_SayHello = new Method( + MethodType.Unary, + __ServiceName, + "SayHello", + __Marshaller_HelloRequest, + __Marshaller_HelloReply); + + /// Service descriptor + public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor + { + get { return global::Helloworld.HelloworldReflection.Descriptor.Services[0]; } + } + + /// Base class for server-side implementations of Greeter + public abstract class GreeterBase + { + /// + /// Sends a greeting + /// + public virtual global::System.Threading.Tasks.Task SayHello(global::Helloworld.HelloRequest request, ServerCallContext context) + { + throw new RpcException(new Status(StatusCode.Unimplemented, "")); + } + + } + + /// Client for Greeter + public class GreeterClient : ClientBase + { + /// Creates a new client for Greeter + /// The channel to use to make remote calls. + public GreeterClient(Channel channel) : base(channel) + { + } + /// Creates a new client for Greeter that uses a custom CallInvoker. + /// The callInvoker to use to make remote calls. + public GreeterClient(CallInvoker callInvoker) : base(callInvoker) + { + } + /// Protected parameterless constructor to allow creation of test doubles. + protected GreeterClient() : base() + { + } + /// Protected constructor to allow creation of configured clients. + /// The client configuration. + protected GreeterClient(ClientBaseConfiguration configuration) : base(configuration) + { + } + + /// + /// Sends a greeting + /// + public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return SayHello(request, new CallOptions(headers, deadline, cancellationToken)); + } + /// + /// Sends a greeting + /// + public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, CallOptions options) + { + return CallInvoker.BlockingUnaryCall(__Method_SayHello, null, options, request); + } + /// + /// Sends a greeting + /// + public virtual AsyncUnaryCall SayHelloAsync(global::Helloworld.HelloRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return SayHelloAsync(request, new CallOptions(headers, deadline, cancellationToken)); + } + /// + /// Sends a greeting + /// + public virtual AsyncUnaryCall SayHelloAsync(global::Helloworld.HelloRequest request, CallOptions options) + { + return CallInvoker.AsyncUnaryCall(__Method_SayHello, null, options, request); + } + protected override GreeterClient NewInstance(ClientBaseConfiguration configuration) + { + return new GreeterClient(configuration); + } + } + + /// Creates service definition that can be registered with a server + public static ServerServiceDefinition BindService(GreeterBase serviceImpl) + { + return ServerServiceDefinition.CreateBuilder() + .AddMethod(__Method_SayHello, serviceImpl.SayHello).Build(); + } + + } +} +#endregion diff --git a/examples/csharp/helloworld-from-cli/Greeter/project.json b/examples/csharp/helloworld-from-cli/Greeter/project.json new file mode 100644 index 00000000000..e06854d1226 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/Greeter/project.json @@ -0,0 +1,19 @@ +{ + "title": "Greeter", + "version": "1.0.0-*", + "buildOptions": { + "debugType": "portable", + }, + "dependencies": { + "Google.Protobuf": "3.0.0-beta3", + "Grpc": "1.0.0-pre1", + }, + "frameworks": { + "net45": { + "frameworkAssemblies": { + "System.Runtime": "", + "System.IO": "" + } + } + } +} diff --git a/examples/csharp/helloworld-from-cli/GreeterClient/Program.cs b/examples/csharp/helloworld-from-cli/GreeterClient/Program.cs new file mode 100644 index 00000000000..444d4735095 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/GreeterClient/Program.cs @@ -0,0 +1,53 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using System; +using Grpc.Core; +using Helloworld; + +namespace GreeterClient +{ + class Program + { + public static void Main(string[] args) + { + Channel channel = new Channel("127.0.0.1:50051", ChannelCredentials.Insecure); + + var client = new Greeter.GreeterClient(channel); + String user = "you"; + + var reply = client.SayHello(new HelloRequest { Name = user }); + Console.WriteLine("Greeting: " + reply.Message); + + channel.ShutdownAsync().Wait(); + Console.WriteLine("Press any key to exit..."); + Console.ReadKey(); + } + } +} diff --git a/examples/csharp/helloworld-from-cli/GreeterClient/project.json b/examples/csharp/helloworld-from-cli/GreeterClient/project.json new file mode 100644 index 00000000000..dc72a30eb81 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/GreeterClient/project.json @@ -0,0 +1,23 @@ +{ + "title": "GreeterClient", + "version": "1.0.0-*", + "buildOptions": { + "debugType": "portable", + "emitEntryPoint": "true" + }, + "dependencies": { + "Google.Protobuf": "3.0.0-beta3", + "Grpc": "1.0.0-pre1", + "Greeter": { + "target": "project" + } + }, + "frameworks": { + "net45": { + "frameworkAssemblies": { + "System.Runtime": "", + "System.IO": "" + } + } + } +} diff --git a/examples/csharp/helloworld-from-cli/GreeterServer/Program.cs b/examples/csharp/helloworld-from-cli/GreeterServer/Program.cs new file mode 100644 index 00000000000..fdab379e81d --- /dev/null +++ b/examples/csharp/helloworld-from-cli/GreeterServer/Program.cs @@ -0,0 +1,66 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using System; +using System.Threading.Tasks; +using Grpc.Core; +using Helloworld; + +namespace GreeterServer +{ + class GreeterImpl : Greeter.GreeterBase + { + // Server side handler of the SayHello RPC + public override Task SayHello(HelloRequest request, ServerCallContext context) + { + return Task.FromResult(new HelloReply { Message = "Hello " + request.Name }); + } + } + + class Program + { + const int Port = 50051; + + public static void Main(string[] args) + { + Server server = new Server + { + Services = { Greeter.BindService(new GreeterImpl()) }, + Ports = { new ServerPort("localhost", Port, ServerCredentials.Insecure) } + }; + server.Start(); + + Console.WriteLine("Greeter server listening on port " + Port); + Console.WriteLine("Press any key to stop the server..."); + Console.ReadKey(); + + server.ShutdownAsync().Wait(); + } + } +} diff --git a/examples/csharp/helloworld-from-cli/GreeterServer/project.json b/examples/csharp/helloworld-from-cli/GreeterServer/project.json new file mode 100644 index 00000000000..4bf201bef38 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/GreeterServer/project.json @@ -0,0 +1,23 @@ +{ + "title": "GreeterServer", + "version": "1.0.0-*", + "buildOptions": { + "debugType": "portable", + "emitEntryPoint": "true" + }, + "dependencies": { + "Google.Protobuf": "3.0.0-beta3", + "Grpc": "1.0.0-pre1", + "Greeter": { + "target": "project" + } + }, + "frameworks": { + "net45": { + "frameworkAssemblies": { + "System.Runtime": "", + "System.IO": "" + } + } + } +} diff --git a/examples/csharp/helloworld-from-cli/README.md b/examples/csharp/helloworld-from-cli/README.md new file mode 100644 index 00000000000..4db077631d8 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/README.md @@ -0,0 +1,59 @@ +gRPC in 3 minutes (C#) +======================== + +BACKGROUND +------------- +This is a different version of the helloworld example, using the dotnet sdk +tools to build and run. + +For this sample, we've already generated the server and client stubs from [helloworld.proto][]. + +Example projects in this directory depend on the [Grpc](https://www.nuget.org/packages/Grpc/) +and [Google.Protobuf](https://www.nuget.org/packages/Google.Protobuf/) NuGet packages +which have been already added to the project for you. + +The examples in this directory target .NET 4.5 framework, as .NET Core support is +currently experimental. + +PREREQUISITES +------------- + +- The DotNetCore SDK cli. + +- The .NET 4.5 framework. + +Both are available to download at https://www.microsoft.com/net/download + +BUILD +------- + +From the `examples/csharp/helloworld-from-cli` directory: + +- `dotnet restore` + +- `dotnet build **/project.json` (this will automatically download NuGet dependencies) + +Try it! +------- + +- Run the server + + ``` + > cd GreeterServer + > dotnet run + ``` + +- Run the client + + ``` + > cd GreeterClient + > dotnet run + ``` + +Tutorial +-------- + +You can find a more detailed tutorial about Grpc in [gRPC Basics: C#][] + +[helloworld.proto]:../../protos/helloworld.proto +[gRPC Basics: C#]:http://www.grpc.io/docs/tutorials/basic/csharp.html From 5b4d3625ebbf638080d07b7d267c5d90904d4f14 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 3 Aug 2016 15:33:37 -0700 Subject: [PATCH 25/97] Update ruby examples to use new _pb protobuf format --- examples/ruby/greeter_client.rb | 2 +- examples/ruby/greeter_server.rb | 2 +- examples/ruby/grpc-demo.gemspec | 4 ++-- examples/ruby/lib/{helloworld.rb => helloworld_pb.rb} | 0 .../lib/{helloworld_services.rb => helloworld_services_pb.rb} | 2 +- examples/ruby/lib/{route_guide.rb => route_guide_pb.rb} | 0 .../{route_guide_services.rb => route_guide_services_pb.rb} | 2 +- examples/ruby/route_guide/route_guide_client.rb | 2 +- examples/ruby/route_guide/route_guide_server.rb | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename examples/ruby/lib/{helloworld.rb => helloworld_pb.rb} (100%) rename examples/ruby/lib/{helloworld_services.rb => helloworld_services_pb.rb} (98%) rename examples/ruby/lib/{route_guide.rb => route_guide_pb.rb} (100%) rename examples/ruby/lib/{route_guide_services.rb => route_guide_services_pb.rb} (99%) diff --git a/examples/ruby/greeter_client.rb b/examples/ruby/greeter_client.rb index cb4aa195e77..1cdf79ebf40 100755 --- a/examples/ruby/greeter_client.rb +++ b/examples/ruby/greeter_client.rb @@ -38,7 +38,7 @@ lib_dir = File.join(this_dir, 'lib') $LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) require 'grpc' -require 'helloworld_services' +require 'helloworld_services_pb' def main stub = Helloworld::Greeter::Stub.new('localhost:50051', :this_channel_is_insecure) diff --git a/examples/ruby/greeter_server.rb b/examples/ruby/greeter_server.rb index 622513d3805..6d82043c526 100755 --- a/examples/ruby/greeter_server.rb +++ b/examples/ruby/greeter_server.rb @@ -38,7 +38,7 @@ lib_dir = File.join(this_dir, 'lib') $LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) require 'grpc' -require 'helloworld_services' +require 'helloworld_services_pb' # GreeterServer is simple server that implements the Helloworld Greeter server. class GreeterServer < Helloworld::Greeter::Service diff --git a/examples/ruby/grpc-demo.gemspec b/examples/ruby/grpc-demo.gemspec index b1dfdae6ab3..e1b77a56ac9 100644 --- a/examples/ruby/grpc-demo.gemspec +++ b/examples/ruby/grpc-demo.gemspec @@ -3,7 +3,7 @@ Gem::Specification.new do |s| s.name = 'grpc-demo' - s.version = '0.11.0' + s.version = '1.0.0' s.authors = ['gRPC Authors'] s.email = 'temiola@google.com' s.homepage = 'https://github.com/grpc/grpc' @@ -17,7 +17,7 @@ Gem::Specification.new do |s| s.require_paths = ['lib'] s.platform = Gem::Platform::RUBY - s.add_dependency 'grpc', '~> 0.11' + s.add_dependency 'grpc', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.7' end diff --git a/examples/ruby/lib/helloworld.rb b/examples/ruby/lib/helloworld_pb.rb similarity index 100% rename from examples/ruby/lib/helloworld.rb rename to examples/ruby/lib/helloworld_pb.rb diff --git a/examples/ruby/lib/helloworld_services.rb b/examples/ruby/lib/helloworld_services_pb.rb similarity index 98% rename from examples/ruby/lib/helloworld_services.rb rename to examples/ruby/lib/helloworld_services_pb.rb index fbec6677942..4fee0aa2a91 100644 --- a/examples/ruby/lib/helloworld_services.rb +++ b/examples/ruby/lib/helloworld_services_pb.rb @@ -32,7 +32,7 @@ # require 'grpc' -require 'helloworld' +require 'helloworld_pb' module Helloworld module Greeter diff --git a/examples/ruby/lib/route_guide.rb b/examples/ruby/lib/route_guide_pb.rb similarity index 100% rename from examples/ruby/lib/route_guide.rb rename to examples/ruby/lib/route_guide_pb.rb diff --git a/examples/ruby/lib/route_guide_services.rb b/examples/ruby/lib/route_guide_services_pb.rb similarity index 99% rename from examples/ruby/lib/route_guide_services.rb rename to examples/ruby/lib/route_guide_services_pb.rb index d8f123dd95b..d43fcc64e9a 100644 --- a/examples/ruby/lib/route_guide_services.rb +++ b/examples/ruby/lib/route_guide_services_pb.rb @@ -32,7 +32,7 @@ # require 'grpc' -require 'route_guide' +require 'route_guide_pb' module Routeguide module RouteGuide diff --git a/examples/ruby/route_guide/route_guide_client.rb b/examples/ruby/route_guide/route_guide_client.rb index e7f802c21e6..330725ece0d 100755 --- a/examples/ruby/route_guide/route_guide_client.rb +++ b/examples/ruby/route_guide/route_guide_client.rb @@ -39,7 +39,7 @@ $LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) require 'grpc' require 'multi_json' -require 'route_guide_services' +require 'route_guide_services_pb' include Routeguide diff --git a/examples/ruby/route_guide/route_guide_server.rb b/examples/ruby/route_guide/route_guide_server.rb index bebe49b3beb..a5a73a8bac1 100755 --- a/examples/ruby/route_guide/route_guide_server.rb +++ b/examples/ruby/route_guide/route_guide_server.rb @@ -40,7 +40,7 @@ $LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) require 'grpc' require 'multi_json' -require 'route_guide_services' +require 'route_guide_services_pb' include Routeguide COORD_FACTOR = 1e7 From 7d9f276ea278313f6ba1930931b5dde14167b8ac Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 4 Aug 2016 11:06:49 -0700 Subject: [PATCH 26/97] Change handshaker API to use a read buffer to pass leftover bytes read between handshakers. --- .../chttp2/client/insecure/channel_create.c | 19 +++++++--- .../client/insecure/channel_create_posix.c | 2 +- .../client/secure/secure_channel_create.c | 13 ++++--- .../chttp2/server/insecure/server_chttp2.c | 6 ++- .../server/insecure/server_chttp2_posix.c | 2 +- .../server/secure/server_secure_chttp2.c | 12 +++--- .../chttp2/transport/chttp2_transport.c | 7 +++- .../chttp2/transport/chttp2_transport.h | 4 +- src/core/lib/channel/handshaker.c | 25 ++++++++----- src/core/lib/channel/handshaker.h | 8 +++- .../lib/http/httpcli_security_connector.c | 8 +++- src/core/lib/security/transport/handshake.c | 9 ++++- src/core/lib/security/transport/handshake.h | 7 ++-- .../security/transport/security_connector.c | 37 +++++++++++++------ .../security/transport/security_connector.h | 14 ++++--- 15 files changed, 115 insertions(+), 58 deletions(-) diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index 6f6855584a7..cbaa75a90af 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -88,14 +88,21 @@ static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg, } static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, - grpc_channel_args *args, void *user_data, + grpc_channel_args *args, + gpr_slice_buffer *read_buffer, void *user_data, grpc_error *error) { connector *c = user_data; - c->result->transport = - grpc_create_chttp2_transport(exec_ctx, args, endpoint, 1); - GPR_ASSERT(c->result->transport); - grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL, 0); - c->result->channel_args = args; + if (error != GRPC_ERROR_NONE) { + grpc_channel_args_destroy(args); + gpr_free(read_buffer); + } else { + c->result->transport = + grpc_create_chttp2_transport(exec_ctx, args, endpoint, 1); + GPR_ASSERT(c->result->transport); + grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, + read_buffer); + c->result->channel_args = args; + } grpc_closure *notify = c->notify; c->notify = NULL; grpc_exec_ctx_sched(exec_ctx, notify, error, NULL); diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index ca435c25cec..b2c5e5b088c 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -75,7 +75,7 @@ grpc_channel *grpc_insecure_channel_create_from_fd( grpc_channel *channel = grpc_channel_create( &exec_ctx, target, final_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); grpc_channel_args_destroy(final_args); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index 4e33b6fa612..9e2bdd758f9 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -114,8 +114,7 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, gpr_mu_unlock(&c->mu); c->result->transport = grpc_create_chttp2_transport( exec_ctx, c->args.channel_args, secure_endpoint, 1); - grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL, - 0); + grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL); auth_context_arg = grpc_auth_context_to_arg(auth_context); c->result->channel_args = grpc_channel_args_copy_and_add(c->tmp_args, &auth_context_arg, 1); @@ -126,10 +125,13 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, } static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, - grpc_channel_args *args, void *user_data, + grpc_channel_args *args, + gpr_slice_buffer *read_buffer, void *user_data, grpc_error *error) { connector *c = user_data; + c->tmp_args = args; if (error != GRPC_ERROR_NONE) { + gpr_free(read_buffer); grpc_closure *notify = c->notify; c->notify = NULL; grpc_exec_ctx_sched(exec_ctx, notify, error, NULL); @@ -137,10 +139,9 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, // TODO(roth, jboeuf): Convert security connector handshaking to use new // handshake API, and then move the code from on_secure_handshake_done() // into this function. - c->tmp_args = args; grpc_channel_security_connector_do_handshake( - exec_ctx, c->security_connector, endpoint, c->args.deadline, - on_secure_handshake_done, c); + exec_ctx, c->security_connector, endpoint, read_buffer, + c->args.deadline, on_secure_handshake_done, c); } } diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c index 9cd374777e9..f0e07429faf 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c @@ -55,7 +55,8 @@ typedef struct server_connect_state { } server_connect_state; static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, - grpc_channel_args *args, void *user_data, + grpc_channel_args *args, + gpr_slice_buffer *read_buffer, void *user_data, grpc_error *error) { server_connect_state *state = user_data; if (error != GRPC_ERROR_NONE) { @@ -64,6 +65,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_error_free_string(error_str); GRPC_ERROR_UNREF(error); grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr); + gpr_free(read_buffer); } else { // Beware that the call to grpc_create_chttp2_transport() has to happen // before grpc_tcp_server_destroy(). This is fine here, but similar code @@ -75,7 +77,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_server_setup_transport(exec_ctx, state->server, transport, state->accepting_pollset, grpc_server_get_channel_args(state->server)); - grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(exec_ctx, transport, read_buffer); } // Clean up. grpc_channel_args_destroy(args); diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c index 96bf4d6f308..4350543c27a 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c @@ -67,7 +67,7 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server *server, &exec_ctx, server_args, server_endpoint, 0 /* is_client */); grpc_endpoint_add_to_pollset(&exec_ctx, server_endpoint, grpc_cq_pollset(cq)); grpc_server_setup_transport(&exec_ctx, server, transport, NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c index ccea15a648d..da3e284fcf2 100644 --- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c +++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c @@ -111,7 +111,7 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, grpc_server_setup_transport(exec_ctx, state->state->server, transport, state->accepting_pollset, args_copy); grpc_channel_args_destroy(args_copy); - grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL); } else { /* We need to consume this here, because the server may already have * gone away. */ @@ -128,7 +128,8 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, } static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, - grpc_channel_args *args, void *user_data, + grpc_channel_args *args, + gpr_slice_buffer *read_buffer, void *user_data, grpc_error *error) { server_secure_connect *state = user_data; if (error != GRPC_ERROR_NONE) { @@ -136,9 +137,10 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str); grpc_error_free_string(error_str); GRPC_ERROR_UNREF(error); + grpc_channel_args_destroy(args); + gpr_free(read_buffer); grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr); grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr); - grpc_channel_args_destroy(args); state_unref(state->state); gpr_free(state); return; @@ -150,8 +152,8 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, // into this function. state->args = args; grpc_server_security_connector_do_handshake( - exec_ctx, state->state->sc, state->acceptor, endpoint, state->deadline, - on_secure_handshake_done, state); + exec_ctx, state->state->sc, state->acceptor, endpoint, read_buffer, + state->deadline, on_secure_handshake_done, state); } static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp, diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 6bd65cea02c..f2f5465201d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2538,9 +2538,12 @@ grpc_transport *grpc_create_chttp2_transport( void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - gpr_slice *slices, size_t nslices) { + gpr_slice_buffer *read_buffer) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)transport; REF_TRANSPORT(t, "reading_action"); /* matches unref inside reading_action */ - gpr_slice_buffer_addn(&t->read_buffer, slices, nslices); + if (read_buffer != NULL) { + gpr_slice_buffer_move_into(read_buffer, &t->read_buffer); + gpr_free(read_buffer); + } reading_action(exec_ctx, t, GRPC_ERROR_NONE); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 5da4276f826..4e2d0954bf1 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -44,8 +44,10 @@ grpc_transport *grpc_create_chttp2_transport( grpc_exec_ctx *exec_ctx, const grpc_channel_args *channel_args, grpc_endpoint *ep, int is_client); +/// Takes ownership of \a read_buffer, which (if non-NULL) contains +/// leftover bytes previously read from the endpoint (e.g., by handshakers). void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - gpr_slice *slices, size_t nslices); + gpr_slice_buffer *read_buffer); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H */ diff --git a/src/core/lib/channel/handshaker.c b/src/core/lib/channel/handshaker.c index 6c3ca198b72..01fcbb20235 100644 --- a/src/core/lib/channel/handshaker.c +++ b/src/core/lib/channel/handshaker.c @@ -62,11 +62,13 @@ void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, + gpr_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data) { handshaker->vtable->do_handshake(exec_ctx, handshaker, endpoint, args, - deadline, acceptor, cb, user_data); + read_buffer, deadline, acceptor, cb, + user_data); } // @@ -143,16 +145,17 @@ void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx, // handshakers together. static void call_next_handshaker(grpc_exec_ctx* exec_ctx, grpc_endpoint* endpoint, - grpc_channel_args* args, void* user_data, - grpc_error* error) { + grpc_channel_args* args, + gpr_slice_buffer* read_buffer, + void* user_data, grpc_error* error) { grpc_handshake_manager* mgr = user_data; GPR_ASSERT(mgr->state != NULL); GPR_ASSERT(mgr->state->index < mgr->count); // If we got an error, skip all remaining handshakers and invoke the // caller-supplied callback immediately. if (error != GRPC_ERROR_NONE) { - mgr->state->final_cb(exec_ctx, endpoint, args, mgr->state->final_user_data, - error); + mgr->state->final_cb(exec_ctx, endpoint, args, read_buffer, + mgr->state->final_user_data, error); return; } grpc_handshaker_done_cb cb = call_next_handshaker; @@ -164,8 +167,9 @@ static void call_next_handshaker(grpc_exec_ctx* exec_ctx, } // Invoke handshaker. grpc_handshaker_do_handshake(exec_ctx, mgr->handshakers[mgr->state->index], - endpoint, args, mgr->state->deadline, - mgr->state->acceptor, cb, user_data); + endpoint, args, read_buffer, + mgr->state->deadline, mgr->state->acceptor, + cb, user_data); ++mgr->state->index; // If this is the last handshaker, clean up state. if (mgr->state->index == mgr->count) { @@ -180,10 +184,12 @@ void grpc_handshake_manager_do_handshake( gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data) { grpc_channel_args* args_copy = grpc_channel_args_copy(args); + gpr_slice_buffer* read_buffer = malloc(sizeof(*read_buffer)); + gpr_slice_buffer_init(read_buffer); if (mgr->count == 0) { // No handshakers registered, so we just immediately call the done // callback with the passed-in endpoint. - cb(exec_ctx, endpoint, args_copy, user_data, GRPC_ERROR_NONE); + cb(exec_ctx, endpoint, args_copy, read_buffer, user_data, GRPC_ERROR_NONE); } else { GPR_ASSERT(mgr->state == NULL); mgr->state = gpr_malloc(sizeof(struct grpc_handshaker_state)); @@ -192,6 +198,7 @@ void grpc_handshake_manager_do_handshake( mgr->state->acceptor = acceptor; mgr->state->final_cb = cb; mgr->state->final_user_data = user_data; - call_next_handshaker(exec_ctx, endpoint, args_copy, mgr, GRPC_ERROR_NONE); + call_next_handshaker(exec_ctx, endpoint, args_copy, read_buffer, mgr, + GRPC_ERROR_NONE); } } diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h index dfc469c417b..fac4935b16b 100644 --- a/src/core/lib/channel/handshaker.h +++ b/src/core/lib/channel/handshaker.h @@ -36,6 +36,7 @@ #include #include +#include #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/endpoint.h" @@ -56,10 +57,11 @@ typedef struct grpc_handshaker grpc_handshaker; /// Callback type invoked when a handshaker is done. -/// Takes ownership of \a args. +/// Takes ownership of \a args and \a read_buffer. typedef void (*grpc_handshaker_done_cb)(grpc_exec_ctx* exec_ctx, grpc_endpoint* endpoint, grpc_channel_args* args, + gpr_slice_buffer* read_buffer, void* user_data, grpc_error* error); struct grpc_handshaker_vtable { @@ -72,9 +74,12 @@ struct grpc_handshaker_vtable { /// Performs handshaking. When finished, calls \a cb with \a user_data. /// Takes ownership of \a args. + /// Takes ownership of \a read_buffer, which contains leftover bytes read + /// from the endpoint by the previous handshaker. /// \a acceptor will be NULL for client-side handshakers. void (*do_handshake)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, + gpr_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data); @@ -101,6 +106,7 @@ void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, + gpr_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data); diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index a57d93bb7bb..0006e809a6a 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -61,6 +61,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) { static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { @@ -69,6 +70,7 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, tsi_result result = TSI_OK; tsi_handshaker *handshaker; if (c->handshaker_factory == NULL) { + gpr_free(read_buffer); cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); return; } @@ -77,10 +79,12 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", tsi_result_to_string(result)); + gpr_free(read_buffer); cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, - nonsecure_endpoint, deadline, cb, user_data); + nonsecure_endpoint, read_buffer, deadline, cb, + user_data); } } @@ -183,7 +187,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, pem_root_certs, pem_root_certs_size, host, &sc) == GRPC_SECURITY_OK); grpc_channel_security_connector_do_handshake( - exec_ctx, sc, tcp, deadline, on_secure_transport_setup_done, c); + exec_ctx, sc, tcp, NULL, deadline, on_secure_transport_setup_done, c); GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli"); } diff --git a/src/core/lib/security/transport/handshake.c b/src/core/lib/security/transport/handshake.c index 540a17283d3..fbeec312b6f 100644 --- a/src/core/lib/security/transport/handshake.c +++ b/src/core/lib/security/transport/handshake.c @@ -325,8 +325,9 @@ static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { void grpc_do_security_handshake( grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, bool is_client_side, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, - grpc_security_handshake_done_cb cb, void *user_data) { + grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, + void *user_data) { grpc_security_connector_handshake_list *handshake_node; grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake)); memset(h, 0, sizeof(grpc_security_handshake)); @@ -346,6 +347,10 @@ void grpc_do_security_handshake( gpr_slice_buffer_init(&h->left_overs); gpr_slice_buffer_init(&h->outgoing); gpr_slice_buffer_init(&h->incoming); + if (read_buffer != NULL) { + gpr_slice_buffer_move_into(read_buffer, &h->incoming); + gpr_free(read_buffer); + } if (!is_client_side) { grpc_server_security_connector *server_connector = (grpc_server_security_connector *)connector; diff --git a/src/core/lib/security/transport/handshake.h b/src/core/lib/security/transport/handshake.h index c0906dd6af0..53092f54214 100644 --- a/src/core/lib/security/transport/handshake.h +++ b/src/core/lib/security/transport/handshake.h @@ -37,12 +37,13 @@ #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/security/transport/security_connector.h" -/* Calls the callback upon completion. Takes owership of handshaker. */ +/* Calls the callback upon completion. Takes owership of handshaker and + * read_buffer. */ void grpc_do_security_handshake( grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, bool is_client_side, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, - grpc_security_handshake_done_cb cb, void *user_data); + grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake); diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index f0ee6770e5f..1ee12319005 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -127,25 +127,29 @@ void grpc_server_security_connector_shutdown( void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, - grpc_security_handshake_done_cb cb, void *user_data) { + grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, + void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { + gpr_free(read_buffer); cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, deadline, cb, user_data); + sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, read_buffer, deadline, + cb, user_data); } } void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_timespec deadline, grpc_security_handshake_done_cb cb, - void *user_data) { + gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { + gpr_free(read_buffer); cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, deadline, cb, - user_data); + sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, read_buffer, + deadline, cb, user_data); } } @@ -312,23 +316,26 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base, - true, nonsecure_endpoint, deadline, cb, user_data); + true, nonsecure_endpoint, read_buffer, deadline, + cb, user_data); } static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base, - false, nonsecure_endpoint, deadline, cb, - user_data); + false, nonsecure_endpoint, read_buffer, deadline, + cb, user_data); } static grpc_security_connector_vtable fake_channel_vtable = { @@ -418,6 +425,7 @@ static grpc_security_status ssl_create_handshaker( static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { @@ -430,10 +438,12 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, : c->target_name, &handshaker); if (status != GRPC_SECURITY_OK) { + gpr_free(read_buffer); cb(exec_ctx, user_data, status, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, - nonsecure_endpoint, deadline, cb, user_data); + nonsecure_endpoint, read_buffer, deadline, cb, + user_data); } } @@ -441,6 +451,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { @@ -450,10 +461,12 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, grpc_security_status status = ssl_create_handshaker(c->handshaker_factory, false, NULL, &handshaker); if (status != GRPC_SECURITY_OK) { + gpr_free(read_buffer); cb(exec_ctx, user_data, status, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false, - nonsecure_endpoint, deadline, cb, user_data); + nonsecure_endpoint, read_buffer, deadline, cb, + user_data); } } diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index c2ddf5ee1eb..917100f8025 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -143,7 +143,8 @@ struct grpc_channel_security_connector { grpc_security_call_host_check_cb cb, void *user_data); void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; @@ -156,8 +157,9 @@ void grpc_channel_security_connector_check_call_host( /* Handshake. */ void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, - grpc_security_handshake_done_cb cb, void *user_data); + grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, + void *user_data); /* --- server_security_connector object. --- @@ -174,14 +176,16 @@ struct grpc_server_security_connector { void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, - grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); + gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data); void grpc_server_security_connector_shutdown( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector); From 28ee43b9c83a82614d0f09e5b214ab4ae771584c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 4 Aug 2016 11:11:46 -0700 Subject: [PATCH 27/97] Updated tests. --- test/core/bad_client/bad_client.c | 2 +- test/core/end2end/fixtures/h2_sockpair+trace.c | 4 ++-- test/core/end2end/fixtures/h2_sockpair.c | 4 ++-- test/core/end2end/fixtures/h2_sockpair_1byte.c | 4 ++-- test/core/end2end/fuzzers/api_fuzzer.c | 2 +- test/core/end2end/fuzzers/client_fuzzer.c | 2 +- test/core/end2end/fuzzers/server_fuzzer.c | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index 24ee3387a0f..be88d4a69a9 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -130,7 +130,7 @@ void grpc_run_bad_client_test( grpc_server_start(a.server); transport = grpc_create_chttp2_transport(&exec_ctx, NULL, sfd.server, 0); server_setup_transport(&a, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); /* Bind everything into the same pollset */ diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.c b/test/core/end2end/fixtures/h2_sockpair+trace.c index 6b0769b6088..b8a5257ab2a 100644 --- a/test/core/end2end/fixtures/h2_sockpair+trace.c +++ b/test/core/end2end/fixtures/h2_sockpair+trace.c @@ -108,7 +108,7 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, grpc_create_chttp2_transport(&exec_ctx, client_args, sfd->client, 1); client_setup_transport(&exec_ctx, &cs, transport); GPR_ASSERT(f->client); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } @@ -124,7 +124,7 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, transport = grpc_create_chttp2_transport(&exec_ctx, server_args, sfd->server, 0); server_setup_transport(f, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/end2end/fixtures/h2_sockpair.c b/test/core/end2end/fixtures/h2_sockpair.c index 7be88f8a68e..a57990d6e73 100644 --- a/test/core/end2end/fixtures/h2_sockpair.c +++ b/test/core/end2end/fixtures/h2_sockpair.c @@ -107,7 +107,7 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, grpc_create_chttp2_transport(&exec_ctx, client_args, sfd->client, 1); client_setup_transport(&exec_ctx, &cs, transport); GPR_ASSERT(f->client); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } @@ -123,7 +123,7 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, transport = grpc_create_chttp2_transport(&exec_ctx, server_args, sfd->server, 0); server_setup_transport(f, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.c b/test/core/end2end/fixtures/h2_sockpair_1byte.c index 166654bcbfd..50aac8045a9 100644 --- a/test/core/end2end/fixtures/h2_sockpair_1byte.c +++ b/test/core/end2end/fixtures/h2_sockpair_1byte.c @@ -107,7 +107,7 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, grpc_create_chttp2_transport(&exec_ctx, client_args, sfd->client, 1); client_setup_transport(&exec_ctx, &cs, transport); GPR_ASSERT(f->client); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } @@ -123,7 +123,7 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, transport = grpc_create_chttp2_transport(&exec_ctx, server_args, sfd->server, 0); server_setup_transport(f, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 13b8bf7561a..96ea82d95ea 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -258,7 +258,7 @@ static void do_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_transport *transport = grpc_create_chttp2_transport(exec_ctx, NULL, server, 0); grpc_server_setup_transport(exec_ctx, g_server, transport, NULL, NULL); - grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL); grpc_exec_ctx_sched(exec_ctx, fc->closure, GRPC_ERROR_NONE, NULL); } else { diff --git a/test/core/end2end/fuzzers/client_fuzzer.c b/test/core/end2end/fuzzers/client_fuzzer.c index 79b23d78569..00e650a30b9 100644 --- a/test/core/end2end/fuzzers/client_fuzzer.c +++ b/test/core/end2end/fuzzers/client_fuzzer.c @@ -63,7 +63,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_completion_queue *cq = grpc_completion_queue_create(NULL); grpc_transport *transport = grpc_create_chttp2_transport(&exec_ctx, NULL, mock_endpoint, 1); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_channel *channel = grpc_channel_create( &exec_ctx, "test-target", NULL, GRPC_CLIENT_DIRECT_CHANNEL, transport); diff --git a/test/core/end2end/fuzzers/server_fuzzer.c b/test/core/end2end/fuzzers/server_fuzzer.c index 80f568ac927..79eaad70c5d 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.c +++ b/test/core/end2end/fuzzers/server_fuzzer.c @@ -71,7 +71,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_transport *transport = grpc_create_chttp2_transport(&exec_ctx, NULL, mock_endpoint, 0); grpc_server_setup_transport(&exec_ctx, server, transport, NULL, NULL); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); grpc_call *call1 = NULL; grpc_call_details call_details1; From a3e7bd85c064da9a1011fd31209e5af6a7ff8f38 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 4 Aug 2016 11:17:22 -0700 Subject: [PATCH 28/97] clang-format --- src/core/lib/channel/handshaker.c | 11 ++++---- src/core/lib/channel/handshaker.h | 3 +-- .../security/transport/security_connector.c | 26 +++++++------------ .../security/transport/security_connector.h | 3 +-- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/core/lib/channel/handshaker.c b/src/core/lib/channel/handshaker.c index 01fcbb20235..c0979f5e806 100644 --- a/src/core/lib/channel/handshaker.c +++ b/src/core/lib/channel/handshaker.c @@ -146,8 +146,8 @@ void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx, static void call_next_handshaker(grpc_exec_ctx* exec_ctx, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, - void* user_data, grpc_error* error) { + gpr_slice_buffer* read_buffer, void* user_data, + grpc_error* error) { grpc_handshake_manager* mgr = user_data; GPR_ASSERT(mgr->state != NULL); GPR_ASSERT(mgr->state->index < mgr->count); @@ -166,10 +166,9 @@ static void call_next_handshaker(grpc_exec_ctx* exec_ctx, user_data = mgr->state->final_user_data; } // Invoke handshaker. - grpc_handshaker_do_handshake(exec_ctx, mgr->handshakers[mgr->state->index], - endpoint, args, read_buffer, - mgr->state->deadline, mgr->state->acceptor, - cb, user_data); + grpc_handshaker_do_handshake( + exec_ctx, mgr->handshakers[mgr->state->index], endpoint, args, + read_buffer, mgr->state->deadline, mgr->state->acceptor, cb, user_data); ++mgr->state->index; // If this is the last handshaker, clean up state. if (mgr->state->index == mgr->count) { diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h index fac4935b16b..b276f6028c1 100644 --- a/src/core/lib/channel/handshaker.h +++ b/src/core/lib/channel/handshaker.h @@ -79,8 +79,7 @@ struct grpc_handshaker_vtable { /// \a acceptor will be NULL for client-side handshakers. void (*do_handshake)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, - gpr_timespec deadline, + gpr_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data); }; diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index 1ee12319005..0eca46eb525 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -325,14 +325,11 @@ static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, cb, user_data); } -static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_server_security_connector *sc, - grpc_tcp_server_acceptor *acceptor, - grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, - gpr_timespec deadline, - grpc_security_handshake_done_cb cb, - void *user_data) { +static void fake_server_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base, false, nonsecure_endpoint, read_buffer, deadline, cb, user_data); @@ -447,14 +444,11 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, } } -static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_server_security_connector *sc, - grpc_tcp_server_acceptor *acceptor, - grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, - gpr_timespec deadline, - grpc_security_handshake_done_cb cb, - void *user_data) { +static void ssl_server_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data) { grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; tsi_handshaker *handshaker; diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index 917100f8025..0b5b44bf1a0 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -158,8 +158,7 @@ void grpc_channel_security_connector_check_call_host( void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector, grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, - gpr_timespec deadline, grpc_security_handshake_done_cb cb, - void *user_data); + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); /* --- server_security_connector object. --- From 76db7f9a7cdf6b5fd6706037c72de1dd131b181e Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 11:56:54 -0700 Subject: [PATCH 29/97] use boost::make_shared --- include/grpc++/impl/codegen/thrift_serializer.h | 14 ++++++-------- tools/grift/README.md | 3 +-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 04dcc699deb..fcb0ffaad6e 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -111,8 +111,8 @@ class ThriftSerializer { Serialize(fields, &byte_buffer, &byte_buffer_size); - gpr_slice slice = - gpr_slice_from_copied_buffer((char*)byte_buffer, byte_buffer_size); + gpr_slice slice = gpr_slice_from_copied_buffer( + reinterpret_cast(byte_buffer), byte_buffer_size); *bp = grpc_raw_byte_buffer_create(&slice, 1); @@ -131,12 +131,12 @@ class ThriftSerializer { last_deserialized_ = true; // reset buffer transport - buffer_->resetBuffer((uint8_t*)serialized_buffer, length); + buffer_->resetBuffer(const_cast(serialized_buffer), length); // read the protocol version if necessary if (serialize_version_) { std::string name = ""; - TMessageType mt = (TMessageType)0; + TMessageType mt = static_cast(0); int32_t seq_id = 0; protocol_->readMessageBegin(name, mt, seq_id); } @@ -200,11 +200,9 @@ class ThriftSerializer { bool serialize_version_; void prepare() { - - buffer_.reset(new TMemoryBuffer()); - + buffer_ = boost::make_shared(*(new TMemoryBuffer())); // create a protocol for the memory buffer transport - protocol_.reset(new Protocol(buffer_)); + protocol_ = std::make_shared(*(new Protocol(buffer_))); prepared_ = true; } diff --git a/tools/grift/README.md b/tools/grift/README.md index 2525f9b83dd..7cbbdc567bf 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -23,5 +23,4 @@ grift uses Compact Protocol to serialize thrift messages. #Installation Before Installing thrift make sure to apply this [patch](grpc_plugins_generator.patch) to third_party/thrift. -Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to -install thrift. \ No newline at end of file +Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to install thrift with commit id bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c. \ No newline at end of file From 2135a1b557f8b992186d5317cb767ac4dbcdfe5c Mon Sep 17 00:00:00 2001 From: siddharthshukla Date: Thu, 4 Aug 2016 02:11:53 +0200 Subject: [PATCH 30/97] add PyPy to testing toolchain --- tools/run_tests/run_tests.py | 92 +++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 542415d9085..fde9297beb5 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -139,6 +139,53 @@ def _is_use_docker_child(): return True if os.getenv('RUN_TESTS_COMMAND') else False +_PythonConfigVars = collections.namedtuple( + '_ConfigVars', ['shell', 'builder', 'builder_prefix_arguments', + 'venv_relative_python', 'toolchain', 'runner']) + + +def _python_config_generator(name, major, minor, bits, config_vars): + return PythonConfig( + name, + config_vars.shell + config_vars.builder + config_vars.builder_prefix_arguments + [ + _python_pattern_function(major=major, minor=minor, bits=bits)] + [ + name] + config_vars.venv_relative_python + config_vars.toolchain, + config_vars.shell + config_vars.runner + [ + os.path.join(name, config_vars.venv_relative_python[0])]) + + +def _pypy_config_generator(name, major, config_vars): + return PythonConfig( + name, + config_vars.shell + config_vars.builder + config_vars.builder_prefix_arguments + [ + _pypy_pattern_function(major=major)] + [ + name] + config_vars.venv_relative_python + config_vars.toolchain, + config_vars.shell + config_vars.runner + [ + os.path.join(name, config_vars.venv_relative_python[0])]) + + +def _python_pattern_function(major, minor, bits): + # Bit-ness is handled by the test machine's environment + if os.name == "nt": + if bits == "64": + return '/c/Python{major}{minor}/python.exe'.format( + major=major, minor=minor, bits=bits) + else: + return '/c/Python{major}{minor}_{bits}bits/python.exe'.format( + major=major, minor=minor, bits=bits) + else: + return 'python{major}.{minor}'.format(major=major, minor=minor) + + +def _pypy_pattern_function(major): + if major == '2': + return 'pypy' + elif major == '3': + return 'pypy3' + else: + raise ValueError("Unknown PyPy major version") + + class CLanguage(object): def __init__(self, make_target, test_lang): @@ -471,36 +518,40 @@ class PythonLanguage(object): bits = '32' else: bits = '64' + if os.name == 'nt': shell = ['bash'] builder = [os.path.abspath('tools/run_tests/build_python_msys2.sh')] builder_prefix_arguments = ['MINGW{}'.format(bits)] venv_relative_python = ['Scripts/python.exe'] toolchain = ['mingw32'] - python_pattern_function = lambda major, minor, bits: ( - '/c/Python{major}{minor}/python.exe'.format(major=major, minor=minor, bits=bits) - if bits == '64' else - '/c/Python{major}{minor}_{bits}bits/python.exe'.format( - major=major, minor=minor, bits=bits)) else: shell = [] builder = [os.path.abspath('tools/run_tests/build_python.sh')] builder_prefix_arguments = [] venv_relative_python = ['bin/python'] toolchain = ['unix'] - # Bit-ness is handled by the test machine's environment - python_pattern_function = lambda major, minor, bits: 'python{major}.{minor}'.format(major=major, minor=minor) + runner = [os.path.abspath('tools/run_tests/run_python.sh')] - python_config_generator = lambda name, major, minor, bits: PythonConfig( - name, - shell + builder + builder_prefix_arguments - + [python_pattern_function(major=major, minor=minor, bits=bits)] - + [name] + venv_relative_python + toolchain, - shell + runner + [os.path.join(name, venv_relative_python[0])]) - python27_config = python_config_generator(name='py27', major='2', minor='7', bits=bits) - python34_config = python_config_generator(name='py34', major='3', minor='4', bits=bits) - python35_config = python_config_generator(name='py35', major='3', minor='5', bits=bits) - python36_config = python_config_generator(name='py36', major='3', minor='6', bits=bits) + config_vars = _PythonConfigVars(shell, builder, builder_prefix_arguments, + venv_relative_python, toolchain, runner) + python27_config = _python_config_generator(name='py27', major='2', + minor='7', bits=bits, + config_vars=config_vars) + python34_config = _python_config_generator(name='py34', major='3', + minor='4', bits=bits, + config_vars=config_vars) + python35_config = _python_config_generator(name='py35', major='3', + minor='5', bits=bits, + config_vars=config_vars) + python36_config = _python_config_generator(name='py36', major='3', + minor='6', bits=bits, + config_vars=config_vars) + pypy27_config = _pypy_config_generator(name='pypy', major='2', + config_vars=config_vars) + pypy32_config = _pypy_config_generator(name='pypy3', major='3', + config_vars=config_vars) + if args.compiler == 'default': if os.name == 'nt': return (python27_config,) @@ -514,6 +565,10 @@ class PythonLanguage(object): return (python35_config,) elif args.compiler == 'python3.6': return (python36_config,) + elif args.compiler == 'pypy': + return (pypy27_config,) + elif args.compiler == 'pypy3': + return (pypy32_config,) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -893,6 +948,7 @@ def runs_per_test_type(arg_str): msg = '\'{}\' is not a positive integer or \'inf\''.format(arg_str) raise argparse.ArgumentTypeError(msg) + # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', @@ -946,7 +1002,7 @@ argp.add_argument('--compiler', 'gcc4.4', 'gcc4.6', 'gcc4.9', 'gcc5.3', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'vs2010', 'vs2013', 'vs2015', - 'python2.7', 'python3.4', 'python3.5', 'python3.6', + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'node0.12', 'node4', 'node5', 'coreclr'], default='default', From a7ee93864a2e822ec510dccec19012e59acdeb7d Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 14:42:35 -0700 Subject: [PATCH 31/97] remove const in Deserialize --- include/grpc++/impl/codegen/thrift_serializer.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index fcb0ffaad6e..7308a1577c8 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -122,8 +122,7 @@ class ThriftSerializer { // Deserialize the passed char array into the passed type, returns the number // of bytes that have been consumed from the passed string. template - uint32_t Deserialize(const uint8_t* serialized_buffer, size_t length, - T* fields) { + uint32_t Deserialize(uint8_t* serialized_buffer, size_t length, T* fields) { // prepare buffer if necessary if (!prepared_) { prepare(); @@ -131,7 +130,7 @@ class ThriftSerializer { last_deserialized_ = true; // reset buffer transport - buffer_->resetBuffer(const_cast(serialized_buffer), length); + buffer_->resetBuffer(serialized_buffer, length); // read the protocol version if necessary if (serialize_version_) { @@ -200,10 +199,9 @@ class ThriftSerializer { bool serialize_version_; void prepare() { - buffer_ = boost::make_shared(*(new TMemoryBuffer())); + buffer_ = boost::make_shared(); // create a protocol for the memory buffer transport - protocol_ = std::make_shared(*(new Protocol(buffer_))); - + protocol_ = std::make_shared(buffer_); prepared_ = true; } From d07c17e3430bbf1cc1a802b76cf57175bbb6603c Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 17:52:00 -0700 Subject: [PATCH 32/97] fix Dockerfile --- tools/grift/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile index 954640f0df0..223ee939927 100644 --- a/tools/grift/Dockerfile +++ b/tools/grift/Dockerfile @@ -43,21 +43,25 @@ RUN apt-get update && \ cmake \ libiberty-dev \ g++ unzip \ - curl make automake libtool + curl make automake libtool libboost-dev # Configure git RUN git config --global user.name "Jenkins" && \ git config --global user.email "jenkins@grpc" +# Clone gRPC RUN git clone https://github.com/grpc/grpc +# Update Submodules RUN cd grpc && git submodule update --init -RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch - +# Install protobuf RUN cd grpc/third_party/protobuf && ./autogen.sh && ./configure && \ make -j && make check -j && make install && ldconfig +# Install gRPC RUN cd grpc && make -j && make install -RUN cd grpc/third_party/thrift && ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file +# Install thrift +RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch && \ + ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file From 4f17395b81508ceedc824602e52876ae94d13c43 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 5 Aug 2016 01:32:34 -0700 Subject: [PATCH 33/97] Properly use unique_ptr rather than explicitly deleting server context wrappers in QPS test --- test/cpp/qps/server_async.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index dea87463312..082b4bc72fe 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -108,14 +108,14 @@ class AsyncQpsServerTest : public Server { auto request_unary = std::bind(request_unary_function, &async_service_, _1, _2, _3, srv_cqs_[j].get(), srv_cqs_[j].get(), _4); - contexts_.push_front( + contexts_.emplace_back( new ServerRpcContextUnaryImpl(request_unary, process_rpc_bound)); } if (request_streaming_function) { auto request_streaming = std::bind(request_streaming_function, &async_service_, _1, _2, srv_cqs_[j].get(), srv_cqs_[j].get(), _3); - contexts_.push_front(new ServerRpcContextStreamingImpl( + contexts_.emplace_back(new ServerRpcContextStreamingImpl( request_streaming, process_rpc_bound)); } } @@ -146,10 +146,6 @@ class AsyncQpsServerTest : public Server { while ((*cq)->Next(&got_tag, &ok)) ; } - while (!contexts_.empty()) { - delete contexts_.front(); - contexts_.pop_front(); - } } private: @@ -336,7 +332,7 @@ class AsyncQpsServerTest : public Server { std::unique_ptr server_; std::vector> srv_cqs_; ServiceType async_service_; - std::forward_list contexts_; + std::vector> contexts_; struct PerThreadShutdownState { mutable std::mutex mutex; From 773ecd62ddeea4c483d3d248fa30a978d2520ee8 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 5 Aug 2016 09:26:56 -0700 Subject: [PATCH 34/97] Dramatically reduce time required to complete sync test when running with lots of threads (by parallelizing shutdown of course) --- test/cpp/qps/client.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 4045e13460f..8a750196b26 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -169,6 +169,7 @@ class Client { // Must call AwaitThreadsCompletion before destructor to avoid a race // between destructor and invocation of virtual ThreadFunc void AwaitThreadsCompletion() { + gpr_atm_rel_store(&thread_pool_done_, static_cast(1)); DestroyMultithreading(); std::unique_lock g(thread_completion_mu_); while (threads_remaining_ != 0) { @@ -180,6 +181,7 @@ class Client { bool closed_loop_; void StartThreads(size_t num_threads) { + gpr_atm_rel_store(&thread_pool_done_, static_cast(0)); threads_remaining_ = num_threads; for (size_t i = 0; i < num_threads; i++) { threads_.emplace_back(new Thread(this, i)); @@ -241,16 +243,11 @@ class Client { class Thread { public: Thread(Client* client, size_t idx) - : done_(false), - client_(client), + : client_(client), idx_(idx), impl_(&Thread::ThreadFunc, this) {} ~Thread() { - { - std::lock_guard g(mu_); - done_ = true; - } impl_.join(); } @@ -280,11 +277,14 @@ class Client { if (entry.used()) { histogram_.Add(entry.value()); } + bool done = false; if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); - done_ = true; + done = true; } - if (done_) { + done = done || (gpr_atm_acq_load(&client_->thread_pool_done_) != + static_cast(0)); + if (done) { client_->CompleteThread(); return; } @@ -292,7 +292,6 @@ class Client { } std::mutex mu_; - bool done_; Histogram histogram_; Client* client_; const size_t idx_; @@ -305,6 +304,7 @@ class Client { InterarrivalTimer interarrival_timer_; std::vector next_time_; + gpr_atm thread_pool_done_; std::mutex thread_completion_mu_; size_t threads_remaining_; std::condition_variable threads_complete_; From 25128f1adf836e63cf76b6666fba8bcf31d7a5a8 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 5 Aug 2016 09:45:03 -0700 Subject: [PATCH 35/97] Better ending for open-loop tests: never wait more than 1 second if we are in termination mode --- test/cpp/qps/client.h | 2 +- test/cpp/qps/client_sync.cc | 31 +++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 8a750196b26..1f98a7fc286 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -179,6 +179,7 @@ class Client { protected: bool closed_loop_; + gpr_atm thread_pool_done_; void StartThreads(size_t num_threads) { gpr_atm_rel_store(&thread_pool_done_, static_cast(0)); @@ -304,7 +305,6 @@ class Client { InterarrivalTimer interarrival_timer_; std::vector next_time_; - gpr_atm thread_pool_done_; std::mutex thread_completion_mu_; size_t threads_remaining_; std::condition_variable threads_complete_; diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index 25c78235532..b677fc1070e 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -79,10 +79,29 @@ class SynchronousClient virtual ~SynchronousClient(){}; protected: - void WaitToIssue(int thread_idx) { + // WaitToIssue returns false if we realize that we need to break out + bool WaitToIssue(int thread_idx) { if (!closed_loop_) { - gpr_sleep_until(NextIssueTime(thread_idx)); + gpr_timespec next_issue_time = NextIssueTime(thread_idx); + // Avoid sleeping for too long continuously because we might + // need to terminate before then. This is an issue since + // exponential distribution can occasionally produce bad outliers + while (true) { + gpr_timespec one_sec_delay = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(1, GPR_TIMESPAN)); + if (gpr_time_cmp(next_issue_time, one_sec_delay) <= 0) { + gpr_sleep_until(next_issue_time); + return true; + } else { + gpr_sleep_until(one_sec_delay); + if (gpr_atm_acq_load(&thread_pool_done_) != static_cast(0)) { + return false; + } + } + } } + return true; } size_t num_threads_; @@ -101,7 +120,9 @@ class SynchronousUnaryClient GRPC_FINAL : public SynchronousClient { ~SynchronousUnaryClient() {} bool ThreadFunc(HistogramEntry* entry, size_t thread_idx) GRPC_OVERRIDE { - WaitToIssue(thread_idx); + if (!WaitToIssue(thread_idx)) { + return true; + } auto* stub = channels_[thread_idx % channels_.size()].get_stub(); double start = UsageTimer::Now(); GPR_TIMER_SCOPE("SynchronousUnaryClient::ThreadFunc", 0); @@ -144,7 +165,9 @@ class SynchronousStreamingClient GRPC_FINAL : public SynchronousClient { } bool ThreadFunc(HistogramEntry* entry, size_t thread_idx) GRPC_OVERRIDE { - WaitToIssue(thread_idx); + if (!WaitToIssue(thread_idx)) { + return true; + } GPR_TIMER_SCOPE("SynchronousStreamingClient::ThreadFunc", 0); double start = UsageTimer::Now(); if (stream_[thread_idx]->Write(request_) && From d02988d6b55c7724548ed556c319222f5e2cf665 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 5 Aug 2016 09:47:47 -0700 Subject: [PATCH 36/97] clang-format --- test/cpp/qps/client.h | 14 +++++--------- test/cpp/qps/client_sync.cc | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 1f98a7fc286..5b1a1c3b827 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -244,13 +244,9 @@ class Client { class Thread { public: Thread(Client* client, size_t idx) - : client_(client), - idx_(idx), - impl_(&Thread::ThreadFunc, this) {} + : client_(client), idx_(idx), impl_(&Thread::ThreadFunc, this) {} - ~Thread() { - impl_.join(); - } + ~Thread() { impl_.join(); } void BeginSwap(Histogram* n) { std::lock_guard g(mu_); @@ -278,13 +274,13 @@ class Client { if (entry.used()) { histogram_.Add(entry.value()); } - bool done = false; + bool done = false; if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); done = true; } - done = done || (gpr_atm_acq_load(&client_->thread_pool_done_) != - static_cast(0)); + done = done || (gpr_atm_acq_load(&client_->thread_pool_done_) != + static_cast(0)); if (done) { client_->CompleteThread(); return; diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index b677fc1070e..53e004ffa02 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -87,18 +87,18 @@ class SynchronousClient // need to terminate before then. This is an issue since // exponential distribution can occasionally produce bad outliers while (true) { - gpr_timespec one_sec_delay = - gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_seconds(1, GPR_TIMESPAN)); - if (gpr_time_cmp(next_issue_time, one_sec_delay) <= 0) { - gpr_sleep_until(next_issue_time); - return true; - } else { - gpr_sleep_until(one_sec_delay); - if (gpr_atm_acq_load(&thread_pool_done_) != static_cast(0)) { - return false; - } - } + gpr_timespec one_sec_delay = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(1, GPR_TIMESPAN)); + if (gpr_time_cmp(next_issue_time, one_sec_delay) <= 0) { + gpr_sleep_until(next_issue_time); + return true; + } else { + gpr_sleep_until(one_sec_delay); + if (gpr_atm_acq_load(&thread_pool_done_) != static_cast(0)) { + return false; + } + } } } return true; From 9eedb4ffd74aed8d246a07f8007960b2bc167f55 Mon Sep 17 00:00:00 2001 From: siddharthshukla Date: Tue, 12 Jul 2016 14:02:12 +0200 Subject: [PATCH 37/97] Switch init/shutdown: lib-wide -> per-object Incremental changes towards PyPy support. --- .../grpcio/grpc/_cython/_cygrpc/call.pyx.pxi | 2 ++ .../grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi | 2 ++ .../grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 2 ++ .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 14 ++++++++++++++ .../grpcio/grpc/_cython/_cygrpc/records.pyx.pxi | 12 ++++++++++++ .../grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 2 ++ src/python/grpcio/grpc/_cython/cygrpc.pyx | 4 ---- 7 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi index ba60986143c..cc3bd7a0672 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi @@ -34,6 +34,7 @@ cdef class Call: def __cinit__(self): # Create an *empty* call + grpc_init() self.c_call = NULL self.references = [] @@ -106,6 +107,7 @@ cdef class Call: def __dealloc__(self): if self.c_call != NULL: grpc_call_destroy(self.c_call) + grpc_shutdown() # The object *should* always be valid from Python. Used for debugging. @property diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 54164014313..3df937eb14f 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -34,6 +34,7 @@ cdef class Channel: def __cinit__(self, bytes target, ChannelArgs arguments=None, ChannelCredentials channel_credentials=None): + grpc_init() cdef grpc_channel_args *c_arguments = NULL cdef char *c_target = NULL self.c_channel = NULL @@ -103,3 +104,4 @@ cdef class Channel: def __dealloc__(self): if self.c_channel != NULL: grpc_channel_destroy(self.c_channel) + grpc_shutdown() diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 5955021ceb7..a258ba40639 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -38,6 +38,7 @@ cdef int _INTERRUPT_CHECK_PERIOD_MS = 200 cdef class CompletionQueue: def __cinit__(self): + grpc_init() with nogil: self.c_completion_queue = grpc_completion_queue_create(NULL) self.is_shutting_down = False @@ -129,3 +130,4 @@ cdef class CompletionQueue: self.c_completion_queue, c_deadline, NULL) self._interpret_event(event) grpc_completion_queue_destroy(self.c_completion_queue) + grpc_shutdown() diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 035ac49a8bf..04872b9c09a 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -33,6 +33,7 @@ cimport cpython cdef class ChannelCredentials: def __cinit__(self): + grpc_init() self.c_credentials = NULL self.c_ssl_pem_key_cert_pair.private_key = NULL self.c_ssl_pem_key_cert_pair.certificate_chain = NULL @@ -47,11 +48,13 @@ cdef class ChannelCredentials: def __dealloc__(self): if self.c_credentials != NULL: grpc_channel_credentials_release(self.c_credentials) + grpc_shutdown() cdef class CallCredentials: def __cinit__(self): + grpc_init() self.c_credentials = NULL self.references = [] @@ -64,17 +67,20 @@ cdef class CallCredentials: def __dealloc__(self): if self.c_credentials != NULL: grpc_call_credentials_release(self.c_credentials) + grpc_shutdown() cdef class ServerCredentials: def __cinit__(self): + grpc_init() self.c_credentials = NULL self.references = [] def __dealloc__(self): if self.c_credentials != NULL: grpc_server_credentials_release(self.c_credentials) + grpc_shutdown() cdef class CredentialsMetadataPlugin: @@ -90,6 +96,7 @@ cdef class CredentialsMetadataPlugin: successful). name (bytes): Plugin name. """ + grpc_init() if not callable(plugin_callback): raise ValueError('expected callable plugin_callback') self.plugin_callback = plugin_callback @@ -105,10 +112,14 @@ cdef class CredentialsMetadataPlugin: cpython.Py_INCREF(self) return result + def __dealloc__(self): + grpc_shutdown() + cdef class AuthMetadataContext: def __cinit__(self): + grpc_init() self.context.service_url = NULL self.context.method_name = NULL @@ -120,6 +131,9 @@ cdef class AuthMetadataContext: def method_name(self): return self.context.method_name + def __dealloc__(self): + grpc_shutdown() + cdef void plugin_get_metadata( void *state, grpc_auth_metadata_context context, diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 54b3d00dfc7..834a44123d4 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -176,12 +176,14 @@ cdef class Timespec: cdef class CallDetails: def __cinit__(self): + grpc_init() with nogil: grpc_call_details_init(&self.c_details) def __dealloc__(self): with nogil: grpc_call_details_destroy(&self.c_details) + grpc_shutdown() @property def method(self): @@ -232,6 +234,7 @@ cdef class Event: cdef class ByteBuffer: def __cinit__(self, bytes data): + grpc_init() if data is None: self.c_byte_buffer = NULL return @@ -288,6 +291,7 @@ cdef class ByteBuffer: def __dealloc__(self): if self.c_byte_buffer != NULL: grpc_byte_buffer_destroy(self.c_byte_buffer) + grpc_shutdown() cdef class SslPemKeyCertPair: @@ -319,6 +323,7 @@ cdef class ChannelArg: cdef class ChannelArgs: def __cinit__(self, args): + grpc_init() self.args = list(args) for arg in self.args: if not isinstance(arg, ChannelArg): @@ -333,6 +338,7 @@ cdef class ChannelArgs: def __dealloc__(self): with nogil: gpr_free(self.c_args.arguments) + grpc_shutdown() def __len__(self): # self.args is never stale; it's only updated from this file @@ -399,6 +405,7 @@ cdef class _MetadataIterator: cdef class Metadata: def __cinit__(self, metadata): + grpc_init() self.metadata = list(metadata) for metadatum in metadata: if not isinstance(metadatum, Metadatum): @@ -420,6 +427,7 @@ cdef class Metadata: # it'd be nice if that were documented somewhere...) # TODO(atash): document this in the C core grpc_metadata_array_destroy(&self.c_metadata_array) + grpc_shutdown() def __len__(self): return self.c_metadata_array.count @@ -437,6 +445,7 @@ cdef class Metadata: cdef class Operation: def __cinit__(self): + grpc_init() self.references = [] self._received_status_details = NULL self._received_status_details_capacity = 0 @@ -529,6 +538,7 @@ cdef class Operation: # This means that we need to clean up after receive_status_on_client. if self.c_op.type == GRPC_OP_RECV_STATUS_ON_CLIENT: gpr_free(self._received_status_details) + grpc_shutdown() def operation_send_initial_metadata(Metadata metadata, int flags): cdef Operation op = Operation() @@ -645,6 +655,7 @@ cdef class _OperationsIterator: cdef class Operations: def __cinit__(self, operations): + grpc_init() self.operations = list(operations) # normalize iterable self.c_ops = NULL self.c_nops = 0 @@ -667,6 +678,7 @@ cdef class Operations: def __dealloc__(self): with nogil: gpr_free(self.c_ops) + grpc_shutdown() def __iter__(self): return _OperationsIterator(self) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 4f2d51b03f5..ca2b8311147 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -35,6 +35,7 @@ import time cdef class Server: def __cinit__(self, ChannelArgs arguments=None): + grpc_init() cdef grpc_channel_args *c_arguments = NULL self.references = [] self.registered_completion_queues = [] @@ -172,3 +173,4 @@ cdef class Server: while not self.is_shutdown: time.sleep(0) grpc_server_destroy(self.c_server) + grpc_shutdown() diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index a9520b9c0fa..08089994a95 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -55,12 +55,8 @@ cdef extern from "Python.h": def _initialize(): - grpc_init() grpc_set_ssl_roots_override_callback( ssl_roots_override_callback) - if Py_AtExit(grpc_shutdown) != 0: - raise ImportError('failed to register gRPC library shutdown callbacks') - _initialize() From 7e024be839687470bd1343f70f08ec1e703eb9ec Mon Sep 17 00:00:00 2001 From: chedeti Date: Fri, 5 Aug 2016 11:15:37 -0700 Subject: [PATCH 38/97] fix multilevel inheritence codegen --- tools/grift/grpc_plugins_generator.patch | 75 ++++++++++++++---------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index a1d4cecde04..de82a01f625 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -59,21 +59,21 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From e724d3abf096278615085bd58217321e32b43fd8 Mon Sep 17 00:00:00 2001 +From 387e4300bc9d98176a92a7c010621443a538e7f2 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 Subject: [PATCH 2/3] grpc cpp plugins generator with example --- - compiler/cpp/src/generate/t_cpp_generator.cc | 478 +++++++++++++++++++++++---- + compiler/cpp/src/generate/t_cpp_generator.cc | 489 +++++++++++++++++++++++---- tutorial/cpp/CMakeLists.txt | 53 --- tutorial/cpp/CppClient.cpp | 80 ----- tutorial/cpp/CppServer.cpp | 181 ---------- - tutorial/cpp/GriftClient.cpp | 93 ++++++ - tutorial/cpp/GriftServer.cpp | 93 ++++++ + tutorial/cpp/GriftClient.cpp | 93 +++++ + tutorial/cpp/GriftServer.cpp | 93 +++++ tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 8 files changed, 641 insertions(+), 416 deletions(-) + 8 files changed, 652 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt delete mode 100644 tutorial/cpp/CppClient.cpp delete mode 100644 tutorial/cpp/CppServer.cpp @@ -82,7 +82,7 @@ Subject: [PATCH 2/3] grpc cpp plugins generator with example create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 6c04899..4e00129 100644 +index 6c04899..1557241 100644 --- a/compiler/cpp/src/generate/t_cpp_generator.cc +++ b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -162,6 +162,8 @@ public: @@ -328,7 +328,7 @@ index 6c04899..4e00129 100644 << endl; f_service_tcc_ << "#ifndef " << svcname << "_TCC" << endl << "#define " << svcname << "_TCC" -@@ -1663,19 +1704,66 @@ void t_cpp_generator::generate_service(t_service* tservice) { +@@ -1663,19 +1704,69 @@ void t_cpp_generator::generate_service(t_service* tservice) { } } @@ -361,15 +361,18 @@ index 6c04899..4e00129 100644 + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + -+ if (extends_service) { -+ vector functions = extends_service->get_functions(); ++ ++ t_service* service_iter = extends_service; ++ while (service_iter) { ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + + for ( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << -+ indent() << "\"/" << extends_service->get_program()->get_namespace("cpp") << -+ "." << extends_service->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; ++ indent() << "\"/" << service_iter->get_program()->get_namespace("cpp") << ++ "." << service_iter->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; + } ++ service_iter = service_iter->get_extends(); + } + + indent_down(); @@ -403,7 +406,7 @@ index 6c04899..4e00129 100644 // Generate all the cob components if (gen_cob_style_) { -@@ -1688,10 +1776,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { +@@ -1688,10 +1779,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { generate_service_async_skeleton(tservice); } @@ -418,7 +421,7 @@ index 6c04899..4e00129 100644 // Close the namespace f_service_ << ns_close_ << endl << endl; f_service_tcc_ << ns_close_ << endl << endl; -@@ -1729,15 +1821,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { +@@ -1729,15 +1824,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { string name_orig = ts->get_name(); // TODO(dreiss): Why is this stuff not in generate_function_helpers? @@ -436,7 +439,7 @@ index 6c04899..4e00129 100644 ts->set_name(name_orig); generate_function_helpers(tservice, *f_iter); -@@ -1745,13 +1833,210 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { +@@ -1745,13 +1836,218 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { } /** @@ -497,9 +500,10 @@ index 6c04899..4e00129 100644 + } + + t_service* extends_service = tservice->get_extends(); -+ if (extends_service) { ++ t_service* service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string function_name = (*f_iter)->get_name(); @@ -508,6 +512,7 @@ index 6c04899..4e00129 100644 + "(::grpc::ClientContext* context, const " << function_name << + "Req& request, " << function_name << "Resp* response) override;" << endl; + } ++ service_iter = service_iter->get_extends(); + } + + f_header_ << @@ -521,14 +526,16 @@ index 6c04899..4e00129 100644 + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + -+ if (extends_service) { ++ service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_header_ << + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } ++ service_iter = service_iter->get_extends(); + } + + indent_down(); @@ -551,9 +558,10 @@ index 6c04899..4e00129 100644 + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + -+ if (extends_service) { ++ service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { + f_service_ << @@ -561,6 +569,7 @@ index 6c04899..4e00129 100644 + ", rpcmethod_" << (*f_iter)->get_name() << "_(" << + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } ++ service_iter = service_iter->get_extends(); + } + f_service_ << + indent() << "{}" << endl; @@ -609,8 +618,9 @@ index 6c04899..4e00129 100644 + + } + -+ if (extends_service) { -+ vector functions = extends_service->get_functions(); ++ service_iter = extends_service; ++ while (service_iter) { ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string function_name = (*f_iter)->get_name(); @@ -631,6 +641,7 @@ index 6c04899..4e00129 100644 + "}" << endl; + + } ++ service_iter = service_iter->get_extends(); + } + +} @@ -648,7 +659,7 @@ index 6c04899..4e00129 100644 if (style == "CobCl") { // Forward declare the client. string client_name = service_name_ + "CobClient"; -@@ -1764,13 +2049,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1764,13 +2060,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty } string extends = ""; @@ -666,7 +677,7 @@ index 6c04899..4e00129 100644 } if (style == "CobCl" && gen_templates_) { -@@ -1778,7 +2065,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1778,7 +2076,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty } f_header_ << "class " << service_if_name << extends << " {" << endl << " public:" << endl; indent_up(); @@ -677,7 +688,7 @@ index 6c04899..4e00129 100644 vector functions = tservice->get_functions(); vector::iterator f_iter; -@@ -1786,7 +2075,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1786,7 +2086,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty if ((*f_iter)->has_doc()) f_header_ << endl; generate_java_doc(f_header_, *f_iter); @@ -691,7 +702,7 @@ index 6c04899..4e00129 100644 } indent_down(); f_header_ << "};" << endl << endl; -@@ -1797,6 +2091,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1797,6 +2102,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty f_header_ << "typedef " << service_if_name << "< ::apache::thrift::protocol::TProtocol> " << service_name_ << style << "If;" << endl << endl; } @@ -758,7 +769,7 @@ index 6c04899..4e00129 100644 } /** -@@ -3095,7 +3449,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* +@@ -3095,7 +3460,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* std::ofstream& out = (gen_templates_ ? f_service_tcc_ : f_service_); @@ -767,7 +778,7 @@ index 6c04899..4e00129 100644 t_field success(tfunction->get_returntype(), "success", 0); if (!tfunction->get_returntype()->is_void()) { result.append(&success); -@@ -3109,17 +3463,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* +@@ -3109,17 +3474,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* } generate_struct_declaration(f_header_, &result, false); @@ -786,7 +797,7 @@ index 6c04899..4e00129 100644 } /** -@@ -3162,8 +3508,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3162,8 +3519,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, << endl; scope_up(out); @@ -797,7 +808,7 @@ index 6c04899..4e00129 100644 if (tfunction->is_oneway() && !unnamed_oprot_seqid) { out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; -@@ -3320,7 +3666,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3320,7 +3677,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; } @@ -806,7 +817,7 @@ index 6c04899..4e00129 100644 << indent() << "void* ctx = NULL;" << endl << indent() << "if (this->eventHandler_.get() != NULL) {" << endl << indent() << " ctx = this->eventHandler_->getContext(" << service_func_name << ", NULL);" << endl -@@ -3487,7 +3833,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3487,7 +3844,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, << "this->eventHandler_.get(), ctx, " << service_func_name << ");" << endl << endl; // Throw the TDelayedException, and catch the result @@ -1475,7 +1486,7 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From f991f33dd6461eae197b6ad0e7088b571f2a7b22 Mon Sep 17 00:00:00 2001 +From 3e4d75a2e2c474ee7700e7c9acaf89fdb768bedc Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 Subject: [PATCH 3/3] grpc java plugins generator From f50020ce038411b2a0864cb61296b67ac1cc032e Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 8 Aug 2016 07:29:31 -0700 Subject: [PATCH 39/97] Appease the const gods, improve readability, stop using 0 and 1 as proxies for false and true. --- test/cpp/qps/client.h | 11 ++++------- test/cpp/qps/client_sync.cc | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 5b1a1c3b827..fada4ba7679 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -169,7 +169,7 @@ class Client { // Must call AwaitThreadsCompletion before destructor to avoid a race // between destructor and invocation of virtual ThreadFunc void AwaitThreadsCompletion() { - gpr_atm_rel_store(&thread_pool_done_, static_cast(1)); + gpr_atm_rel_store(&thread_pool_done_, static_cast(true)); DestroyMultithreading(); std::unique_lock g(thread_completion_mu_); while (threads_remaining_ != 0) { @@ -182,7 +182,7 @@ class Client { gpr_atm thread_pool_done_; void StartThreads(size_t num_threads) { - gpr_atm_rel_store(&thread_pool_done_, static_cast(0)); + gpr_atm_rel_store(&thread_pool_done_, static_cast(false)); threads_remaining_ = num_threads; for (size_t i = 0; i < num_threads; i++) { threads_.emplace_back(new Thread(this, i)); @@ -274,14 +274,11 @@ class Client { if (entry.used()) { histogram_.Add(entry.value()); } - bool done = false; if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); - done = true; } - done = done || (gpr_atm_acq_load(&client_->thread_pool_done_) != - static_cast(0)); - if (done) { + if (!thread_still_ok || + static_cast(gpr_atm_acq_load(&client_->thread_pool_done_))) { client_->CompleteThread(); return; } diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index 53e004ffa02..8062424a1fb 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -82,12 +82,12 @@ class SynchronousClient // WaitToIssue returns false if we realize that we need to break out bool WaitToIssue(int thread_idx) { if (!closed_loop_) { - gpr_timespec next_issue_time = NextIssueTime(thread_idx); + const gpr_timespec next_issue_time = NextIssueTime(thread_idx); // Avoid sleeping for too long continuously because we might // need to terminate before then. This is an issue since // exponential distribution can occasionally produce bad outliers while (true) { - gpr_timespec one_sec_delay = + const gpr_timespec one_sec_delay = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)); if (gpr_time_cmp(next_issue_time, one_sec_delay) <= 0) { From dc3d561f4a470490d75a3018683f4a67705f1250 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 8 Aug 2016 16:30:10 -0700 Subject: [PATCH 40/97] remove dedicated thread for read loop in ruby bidi calls --- src/ruby/lib/grpc/generic/bidi_call.rb | 91 ++++++++++---------------- 1 file changed, 34 insertions(+), 57 deletions(-) diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index c2ac3c4dafe..75ddff0bfd9 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -61,7 +61,6 @@ module GRPC @call = call @marshal = marshal @op_notifier = nil # signals completion on clients - @readq = Queue.new @unmarshal = unmarshal @metadata_received = metadata_received @reads_complete = false @@ -81,8 +80,7 @@ module GRPC def run_on_client(requests, op_notifier, &blk) @op_notifier = op_notifier @enq_th = Thread.new { write_loop(requests) } - @loop_th = start_read_loop - each_queued_msg(&blk) + read_loop(&blk) end # Begins orchestration of the Bidi stream for a server generating replies. @@ -97,8 +95,7 @@ module GRPC # # @param gen_each_reply [Proc] generates the BiDi stream replies. def run_on_server(gen_each_reply) - replys = gen_each_reply.call(each_queued_msg) - @loop_th = start_read_loop(is_client: false) + replys = gen_each_reply.call(read_loop(is_client: false)) write_loop(replys, is_client: false) end @@ -135,24 +132,6 @@ module GRPC batch_result end - # each_queued_msg yields each message on this instances readq - # - # - messages are added to the readq by #read_loop - # - iteration ends when the instance itself is added - def each_queued_msg - return enum_for(:each_queued_msg) unless block_given? - count = 0 - loop do - GRPC.logger.debug("each_queued_msg: waiting##{count}") - count += 1 - req = @readq.pop - GRPC.logger.debug("each_queued_msg: req = #{req}") - fail req if req.is_a? StandardError - break if req.equal?(END_OF_READS) - yield req - end - end - def write_loop(requests, is_client: true) GRPC.logger.debug('bidi-write-loop: starting') count = 0 @@ -190,47 +169,45 @@ module GRPC raise e end - # starts the read loop - def start_read_loop(is_client: true) - Thread.new do - GRPC.logger.debug('bidi-read-loop: starting') - begin - count = 0 - # queue the initial read before beginning the loop - loop do - GRPC.logger.debug("bidi-read-loop: #{count}") - count += 1 - batch_result = read_using_run_batch + # Provides an enumerator that yields results of remote reads + def read_loop(is_client: true) + return enum_for(:read_loop, + is_client: is_client) unless block_given? + GRPC.logger.debug('bidi-read-loop: starting') + begin + count = 0 + # queue the initial read before beginning the loop + loop do + GRPC.logger.debug("bidi-read-loop: #{count}") + count += 1 + batch_result = read_using_run_batch - # handle the next message - if batch_result.message.nil? - GRPC.logger.debug("bidi-read-loop: null batch #{batch_result}") + # handle the next message + if batch_result.message.nil? + GRPC.logger.debug("bidi-read-loop: null batch #{batch_result}") - if is_client - batch_result = @call.run_batch(RECV_STATUS_ON_CLIENT => nil) - @call.status = batch_result.status - batch_result.check_status - GRPC.logger.debug("bidi-read-loop: done status #{@call.status}") - end - - @readq.push(END_OF_READS) - GRPC.logger.debug('bidi-read-loop: done reading!') - break + if is_client + batch_result = @call.run_batch(RECV_STATUS_ON_CLIENT => nil) + @call.status = batch_result.status + batch_result.check_status + GRPC.logger.debug("bidi-read-loop: done status #{@call.status}") end - # push the latest read onto the queue and continue reading - res = @unmarshal.call(batch_result.message) - @readq.push(res) + GRPC.logger.debug('bidi-read-loop: done reading!') + break end - rescue StandardError => e - GRPC.logger.warn('bidi: read-loop failed') - GRPC.logger.warn(e) - @readq.push(e) # let each_queued_msg terminate with this error + + res = @unmarshal.call(batch_result.message) + yield res end - GRPC.logger.debug('bidi-read-loop: finished') - @reads_complete = true - finished + rescue StandardError => e + GRPC.logger.warn('bidi: read-loop failed') + GRPC.logger.warn(e) + raise e end + GRPC.logger.debug('bidi-read-loop: finished') + @reads_complete = true + finished end end end From f0f58e68738abbc317f7f449c5104f7fbbff26bd Mon Sep 17 00:00:00 2001 From: siddharthshukla Date: Mon, 8 Aug 2016 22:46:50 +0200 Subject: [PATCH 41/97] skip test run if running with pypy don't run cygrpc_test.TypeSmokeTest.testCallCredentialsFromPluginUpdown if the interpreter is PyPy --- src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py index f9a8e2401bb..2f50263730f 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py @@ -30,6 +30,7 @@ import time import threading import unittest +import platform from grpc._cython import cygrpc from tests.unit._cython import test_utilities @@ -113,6 +114,9 @@ class TypeSmokeTest(unittest.TestCase): lambda ignored_a, ignored_b: None, b'') del plugin + @unittest.skipIf( + platform.python_implementation() == "PyPy", + 'TODO(issue 7672): figure out why this fails on PyPy') def testCallCredentialsFromPluginUpDown(self): plugin = cygrpc.CredentialsMetadataPlugin(_metadata_plugin_callback, b'') call_credentials = cygrpc.call_credentials_metadata_plugin(plugin) From a6bdb30311383a49337529ebde7976ba9239dab9 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 9 Aug 2016 08:54:39 -0700 Subject: [PATCH 42/97] fix jenkins linux image in create script --- tools/gce/create_linux_worker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index 8a2df40859b..7bf8b240818 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -43,8 +43,8 @@ gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ --machine-type n1-standard-8 \ - --image-family=ubuntu-1510 \ - --image-project=ubuntu-os-cloud \ + --image=ubuntu-1510 \ + --image-project=grpc-testing \ --boot-disk-size 1000 echo 'Created GCE instance, waiting 60 seconds for it to come online.' From 80db7f8c6e927527ce9d3bf8c79cf1175845ad90 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 9 Aug 2016 11:35:19 -0700 Subject: [PATCH 43/97] Make Node grpc-tools protoc automatically call the plugin --- src/node/tools/bin/protoc.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/node/tools/bin/protoc.js b/src/node/tools/bin/protoc.js index 53fc5dc4280..7f8356867aa 100755 --- a/src/node/tools/bin/protoc.js +++ b/src/node/tools/bin/protoc.js @@ -47,7 +47,11 @@ var exe_ext = process.platform === 'win32' ? '.exe' : ''; var protoc = path.resolve(__dirname, 'protoc' + exe_ext); -var child_process = execFile(protoc, process.argv.slice(2), function(error, stdout, stderr) { +var plugin = path.resolve(__dirname, 'grpc_node_plugin' + exe_ext); + +var args = ['--plugin=protoc-gen-grpc=' + plugin].concat(process.argv.slice(2)); + +var child_process = execFile(protoc, args, function(error, stdout, stderr) { if (error) { throw error; } From 6bfe7dadf71ce6637adef1adc990a653849f56c1 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 9 Aug 2016 11:40:05 -0700 Subject: [PATCH 44/97] add all runtime nodes back into testing packages --- src/csharp/Grpc.Examples.MathClient/project.json | 5 +++++ src/csharp/Grpc.Examples.MathServer/project.json | 5 +++++ src/csharp/Grpc.Examples/project.json | 3 ++- src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json | 5 +++++ src/csharp/Grpc.IntegrationTesting.StressClient/project.json | 5 +++++ .../csharp/Grpc.Examples.MathClient/project.json.template | 2 +- .../csharp/Grpc.Examples.MathServer/project.json.template | 2 +- templates/src/csharp/Grpc.Examples/project.json.template | 3 ++- .../Grpc.IntegrationTesting.QpsWorker/project.json.template | 2 +- .../project.json.template | 2 +- templates/src/csharp/build_options.include | 4 +--- 11 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/csharp/Grpc.Examples.MathClient/project.json b/src/csharp/Grpc.Examples.MathClient/project.json index 764a335ddfb..ad319478ab0 100644 --- a/src/csharp/Grpc.Examples.MathClient/project.json +++ b/src/csharp/Grpc.Examples.MathClient/project.json @@ -42,6 +42,11 @@ } } }, + "runtimes": { + "win7-x64": { }, + "debian.8-x64": { }, + "osx.10.11-x64": { } + }, "dependencies": { "Grpc.Examples": { diff --git a/src/csharp/Grpc.Examples.MathServer/project.json b/src/csharp/Grpc.Examples.MathServer/project.json index 764a335ddfb..ad319478ab0 100644 --- a/src/csharp/Grpc.Examples.MathServer/project.json +++ b/src/csharp/Grpc.Examples.MathServer/project.json @@ -42,6 +42,11 @@ } } }, + "runtimes": { + "win7-x64": { }, + "debian.8-x64": { }, + "osx.10.11-x64": { } + }, "dependencies": { "Grpc.Examples": { diff --git a/src/csharp/Grpc.Examples/project.json b/src/csharp/Grpc.Examples/project.json index 5329f390e4e..98bd5d852c4 100644 --- a/src/csharp/Grpc.Examples/project.json +++ b/src/csharp/Grpc.Examples/project.json @@ -20,11 +20,12 @@ "System.IO": "" } }, - "netstandard1.5": { + "netcoreapp1.0": { "imports": [ "portable-net45" ], "dependencies": { + "Microsoft.NETCore.App": "1.0.0", "NETStandard.Library": "1.6.0" } } diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json index 9afcb306b40..287950720fe 100644 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json @@ -44,6 +44,11 @@ } } }, + "runtimes": { + "win7-x64": { }, + "debian.8-x64": { }, + "osx.10.11-x64": { } + }, "dependencies": { "Grpc.IntegrationTesting": { diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json index 9afcb306b40..287950720fe 100644 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json @@ -44,6 +44,11 @@ } } }, + "runtimes": { + "win7-x64": { }, + "debian.8-x64": { }, + "osx.10.11-x64": { } + }, "dependencies": { "Grpc.IntegrationTesting": { diff --git a/templates/src/csharp/Grpc.Examples.MathClient/project.json.template b/templates/src/csharp/Grpc.Examples.MathClient/project.json.template index 51c5e85c66d..67151dbcfa8 100644 --- a/templates/src/csharp/Grpc.Examples.MathClient/project.json.template +++ b/templates/src/csharp/Grpc.Examples.MathClient/project.json.template @@ -1,7 +1,7 @@ %YAML 1.2 --- | { - <%include file="../build_options.include" args="executable=True,includeRuntimes=False"/> + <%include file="../build_options.include" args="executable=True"/> "dependencies": { "Grpc.Examples": { "target": "project" diff --git a/templates/src/csharp/Grpc.Examples.MathServer/project.json.template b/templates/src/csharp/Grpc.Examples.MathServer/project.json.template index 51c5e85c66d..67151dbcfa8 100644 --- a/templates/src/csharp/Grpc.Examples.MathServer/project.json.template +++ b/templates/src/csharp/Grpc.Examples.MathServer/project.json.template @@ -1,7 +1,7 @@ %YAML 1.2 --- | { - <%include file="../build_options.include" args="executable=True,includeRuntimes=False"/> + <%include file="../build_options.include" args="executable=True"/> "dependencies": { "Grpc.Examples": { "target": "project" diff --git a/templates/src/csharp/Grpc.Examples/project.json.template b/templates/src/csharp/Grpc.Examples/project.json.template index d5d63f658e4..117f842e01e 100644 --- a/templates/src/csharp/Grpc.Examples/project.json.template +++ b/templates/src/csharp/Grpc.Examples/project.json.template @@ -15,11 +15,12 @@ "System.IO": "" } }, - "netstandard1.5": { + "netcoreapp1.0": { "imports": [ "portable-net45" ], "dependencies": { + "Microsoft.NETCore.App": "1.0.0", "NETStandard.Library": "1.6.0" } } diff --git a/templates/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json.template b/templates/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json.template index af1ee425096..93151f2b89e 100644 --- a/templates/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json.template +++ b/templates/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json.template @@ -1,7 +1,7 @@ %YAML 1.2 --- | { - <%include file="../build_options.include" args="executable=True,includeData=True,includeRuntimes=False"/> + <%include file="../build_options.include" args="executable=True,includeData=True"/> "dependencies": { "Grpc.IntegrationTesting": { "target": "project" diff --git a/templates/src/csharp/Grpc.IntegrationTesting.StressClient/project.json.template b/templates/src/csharp/Grpc.IntegrationTesting.StressClient/project.json.template index af1ee425096..93151f2b89e 100644 --- a/templates/src/csharp/Grpc.IntegrationTesting.StressClient/project.json.template +++ b/templates/src/csharp/Grpc.IntegrationTesting.StressClient/project.json.template @@ -1,7 +1,7 @@ %YAML 1.2 --- | { - <%include file="../build_options.include" args="executable=True,includeData=True,includeRuntimes=False"/> + <%include file="../build_options.include" args="executable=True,includeData=True"/> "dependencies": { "Grpc.IntegrationTesting": { "target": "project" diff --git a/templates/src/csharp/build_options.include b/templates/src/csharp/build_options.include index a200897e0f2..8597ae33675 100644 --- a/templates/src/csharp/build_options.include +++ b/templates/src/csharp/build_options.include @@ -1,4 +1,4 @@ -<%page args="executable=False,includeData=False,includeRuntimes=True"/>\ +<%page args="executable=False,includeData=False"/>\ "buildOptions": { % if executable: "emitEntryPoint": true @@ -52,10 +52,8 @@ } }, %endif - % if includeRuntimes: "runtimes": { "win7-x64": { }, "debian.8-x64": { }, "osx.10.11-x64": { } }, - % endif \ No newline at end of file From f07506438c012f1f466670f05284594ca6808a26 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 9 Aug 2016 14:25:08 -0700 Subject: [PATCH 45/97] Work in progress. Do not check in yet. --- .../cronet/transport/cronet_transport.c | 543 ++++++++++++------ 1 file changed, 360 insertions(+), 183 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 694d346fc33..e8746b4e6e6 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -51,6 +51,20 @@ #define GRPC_HEADER_SIZE_IN_BYTES 5 +#define CRONET_LOG(...) {if (1) gpr_log(__VA_ARGS__);} + +enum OP_RESULT { + ACTION_TAKEN_WITH_CALLBACK, + ACTION_TAKEN_NO_CALLBACK, + NO_ACTION_POSSIBLE +}; + +const char *OP_RESULT_STRING[] = { + "ACTION_TAKEN_WITH_CALLBACK", + "ACTION_TAKEN_NO_CALLBACK", + "NO_ACTION_POSSIBLE" +}; + enum OP_ID { OP_SEND_INITIAL_METADATA = 0, OP_SEND_MESSAGE, @@ -60,9 +74,29 @@ enum OP_ID { OP_RECV_TRAILING_METADATA, OP_CANCEL_ERROR, OP_ON_COMPLETE, + OP_FAILED, + OP_CANCELED, + OP_RECV_MESSAGE_AND_ON_COMPLETE, + OP_READ_REQ_MADE, OP_NUM_OPS }; +const char *op_id_string[] = { + "OP_SEND_INITIAL_METADATA", + "OP_SEND_MESSAGE", + "OP_SEND_TRAILING_METADATA", + "OP_RECV_MESSAGE", + "OP_RECV_INITIAL_METADATA", + "OP_RECV_TRAILING_METADATA", + "OP_CANCEL_ERROR", + "OP_ON_COMPLETE", + "OP_FAILED", + "OP_CANCELED", + "OP_RECV_MESSAGE_AND_ON_COMPLETE", + "OP_READ_REQ_MADE", + "OP_NUM_OPS" +}; + /* Cronet callbacks */ static void on_request_headers_sent(cronet_bidirectional_stream *); @@ -75,7 +109,7 @@ static void on_response_trailers_received(cronet_bidirectional_stream *, const cronet_bidirectional_stream_header_array *); static void on_succeeded(cronet_bidirectional_stream *); static void on_failed(cronet_bidirectional_stream *, int); -//static void on_canceled(cronet_bidirectional_stream *); +static void on_canceled(cronet_bidirectional_stream *); static cronet_bidirectional_stream_callback cronet_callbacks = { on_request_headers_sent, on_response_headers_received, @@ -84,7 +118,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_response_trailers_received, on_succeeded, on_failed, - NULL //on_canceled + on_canceled }; // Cronet transport object @@ -121,30 +155,49 @@ struct write_state { char *write_buffer; }; -#define MAX_PENDING_OPS 10 +// maximum ops in a batch.. There is not much thinking behind this limit, except +// that it seems to be enough for most use cases. +#define MAX_PENDING_OPS 100 + +struct op_state { + bool state_op_done[OP_NUM_OPS]; + bool state_callback_received[OP_NUM_OPS]; + // data structure for storing data coming from server + struct read_state rs; + // data structure for storing data going to the server + struct write_state ws; +}; + +struct op_and_state { + grpc_transport_stream_op op; + struct op_state state; + bool done; + struct stream_obj *s; // Pointer back to the stream object +}; + struct op_storage { - grpc_transport_stream_op pending_ops[MAX_PENDING_OPS]; + struct op_and_state pending_ops[MAX_PENDING_OPS]; int wrptr; int rdptr; int num_pending_ops; }; struct stream_obj { + struct op_and_state *oas; grpc_transport_stream_op *curr_op; grpc_cronet_transport curr_ct; grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; - // TODO (makdharma) : make a sub structure for tracking state - bool state_op_done[OP_NUM_OPS]; - bool state_callback_received[OP_NUM_OPS]; + // This holds the state that is at stream level (response and req metadata) + struct op_state state; - // Read state - struct read_state rs; - // Write state - struct write_state ws; + //struct op_state state; // OP storage struct op_storage storage; + + // Mutex to protect execute_curr_streaming_op + gpr_mu mu; }; typedef struct stream_obj stream_obj; @@ -152,123 +205,155 @@ typedef struct stream_obj stream_obj; cronet_bidirectional_stream_header_array header_array; // -static void execute_curr_stream_op(stream_obj *s); +static enum OP_RESULT execute_stream_op(struct op_and_state *oas); /************************************************************* Op Storage */ -static void add_pending_op(struct op_storage *storage, grpc_transport_stream_op *op) { +static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { + struct op_storage *storage = &s->storage; GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); storage->num_pending_ops++; - gpr_log(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", + CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr], op, sizeof(grpc_transport_stream_op)); + memcpy(&storage->pending_ops[storage->wrptr].op, op, sizeof(grpc_transport_stream_op)); + memset(&storage->pending_ops[storage->wrptr].state, 0, sizeof(storage->pending_ops[storage->wrptr].state)); + storage->pending_ops[storage->wrptr].done = false; + storage->pending_ops[storage->wrptr].s = s; storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; } -static grpc_transport_stream_op *pop_pending_op(struct op_storage *storage) { - if (storage->num_pending_ops == 0) return NULL; - grpc_transport_stream_op *result = &storage->pending_ops[storage->rdptr]; - storage->rdptr = (storage->rdptr + 1) % MAX_PENDING_OPS; - storage->num_pending_ops--; - gpr_log(GPR_DEBUG, "popping op @rdptr=%d. %d more left in queue", - storage->rdptr, storage->num_pending_ops); - return result; +static void execute_from_storage(stream_obj *s) { + // Cycle through ops and try to take next action. Break when either + // an action with callback is taken, or no action is possible. + gpr_mu_lock(&s->mu); + for (int i = 0; i < s->storage.wrptr; ) { + CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); + if (s->storage.pending_ops[i].done) { + i++; + continue; + } + enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); + CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", OP_RESULT_STRING[result], i); + if (result == NO_ACTION_POSSIBLE) { + i++; + } else if (result == ACTION_TAKEN_WITH_CALLBACK) { + break; + } + } + gpr_mu_unlock(&s->mu); } + /************************************************************* Cronet Callback Ipmlementation */ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); + CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_FAILED] = true; + s->cbs = NULL; + execute_from_storage(s); +} + +static void on_canceled(cronet_bidirectional_stream *stream) { + CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_CANCELED] = true; + s->cbs = NULL; + execute_from_storage(s); } static void on_succeeded(cronet_bidirectional_stream *stream) { - gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); + CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->cbs = NULL; } - static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); + CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; - s->state_op_done[OP_SEND_INITIAL_METADATA] = true; - s->state_callback_received[OP_SEND_INITIAL_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; + s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true; + execute_from_storage(s); } static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, headers, negotiated_protocol); - stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->rs.initial_metadata, 0, sizeof(s->rs.initial_metadata)); - grpc_chttp2_incoming_metadata_buffer_init(&s->rs.initial_metadata); + memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); unsigned int i = 0; for (i = 0; i < headers->count; i++) { grpc_chttp2_incoming_metadata_buffer_add( - &s->rs.initial_metadata, + &s->state.rs.initial_metadata, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(headers->headers[i].key), grpc_mdstr_from_string(headers->headers[i].value))); } - s->state_callback_received[OP_RECV_INITIAL_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true; + execute_from_storage(s); } static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { - gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); stream_obj *s = (stream_obj *)stream->annotation; - if (s->ws.write_buffer) { - gpr_free(s->ws.write_buffer); - s->ws.write_buffer = NULL; + CRONET_LOG(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; } - s->state_callback_received[OP_SEND_MESSAGE] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_SEND_MESSAGE] = true; + execute_from_storage(s); } static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; - gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); if (count > 0) { - s->rs.received_bytes += count; - s->rs.remaining_bytes -= count; - if (s->rs.remaining_bytes > 0) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer + s->rs.received_bytes, s->rs.remaining_bytes); + s->state.rs.received_bytes += count; + s->state.rs.remaining_bytes -= count; + if (s->state.rs.remaining_bytes > 0) { + //CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + s->state.state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); } else { - execute_curr_stream_op(s); + execute_from_storage(s); } - s->state_callback_received[OP_RECV_MESSAGE] = true; + s->state.state_callback_received[OP_RECV_MESSAGE] = true; } } static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, trailers); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->rs.trailing_metadata, 0, sizeof(s->rs.trailing_metadata)); - s->rs.trailing_metadata_valid = false; - grpc_chttp2_incoming_metadata_buffer_init(&s->rs.trailing_metadata); + memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); + s->state.rs.trailing_metadata_valid = false; + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { - gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, trailers->headers[i].value); grpc_chttp2_incoming_metadata_buffer_add( - &s->rs.trailing_metadata, grpc_mdelem_from_metadata_strings( + &s->state.rs.trailing_metadata, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(trailers->headers[i].key), grpc_mdstr_from_string(trailers->headers[i].value))); - s->rs.trailing_metadata_valid = true; + s->state.rs.trailing_metadata_valid = true; } - s->state_callback_received[OP_RECV_TRAILING_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; + execute_from_storage(s); } /************************************************************* @@ -295,10 +380,10 @@ static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, memcpy(p, GPR_SLICE_START_PTR(slice), length); } -static void enqueue_callback(grpc_closure *callback) { +static void enqueue_callback(grpc_closure *callback, grpc_error *error) { GPR_ASSERT(callback); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(&exec_ctx, callback, error, NULL); grpc_exec_ctx_finish(&exec_ctx); } @@ -342,6 +427,7 @@ static void convert_metadata_to_cronet_headers( gpr_asprintf(pp_url, "https://%s%s", host, value); continue; } + gpr_log(GPR_DEBUG, "header %s = %s", key, value); headers[num_headers].key = key; headers[num_headers].value = value; num_headers++; @@ -365,165 +451,252 @@ static int parse_grpc_header(const uint8_t *data) { /* Op Execution */ - -static bool op_can_be_run(stream_obj *s, enum OP_ID op_id) { - if (op_id == OP_SEND_INITIAL_METADATA) { +static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *stream_state, struct op_state *op_state, enum OP_ID op_id) { + // We use op's own state for most state, except for metadata related callbacks, which + // are at the stream level. TODO: WTF does this comment mean? + bool result = true; + // When call is canceled, every op can be run + if (stream_state->state_op_done[OP_CANCEL_ERROR] || stream_state->state_callback_received[OP_FAILED]) { + if (op_id == OP_SEND_INITIAL_METADATA) result = false; + if (op_id == OP_SEND_MESSAGE) result = false; + if (op_id == OP_SEND_TRAILING_METADATA) result = false; + if (op_id == OP_CANCEL_ERROR) result = false; // already executed - if (s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_INITIAL_METADATA) { + if (op_id == OP_RECV_INITIAL_METADATA && stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; + if (op_id == OP_RECV_MESSAGE && stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + if (op_id == OP_RECV_TRAILING_METADATA && stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + } else if (op_id == OP_SEND_INITIAL_METADATA) { // already executed - if (s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + if (stream_state->state_op_done[OP_SEND_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_INITIAL_METADATA) { + // already executed + if (stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; // we haven't sent headers yet. - if (!s->state_callback_received[OP_SEND_INITIAL_METADATA]) return false; + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_SEND_MESSAGE) { + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + } else if (op_id == OP_SEND_MESSAGE) { // already executed - if (s->state_op_done[OP_SEND_MESSAGE]) return false; + if (stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // we haven't sent headers yet. + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_MESSAGE) { + // already executed + if (op_state->state_op_done[OP_RECV_MESSAGE]) result = false; // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_MESSAGE) { + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_TRAILING_METADATA) { // already executed - if (s->state_op_done[OP_RECV_MESSAGE]) return false; - // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_TRAILING_METADATA) { - // already executed - if (s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + if (stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + // we have asked for but haven't received message yet. + else if (stream_state->state_op_done[OP_READ_REQ_MADE] && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; // we haven't received trailers yet. - if (!s->state_callback_received[OP_RECV_TRAILING_METADATA]) return false; - } - if (op_id == OP_SEND_TRAILING_METADATA) { + else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) result = false; + } else if (op_id == OP_SEND_TRAILING_METADATA) { // already executed - if (s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + if (stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; + // we haven't sent initial metadata yet + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't sent message yet - if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; - } - - if (op_id == OP_ON_COMPLETE) { + else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // we haven't got on_write_completed for the send yet + else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + } else if (op_id == OP_CANCEL_ERROR) { // already executed - if (s->state_op_done[OP_ON_COMPLETE]) return false; + if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; + } else if (op_id == OP_ON_COMPLETE) { + // already executed (note we're checking op specific state, not stream state) + if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; // Check if every op that was asked for is done. - if (s->curr_op->send_initial_metadata && !s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; - if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; - if (s->curr_op->send_trailing_metadata && !s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; - if (s->curr_op->recv_initial_metadata && !s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; - if (s->curr_op->recv_message && !s->state_op_done[OP_RECV_MESSAGE]) return false; - if (s->curr_op->recv_trailing_metadata && !s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; + else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; + else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + else if (curr_op->recv_trailing_metadata) { + // We aren't done with trailing metadata yet + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + // We've asked for actual message in an earlier op, and it hasn't been delivered yet. + // (TODO: What happens when multiple messages are asked for? How do you know when last message arrived?) + else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { + // If this op is not the one asking for read, (which means some earlier op has asked), and the + // read hasn't been delivered. + if(!curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE]) result = false; + } + } + // We should see at least one on_write_completed for the trailers that we sent + else if (curr_op->send_trailing_metadata && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; } - return true; + CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], result? "YES":"NO"); + return result; } -static void execute_curr_stream_op(stream_obj *s) { - if (s->curr_op->send_initial_metadata && op_can_be_run(s, OP_SEND_INITIAL_METADATA)) { +static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { + // TODO TODO : This can be called from network thread and main thread. add a mutex. + grpc_transport_stream_op *stream_op = &oas->op; + struct stream_obj *s = oas->s; + struct op_state *stream_state = &s->state; + //CRONET_LOG(GPR_DEBUG, "execute_stream_op"); + enum OP_RESULT result = NO_ACTION_POSSIBLE; + if (stream_op->send_initial_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_INITIAL_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); // This OP is the beginning. Reset various states - memset(&s->rs, 0, sizeof(s->rs)); - memset(&s->ws, 0, sizeof(s->ws)); - memset(s->state_op_done, 0, sizeof(s->state_op_done)); - memset(s->state_callback_received, 0, sizeof(s->state_callback_received)); + memset(&stream_state->rs, 0, sizeof(stream_state->rs)); + memset(&stream_state->ws, 0, sizeof(stream_state->ws)); + memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); + memset(stream_state->state_callback_received, 0, sizeof(stream_state->state_callback_received)); // Start new cronet stream GPR_ASSERT(s->cbs == NULL); - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_create"); s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); char *url; - convert_metadata_to_cronet_headers(s->curr_op->send_initial_metadata->list.head, + convert_metadata_to_cronet_headers(stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, &header_array.headers, &header_array.count); header_array.capacity = header_array.count; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start %s", url); cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); - s->state_op_done[OP_SEND_INITIAL_METADATA] = true; - } else if (s->curr_op->recv_initial_metadata && - op_can_be_run(s, OP_RECV_INITIAL_METADATA)) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->rs.initial_metadata, - s->curr_op->recv_initial_metadata); - enqueue_callback(s->curr_op->recv_initial_metadata_ready); - s->state_op_done[OP_RECV_INITIAL_METADATA] = true; - // We are ready to execute send_message. - execute_curr_stream_op(s); - } else if (s->curr_op->send_message && op_can_be_run(s, OP_SEND_MESSAGE)) { + stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->recv_initial_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_INITIAL_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); + if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { + grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, + stream_op->recv_initial_metadata); + enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE); + } else { + enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + } + stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; + result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_MESSAGE)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); // TODO (makdharma): Make into a standalone function gpr_slice_buffer write_slice_buffer; gpr_slice slice; gpr_slice_buffer_init(&write_slice_buffer); - grpc_byte_stream_next(NULL, s->curr_op->send_message, &slice, - s->curr_op->send_message->length, NULL); + grpc_byte_stream_next(NULL, stream_op->send_message, &slice, + stream_op->send_message->length, NULL); // Check that compression flag is not ON. We don't support compression yet. // TODO (makdharma): add compression support - GPR_ASSERT(s->curr_op->send_message->flags == 0); + GPR_ASSERT(stream_op->send_message->flags == 0); gpr_slice_buffer_add(&write_slice_buffer, slice); GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet if (write_slice_buffer.count > 0) { int write_buffer_size; - create_grpc_frame(&write_slice_buffer, &s->ws.write_buffer, &write_buffer_size); - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", s->ws.write_buffer); - cronet_bidirectional_stream_write(s->cbs, s->ws.write_buffer, + create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", stream_state->ws.write_buffer); + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, write_buffer_size, false); // TODO: What if this is not the last write? + result = ACTION_TAKEN_WITH_CALLBACK; } - s->state_op_done[OP_SEND_MESSAGE] = true; - } else if (s->curr_op->recv_message && op_can_be_run(s, OP_RECV_MESSAGE)) { - if (s->rs.length_field_received == false) { - if (s->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && s->rs.remaining_bytes == 0) { + stream_state->state_op_done[OP_SEND_MESSAGE] = true; + } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); + if (stream_state->state_op_done[OP_CANCEL_ERROR]) { + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_CANCELLED); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + } else if (stream_state->rs.length_field_received == false) { + if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { // Start a read operation for data - s->rs.length_field_received = true; - s->rs.length_field = s->rs.remaining_bytes = - parse_grpc_header((const uint8_t *)s->rs.read_buffer); - GPR_ASSERT(s->rs.length_field > 0); // Empty message? - gpr_log(GPR_DEBUG, "length field = %d", s->rs.length_field); - s->rs.read_buffer = gpr_malloc(s->rs.length_field); - GPR_ASSERT(s->rs.read_buffer); - s->rs.remaining_bytes = s->rs.length_field; - s->rs.received_bytes = 0; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, - s->rs.remaining_bytes); - } else if (s->rs.remaining_bytes == 0) { + stream_state->rs.length_field_received = true; + stream_state->rs.length_field = stream_state->rs.remaining_bytes = + parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); + CRONET_LOG(GPR_DEBUG, "length field = %d", stream_state->rs.length_field); + if (stream_state->rs.length_field > 0) { + stream_state->rs.read_buffer = gpr_malloc(stream_state->rs.length_field); + GPR_ASSERT(stream_state->rs.read_buffer); + stream_state->rs.remaining_bytes = stream_state->rs.length_field; + stream_state->rs.received_bytes = 0; + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, + stream_state->rs.remaining_bytes); + result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->rs.remaining_bytes = 0; + CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); + gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + result = ACTION_TAKEN_NO_CALLBACK; + } + } else if (stream_state->rs.remaining_bytes == 0) { // Start a read operation for first 5 bytes (GRPC header) - s->rs.read_buffer = s->rs.grpc_header_bytes; - s->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; - s->rs.received_bytes = 0; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, - s->rs.remaining_bytes); + stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; + stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + stream_state->rs.received_bytes = 0; + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, + stream_state->rs.remaining_bytes); } - } else if (s->rs.remaining_bytes == 0) { - gpr_log(GPR_DEBUG, "read operation complete"); - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->rs.length_field); + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_state->rs.remaining_bytes == 0) { + CRONET_LOG(GPR_DEBUG, "read operation complete"); + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)stream_state->rs.length_field); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, s->rs.read_buffer, (size_t)s->rs.length_field); - gpr_slice_buffer_init(&s->rs.read_slice_buffer); - gpr_slice_buffer_add(&s->rs.read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&s->rs.sbs, &s->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)s->curr_op->recv_message) = (grpc_byte_buffer *)&s->rs.sbs; - enqueue_callback(s->curr_op->recv_message_ready); - s->state_op_done[OP_RECV_MESSAGE] = true; - execute_curr_stream_op(s); + memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); + gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); + gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + // Clear read state of the stream, so next read op (if it were to come) will work + stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = stream_state->rs.length_field_received = 0; + result = ACTION_TAKEN_NO_CALLBACK; } - } else if (s->curr_op->recv_trailing_metadata && - op_can_be_run(s, OP_RECV_TRAILING_METADATA)) { - if (s->rs.trailing_metadata_valid) { + } else if (stream_op->recv_trailing_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_TRAILING_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_TRAILING_METADATA", oas); + if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( - &s->rs.trailing_metadata, s->curr_op->recv_trailing_metadata); - s->rs.trailing_metadata_valid = false; + &oas->s->state.rs.trailing_metadata, stream_op->recv_trailing_metadata); + stream_state->rs.trailing_metadata_valid = false; } - s->state_op_done[OP_RECV_TRAILING_METADATA] = true; - execute_curr_stream_op(s); - } else if (s->curr_op->send_trailing_metadata && - op_can_be_run(s, OP_SEND_TRAILING_METADATA)) { - - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; + result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_TRAILING_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); - s->state_op_done[OP_SEND_TRAILING_METADATA] = true; - } else if (op_can_be_run(s, OP_ON_COMPLETE)) { + stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->cancel_error && op_can_be_run(stream_op, stream_state, &oas->state, OP_CANCEL_ERROR)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); + CRONET_LOG(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + // Cancel might have come before creation of stream + if (s->cbs) { + cronet_bidirectional_stream_cancel(s->cbs); + } + stream_state->state_op_done[OP_CANCEL_ERROR] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->on_complete && op_can_be_run(stream_op, stream_state, &oas->state, OP_ON_COMPLETE)) { // All ops are complete. Call the on_complete callback - enqueue_callback(s->curr_op->on_complete); - s->state_op_done[OP_ON_COMPLETE] = true; - cronet_bidirectional_stream_destroy(s->cbs); - s->cbs = NULL; + CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); + //CRONET_LOG(GPR_DEBUG, "calling on_complete"); + enqueue_callback(stream_op->on_complete, GRPC_ERROR_NONE); + // Instead of setting stream state, use the op state as on_complete is on per op basis + oas->state.state_op_done[OP_ON_COMPLETE] = true; + oas->done = true; // Mark this op as completed + // reset any send or receive message state. + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + stream_state->state_op_done[OP_SEND_MESSAGE] = false; + result = ACTION_TAKEN_NO_CALLBACK; + // If this is the on_complete callback being called for a received message - make a note + if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; + } else { + //CRONET_LOG(GPR_DEBUG, "No op ready to run"); + result = NO_ACTION_POSSIBLE; } + return result; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -533,7 +706,14 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, const void *server_data) { stream_obj *s = (stream_obj *)gs; memset(&s->storage, 0, sizeof(s->storage)); + memset(&s->state, 0, sizeof(s->state)); s->curr_op = NULL; + s->cbs = NULL; + memset(&s->state.rs, 0, sizeof(s->state.rs)); + memset(&s->state.ws, 0, sizeof(s->state.ws)); + memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); + memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); + gpr_mu_init(&s->mu); return 0; } @@ -546,15 +726,12 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_transport_stream_op *op) { - gpr_log(GPR_DEBUG, "perform_stream_op"); + CRONET_LOG(GPR_DEBUG, "perform_stream_op"); stream_obj *s = (stream_obj *)gs; - memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); - add_pending_op(&s->storage, op); - if (s->curr_op == NULL) { - s->curr_op = pop_pending_op(&s->storage); - } s->curr_gs = gs; - execute_curr_stream_op(s); + memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); + add_to_storage(s, op); + execute_from_storage(s); } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, From 0068bdb65a1b96c143189170811d004aa8bf0cd2 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 8 Aug 2016 20:17:51 -0700 Subject: [PATCH 46/97] php7: fix ubuntu compile error --- src/php/ext/grpc/php7_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/php/ext/grpc/php7_wrapper.h b/src/php/ext/grpc/php7_wrapper.h index fd8d35636f0..0d40e825073 100644 --- a/src/php/ext/grpc/php7_wrapper.h +++ b/src/php/ext/grpc/php7_wrapper.h @@ -143,7 +143,7 @@ static inline int php_grpc_zend_hash_find(HashTable *ht, char *key, int len, #define PHP_GRPC_RETURN_STRING(val, dup) RETURN_STRING(val) #define PHP_GRPC_MAKE_STD_ZVAL(pzv) \ - zval _stack_zval_##pzv; \ + static zval _stack_zval_##pzv; \ pzv = &(_stack_zval_##pzv) #define PHP_GRPC_DELREF(zv) From 83b34c2285d104ab60a40e6ad88cc32c5815e7f7 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 9 Aug 2016 12:04:49 -0700 Subject: [PATCH 47/97] php: use emalloc to replicate MAKE_STD_ZVAL --- src/php/ext/grpc/php7_wrapper.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/php/ext/grpc/php7_wrapper.h b/src/php/ext/grpc/php7_wrapper.h index 0d40e825073..1d7824113fa 100644 --- a/src/php/ext/grpc/php7_wrapper.h +++ b/src/php/ext/grpc/php7_wrapper.h @@ -143,8 +143,7 @@ static inline int php_grpc_zend_hash_find(HashTable *ht, char *key, int len, #define PHP_GRPC_RETURN_STRING(val, dup) RETURN_STRING(val) #define PHP_GRPC_MAKE_STD_ZVAL(pzv) \ - static zval _stack_zval_##pzv; \ - pzv = &(_stack_zval_##pzv) + pzv = (zval *)emalloc(sizeof(zval)); #define PHP_GRPC_DELREF(zv) #define PHP_GRPC_WRAP_OBJECT_START(name) \ From 7e8efc3ba8038ddafcf9018404c499540b44c471 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 9 Aug 2016 21:47:46 -0700 Subject: [PATCH 48/97] php: update package.xml --- package.xml | 19 +++++++++++++++++-- templates/package.xml.template | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/package.xml b/package.xml index 65764e88889..1bf57f28cc8 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-07-28 + 2016-08-09 1.0.0 @@ -22,7 +22,7 @@ BSD -- PHP7 Support continued, reduce code duplication #7543 +- Fixed Ubuntu compile error #7571, #7642 @@ -1131,5 +1131,20 @@ Update to wrap gRPC C Core version 0.10.0 - PHP7 Support continued, reduce code duplication #7543 + + + 1.0.0RC4 + 1.0.0RC4 + + + stable + stable + + 2016-08-09 + BSD + +- Fixed Ubuntu compile error #7571, #7642 + + diff --git a/templates/package.xml.template b/templates/package.xml.template index 87b10389598..43d3aa2a584 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ grpc-packages@google.com yes - 2016-07-28 + 2016-08-09 ${settings.php_version.php()} @@ -24,7 +24,7 @@ BSD - - PHP7 Support continued, reduce code duplication #7543 + - Fixed Ubuntu compile error #7571, #7642 @@ -249,5 +249,20 @@ - PHP7 Support continued, reduce code duplication #7543 + + + 1.0.0RC4 + 1.0.0RC4 + + + stable + stable + + 2016-08-09 + BSD + + - Fixed Ubuntu compile error #7571, #7642 + + From f789facafd66a4f1d789d5062f1dd17c0142c006 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 9 Aug 2016 22:01:17 -0700 Subject: [PATCH 49/97] minor text change --- examples/php/route_guide/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/php/route_guide/README.md b/examples/php/route_guide/README.md index 4e74a79f136..26f1704f122 100644 --- a/examples/php/route_guide/README.md +++ b/examples/php/route_guide/README.md @@ -1,6 +1,6 @@ #gRPC Basics: PHP sample code The files in this folder are the samples used in [gRPC Basics: PHP][], -a detailed tutorial for using gRPC in Ruby. +a detailed tutorial for using gRPC in PHP. [gRPC Basics: PHP]:http://www.grpc.io/docs/tutorials/basic/php.html From 5312815ff212e6d4bbea1f1f6264b1295ca0106c Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 9 Aug 2016 09:40:16 -0700 Subject: [PATCH 50/97] add netcore portability package dependency --- examples/csharp/helloworld-from-cli/Greeter/project.json | 3 +++ examples/csharp/helloworld-from-cli/GreeterClient/project.json | 3 +++ examples/csharp/helloworld-from-cli/GreeterServer/project.json | 3 +++ 3 files changed, 9 insertions(+) diff --git a/examples/csharp/helloworld-from-cli/Greeter/project.json b/examples/csharp/helloworld-from-cli/Greeter/project.json index e06854d1226..87749418100 100644 --- a/examples/csharp/helloworld-from-cli/Greeter/project.json +++ b/examples/csharp/helloworld-from-cli/Greeter/project.json @@ -13,6 +13,9 @@ "frameworkAssemblies": { "System.Runtime": "", "System.IO": "" + }, + "dependencies": { + "Microsoft.NETCore.Platforms": "1.0.1" } } } diff --git a/examples/csharp/helloworld-from-cli/GreeterClient/project.json b/examples/csharp/helloworld-from-cli/GreeterClient/project.json index dc72a30eb81..c2bf694cd86 100644 --- a/examples/csharp/helloworld-from-cli/GreeterClient/project.json +++ b/examples/csharp/helloworld-from-cli/GreeterClient/project.json @@ -17,6 +17,9 @@ "frameworkAssemblies": { "System.Runtime": "", "System.IO": "" + }, + "dependencies": { + "Microsoft.NETCore.Platforms": "1.0.1" } } } diff --git a/examples/csharp/helloworld-from-cli/GreeterServer/project.json b/examples/csharp/helloworld-from-cli/GreeterServer/project.json index 4bf201bef38..29a10670f43 100644 --- a/examples/csharp/helloworld-from-cli/GreeterServer/project.json +++ b/examples/csharp/helloworld-from-cli/GreeterServer/project.json @@ -17,6 +17,9 @@ "frameworkAssemblies": { "System.Runtime": "", "System.IO": "" + }, + "dependencies": { + "Microsoft.NETCore.Platforms": "1.0.1" } } } From 4ce458339a556fe4890e66a2672a3414ad4647ef Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 10 Aug 2016 12:38:03 -0700 Subject: [PATCH 51/97] Remove warning about protobuf 3.0.0+ from the Makefile --- Makefile | 29 ----------------------------- templates/Makefile.template | 29 ----------------------------- 2 files changed, 58 deletions(-) diff --git a/Makefile b/Makefile index ad2ff32d419..e75dc3c2ea3 100644 --- a/Makefile +++ b/Makefile @@ -767,13 +767,6 @@ ifeq ($(MAKECMDGOALS),clean) NO_DEPS = true endif -INSTALL_OK = false -ifeq ($(HAS_VALID_PROTOC),true) -ifeq ($(HAS_SYSTEM_PROTOBUF_VERIFY),true) -INSTALL_OK = true -endif -endif - .SECONDARY = %.pb.h %.pb.cc PROTOC_PLUGINS = $(BINDIR)/$(CONFIG)/grpc_cpp_plugin $(BINDIR)/$(CONFIG)/grpc_csharp_plugin $(BINDIR)/$(CONFIG)/grpc_node_plugin $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin $(BINDIR)/$(CONFIG)/grpc_python_plugin $(BINDIR)/$(CONFIG)/grpc_ruby_plugin @@ -2285,28 +2278,6 @@ install-certs: etc/roots.pem $(Q) $(INSTALL) -d $(prefix)/share/grpc $(Q) $(INSTALL) etc/roots.pem $(prefix)/share/grpc/roots.pem -verify-install: -ifeq ($(INSTALL_OK),true) - @echo "Your system looks ready to go." - @echo -else - @echo "Warning: it looks like protoc 3.0.0+ isn't installed on your system," - @echo "which means that you won't be able to compile .proto files for use" - @echo "with gRPC." - @echo - @echo "If you are just using pre-compiled protocol buffers, or you otherwise" - @echo "have no need to compile .proto files, you can ignore this." - @echo - @echo "If you do need protobuf for some reason, you can download and install" - @echo "it from:" - @echo - @echo " https://github.com/google/protobuf/releases" - @echo - @echo "Once you've done so, you can re-run this check by doing:" - @echo - @echo " make verify-install" -endif - clean: $(E) "[CLEAN] Cleaning build directories." $(Q) $(RM) -rf $(OBJDIR) $(LIBDIR) $(BINDIR) $(GENDIR) cache.mk diff --git a/templates/Makefile.template b/templates/Makefile.template index 0cbd8bfdd55..bfa99d66d28 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -655,13 +655,6 @@ NO_DEPS = true endif - INSTALL_OK = false - ifeq ($(HAS_VALID_PROTOC),true) - ifeq ($(HAS_SYSTEM_PROTOBUF_VERIFY),true) - INSTALL_OK = true - endif - endif - .SECONDARY = %.pb.h %.pb.cc PROTOC_PLUGINS =\ @@ -1279,28 +1272,6 @@ $(Q) $(INSTALL) -d $(prefix)/share/grpc $(Q) $(INSTALL) etc/roots.pem $(prefix)/share/grpc/roots.pem - verify-install: - ifeq ($(INSTALL_OK),true) - @echo "Your system looks ready to go." - @echo - else - @echo "Warning: it looks like protoc 3.0.0+ isn't installed on your system," - @echo "which means that you won't be able to compile .proto files for use" - @echo "with gRPC." - @echo - @echo "If you are just using pre-compiled protocol buffers, or you otherwise" - @echo "have no need to compile .proto files, you can ignore this." - @echo - @echo "If you do need protobuf for some reason, you can download and install" - @echo "it from:" - @echo - @echo " https://github.com/google/protobuf/releases" - @echo - @echo "Once you've done so, you can re-run this check by doing:" - @echo - @echo " make verify-install" - endif - clean: $(E) "[CLEAN] Cleaning build directories." $(Q) $(RM) -rf $(OBJDIR) $(LIBDIR) $(BINDIR) $(GENDIR) cache.mk From a3c5535c64445ed77acbadde627851b6a417790a Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 10 Aug 2016 13:41:31 -0700 Subject: [PATCH 52/97] Remove use of verify-install target in Makefile --- Makefile | 2 +- templates/Makefile.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e75dc3c2ea3..f5de4d129ca 100644 --- a/Makefile +++ b/Makefile @@ -2095,7 +2095,7 @@ $(OBJDIR)/$(CONFIG)/%.o : %.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< -install: install_c install_cxx install-plugins install-certs verify-install +install: install_c install_cxx install-plugins install-certs install_c: install-headers_c install-static_c install-shared_c diff --git a/templates/Makefile.template b/templates/Makefile.template index bfa99d66d28..6f768af31e0 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -1154,7 +1154,7 @@ $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< - install: install_c install_cxx install-plugins install-certs verify-install + install: install_c install_cxx install-plugins install-certs install_c: install-headers_c install-static_c install-shared_c From 198f8b01946ac107700958ac6c2ea11ed523f2a5 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 13:44:13 -0700 Subject: [PATCH 53/97] more fixes --- .../cronet/transport/cronet_transport.c | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index e8746b4e6e6..dc6a780edaf 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -75,6 +75,7 @@ enum OP_ID { OP_CANCEL_ERROR, OP_ON_COMPLETE, OP_FAILED, + OP_SUCCEEDED, OP_CANCELED, OP_RECV_MESSAGE_AND_ON_COMPLETE, OP_READ_REQ_MADE, @@ -91,6 +92,7 @@ const char *op_id_string[] = { "OP_CANCEL_ERROR", "OP_ON_COMPLETE", "OP_FAILED", + "OP_SUCCEEDED", "OP_CANCELED", "OP_RECV_MESSAGE_AND_ON_COMPLETE", "OP_READ_REQ_MADE", @@ -189,6 +191,8 @@ struct stream_obj { grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; + // Used for executing callbacks for ops + grpc_exec_ctx exec_ctx; // This holds the state that is at stream level (response and req metadata) struct op_state state; @@ -227,7 +231,10 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { static void execute_from_storage(stream_obj *s) { // Cycle through ops and try to take next action. Break when either // an action with callback is taken, or no action is possible. - gpr_mu_lock(&s->mu); + // This can be executed from the Cronet network thread via cronet callback + // or on the application supplied thread via the perform_stream_op function. + if (1) {//gpr_mu_lock(&s->mu) == 0) { + gpr_mu_lock(&s->mu); for (int i = 0; i < s->storage.wrptr; ) { CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); if (s->storage.pending_ops[i].done) { @@ -242,7 +249,9 @@ static void execute_from_storage(stream_obj *s) { break; } } - gpr_mu_unlock(&s->mu); + gpr_mu_unlock(&s->mu); + } + grpc_exec_ctx_finish(&s->exec_ctx); } @@ -271,7 +280,9 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_FAILED] = true; s->cbs = NULL; + execute_from_storage(s); } static void on_request_headers_sent(cronet_bidirectional_stream *stream) { @@ -380,13 +391,6 @@ static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, memcpy(p, GPR_SLICE_START_PTR(slice), length); } -static void enqueue_callback(grpc_closure *callback, grpc_error *error) { - GPR_ASSERT(callback); - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_exec_ctx_sched(&exec_ctx, callback, error, NULL); - grpc_exec_ctx_finish(&exec_ctx); -} - static void convert_metadata_to_cronet_headers( grpc_linked_mdelem *head, const char *host, @@ -498,9 +502,10 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // we haven't sent initial metadata yet else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't sent message yet + // TODO: Streaming Write case is a problem. What if there is an outstanding write (2nd, 3rd,..) present. else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; // we haven't got on_write_completed for the send yet - else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + else if (stream_state->state_op_done[OP_SEND_MESSAGE] && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; } else if (op_id == OP_CANCEL_ERROR) { // already executed if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; @@ -510,10 +515,12 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // Check if every op that was asked for is done. else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; else if (curr_op->recv_trailing_metadata) { + //if (!stream_state->state_op_done[OP_SUCCEEDED]) result = false; gpr_log(GPR_DEBUG, "HACK!!"); // We aren't done with trailing metadata yet if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; // We've asked for actual message in an earlier op, and it hasn't been delivered yet. @@ -521,7 +528,7 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { // If this op is not the one asking for read, (which means some earlier op has asked), and the // read hasn't been delivered. - if(!curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE]) result = false; + if(!curr_op->recv_message && !stream_state->state_op_done[OP_SUCCEEDED]) result = false; } } // We should see at least one on_write_completed for the trailers that we sent @@ -563,9 +570,9 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); - enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE, NULL); } else { - enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED, NULL); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; @@ -595,7 +602,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_CANCELLED); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { @@ -620,7 +627,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. result = ACTION_TAKEN_NO_CALLBACK; @@ -645,7 +652,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. // Clear read state of the stream, so next read op (if it were to come) will work @@ -682,7 +689,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { // All ops are complete. Call the on_complete callback CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); //CRONET_LOG(GPR_DEBUG, "calling on_complete"); - enqueue_callback(stream_op->on_complete, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, NULL); // Instead of setting stream state, use the op state as on_complete is on per op basis oas->state.state_op_done[OP_ON_COMPLETE] = true; oas->done = true; // Mark this op as completed @@ -714,6 +721,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); gpr_mu_init(&s->mu); + s->exec_ctx = *exec_ctx; return 0; } From 35da822b442a60d5e9d0711cc046f9c578d63ad9 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 14:53:40 -0700 Subject: [PATCH 54/97] WIP --- src/core/ext/transport/cronet/transport/cronet_transport.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index dc6a780edaf..7e65def4de1 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -280,7 +280,7 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; cronet_bidirectional_stream_destroy(s->cbs); - s->state.state_callback_received[OP_FAILED] = true; + s->state.state_callback_received[OP_SUCCEEDED] = true; s->cbs = NULL; execute_from_storage(s); } @@ -480,8 +480,8 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // we haven't received headers yet. else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; } else if (op_id == OP_SEND_MESSAGE) { - // already executed - if (stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // already executed (note we're checking op specific state, not stream state) + if (op_state->state_op_done[OP_SEND_MESSAGE]) result = false; // we haven't sent headers yet. else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; } else if (op_id == OP_RECV_MESSAGE) { @@ -599,6 +599,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_MESSAGE] = true; + oas->state.state_op_done[OP_SEND_MESSAGE] = true; } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { From feef8081c1b700cd640d57496ef27901841c312e Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 10 Aug 2016 15:39:16 -0700 Subject: [PATCH 55/97] update original helloworld README --- examples/csharp/helloworld/README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/csharp/helloworld/README.md b/examples/csharp/helloworld/README.md index d13c9ac9db4..71840ad48bf 100644 --- a/examples/csharp/helloworld/README.md +++ b/examples/csharp/helloworld/README.md @@ -13,7 +13,7 @@ PREREQUISITES ------------- - Windows: .NET Framework 4.5+, Visual Studio 2013 or 2015 -- Linux: Mono 4+, MonoDevelop 5.9+ (with NuGet add-in installed) +- Linux: Mono 4+, MonoDevelop 5.9+ - Mac OS X: Xamarin Studio 5.9+ BUILD @@ -21,7 +21,20 @@ BUILD - Open solution `Greeter.sln` with Visual Studio, Monodevelop (on Linux) or Xamarin Studio (on Mac OS X) -- Build the solution (this will automatically download NuGet dependencies) +# Using Visual Studio + +* Build the solution (this will automatically download NuGet dependencies) + +# Using Monodevelop or Xamarin Studio + +The nuget add-in available for Xamarin Studio and Monodevelop IDEs is too old to +download all of the nuget dependencies of gRPC. One alternative to is to use the dotnet command line tools instead (see [helloworld-from-cli]). + +Using these IDEs, a workaround is as follows: +* Obtain a nuget executable for your platform and update it with + `nuget update -self`. +* Navigate to this directory and run `nuget restore`. +* Now that packages have been restored into their proper package folder, build the solution from your IDE. Try it! ------- @@ -49,5 +62,6 @@ Tutorial You can find a more detailed tutorial in [gRPC Basics: C#][] +[helloworld-from-cli]:../helloworld-from-cli/README.md [helloworld.proto]:../../protos/helloworld.proto [gRPC Basics: C#]:http://www.grpc.io/docs/tutorials/basic/csharp.html From f66a37b63aed77732c8d7d253af89296a23bb72e Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 17:47:25 -0700 Subject: [PATCH 56/97] WIP --- .../cronet/transport/cronet_transport.c | 597 +++++++++++------- 1 file changed, 356 insertions(+), 241 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 7e65def4de1..1d756039809 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -50,8 +50,17 @@ #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 +// maximum ops in a batch. There is not much thinking behind this limit, except +// that it seems to be enough for most use cases. +#define MAX_PENDING_OPS 100 -#define CRONET_LOG(...) {if (1) gpr_log(__VA_ARGS__);} +#define CRONET_LOG(...) \ + { \ + if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ + } + +// TODO (makdharma): Hook up into the wider tracing mechanism +int grpc_cronet_trace = 1; enum OP_RESULT { ACTION_TAKEN_WITH_CALLBACK, @@ -59,11 +68,10 @@ enum OP_RESULT { NO_ACTION_POSSIBLE }; -const char *OP_RESULT_STRING[] = { - "ACTION_TAKEN_WITH_CALLBACK", - "ACTION_TAKEN_NO_CALLBACK", - "NO_ACTION_POSSIBLE" -}; +// Used for printing debug +const char *op_result_string[] = {"ACTION_TAKEN_WITH_CALLBACK", + "ACTION_TAKEN_NO_CALLBACK", + "NO_ACTION_POSSIBLE"}; enum OP_ID { OP_SEND_INITIAL_METADATA = 0, @@ -82,32 +90,31 @@ enum OP_ID { OP_NUM_OPS }; -const char *op_id_string[] = { - "OP_SEND_INITIAL_METADATA", - "OP_SEND_MESSAGE", - "OP_SEND_TRAILING_METADATA", - "OP_RECV_MESSAGE", - "OP_RECV_INITIAL_METADATA", - "OP_RECV_TRAILING_METADATA", - "OP_CANCEL_ERROR", - "OP_ON_COMPLETE", - "OP_FAILED", - "OP_SUCCEEDED", - "OP_CANCELED", - "OP_RECV_MESSAGE_AND_ON_COMPLETE", - "OP_READ_REQ_MADE", - "OP_NUM_OPS" -}; +const char *op_id_string[] = {"OP_SEND_INITIAL_METADATA", + "OP_SEND_MESSAGE", + "OP_SEND_TRAILING_METADATA", + "OP_RECV_MESSAGE", + "OP_RECV_INITIAL_METADATA", + "OP_RECV_TRAILING_METADATA", + "OP_CANCEL_ERROR", + "OP_ON_COMPLETE", + "OP_FAILED", + "OP_SUCCEEDED", + "OP_CANCELED", + "OP_RECV_MESSAGE_AND_ON_COMPLETE", + "OP_READ_REQ_MADE", + "OP_NUM_OPS"}; /* Cronet callbacks */ static void on_request_headers_sent(cronet_bidirectional_stream *); -static void on_response_headers_received(cronet_bidirectional_stream *, - const cronet_bidirectional_stream_header_array *, - const char *); +static void on_response_headers_received( + cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *, const char *); static void on_write_completed(cronet_bidirectional_stream *, const char *); static void on_read_completed(cronet_bidirectional_stream *, char *, int); -static void on_response_trailers_received(cronet_bidirectional_stream *, +static void on_response_trailers_received( + cronet_bidirectional_stream *, const cronet_bidirectional_stream_header_array *); static void on_succeeded(cronet_bidirectional_stream *); static void on_failed(cronet_bidirectional_stream *, int); @@ -120,8 +127,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_response_trailers_received, on_succeeded, on_failed, - on_canceled -}; + on_canceled}; // Cronet transport object struct grpc_cronet_transport { @@ -132,7 +138,7 @@ struct grpc_cronet_transport { typedef struct grpc_cronet_transport grpc_cronet_transport; struct read_state { - // vars to store data coming from cronet + /* vars to store data coming from server */ char *read_buffer; bool length_field_received; int received_bytes; @@ -141,15 +147,15 @@ struct read_state { char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; char *payload_field; - // vars for holding data destined for the application + /* vars for holding data destined for the application */ struct grpc_slice_buffer_stream sbs; gpr_slice_buffer read_slice_buffer; - // vars for trailing metadata + /* vars for trailing metadata */ grpc_chttp2_incoming_metadata_buffer trailing_metadata; bool trailing_metadata_valid; - // vars for initial metadata + /* vars for initial metadata */ grpc_chttp2_incoming_metadata_buffer initial_metadata; }; @@ -157,16 +163,13 @@ struct write_state { char *write_buffer; }; -// maximum ops in a batch.. There is not much thinking behind this limit, except -// that it seems to be enough for most use cases. -#define MAX_PENDING_OPS 100 - +/* track state of one stream op */ struct op_state { bool state_op_done[OP_NUM_OPS]; bool state_callback_received[OP_NUM_OPS]; - // data structure for storing data coming from server + /* data structure for storing data coming from server */ struct read_state rs; - // data structure for storing data going to the server + /* data structure for storing data going to the server */ struct write_state ws; }; @@ -174,7 +177,7 @@ struct op_and_state { grpc_transport_stream_op op; struct op_state state; bool done; - struct stream_obj *s; // Pointer back to the stream object + struct stream_obj *s; /* Pointer back to the stream object */ }; struct op_storage { @@ -190,73 +193,74 @@ struct stream_obj { grpc_cronet_transport curr_ct; grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; + cronet_bidirectional_stream_header_array header_array; - // Used for executing callbacks for ops + /* Used for executing callbacks for ops */ grpc_exec_ctx exec_ctx; - // This holds the state that is at stream level (response and req metadata) + /* Stream level state. Some state will be tracked both at stream and stream_op + * level */ struct op_state state; - //struct op_state state; - // OP storage + /* OP storage */ struct op_storage storage; - // Mutex to protect execute_curr_streaming_op + /* Mutex to protect storage */ gpr_mu mu; }; typedef struct stream_obj stream_obj; -/* Globals */ -cronet_bidirectional_stream_header_array header_array; - -// static enum OP_RESULT execute_stream_op(struct op_and_state *oas); -/************************************************************* - Op Storage +/* + Add a new stream op to op storage. */ - static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { + gpr_mu_lock(&s->mu); struct op_storage *storage = &s->storage; GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); storage->num_pending_ops++; CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", - storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr].op, op, sizeof(grpc_transport_stream_op)); - memset(&storage->pending_ops[storage->wrptr].state, 0, sizeof(storage->pending_ops[storage->wrptr].state)); + storage->wrptr, storage->num_pending_ops); + memcpy(&storage->pending_ops[storage->wrptr].op, op, + sizeof(grpc_transport_stream_op)); + memset(&storage->pending_ops[storage->wrptr].state, 0, + sizeof(storage->pending_ops[storage->wrptr].state)); storage->pending_ops[storage->wrptr].done = false; storage->pending_ops[storage->wrptr].s = s; - storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + gpr_mu_unlock(&s->mu); } +/* + Cycle through ops and try to take next action. Break when either + an action with callback is taken, or no action is possible. + This can be executed from the Cronet network thread via cronet callback + or on the application supplied thread via the perform_stream_op function. +*/ static void execute_from_storage(stream_obj *s) { - // Cycle through ops and try to take next action. Break when either - // an action with callback is taken, or no action is possible. - // This can be executed from the Cronet network thread via cronet callback - // or on the application supplied thread via the perform_stream_op function. - if (1) {//gpr_mu_lock(&s->mu) == 0) { - gpr_mu_lock(&s->mu); - for (int i = 0; i < s->storage.wrptr; ) { - CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); - if (s->storage.pending_ops[i].done) { - i++; - continue; - } - enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); - CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", OP_RESULT_STRING[result], i); - if (result == NO_ACTION_POSSIBLE) { - i++; - } else if (result == ACTION_TAKEN_WITH_CALLBACK) { - break; - } + gpr_mu_lock(&s->mu); + for (int i = 0; i < s->storage.wrptr;) { + CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, + s->storage.pending_ops[i].done); + if (s->storage.pending_ops[i].done) { + i++; + continue; + } + enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); + CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", + op_result_string[result], i); + if (result == NO_ACTION_POSSIBLE) { + i++; + } else if (result == ACTION_TAKEN_WITH_CALLBACK) { + break; } - gpr_mu_unlock(&s->mu); } + gpr_mu_unlock(&s->mu); grpc_exec_ctx_finish(&s->exec_ctx); } - -/************************************************************* -Cronet Callback Ipmlementation +/* + Cronet callback */ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); @@ -267,6 +271,9 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_canceled(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; @@ -276,6 +283,9 @@ static void on_canceled(cronet_bidirectional_stream *stream) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; @@ -285,22 +295,33 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true; + /* Free the memory allocated for headers */ + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + } execute_from_storage(s); } +/* + Cronet callback +*/ static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, - headers, negotiated_protocol); + headers, negotiated_protocol); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); + memset(&s->state.rs.initial_metadata, 0, + sizeof(s->state.rs.initial_metadata)); grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); unsigned int i = 0; for (i = 0; i < headers->count; i++) { @@ -314,6 +335,9 @@ static void on_response_headers_received( execute_from_storage(s); } +/* + Cronet callback +*/ static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { stream_obj *s = (stream_obj *)stream->annotation; @@ -326,17 +350,23 @@ static void on_write_completed(cronet_bidirectional_stream *stream, execute_from_storage(s); } +/* + Cronet callback +*/ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; - CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, + count); if (count > 0) { s->state.rs.received_bytes += count; s->state.rs.remaining_bytes -= count; if (s->state.rs.remaining_bytes > 0) { - //CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - s->state.state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made - cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + s->state.state_op_done[OP_READ_REQ_MADE] = true; + cronet_bidirectional_stream_read( + s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, + s->state.rs.remaining_bytes); } else { execute_from_storage(s); } @@ -344,76 +374,82 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, } } +/* + Cronet callback +*/ static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, - trailers); + trailers); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); + memset(&s->state.rs.trailing_metadata, 0, + sizeof(s->state.rs.trailing_metadata)); s->state.rs.trailing_metadata_valid = false; grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, - trailers->headers[i].value); + trailers->headers[i].value); grpc_chttp2_incoming_metadata_buffer_add( - &s->state.rs.trailing_metadata, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); + &s->state.rs.trailing_metadata, + grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(trailers->headers[i].key), + grpc_mdstr_from_string(trailers->headers[i].value))); s->state.rs.trailing_metadata_valid = true; } s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; execute_from_storage(s); } -/************************************************************* -Utility functions. Can be in their own file +/* + Utility function that takes the data from s->write_slice_buffer and assembles + into a contiguous byte stream with 5 byte gRPC header prepended. */ -// This function takes the data from s->write_slice_buffer and assembles into -// a contiguous byte stream with 5 byte gRPC header prepended. static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, - char **pp_write_buffer, int *p_write_buffer_size) { + char **pp_write_buffer, + int *p_write_buffer_size) { gpr_slice slice = gpr_slice_buffer_take_first(write_slice_buffer); size_t length = GPR_SLICE_LENGTH(slice); - // TODO (makdharma): FREE THIS!! HACK! *p_write_buffer_size = (int)length + GRPC_HEADER_SIZE_IN_BYTES; + /* This is freed in the on_write_completed callback */ char *write_buffer = gpr_malloc(length + GRPC_HEADER_SIZE_IN_BYTES); *pp_write_buffer = write_buffer; uint8_t *p = (uint8_t *)write_buffer; - // Append 5 byte header + /* Append 5 byte header */ *p++ = 0; *p++ = (uint8_t)(length >> 24); *p++ = (uint8_t)(length >> 16); *p++ = (uint8_t)(length >> 8); *p++ = (uint8_t)(length); - // append actual data + /* append actual data */ memcpy(p, GPR_SLICE_START_PTR(slice), length); } +/* + Convert metadata in a format that Cronet can consume +*/ static void convert_metadata_to_cronet_headers( - grpc_linked_mdelem *head, - const char *host, - char **pp_url, - cronet_bidirectional_stream_header **pp_headers, - size_t *p_num_headers) { + grpc_linked_mdelem *head, const char *host, char **pp_url, + cronet_bidirectional_stream_header **pp_headers, size_t *p_num_headers) { grpc_linked_mdelem *curr = head; - // Walk the linked list and get number of header fields + /* Walk the linked list and get number of header fields */ uint32_t num_headers_available = 0; while (curr != NULL) { curr = curr->next; num_headers_available++; } - // Allocate enough memory. TODO (makdharma): FREE MEMORY! HACK HACK - cronet_bidirectional_stream_header *headers = - (cronet_bidirectional_stream_header *)gpr_malloc( - sizeof(cronet_bidirectional_stream_header) * num_headers_available); + /* Allocate enough memory. It is freed in the on_request_headers_sent callback + */ + cronet_bidirectional_stream_header *headers = + (cronet_bidirectional_stream_header *)gpr_malloc( + sizeof(cronet_bidirectional_stream_header) * num_headers_available); *pp_headers = headers; - // Walk the linked list again, this time copying the header fields. - // s->num_headers - // can be less than num_headers_available, as some headers are not used for - // cronet + /* Walk the linked list again, this time copying the header fields. + s->num_headers can be less than num_headers_available, as some headers + are not used for cronet + */ curr = head; int num_headers = 0; while (num_headers < num_headers_available) { @@ -423,15 +459,15 @@ static void convert_metadata_to_cronet_headers( const char *value = grpc_mdstr_as_c_string(mdelem->value); if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || strcmp(key, ":authority") == 0) { - // Cronet populates these fields on its own. + /* Cronet populates these fields on its own */ continue; } if (strcmp(key, ":path") == 0) { - // Create URL by appending :path value to the hostname + /* Create URL by appending :path value to the hostname */ gpr_asprintf(pp_url, "https://%s%s", host, value); continue; } - gpr_log(GPR_DEBUG, "header %s = %s", key, value); + CRONET_LOG(GPR_DEBUG, "header %s = %s", key, value); headers[num_headers].key = key; headers[num_headers].value = value; num_headers++; @@ -453,261 +489,342 @@ static int parse_grpc_header(const uint8_t *data) { } /* -Op Execution + Op Execution: Decide if one of the actions contained in the stream op can be + executed. This is the heart of the state machine. */ -static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *stream_state, struct op_state *op_state, enum OP_ID op_id) { - // We use op's own state for most state, except for metadata related callbacks, which - // are at the stream level. TODO: WTF does this comment mean? +static bool op_can_be_run(grpc_transport_stream_op *curr_op, + struct op_state *stream_state, + struct op_state *op_state, enum OP_ID op_id) { bool result = true; - // When call is canceled, every op can be run - if (stream_state->state_op_done[OP_CANCEL_ERROR] || stream_state->state_callback_received[OP_FAILED]) { + /* When call is canceled, every op can be run, except under following + conditions + */ + if (stream_state->state_op_done[OP_CANCEL_ERROR] || + stream_state->state_callback_received[OP_FAILED]) { if (op_id == OP_SEND_INITIAL_METADATA) result = false; if (op_id == OP_SEND_MESSAGE) result = false; if (op_id == OP_SEND_TRAILING_METADATA) result = false; if (op_id == OP_CANCEL_ERROR) result = false; - // already executed - if (op_id == OP_RECV_INITIAL_METADATA && stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - if (op_id == OP_RECV_MESSAGE && stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; - if (op_id == OP_RECV_TRAILING_METADATA && stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + /* already executed */ + if (op_id == OP_RECV_INITIAL_METADATA && + stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + result = false; + if (op_id == OP_RECV_MESSAGE && + stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; + if (op_id == OP_RECV_TRAILING_METADATA && + stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + result = false; } else if (op_id == OP_SEND_INITIAL_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_SEND_INITIAL_METADATA]) result = false; } else if (op_id == OP_RECV_INITIAL_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - // we haven't sent headers yet. - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - // we haven't received headers yet. - else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + /* we haven't sent headers yet. */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + /* we haven't received headers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) + result = false; } else if (op_id == OP_SEND_MESSAGE) { - // already executed (note we're checking op specific state, not stream state) + /* already executed (note we're checking op specific state, not stream + state) */ if (op_state->state_op_done[OP_SEND_MESSAGE]) result = false; - // we haven't sent headers yet. - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + /* we haven't sent headers yet. */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; } else if (op_id == OP_RECV_MESSAGE) { - // already executed + /* already executed */ if (op_state->state_op_done[OP_RECV_MESSAGE]) result = false; - // we haven't received headers yet. - else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + /* we haven't received headers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) + result = false; } else if (op_id == OP_RECV_TRAILING_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; - // we have asked for but haven't received message yet. - else if (stream_state->state_op_done[OP_READ_REQ_MADE] && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; - // we haven't received trailers yet. - else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) result = false; + /* we have asked for but haven't received message yet. */ + else if (stream_state->state_op_done[OP_READ_REQ_MADE] && + !stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; + /* we haven't received trailers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) + result = false; } else if (op_id == OP_SEND_TRAILING_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; - // we haven't sent initial metadata yet - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - // we haven't sent message yet - // TODO: Streaming Write case is a problem. What if there is an outstanding write (2nd, 3rd,..) present. - else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; - // we haven't got on_write_completed for the send yet - else if (stream_state->state_op_done[OP_SEND_MESSAGE] && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + /* we haven't sent initial metadata yet */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + /* we haven't sent message yet */ + else if (curr_op->send_message && + !stream_state->state_op_done[OP_SEND_MESSAGE]) + result = false; + /* we haven't got on_write_completed for the send yet */ + else if (stream_state->state_op_done[OP_SEND_MESSAGE] && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; } else if (op_id == OP_CANCEL_ERROR) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; } else if (op_id == OP_ON_COMPLETE) { - // already executed (note we're checking op specific state, not stream state) + /* already executed (note we're checking op specific state, not stream + state) */ if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; - // Check if every op that was asked for is done. - else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; - else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; - else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; - else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + /* Check if every op that was asked for is done. */ + else if (curr_op->send_initial_metadata && + !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + else if (curr_op->send_message && + !stream_state->state_op_done[OP_SEND_MESSAGE]) + result = false; + else if (curr_op->send_message && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; + else if (curr_op->send_trailing_metadata && + !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) + result = false; + else if (curr_op->recv_initial_metadata && + !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + result = false; + else if (curr_op->recv_message && + !stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; else if (curr_op->recv_trailing_metadata) { - //if (!stream_state->state_op_done[OP_SUCCEEDED]) result = false; gpr_log(GPR_DEBUG, "HACK!!"); - // We aren't done with trailing metadata yet - if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; - // We've asked for actual message in an earlier op, and it hasn't been delivered yet. - // (TODO: What happens when multiple messages are asked for? How do you know when last message arrived?) + /* We aren't done with trailing metadata yet */ + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + result = false; + /* We've asked for actual message in an earlier op, and it hasn't been + delivered yet. */ else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { - // If this op is not the one asking for read, (which means some earlier op has asked), and the - // read hasn't been delivered. - if(!curr_op->recv_message && !stream_state->state_op_done[OP_SUCCEEDED]) result = false; + /* If this op is not the one asking for read, (which means some earlier + op has asked), and the read hasn't been delivered. */ + if (!curr_op->recv_message && + !stream_state->state_op_done[OP_SUCCEEDED]) + result = false; } } - // We should see at least one on_write_completed for the trailers that we sent - else if (curr_op->send_trailing_metadata && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + /* We should see at least one on_write_completed for the trailers that we + sent */ + else if (curr_op->send_trailing_metadata && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; } - CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], result? "YES":"NO"); + CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], + result ? "YES" : "NO"); return result; } static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { - // TODO TODO : This can be called from network thread and main thread. add a mutex. grpc_transport_stream_op *stream_op = &oas->op; struct stream_obj *s = oas->s; struct op_state *stream_state = &s->state; - //CRONET_LOG(GPR_DEBUG, "execute_stream_op"); enum OP_RESULT result = NO_ACTION_POSSIBLE; - if (stream_op->send_initial_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_INITIAL_METADATA)) { + if (stream_op->send_initial_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); - // This OP is the beginning. Reset various states + /* This OP is the beginning. Reset various states */ memset(&stream_state->rs, 0, sizeof(stream_state->rs)); memset(&stream_state->ws, 0, sizeof(stream_state->ws)); memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); - memset(stream_state->state_callback_received, 0, sizeof(stream_state->state_callback_received)); - // Start new cronet stream + memset(stream_state->state_callback_received, 0, + sizeof(stream_state->state_callback_received)); + /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled, + * on_failed */ GPR_ASSERT(s->cbs == NULL); CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_create"); - s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); + s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, + &cronet_callbacks); char *url; - convert_metadata_to_cronet_headers(stream_op->send_initial_metadata->list.head, - s->curr_ct.host, &url, &header_array.headers, &header_array.count); - header_array.capacity = header_array.count; + convert_metadata_to_cronet_headers( + stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, + &s->header_array.headers, &s->header_array.count); + s->header_array.capacity = s->header_array.count; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start %s", url); - cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); + cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &s->header_array, + false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->recv_initial_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_INITIAL_METADATA)) { + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, - stream_op->recv_initial_metadata); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE, NULL); + grpc_chttp2_incoming_metadata_buffer_publish( + &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_NONE, NULL); } else { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_CANCELLED, NULL); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_MESSAGE)) { + } else if (stream_op->send_message && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); - // TODO (makdharma): Make into a standalone function gpr_slice_buffer write_slice_buffer; gpr_slice slice; gpr_slice_buffer_init(&write_slice_buffer); grpc_byte_stream_next(NULL, stream_op->send_message, &slice, stream_op->send_message->length, NULL); - // Check that compression flag is not ON. We don't support compression yet. - // TODO (makdharma): add compression support + /* Check that compression flag is OFF. We don't support compression yet. */ GPR_ASSERT(stream_op->send_message->flags == 0); gpr_slice_buffer_add(&write_slice_buffer, slice); - GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet + GPR_ASSERT(write_slice_buffer.count == + 1); /* Empty request not handled yet */ if (write_slice_buffer.count > 0) { int write_buffer_size; - create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", stream_state->ws.write_buffer); + create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, + &write_buffer_size); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", + stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, - write_buffer_size, false); // TODO: What if this is not the last write? + write_buffer_size, false); result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; - } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { + } else if (stream_op->recv_message && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { - if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { - // Start a read operation for data + if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && + stream_state->rs.remaining_bytes == 0) { + /* Start a read operation for data */ stream_state->rs.length_field_received = true; stream_state->rs.length_field = stream_state->rs.remaining_bytes = - parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); - CRONET_LOG(GPR_DEBUG, "length field = %d", stream_state->rs.length_field); + parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); + CRONET_LOG(GPR_DEBUG, "length field = %d", + stream_state->rs.length_field); if (stream_state->rs.length_field > 0) { - stream_state->rs.read_buffer = gpr_malloc(stream_state->rs.length_field); + stream_state->rs.read_buffer = + gpr_malloc(stream_state->rs.length_field); GPR_ASSERT(stream_state->rs.read_buffer); stream_state->rs.remaining_bytes = stream_state->rs.length_field; stream_state->rs.received_bytes = 0; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + stream_state->state_op_done[OP_READ_REQ_MADE] = + true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, - stream_state->rs.remaining_bytes); + stream_state->rs.remaining_bytes); result = ACTION_TAKEN_WITH_CALLBACK; } else { stream_state->rs.remaining_bytes = 0; CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); - grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, + &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = + (grpc_byte_buffer *)&stream_state->rs.sbs; + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; - oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + oas->state.state_op_done[OP_RECV_MESSAGE] = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { - // Start a read operation for first 5 bytes (GRPC header) + /* Start a read operation for first 5 bytes (GRPC header) */ stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.received_bytes = 0; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + stream_state->state_op_done[OP_READ_REQ_MADE] = + true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); } result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_state->rs.remaining_bytes == 0) { CRONET_LOG(GPR_DEBUG, "read operation complete"); - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)stream_state->rs.length_field); + gpr_slice read_data_slice = + gpr_slice_malloc((uint32_t)stream_state->rs.length_field); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); + memcpy(dst_p, stream_state->rs.read_buffer, + (size_t)stream_state->rs.length_field); gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); - gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); + gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, + read_data_slice); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, + &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = + (grpc_byte_buffer *)&stream_state->rs.sbs; + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; - oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. - // Clear read state of the stream, so next read op (if it were to come) will work - stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = stream_state->rs.length_field_received = 0; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; + /* Clear read state of the stream, so next read op (if it were to come) + * will work */ + stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = + stream_state->rs.length_field_received = 0; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_TRAILING_METADATA)) { + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_TRAILING_METADATA", oas); if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( - &oas->s->state.rs.trailing_metadata, stream_op->recv_trailing_metadata); + &oas->s->state.rs.trailing_metadata, + stream_op->recv_trailing_metadata); stream_state->rs.trailing_metadata_valid = false; } stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_TRAILING_METADATA)) { + } else if (stream_op->send_trailing_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; result = ACTION_TAKEN_WITH_CALLBACK; - } else if (stream_op->cancel_error && op_can_be_run(stream_op, stream_state, &oas->state, OP_CANCEL_ERROR)) { + } else if (stream_op->cancel_error && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_CANCEL_ERROR)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); CRONET_LOG(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - // Cancel might have come before creation of stream if (s->cbs) { cronet_bidirectional_stream_cancel(s->cbs); } stream_state->state_op_done[OP_CANCEL_ERROR] = true; result = ACTION_TAKEN_WITH_CALLBACK; - } else if (stream_op->on_complete && op_can_be_run(stream_op, stream_state, &oas->state, OP_ON_COMPLETE)) { - // All ops are complete. Call the on_complete callback + } else if (stream_op->on_complete && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_ON_COMPLETE)) { + /* All actions in this stream_op are complete. Call the on_complete callback + */ CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); - //CRONET_LOG(GPR_DEBUG, "calling on_complete"); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, NULL); - // Instead of setting stream state, use the op state as on_complete is on per op basis + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, + NULL); oas->state.state_op_done[OP_ON_COMPLETE] = true; - oas->done = true; // Mark this op as completed - // reset any send or receive message state. + oas->done = true; + /* reset any send or receive message state. */ stream_state->state_callback_received[OP_SEND_MESSAGE] = false; stream_state->state_op_done[OP_SEND_MESSAGE] = false; result = ACTION_TAKEN_NO_CALLBACK; - // If this is the on_complete callback being called for a received message - make a note - if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; + /* If this is the on_complete callback being called for a received message - + make a note */ + if (stream_op->recv_message) + stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; } else { - //CRONET_LOG(GPR_DEBUG, "No op ready to run"); result = NO_ACTION_POSSIBLE; } return result; } -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/* + Functions used by upper layers to access transport functionality. +*/ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, @@ -720,7 +837,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, memset(&s->state.rs, 0, sizeof(s->state.rs)); memset(&s->state.ws, 0, sizeof(s->state.ws)); memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); - memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); + memset(s->state.state_callback_received, 0, + sizeof(s->state.state_callback_received)); gpr_mu_init(&s->mu); s->exec_ctx = *exec_ctx; return 0; @@ -744,19 +862,16 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { -} + grpc_stream *gs, void *and_free_memory) {} -static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { -} +static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {} static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { return NULL; } static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_transport_op *op) { -} + grpc_transport_op *op) {} const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), "cronet_http", From 5450f05e0c789ea48ec90483a8520062b8d828c0 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 11 Aug 2016 02:24:46 +0000 Subject: [PATCH 57/97] Migrate distrib, interop, and stress to GA API --- .../tests/interop/_insecure_interop_test.py | 14 +- .../tests/interop/_secure_interop_test.py | 24 +- .../grpcio_tests/tests/interop/client.py | 56 ++-- .../grpcio_tests/tests/interop/methods.py | 266 +++++++++--------- .../grpcio_tests/tests/interop/server.py | 12 +- .../grpcio_tests/tests/stress/client.py | 21 +- .../tests/stress/metrics_server.py | 2 +- test/distrib/python/distribtest.py | 4 +- 8 files changed, 198 insertions(+), 201 deletions(-) diff --git a/src/python/grpcio_tests/tests/interop/_insecure_interop_test.py b/src/python/grpcio_tests/tests/interop/_insecure_interop_test.py index c753d6faf05..936c895bd2e 100644 --- a/src/python/grpcio_tests/tests/interop/_insecure_interop_test.py +++ b/src/python/grpcio_tests/tests/interop/_insecure_interop_test.py @@ -29,9 +29,10 @@ """Insecure client-server interoperability as a unit test.""" +from concurrent import futures import unittest -from grpc.beta import implementations +import grpc from src.proto.grpc.testing import test_pb2 from tests.interop import _interop_test_case @@ -44,14 +45,13 @@ class InsecureInteropTest( unittest.TestCase): def setUp(self): - self.server = test_pb2.beta_create_TestService_server(methods.TestService()) + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + test_pb2.add_TestServiceServicer_to_server( + methods.TestService(), self.server) port = self.server.add_insecure_port('[::]:0') self.server.start() - self.stub = test_pb2.beta_create_TestService_stub( - implementations.insecure_channel('localhost', port)) - - def tearDown(self): - self.server.stop(0) + self.stub = test_pb2.TestServiceStub( + grpc.insecure_channel('localhost:{}'.format(port))) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/interop/_secure_interop_test.py b/src/python/grpcio_tests/tests/interop/_secure_interop_test.py index cb09f54a347..eaca553e1b8 100644 --- a/src/python/grpcio_tests/tests/interop/_secure_interop_test.py +++ b/src/python/grpcio_tests/tests/interop/_secure_interop_test.py @@ -29,17 +29,16 @@ """Secure client-server interoperability as a unit test.""" +from concurrent import futures import unittest -from grpc.beta import implementations +import grpc from src.proto.grpc.testing import test_pb2 from tests.interop import _interop_test_case from tests.interop import methods from tests.interop import resources -from tests.unit.beta import test_utilities - _SERVER_HOST_OVERRIDE = 'foo.test.google.fr' @@ -48,19 +47,18 @@ class SecureInteropTest( unittest.TestCase): def setUp(self): - self.server = test_pb2.beta_create_TestService_server(methods.TestService()) + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + test_pb2.add_TestServiceServicer_to_server( + methods.TestService(), self.server) port = self.server.add_secure_port( - '[::]:0', implementations.ssl_server_credentials( + '[::]:0', grpc.ssl_server_credentials( [(resources.private_key(), resources.certificate_chain())])) self.server.start() - self.stub = test_pb2.beta_create_TestService_stub( - test_utilities.not_really_secure_channel( - 'localhost', port, implementations.ssl_channel_credentials( - resources.test_root_certificates()), - _SERVER_HOST_OVERRIDE)) - - def tearDown(self): - self.server.stop(0) + self.stub = test_pb2.TestServiceStub( + grpc.secure_channel( + 'localhost:{}'.format(port), + grpc.ssl_channel_credentials(resources.test_root_certificates()), + (('grpc.ssl_target_name_override', _SERVER_HOST_OVERRIDE,),))) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 8aa1ce30c1a..9d61d189759 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -32,14 +32,12 @@ import argparse from oauth2client import client as oauth2client_client +import grpc from grpc.beta import implementations from src.proto.grpc.testing import test_pb2 from tests.interop import methods from tests.interop import resources -from tests.unit.beta import test_utilities - -_ONE_DAY_IN_SECONDS = 60 * 60 * 24 def _args(): @@ -66,41 +64,49 @@ def _args(): return parser.parse_args() +def _application_default_credentials(): + return oauth2client_client.GoogleCredentials.get_application_default() + + def _stub(args): + target = '{}:{}'.format(args.server_host, args.server_port) if args.test_case == 'oauth2_auth_token': - creds = oauth2client_client.GoogleCredentials.get_application_default() - scoped_creds = creds.create_scoped([args.oauth_scope]) - access_token = scoped_creds.get_access_token().access_token - call_creds = implementations.access_token_call_credentials(access_token) + google_credentials = _application_default_credentials() + scoped_credentials = google_credentials.create_scoped([args.oauth_scope]) + access_token = scoped_credentials.get_access_token().access_token + call_credentials = grpc.access_token_call_credentials(access_token) elif args.test_case == 'compute_engine_creds': - creds = oauth2client_client.GoogleCredentials.get_application_default() - scoped_creds = creds.create_scoped([args.oauth_scope]) - call_creds = implementations.google_call_credentials(scoped_creds) + google_credentials = _application_default_credentials() + scoped_credentials = google_credentials.create_scoped([args.oauth_scope]) + # TODO(https://github.com/grpc/grpc/issues/6799): Eliminate this last + # remaining use of the Beta API. + call_credentials = implementations.google_call_credentials( + scoped_credentials) elif args.test_case == 'jwt_token_creds': - creds = oauth2client_client.GoogleCredentials.get_application_default() - call_creds = implementations.google_call_credentials(creds) + google_credentials = _application_default_credentials() + # TODO(https://github.com/grpc/grpc/issues/6799): Eliminate this last + # remaining use of the Beta API. + call_credentials = implementations.google_call_credentials( + google_credentials) else: - call_creds = None + call_credentials = None if args.use_tls: if args.use_test_ca: root_certificates = resources.test_root_certificates() else: root_certificates = None # will load default roots. - channel_creds = implementations.ssl_channel_credentials(root_certificates) - if call_creds is not None: - channel_creds = implementations.composite_channel_credentials( - channel_creds, call_creds) + channel_credentials = grpc.ssl_channel_credentials(root_certificates) + if call_credentials is not None: + channel_credentials = grpc.composite_channel_credentials( + channel_credentials, call_credentials) - channel = test_utilities.not_really_secure_channel( - args.server_host, args.server_port, channel_creds, - args.server_host_override) - stub = test_pb2.beta_create_TestService_stub(channel) + channel = grpc.secure_channel( + target, channel_credentials, + (('grpc.ssl_target_name_override', args.server_host_override,),)) else: - channel = implementations.insecure_channel( - args.server_host, args.server_port) - stub = test_pb2.beta_create_TestService_stub(channel) - return stub + channel = grpc.insecure_channel(target) + return test_pb2.TestServiceStub(channel) def _test_case_from_arg(test_case_arg): diff --git a/src/python/grpcio_tests/tests/interop/methods.py b/src/python/grpcio_tests/tests/interop/methods.py index 97e6c9e27ef..7edd75c56c9 100644 --- a/src/python/grpcio_tests/tests/interop/methods.py +++ b/src/python/grpcio_tests/tests/interop/methods.py @@ -29,8 +29,6 @@ """Implementations of interoperability test methods.""" -from __future__ import print_function - import enum import json import os @@ -41,26 +39,21 @@ from oauth2client import client as oauth2client_client import grpc from grpc.beta import implementations -from grpc.beta import interfaces -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import face from src.proto.grpc.testing import empty_pb2 from src.proto.grpc.testing import messages_pb2 from src.proto.grpc.testing import test_pb2 -_TIMEOUT = 7 - -class TestService(test_pb2.BetaTestServiceServicer): +class TestService(test_pb2.TestServiceServicer): def EmptyCall(self, request, context): return empty_pb2.Empty() def UnaryCall(self, request, context): if request.HasField('response_status'): - context.code(request.response_status.code) - context.details(request.response_status.message) + context.set_code(request.response_status.code) + context.set_details(request.response_status.message) return messages_pb2.SimpleResponse( payload=messages_pb2.Payload( type=messages_pb2.COMPRESSABLE, @@ -68,8 +61,8 @@ class TestService(test_pb2.BetaTestServiceServicer): def StreamingOutputCall(self, request, context): if request.HasField('response_status'): - context.code(request.response_status.code) - context.details(request.response_status.message) + context.set_code(request.response_status.code) + context.set_details(request.response_status.message) for response_parameters in request.response_parameters: yield messages_pb2.StreamingOutputCallResponse( payload=messages_pb2.Payload( @@ -79,7 +72,7 @@ class TestService(test_pb2.BetaTestServiceServicer): def StreamingInputCall(self, request_iterator, context): aggregate_size = 0 for request in request_iterator: - if request.payload and request.payload.body: + if request.payload is not None and request.payload.body: aggregate_size += len(request.payload.body) return messages_pb2.StreamingInputCallResponse( aggregated_payload_size=aggregate_size) @@ -87,8 +80,8 @@ class TestService(test_pb2.BetaTestServiceServicer): def FullDuplexCall(self, request_iterator, context): for request in request_iterator: if request.HasField('response_status'): - context.code(request.response_status.code) - context.details(request.response_status.message) + context.set_code(request.response_status.code) + context.set_details(request.response_status.message) for response_parameters in request.response_parameters: yield messages_pb2.StreamingOutputCallResponse( payload=messages_pb2.Payload( @@ -101,83 +94,80 @@ class TestService(test_pb2.BetaTestServiceServicer): return self.FullDuplexCall(request_iterator, context) -def _large_unary_common_behavior(stub, fill_username, fill_oauth_scope, - protocol_options=None): - with stub: - request = messages_pb2.SimpleRequest( - response_type=messages_pb2.COMPRESSABLE, response_size=314159, - payload=messages_pb2.Payload(body=b'\x00' * 271828), - fill_username=fill_username, fill_oauth_scope=fill_oauth_scope) - response_future = stub.UnaryCall.future(request, _TIMEOUT, - protocol_options=protocol_options) - response = response_future.result() - if response.payload.type is not messages_pb2.COMPRESSABLE: - raise ValueError( - 'response payload type is "%s"!' % type(response.payload.type)) - if len(response.payload.body) != 314159: - raise ValueError( - 'response body of incorrect size %d!' % len(response.payload.body)) +def _large_unary_common_behavior( + stub, fill_username, fill_oauth_scope, call_credentials): + request = messages_pb2.SimpleRequest( + response_type=messages_pb2.COMPRESSABLE, response_size=314159, + payload=messages_pb2.Payload(body=b'\x00' * 271828), + fill_username=fill_username, fill_oauth_scope=fill_oauth_scope) + response_future = stub.UnaryCall.future( + request, credentials=call_credentials) + response = response_future.result() + if response.payload.type is not messages_pb2.COMPRESSABLE: + raise ValueError( + 'response payload type is "%s"!' % type(response.payload.type)) + elif len(response.payload.body) != 314159: + raise ValueError( + 'response body of incorrect size %d!' % len(response.payload.body)) + else: return response def _empty_unary(stub): - with stub: - response = stub.EmptyCall(empty_pb2.Empty(), _TIMEOUT) - if not isinstance(response, empty_pb2.Empty): - raise TypeError( - 'response is of type "%s", not empty_pb2.Empty!', type(response)) + response = stub.EmptyCall(empty_pb2.Empty()) + if not isinstance(response, empty_pb2.Empty): + raise TypeError( + 'response is of type "%s", not empty_pb2.Empty!', type(response)) def _large_unary(stub): - _large_unary_common_behavior(stub, False, False) + _large_unary_common_behavior(stub, False, False, None) def _client_streaming(stub): - with stub: - payload_body_sizes = (27182, 8, 1828, 45904) - payloads = ( - messages_pb2.Payload(body=b'\x00' * size) - for size in payload_body_sizes) - requests = ( - messages_pb2.StreamingInputCallRequest(payload=payload) - for payload in payloads) - response = stub.StreamingInputCall(requests, _TIMEOUT) - if response.aggregated_payload_size != 74922: - raise ValueError( - 'incorrect size %d!' % response.aggregated_payload_size) + payload_body_sizes = (27182, 8, 1828, 45904,) + payloads = ( + messages_pb2.Payload(body=b'\x00' * size) + for size in payload_body_sizes) + requests = ( + messages_pb2.StreamingInputCallRequest(payload=payload) + for payload in payloads) + response = stub.StreamingInputCall(requests) + if response.aggregated_payload_size != 74922: + raise ValueError( + 'incorrect size %d!' % response.aggregated_payload_size) def _server_streaming(stub): - sizes = (31415, 9, 2653, 58979) + sizes = (31415, 9, 2653, 58979,) - with stub: - request = messages_pb2.StreamingOutputCallRequest( - response_type=messages_pb2.COMPRESSABLE, - response_parameters=( - messages_pb2.ResponseParameters(size=sizes[0]), - messages_pb2.ResponseParameters(size=sizes[1]), - messages_pb2.ResponseParameters(size=sizes[2]), - messages_pb2.ResponseParameters(size=sizes[3]), - )) - response_iterator = stub.StreamingOutputCall(request, _TIMEOUT) - for index, response in enumerate(response_iterator): - if response.payload.type != messages_pb2.COMPRESSABLE: - raise ValueError( - 'response body of invalid type %s!' % response.payload.type) - if len(response.payload.body) != sizes[index]: - raise ValueError( - 'response body of invalid size %d!' % len(response.payload.body)) + request = messages_pb2.StreamingOutputCallRequest( + response_type=messages_pb2.COMPRESSABLE, + response_parameters=( + messages_pb2.ResponseParameters(size=sizes[0]), + messages_pb2.ResponseParameters(size=sizes[1]), + messages_pb2.ResponseParameters(size=sizes[2]), + messages_pb2.ResponseParameters(size=sizes[3]), + ) + ) + response_iterator = stub.StreamingOutputCall(request) + for index, response in enumerate(response_iterator): + if response.payload.type != messages_pb2.COMPRESSABLE: + raise ValueError( + 'response body of invalid type %s!' % response.payload.type) + elif len(response.payload.body) != sizes[index]: + raise ValueError( + 'response body of invalid size %d!' % len(response.payload.body)) def _cancel_after_begin(stub): - with stub: - sizes = (27182, 8, 1828, 45904) - payloads = [messages_pb2.Payload(body=b'\x00' * size) for size in sizes] - requests = [messages_pb2.StreamingInputCallRequest(payload=payload) - for payload in payloads] - responses = stub.StreamingInputCall.future(requests, _TIMEOUT) - responses.cancel() - if not responses.cancelled(): - raise ValueError('expected call to be cancelled') + sizes = (27182, 8, 1828, 45904,) + payloads = (messages_pb2.Payload(body=b'\x00' * size) for size in sizes) + requests = (messages_pb2.StreamingInputCallRequest(payload=payload) + for payload in payloads) + response_future = stub.StreamingInputCall.future(requests) + response_future.cancel() + if not response_future.cancelled(): + raise ValueError('expected call to be cancelled') class _Pipe(object): @@ -220,18 +210,17 @@ class _Pipe(object): def _ping_pong(stub): - request_response_sizes = (31415, 9, 2653, 58979) - request_payload_sizes = (27182, 8, 1828, 45904) + request_response_sizes = (31415, 9, 2653, 58979,) + request_payload_sizes = (27182, 8, 1828, 45904,) - with stub, _Pipe() as pipe: - response_iterator = stub.FullDuplexCall(pipe, _TIMEOUT) - print('Starting ping-pong with response iterator %s' % response_iterator) + with _Pipe() as pipe: + response_iterator = stub.FullDuplexCall(pipe) for response_size, payload_size in zip( request_response_sizes, request_payload_sizes): request = messages_pb2.StreamingOutputCallRequest( response_type=messages_pb2.COMPRESSABLE, - response_parameters=(messages_pb2.ResponseParameters( - size=response_size),), + response_parameters=( + messages_pb2.ResponseParameters(size=response_size),), payload=messages_pb2.Payload(body=b'\x00' * payload_size)) pipe.add(request) response = next(response_iterator) @@ -244,17 +233,17 @@ def _ping_pong(stub): def _cancel_after_first_response(stub): - request_response_sizes = (31415, 9, 2653, 58979) - request_payload_sizes = (27182, 8, 1828, 45904) - with stub, _Pipe() as pipe: - response_iterator = stub.FullDuplexCall(pipe, _TIMEOUT) + request_response_sizes = (31415, 9, 2653, 58979,) + request_payload_sizes = (27182, 8, 1828, 45904,) + with _Pipe() as pipe: + response_iterator = stub.FullDuplexCall(pipe) response_size = request_response_sizes[0] payload_size = request_payload_sizes[0] request = messages_pb2.StreamingOutputCallRequest( response_type=messages_pb2.COMPRESSABLE, - response_parameters=(messages_pb2.ResponseParameters( - size=response_size),), + response_parameters=( + messages_pb2.ResponseParameters(size=response_size),), payload=messages_pb2.Payload(body=b'\x00' * payload_size)) pipe.add(request) response = next(response_iterator) @@ -264,16 +253,17 @@ def _cancel_after_first_response(stub): try: next(response_iterator) - except Exception: - pass + except grpc.RpcError as rpc_error: + if rpc_error.code() is not grpc.StatusCode.CANCELLED: + raise else: raise ValueError('expected call to be cancelled') def _timeout_on_sleeping_server(stub): request_payload_size = 27182 - with stub, _Pipe() as pipe: - response_iterator = stub.FullDuplexCall(pipe, 0.001) + with _Pipe() as pipe: + response_iterator = stub.FullDuplexCall(pipe, timeout=0.001) request = messages_pb2.StreamingOutputCallRequest( response_type=messages_pb2.COMPRESSABLE, @@ -282,15 +272,16 @@ def _timeout_on_sleeping_server(stub): time.sleep(0.1) try: next(response_iterator) - except face.ExpirationError: - pass + except grpc.RpcError as rpc_error: + if rpc_error.code() is not grpc.StatusCode.DEADLINE_EXCEEDED: + raise else: raise ValueError('expected call to exceed deadline') def _empty_stream(stub): - with stub, _Pipe() as pipe: - response_iterator = stub.FullDuplexCall(pipe, _TIMEOUT) + with _Pipe() as pipe: + response_iterator = stub.FullDuplexCall(pipe) pipe.close() try: next(response_iterator) @@ -300,65 +291,64 @@ def _empty_stream(stub): def _status_code_and_message(stub): - with stub: - message = 'test status message' - code = 2 - status = grpc.StatusCode.UNKNOWN # code = 2 - request = messages_pb2.SimpleRequest( - response_type=messages_pb2.COMPRESSABLE, - response_size=1, - payload=messages_pb2.Payload(body=b'\x00'), - response_status=messages_pb2.EchoStatus(code=code, message=message) - ) - response_future = stub.UnaryCall.future(request, _TIMEOUT) - if response_future.code() != status: - raise ValueError( - 'expected code %s, got %s' % (status, response_future.code())) - if response_future.details() != message: - raise ValueError( - 'expected message %s, got %s' % (message, response_future.details())) + message = 'test status message' + code = 2 + status = grpc.StatusCode.UNKNOWN # code = 2 + request = messages_pb2.SimpleRequest( + response_type=messages_pb2.COMPRESSABLE, + response_size=1, + payload=messages_pb2.Payload(body=b'\x00'), + response_status=messages_pb2.EchoStatus(code=code, message=message) + ) + response_future = stub.UnaryCall.future(request) + if response_future.code() != status: + raise ValueError( + 'expected code %s, got %s' % (status, response_future.code())) + elif response_future.details() != message: + raise ValueError( + 'expected message %s, got %s' % (message, response_future.details())) - request = messages_pb2.StreamingOutputCallRequest( - response_type=messages_pb2.COMPRESSABLE, - response_parameters=( - messages_pb2.ResponseParameters(size=1),), - response_status=messages_pb2.EchoStatus(code=code, message=message)) - response_iterator = stub.StreamingOutputCall(request, _TIMEOUT) - if response_future.code() != status: - raise ValueError( - 'expected code %s, got %s' % (status, response_iterator.code())) - if response_future.details() != message: - raise ValueError( - 'expected message %s, got %s' % (message, response_iterator.details())) + request = messages_pb2.StreamingOutputCallRequest( + response_type=messages_pb2.COMPRESSABLE, + response_parameters=( + messages_pb2.ResponseParameters(size=1),), + response_status=messages_pb2.EchoStatus(code=code, message=message)) + response_iterator = stub.StreamingOutputCall(request) + if response_future.code() != status: + raise ValueError( + 'expected code %s, got %s' % (status, response_iterator.code())) + elif response_future.details() != message: + raise ValueError( + 'expected message %s, got %s' % (message, response_iterator.details())) def _compute_engine_creds(stub, args): - response = _large_unary_common_behavior(stub, True, True) + response = _large_unary_common_behavior(stub, True, True, None) if args.default_service_account != response.username: raise ValueError( - 'expected username %s, got %s' % (args.default_service_account, - response.username)) + 'expected username %s, got %s' % ( + args.default_service_account, response.username)) def _oauth2_auth_token(stub, args): json_key_filename = os.environ[ oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS] wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] - response = _large_unary_common_behavior(stub, True, True) + response = _large_unary_common_behavior(stub, True, True, None) if wanted_email != response.username: raise ValueError( 'expected username %s, got %s' % (wanted_email, response.username)) if args.oauth_scope.find(response.oauth_scope) == -1: raise ValueError( - 'expected to find oauth scope "%s" in received "%s"' % - (response.oauth_scope, args.oauth_scope)) + 'expected to find oauth scope "{}" in received "{}"'.format( + response.oauth_scope, args.oauth_scope)) def _jwt_token_creds(stub, args): json_key_filename = os.environ[ oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS] wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] - response = _large_unary_common_behavior(stub, True, False) + response = _large_unary_common_behavior(stub, True, False, None) if wanted_email != response.username: raise ValueError( 'expected username %s, got %s' % (wanted_email, response.username)) @@ -370,11 +360,11 @@ def _per_rpc_creds(stub, args): wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] credentials = oauth2client_client.GoogleCredentials.get_application_default() scoped_credentials = credentials.create_scoped([args.oauth_scope]) - call_creds = implementations.google_call_credentials(scoped_credentials) - options = interfaces.grpc_call_options(disable_compression=False, - credentials=call_creds) - response = _large_unary_common_behavior(stub, True, False, - protocol_options=options) + # TODO(https://github.com/grpc/grpc/issues/6799): Eliminate this last + # remaining use of the Beta API. + call_credentials = implementations.google_call_credentials( + scoped_credentials) + response = _large_unary_common_behavior(stub, True, False, call_credentials) if wanted_email != response.username: raise ValueError( 'expected username %s, got %s' % (wanted_email, response.username)) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index ab2c3c708f4..1ae83bc57d0 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -30,10 +30,11 @@ """The Python implementation of the GRPC interoperability test server.""" import argparse +from concurrent import futures import logging import time -from grpc.beta import implementations +import grpc from src.proto.grpc.testing import test_pb2 from tests.interop import methods @@ -51,12 +52,13 @@ def serve(): default=False, type=resources.parse_bool) args = parser.parse_args() - server = test_pb2.beta_create_TestService_server(methods.TestService()) + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + test_pb2.add_TestServiceServicer_to_server(methods.TestService(), server) if args.use_tls: private_key = resources.private_key() certificate_chain = resources.certificate_chain() - credentials = implementations.ssl_server_credentials( - [(private_key, certificate_chain)]) + credentials = grpc.ssl_server_credentials( + ((private_key, certificate_chain),)) server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) @@ -68,7 +70,7 @@ def serve(): time.sleep(_ONE_DAY_IN_SECONDS) except BaseException as e: logging.info('Caught exception "%s"; stopping server...', e) - server.stop(0) + server.stop(None) logging.info('Server stopped; exiting.') if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/stress/client.py b/src/python/grpcio_tests/tests/stress/client.py index 0de2532cd88..975f33b4c16 100644 --- a/src/python/grpcio_tests/tests/stress/client.py +++ b/src/python/grpcio_tests/tests/stress/client.py @@ -30,9 +30,10 @@ """Entry point for running stress tests.""" import argparse +from concurrent import futures import threading -from grpc.beta import implementations +import grpc from six.moves import queue from src.proto.grpc.testing import metrics_pb2 from src.proto.grpc.testing import test_pb2 @@ -92,24 +93,24 @@ def _parse_weighted_test_cases(test_case_args): def run_test(args): test_cases = _parse_weighted_test_cases(args.test_cases) - test_servers = args.server_addresses.split(',') + test_server_targets = args.server_addresses.split(',') # Propagate any client exceptions with a queue exception_queue = queue.Queue() stop_event = threading.Event() hist = histogram.Histogram(1, 1) runners = [] - server = metrics_pb2.beta_create_MetricsService_server( - metrics_server.MetricsServer(hist)) + server = grpc.server(futures.ThreadPoolExecutor(max_workers=25)) + metrics_pb2.add_MetricsServiceServicer_to_server( + metrics_server.MetricsServer(hist), server) server.add_insecure_port('[::]:{}'.format(args.metrics_port)) server.start() - for test_server in test_servers: - host, port = test_server.split(':', 1) + for test_server_target in test_server_targets: for _ in xrange(args.num_channels_per_server): - channel = implementations.insecure_channel(host, int(port)) + channel = grpc.insecure_channel(test_server_target) for _ in xrange(args.num_stubs_per_channel): - stub = test_pb2.beta_create_TestService_stub(channel) + stub = test_pb2.TestServiceStub(channel) runner = test_runner.TestRunner(stub, test_cases, hist, exception_queue, stop_event) runners.append(runner) @@ -128,8 +129,8 @@ def run_test(args): stop_event.set() for runner in runners: runner.join() - runner = None - server.stop(0) + runner = None + server.stop(None) if __name__ == '__main__': run_test(_args()) diff --git a/src/python/grpcio_tests/tests/stress/metrics_server.py b/src/python/grpcio_tests/tests/stress/metrics_server.py index b994e4643e5..33dd1d6f2ad 100644 --- a/src/python/grpcio_tests/tests/stress/metrics_server.py +++ b/src/python/grpcio_tests/tests/stress/metrics_server.py @@ -36,7 +36,7 @@ from src.proto.grpc.testing import metrics_pb2 GAUGE_NAME = 'python_overall_qps' -class MetricsServer(metrics_pb2.BetaMetricsServiceServicer): +class MetricsServer(metrics_pb2.MetricsServiceServicer): def __init__(self, histogram): self._start_time = time.time() diff --git a/test/distrib/python/distribtest.py b/test/distrib/python/distribtest.py index dc206881409..0125ee6a567 100644 --- a/test/distrib/python/distribtest.py +++ b/test/distrib/python/distribtest.py @@ -27,10 +27,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from grpc.beta import implementations +import grpc # This code doesn't do much but makes sure the native extension is loaded # which is what we are testing here. -channel = implementations.insecure_channel('localhost', 1000) +channel = grpc.insecure_channel('localhost:1000') del channel print 'Success!' From a6624bb0b59b6019adefb9e29a23d0336222ce46 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sun, 7 Aug 2016 07:59:41 +0800 Subject: [PATCH 58/97] upgrade Google.Apis.Auth dependency to 0.15.0 --- src/csharp/Grpc.Auth/project.json | 4 ++-- templates/src/csharp/Grpc.Auth/project.json.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 6a1c9fc9900..ffa4e7cb454 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -23,13 +23,13 @@ }, "dependencies": { "Grpc.Core": "1.0.0-pre2", - "Google.Apis.Auth": "1.11.1" + "Google.Apis.Auth": "1.15.0" }, "frameworks": { "net45": { }, "netstandard1.5": { "imports": [ - "net45" + "portable-net45" ], "dependencies": { "Microsoft.NETCore.Portable.Compatibility": "1.0.1", diff --git a/templates/src/csharp/Grpc.Auth/project.json.template b/templates/src/csharp/Grpc.Auth/project.json.template index b3244e4d3c9..19d4b42cf06 100644 --- a/templates/src/csharp/Grpc.Auth/project.json.template +++ b/templates/src/csharp/Grpc.Auth/project.json.template @@ -25,13 +25,13 @@ }, "dependencies": { "Grpc.Core": "${settings.csharp_version}", - "Google.Apis.Auth": "1.11.1" + "Google.Apis.Auth": "1.15.0" }, "frameworks": { "net45": { }, "netstandard1.5": { "imports": [ - "net45" + "portable-net45" ], "dependencies": { "Microsoft.NETCore.Portable.Compatibility": "1.0.1", From 9490daf67fdf42e4308889eb6f486c88343aa6d7 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sun, 7 Aug 2016 21:10:25 +0800 Subject: [PATCH 59/97] remove Google.Apis.Auth related CoreCLR todos --- .../Grpc.IntegrationTesting/InteropClient.cs | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs index ec407d3fcf4..2747c0e5b76 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs @@ -145,26 +145,16 @@ namespace Grpc.IntegrationTesting if (options.TestCase == "jwt_token_creds") { -#if !NETCOREAPP1_0 var googleCredential = await GoogleCredential.GetApplicationDefaultAsync(); Assert.IsTrue(googleCredential.IsCreateScopedRequired); credentials = ChannelCredentials.Create(credentials, googleCredential.ToCallCredentials()); -#else - // TODO(jtattermusch): implement this - throw new NotImplementedException("Not supported on CoreCLR yet"); -#endif } if (options.TestCase == "compute_engine_creds") { -#if !NETCOREAPP1_0 var googleCredential = await GoogleCredential.GetApplicationDefaultAsync(); Assert.IsFalse(googleCredential.IsCreateScopedRequired); credentials = ChannelCredentials.Create(credentials, googleCredential.ToCallCredentials()); -#else - // TODO(jtattermusch): implement this - throw new NotImplementedException("Not supported on CoreCLR yet"); -#endif } return credentials; } @@ -395,7 +385,6 @@ namespace Grpc.IntegrationTesting public static async Task RunOAuth2AuthTokenAsync(TestService.TestServiceClient client, string oauthScope) { -#if !NETCOREAPP1_0 Console.WriteLine("running oauth2_auth_token"); ITokenAccess credential = (await GoogleCredential.GetApplicationDefaultAsync()).CreateScoped(new[] { oauthScope }); string oauth2Token = await credential.GetAccessTokenForRequestAsync(); @@ -413,15 +402,10 @@ namespace Grpc.IntegrationTesting Assert.True(oauthScope.Contains(response.OauthScope)); Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username); Console.WriteLine("Passed!"); -#else - // TODO(jtattermusch): implement this - throw new NotImplementedException("Not supported on CoreCLR yet"); -#endif } public static async Task RunPerRpcCredsAsync(TestService.TestServiceClient client, string oauthScope) { -#if !NETCOREAPP1_0 Console.WriteLine("running per_rpc_creds"); ITokenAccess googleCredential = await GoogleCredential.GetApplicationDefaultAsync(); @@ -435,10 +419,6 @@ namespace Grpc.IntegrationTesting Assert.AreEqual(GetEmailFromServiceAccountFile(), response.Username); Console.WriteLine("Passed!"); -#else - // TODO(jtattermusch): implement this - throw new NotImplementedException("Not supported on CoreCLR yet"); -#endif } public static async Task RunCancelAfterBeginAsync(TestService.TestServiceClient client) @@ -731,17 +711,12 @@ namespace Grpc.IntegrationTesting // extracts the client_email field from service account file used for auth test cases private static string GetEmailFromServiceAccountFile() { -#if !NETCOREAPP1_0 string keyFile = Environment.GetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS"); Assert.IsNotNull(keyFile); var jobject = JObject.Parse(File.ReadAllText(keyFile)); string email = jobject.GetValue("client_email").Value(); Assert.IsTrue(email.Length > 0); // spec requires nonempty client email. return email; -#else - // TODO(jtattermusch): implement this - throw new NotImplementedException("Not supported on CoreCLR yet"); -#endif } private static Metadata CreateTestMetadata() From 07bd74edb88d605ff1d8dca040ce03302c30bc1a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sun, 7 Aug 2016 21:45:44 +0800 Subject: [PATCH 60/97] update csproj project to Google.Apis.Auth 0.15.0 --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 35 ++++++++----------- src/csharp/Grpc.Auth/packages.config | 4 +-- .../Grpc.IntegrationTesting.Client.csproj | 35 ++++++++----------- .../packages.config | 4 +-- .../Grpc.IntegrationTesting.Server.csproj | 35 ++++++++----------- .../packages.config | 4 +-- .../Grpc.IntegrationTesting.csproj | 18 +++++----- .../Grpc.IntegrationTesting/packages.config | 4 +-- 8 files changed, 62 insertions(+), 77 deletions(-) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index 1fa14fc3dfb..7a6955311ac 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -39,30 +39,25 @@ ..\keys\Grpc.snk - - False - ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.PlatformServices.dll - - - False - ..\packages\Google.Apis.Core.1.11.1\lib\net45\Google.Apis.Core.dll - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - + + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + + + ..\packages\Google.Apis.Core.1.15.0\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll + diff --git a/src/csharp/Grpc.Auth/packages.config b/src/csharp/Grpc.Auth/packages.config index c20d9ceed6a..738d3e6f3b6 100644 --- a/src/csharp/Grpc.Auth/packages.config +++ b/src/csharp/Grpc.Auth/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index 91fb3ce5bcf..6816b5c5a2d 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -39,30 +39,25 @@ ..\keys\Grpc.snk - - False - ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.PlatformServices.dll - - - False - ..\packages\Google.Apis.Core.1.11.1\lib\net45\Google.Apis.Core.dll - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - + + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + + + ..\packages\Google.Apis.Core.1.15.0\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll + diff --git a/src/csharp/Grpc.IntegrationTesting.Client/packages.config b/src/csharp/Grpc.IntegrationTesting.Client/packages.config index c20d9ceed6a..738d3e6f3b6 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/packages.config +++ b/src/csharp/Grpc.IntegrationTesting.Client/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index f73d99dbd1a..987387ca259 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -39,30 +39,25 @@ ..\keys\Grpc.snk - - False - ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.dll - - - False - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.PlatformServices.dll - - - False - ..\packages\Google.Apis.Core.1.11.1\lib\net45\Google.Apis.Core.dll - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - + + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + + + ..\packages\Google.Apis.Core.1.15.0\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll + diff --git a/src/csharp/Grpc.IntegrationTesting.Server/packages.config b/src/csharp/Grpc.IntegrationTesting.Server/packages.config index c20d9ceed6a..738d3e6f3b6 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/packages.config +++ b/src/csharp/Grpc.IntegrationTesting.Server/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index 7512d2a5d11..df1d76765e2 100644 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -51,15 +51,6 @@ ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.dll - - - ..\packages\Google.Apis.Auth.1.11.1\lib\net45\Google.Apis.Auth.PlatformServices.dll - - - ..\packages\Google.Apis.Core.1.11.1\lib\net45\Google.Apis.Core.dll - ..\packages\Google.Protobuf.3.0.0-beta3\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll @@ -75,6 +66,15 @@ ..\packages\NUnitLite.3.2.0\lib\net45\nunitlite.dll + + ..\packages\Google.Apis.Core.1.15.0\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll + diff --git a/src/csharp/Grpc.IntegrationTesting/packages.config b/src/csharp/Grpc.IntegrationTesting/packages.config index e6e64e65581..a78af9dc885 100644 --- a/src/csharp/Grpc.IntegrationTesting/packages.config +++ b/src/csharp/Grpc.IntegrationTesting/packages.config @@ -2,8 +2,8 @@ - - + + From 68689229e8d5cecee9d9e534c2c8aec87bf31b27 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Aug 2016 06:52:39 +0800 Subject: [PATCH 61/97] update Grpc.Auth.nuspec --- src/csharp/Grpc.Auth/Grpc.Auth.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.nuspec b/src/csharp/Grpc.Auth/Grpc.Auth.nuspec index 4baed3704c4..a1f5668e2e0 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.nuspec +++ b/src/csharp/Grpc.Auth/Grpc.Auth.nuspec @@ -15,7 +15,7 @@ Copyright 2015, Google Inc. gRPC RPC Protocol HTTP/2 Auth OAuth2 - + From fab839e96774e61567f085d8f961897a1cc3e07b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 Aug 2016 18:54:14 +0800 Subject: [PATCH 62/97] remove leftover app.config --- .../Grpc.IntegrationTesting.QpsWorker.csproj | 1 - .../Grpc.IntegrationTesting.QpsWorker/app.config | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 src/csharp/Grpc.IntegrationTesting.QpsWorker/app.config diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index dda26a68923..593bf0939db 100644 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -58,7 +58,6 @@ - \ No newline at end of file diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/app.config b/src/csharp/Grpc.IntegrationTesting.QpsWorker/app.config deleted file mode 100644 index e204447bb34..00000000000 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/app.config +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file From 3ed00635f91dad88788b99b2e06d9d7e647fb9d5 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 11 Aug 2016 08:46:39 -0700 Subject: [PATCH 63/97] Add optional resource parameters to census resource test --- test/core/census/resource_test.c | 47 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/test/core/census/resource_test.c b/test/core/census/resource_test.c index e1556d7d7b0..cfe4f037fd3 100644 --- a/test/core/census/resource_test.c +++ b/test/core/census/resource_test.c @@ -95,15 +95,13 @@ static void test_define_single_resource(const char *file, const char *name, } // Try deleting various resources (both those that exist and those that don't). -static void test_delete_resource() { +static void test_delete_resource(char* minimal_good, char* full) { initialize_resources(); // Try deleting resource before any are defined. census_delete_resource(0); // Create and check a couple of resources. - int32_t rid1 = define_resource_from_file( - "test/core/census/data/resource_minimal_good.pb"); - int32_t rid2 = - define_resource_from_file("test/core/census/data/resource_full.pb"); + int32_t rid1 = define_resource_from_file(minimal_good); + int32_t rid2 = define_resource_from_file(full); GPR_ASSERT(rid1 >= 0 && rid2 >= 0 && rid1 != rid2); int32_t rid3 = census_resource_id("minimal_good"); int32_t rid4 = census_resource_id("full_resource"); @@ -117,8 +115,7 @@ static void test_delete_resource() { rid3 = census_resource_id("minimal_good"); GPR_ASSERT(rid3 < 0); // Check that re-adding works. - rid1 = define_resource_from_file( - "test/core/census/data/resource_minimal_good.pb"); + rid1 = define_resource_from_file(minimal_good); GPR_ASSERT(rid1 >= 0); rid3 = census_resource_id("minimal_good"); GPR_ASSERT(rid1 == rid3); @@ -136,22 +133,34 @@ static void test_base_resources() { } int main(int argc, char **argv) { + char *resource_empty_name_pb, *resource_full_pb, *resource_minimal_good_pb, + *resource_no_name_pb, *resource_no_numerator_pb, *resource_no_unit_pb; + if (argc >= 7) { + resource_empty_name_pb = argv[1]; + resource_full_pb = argv[2]; + resource_minimal_good_pb = argv[3]; + resource_no_name_pb = argv[4]; + resource_no_numerator_pb = argv[5]; + resource_no_unit_pb = argv[6]; + } else { + resource_empty_name_pb = "test/core/census/data/resource_empty_name.pb"; + resource_full_pb = "test/core/census/data/resource_full.pb"; + resource_minimal_good_pb = "test/core/census/data/resource_minimal_good.pb"; + resource_no_name_pb = "test/core/census/data/resource_no_name.pb"; + resource_no_numerator_pb = "test/core/census/data/resource_no_numerator.pb"; + resource_no_unit_pb = "test/core/census/data/resource_no_unit.pb"; + } grpc_test_init(argc, argv); test_enable_disable(); test_empty_definition(); - test_define_single_resource("test/core/census/data/resource_minimal_good.pb", - "minimal_good", true); - test_define_single_resource("test/core/census/data/resource_full.pb", - "full_resource", true); - test_define_single_resource("test/core/census/data/resource_no_name.pb", - "resource_no_name", false); - test_define_single_resource("test/core/census/data/resource_no_numerator.pb", + test_define_single_resource(resource_minimal_good_pb, "minimal_good", true); + test_define_single_resource(resource_full_pb, "full_resource", true); + test_define_single_resource(resource_no_name_pb, "resource_no_name", false); + test_define_single_resource(resource_no_numerator_pb, "resource_no_numerator", false); - test_define_single_resource("test/core/census/data/resource_no_unit.pb", - "resource_no_unit", false); - test_define_single_resource("test/core/census/data/resource_empty_name.pb", - "resource_empty_name", false); - test_delete_resource(); + test_define_single_resource(resource_no_unit_pb, "resource_no_unit", false); + test_define_single_resource(resource_empty_name_pb, "resource_empty_name", false); + test_delete_resource(resource_minimal_good_pb, resource_full_pb); test_base_resources(); return 0; } From a6402a9d43410057100d90b0498371790c6052e4 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 11 Aug 2016 10:09:37 -0700 Subject: [PATCH 64/97] Made strings constant --- test/core/census/resource_test.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/core/census/resource_test.c b/test/core/census/resource_test.c index cfe4f037fd3..8f5a4387d1a 100644 --- a/test/core/census/resource_test.c +++ b/test/core/census/resource_test.c @@ -95,7 +95,7 @@ static void test_define_single_resource(const char *file, const char *name, } // Try deleting various resources (both those that exist and those that don't). -static void test_delete_resource(char* minimal_good, char* full) { +static void test_delete_resource(const char *minimal_good, const char *full) { initialize_resources(); // Try deleting resource before any are defined. census_delete_resource(0); @@ -133,9 +133,9 @@ static void test_base_resources() { } int main(int argc, char **argv) { - char *resource_empty_name_pb, *resource_full_pb, *resource_minimal_good_pb, + const char *resource_empty_name_pb, *resource_full_pb, *resource_minimal_good_pb, *resource_no_name_pb, *resource_no_numerator_pb, *resource_no_unit_pb; - if (argc >= 7) { + if (argc == 7) { resource_empty_name_pb = argv[1]; resource_full_pb = argv[2]; resource_minimal_good_pb = argv[3]; @@ -143,6 +143,7 @@ int main(int argc, char **argv) { resource_no_numerator_pb = argv[5]; resource_no_unit_pb = argv[6]; } else { + GPR_ASSERT(argc == 1); resource_empty_name_pb = "test/core/census/data/resource_empty_name.pb"; resource_full_pb = "test/core/census/data/resource_full.pb"; resource_minimal_good_pb = "test/core/census/data/resource_minimal_good.pb"; From 530284269a31568fab04c200992857c7c52c6b56 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 11 Aug 2016 11:21:30 -0700 Subject: [PATCH 65/97] Clang format --- test/core/census/resource_test.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/core/census/resource_test.c b/test/core/census/resource_test.c index 8f5a4387d1a..f0e70396156 100644 --- a/test/core/census/resource_test.c +++ b/test/core/census/resource_test.c @@ -133,8 +133,9 @@ static void test_base_resources() { } int main(int argc, char **argv) { - const char *resource_empty_name_pb, *resource_full_pb, *resource_minimal_good_pb, - *resource_no_name_pb, *resource_no_numerator_pb, *resource_no_unit_pb; + const char *resource_empty_name_pb, *resource_full_pb, + *resource_minimal_good_pb, *resource_no_name_pb, + *resource_no_numerator_pb, *resource_no_unit_pb; if (argc == 7) { resource_empty_name_pb = argv[1]; resource_full_pb = argv[2]; @@ -157,10 +158,11 @@ int main(int argc, char **argv) { test_define_single_resource(resource_minimal_good_pb, "minimal_good", true); test_define_single_resource(resource_full_pb, "full_resource", true); test_define_single_resource(resource_no_name_pb, "resource_no_name", false); - test_define_single_resource(resource_no_numerator_pb, - "resource_no_numerator", false); + test_define_single_resource(resource_no_numerator_pb, "resource_no_numerator", + false); test_define_single_resource(resource_no_unit_pb, "resource_no_unit", false); - test_define_single_resource(resource_empty_name_pb, "resource_empty_name", false); + test_define_single_resource(resource_empty_name_pb, "resource_empty_name", + false); test_delete_resource(resource_minimal_good_pb, resource_full_pb); test_base_resources(); return 0; From a9bc030a3a9eb876c4ca00b40d68013ed91a8d66 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 11 Aug 2016 12:05:55 -0700 Subject: [PATCH 66/97] add mutex wrapper around sending and modifying of initial metadata --- src/ruby/lib/grpc/generic/active_call.rb | 37 +++-- src/ruby/spec/generic/active_call_spec.rb | 171 ++++++++-------------- 2 files changed, 86 insertions(+), 122 deletions(-) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index d43a9e7a4b7..f9c41f0c0ed 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -113,11 +113,17 @@ module GRPC fail(ArgumentError, 'Already sent md') if started && metadata_to_send @metadata_to_send = metadata_to_send || {} unless started + @send_initial_md_mutex = Mutex.new end + # Sends the initial metadata that has yet to be sent. + # Fails if metadata has already been sent for this call. def send_initial_metadata - fail 'Already sent metadata' if @metadata_sent - start_call(@metadata_to_send) + @send_initial_md_mutex.synchronize do + fail('Already send initial metadata') if @metadata_sent + @metadata_tag = ActiveCall.client_invoke(@call, @metadata_to_send) + @metadata_sent = true + end end # output_metadata are provides access to hash that can be used to @@ -195,7 +201,7 @@ module GRPC # @param marshalled [false, true] indicates if the object is already # marshalled. def remote_send(req, marshalled = false) - start_call(@metadata_to_send) unless @metadata_sent + send_initial_metadata unless @metadata_sent GRPC.logger.debug("sending #{req}, marshalled? #{marshalled}") payload = marshalled ? req : @marshal.call(req) @call.run_batch(SEND_MESSAGE => payload) @@ -211,7 +217,7 @@ module GRPC # list, mulitple metadata for its key are sent def send_status(code = OK, details = '', assert_finished = false, metadata: {}) - start_call unless @metadata_sent + send_initial_metadata unless @metadata_sent ops = { SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata) } @@ -312,7 +318,8 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Object] the response received from the server def request_response(req, metadata: {}) - start_call(metadata) + merge_metadata_to_send(metadata) && + send_initial_metadata unless @metadata_sent remote_send(req) writes_done(false) response = remote_read @@ -336,7 +343,8 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Object] the response received from the server def client_streamer(requests, metadata: {}) - start_call(metadata) + merge_metadata_to_send(metadata) && + send_initial_metadata unless @metadata_sent requests.each { |r| remote_send(r) } writes_done(false) response = remote_read @@ -362,7 +370,8 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Enumerator|nil] a response Enumerator def server_streamer(req, metadata: {}) - start_call(metadata) + merge_metadata_to_send(metadata) && + send_initial_metadata unless @metadata_sent remote_send(req) writes_done(false) replies = enum_for(:each_remote_read_then_finish) @@ -401,7 +410,8 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Enumerator, nil] a response Enumerator def bidi_streamer(requests, metadata: {}, &blk) - start_call(metadata) unless @metadata_sent + merge_metadata_to_send(metadata) && + send_initial_metadata unless @metadata_sent bd = BidiCall.new(@call, @marshal, @unmarshal, @@ -444,9 +454,14 @@ module GRPC @op_notifier.notify(self) end + # Add to the metadata that will be sent from the server. + # Fails if metadata has already been sent. + # Unused by client calls. def merge_metadata_to_send(new_metadata = {}) - fail('cant change metadata after already sent') if @metadata_sent - @metadata_to_send.merge!(new_metadata) + @send_initial_md_mutex.synchronize do + fail('cant change metadata after already sent') if @metadata_sent + @metadata_to_send.merge!(new_metadata) + end end private @@ -456,7 +471,7 @@ module GRPC # a list, multiple metadata for its key are sent def start_call(metadata = {}) return if @metadata_sent - @metadata_tag = ActiveCall.client_invoke(@call, metadata) + merge_metadata_to_send(metadata) && send_initial_metadata @metadata_sent = true end diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index 0c72be9a98a..79f739e8fa7 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -242,7 +242,12 @@ describe GRPC::ActiveCall do describe '#merge_metadata_to_send', merge_metadata_to_send: true do it 'adds to existing metadata when there is existing metadata to send' do call = make_test_call - starting_metadata = { k1: 'key1_val', k2: 'key2_val' } + starting_metadata = { + k1: 'key1_val', + k2: 'key2_val', + k3: 'key3_val' + } + @client_call = ActiveCall.new( call, @pass_through, @pass_through, @@ -253,13 +258,13 @@ describe GRPC::ActiveCall do expect(@client_call.metadata_to_send).to eq(starting_metadata) @client_call.merge_metadata_to_send( - k3: 'key3_val', + k3: 'key3_new_val', k4: 'key4_val') expected_md_to_send = { k1: 'key1_val', k2: 'key2_val', - k3: 'key3_val', + k3: 'key3_new_val', k4: 'key4_val' } expect(@client_call.metadata_to_send).to eq(expected_md_to_send) @@ -269,23 +274,6 @@ describe GRPC::ActiveCall do expect(@client_call.metadata_to_send).to eq(expected_md_to_send) end - it 'overrides existing metadata if adding metadata with an existing key' do - call = make_test_call - starting_metadata = { k1: 'key1_val', k2: 'key2_val' } - @client_call = ActiveCall.new( - call, - @pass_through, - @pass_through, - deadline, - started: false, - metadata_to_send: starting_metadata) - - expect(@client_call.metadata_to_send).to eq(starting_metadata) - @client_call.merge_metadata_to_send(k1: 'key1_new_val') - expect(@client_call.metadata_to_send).to eq(k1: 'key1_new_val', - k2: 'key2_val') - end - it 'fails when initial metadata has already been sent' do call = make_test_call @client_call = ActiveCall.new( @@ -530,121 +518,82 @@ describe GRPC::ActiveCall do end # Test sending of the initial metadata in #run_server_bidi - # from the server handler both implicitly and explicitly, - # when the server handler function has one argument and two arguments - describe '#run_server_bidi sanity tests', run_server_bidi: true do - it 'sends the initial metadata implicitly if not already sent' do - requests = ['first message', 'second message'] - server_to_client_metadata = { 'test_key' => 'test_val' } - server_status = OK + # from the server handler both implicitly and explicitly. + describe '#run_server_bidi metadata sending tests', run_server_bidi: true do + before(:each) do + @requests = ['first message', 'second message'] + @server_to_client_metadata = { 'test_key' => 'test_val' } + @server_status = OK - client_call = make_test_call - client_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) + @client_call = make_test_call + @client_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) recvd_rpc = @server.request_call recvd_call = recvd_rpc.call - server_call = ActiveCall.new(recvd_call, - @pass_through, - @pass_through, - deadline, - metadata_received: true, - started: false, - metadata_to_send: server_to_client_metadata) + @server_call = ActiveCall.new( + recvd_call, + @pass_through, + @pass_through, + deadline, + metadata_received: true, + started: false, + metadata_to_send: @server_to_client_metadata) + end + after(:each) do + # Send the requests and send a close so the server can send a status + @requests.each do |message| + @client_call.run_batch(CallOps::SEND_MESSAGE => message) + end + @client_call.run_batch(CallOps::SEND_CLOSE_FROM_CLIENT => nil) + + @server_thread.join + + # Expect that initial metadata was sent, + # the requests were echoed, and a status was sent + batch_result = @client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil) + expect(batch_result.metadata).to eq(@server_to_client_metadata) + + @requests.each do |message| + batch_result = @client_call.run_batch( + CallOps::RECV_MESSAGE => nil) + expect(batch_result.message).to eq(message) + end + + batch_result = @client_call.run_batch( + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(batch_result.status.code).to eq(@server_status) + end + + it 'sends the initial metadata implicitly if not already sent' do # Server handler that doesn't have access to a "call" # It echoes the requests fake_gen_each_reply_with_no_call_param = proc do |msgs| msgs end - server_thread = Thread.new do - server_call.run_server_bidi( + @server_thread = Thread.new do + @server_call.run_server_bidi( fake_gen_each_reply_with_no_call_param) - server_call.send_status(server_status) + @server_call.send_status(@server_status) end - - # Send the requests and send a close so the server can send a status - requests.each do |message| - client_call.run_batch(CallOps::SEND_MESSAGE => message) - end - client_call.run_batch(CallOps::SEND_CLOSE_FROM_CLIENT => nil) - - server_thread.join - - # Expect that initial metadata was sent, - # the requests were echoed, and a status was sent - batch_result = client_call.run_batch( - CallOps::RECV_INITIAL_METADATA => nil) - expect(batch_result.metadata).to eq(server_to_client_metadata) - - requests.each do |message| - batch_result = client_call.run_batch( - CallOps::RECV_MESSAGE => nil) - expect(batch_result.message).to eq(message) - end - - batch_result = client_call.run_batch( - CallOps::RECV_STATUS_ON_CLIENT => nil) - expect(batch_result.status.code).to eq(server_status) end it 'sends the metadata when sent explicitly and not already sent' do - requests = ['first message', 'second message'] - server_to_client_metadata = { 'test_key' => 'test_val' } - server_status = OK - - client_call = make_test_call - client_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) - - recvd_rpc = @server.request_call - recvd_call = recvd_rpc.call - server_call = ActiveCall.new(recvd_call, - @pass_through, - @pass_through, - deadline, - metadata_received: true, - started: false) - # Fake server handler that has access to a "call" object and - # uses it to explicitly update and sent the initial metadata + # uses it to explicitly update and send the initial metadata fake_gen_each_reply_with_call_param = proc do |msgs, call_param| - call_param.merge_metadata_to_send(server_to_client_metadata) + call_param.merge_metadata_to_send(@server_to_client_metadata) call_param.send_initial_metadata msgs end - server_thread = Thread.new do - server_call.run_server_bidi( + @server_thread = Thread.new do + @server_call.run_server_bidi( fake_gen_each_reply_with_call_param) - server_call.send_status(server_status) + @server_call.send_status(@server_status) end - - # Send requests and a close from the client so the server - # can send a status - requests.each do |message| - client_call.run_batch( - CallOps::SEND_MESSAGE => message) - end - client_call.run_batch( - CallOps::SEND_CLOSE_FROM_CLIENT => nil) - - server_thread.join - - # Verify that the correct metadata was sent, the requests - # were echoed, and the correct status was sent - batch_result = client_call.run_batch( - CallOps::RECV_INITIAL_METADATA => nil) - expect(batch_result.metadata).to eq(server_to_client_metadata) - - requests.each do |message| - batch_result = client_call.run_batch( - CallOps::RECV_MESSAGE => nil) - expect(batch_result.message).to eq(message) - end - - batch_result = client_call.run_batch( - CallOps::RECV_STATUS_ON_CLIENT => nil) - expect(batch_result.status.code).to eq(server_status) end end From 280885eaa5e48912fe623dcbad8ad398d8eb405d Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Thu, 11 Aug 2016 14:16:36 -0700 Subject: [PATCH 67/97] WIP --- .../cronet/transport/cronet_transport.c | 177 +++++++++++++----- 1 file changed, 128 insertions(+), 49 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 1d756039809..8f11ef73792 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -50,16 +50,13 @@ #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 -// maximum ops in a batch. There is not much thinking behind this limit, except -// that it seems to be enough for most use cases. -#define MAX_PENDING_OPS 100 #define CRONET_LOG(...) \ { \ if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ } -// TODO (makdharma): Hook up into the wider tracing mechanism +/* TODO (makdharma): Hook up into the wider tracing mechanism */ int grpc_cronet_trace = 1; enum OP_RESULT { @@ -68,7 +65,7 @@ enum OP_RESULT { NO_ACTION_POSSIBLE }; -// Used for printing debug +/* Used for printing debug */ const char *op_result_string[] = {"ACTION_TAKEN_WITH_CALLBACK", "ACTION_TAKEN_NO_CALLBACK", "NO_ACTION_POSSIBLE"}; @@ -129,7 +126,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_failed, on_canceled}; -// Cronet transport object +/* Cronet transport object */ struct grpc_cronet_transport { grpc_transport base; /* must be first element in this structure */ cronet_engine *engine; @@ -146,6 +143,7 @@ struct read_state { int length_field; char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; char *payload_field; + bool read_stream_closed; /* vars for holding data destined for the application */ struct grpc_slice_buffer_stream sbs; @@ -177,14 +175,13 @@ struct op_and_state { grpc_transport_stream_op op; struct op_state state; bool done; - struct stream_obj *s; /* Pointer back to the stream object */ + struct stream_obj *s; /* Pointer back to the stream object */ + struct op_and_state *next; /* next op_and_state in the linked list */ }; struct op_storage { - struct op_and_state pending_ops[MAX_PENDING_OPS]; - int wrptr; - int rdptr; int num_pending_ops; + struct op_and_state *head; }; struct stream_obj { @@ -217,20 +214,52 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas); static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { gpr_mu_lock(&s->mu); struct op_storage *storage = &s->storage; - GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); + /* add new op at the beginning of the linked list. The memory is freed + in remove_from_storage */ + struct op_and_state *new_op = gpr_malloc(sizeof(struct op_and_state)); + memcpy(&new_op->op, op, sizeof(grpc_transport_stream_op)); + memset(&new_op->state, 0, sizeof(new_op->state)); + new_op->s = s; + new_op->done = false; + new_op->next = storage->head; + storage->head = new_op; storage->num_pending_ops++; - CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", - storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr].op, op, - sizeof(grpc_transport_stream_op)); - memset(&storage->pending_ops[storage->wrptr].state, 0, - sizeof(storage->pending_ops[storage->wrptr].state)); - storage->pending_ops[storage->wrptr].done = false; - storage->pending_ops[storage->wrptr].s = s; - storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + CRONET_LOG(GPR_DEBUG, "adding new op %p. %d in the queue.", new_op, + storage->num_pending_ops); gpr_mu_unlock(&s->mu); } +/* + Traverse the linked list and delete op and free memory +*/ +static void remove_from_storage(struct stream_obj *s, + struct op_and_state *oas) { + struct op_and_state *curr; + if (s->storage.head == NULL || oas == NULL) { + return; + } + if (s->storage.head == oas) { + s->storage.head = oas->next; + gpr_free(oas); + s->storage.num_pending_ops--; + CRONET_LOG(GPR_DEBUG, "Freed %p. Now %d in the queue", oas, + s->storage.num_pending_ops); + } else { + for (curr = s->storage.head; curr != NULL; curr = curr->next) { + if (curr->next == oas) { + curr->next = oas->next; + s->storage.num_pending_ops--; + CRONET_LOG(GPR_DEBUG, "Freed %p. Now %d in the queue", oas, + s->storage.num_pending_ops); + gpr_free(oas); + break; + } else if (curr->next == NULL) { + CRONET_LOG(GPR_ERROR, "Reached end of LL and did not find op to free"); + } + } + } +} + /* Cycle through ops and try to take next action. Break when either an action with callback is taken, or no action is possible. @@ -239,18 +268,21 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { */ static void execute_from_storage(stream_obj *s) { gpr_mu_lock(&s->mu); - for (int i = 0; i < s->storage.wrptr;) { - CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, - s->storage.pending_ops[i].done); - if (s->storage.pending_ops[i].done) { - i++; - continue; + for (struct op_and_state *curr = s->storage.head; curr != NULL;) { + CRONET_LOG(GPR_DEBUG, "calling op at %p. done = %d", curr, curr->done); + GPR_ASSERT(curr->done == 0); + enum OP_RESULT result = execute_stream_op(curr); + CRONET_LOG(GPR_DEBUG, "execute_stream_op[%p] returns %s", curr, + op_result_string[result]); + /* if this op is done, then remove it and free memory */ + if (curr->done) { + struct op_and_state *next = curr->next; + remove_from_storage(s, curr); + curr = next; } - enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); - CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", - op_result_string[result], i); + /* continue processing the same op if ACTION_TAKEN_WITHOUT_CALLBACK */ if (result == NO_ACTION_POSSIBLE) { - i++; + curr = curr->next; } else if (result == ACTION_TAKEN_WITH_CALLBACK) { break; } @@ -268,6 +300,14 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { cronet_bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_FAILED] = true; s->cbs = NULL; + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + s->header_array.headers = NULL; + } + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; + } execute_from_storage(s); } @@ -280,6 +320,14 @@ static void on_canceled(cronet_bidirectional_stream *stream) { cronet_bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_CANCELED] = true; s->cbs = NULL; + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + s->header_array.headers = NULL; + } + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; + } execute_from_storage(s); } @@ -306,6 +354,7 @@ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { /* Free the memory allocated for headers */ if (s->header_array.headers) { gpr_free(s->header_array.headers); + s->header_array.headers = NULL; } execute_from_storage(s); } @@ -358,6 +407,7 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, stream_obj *s = (stream_obj *)stream->annotation; CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + s->state.state_callback_received[OP_RECV_MESSAGE] = true; if (count > 0) { s->state.rs.received_bytes += count; s->state.rs.remaining_bytes -= count; @@ -370,7 +420,9 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, } else { execute_from_storage(s); } - s->state.state_callback_received[OP_RECV_MESSAGE] = true; + } else { + s->state.rs.read_stream_closed = true; + execute_from_storage(s); } } @@ -570,38 +622,51 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, } else if (op_id == OP_ON_COMPLETE) { /* already executed (note we're checking op specific state, not stream state) */ - if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; + if (op_state->state_op_done[OP_ON_COMPLETE]) { + CRONET_LOG(GPR_DEBUG, "Because"); + result = false; + } /* Check if every op that was asked for is done. */ else if (curr_op->send_initial_metadata && - !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_message && - !stream_state->state_op_done[OP_SEND_MESSAGE]) + } else if (curr_op->send_message && + !op_state->state_op_done[OP_SEND_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_message && - !stream_state->state_callback_received[OP_SEND_MESSAGE]) + } else if (curr_op->send_message && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_trailing_metadata && - !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) + } else if (curr_op->send_trailing_metadata && + !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_initial_metadata && - !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + } else if (curr_op->recv_initial_metadata && + !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_message && - !stream_state->state_op_done[OP_RECV_MESSAGE]) + } else if (curr_op->recv_message && + !stream_state->state_op_done[OP_RECV_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_trailing_metadata) { + } else if (curr_op->recv_trailing_metadata) { /* We aren't done with trailing metadata yet */ - if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; + } /* We've asked for actual message in an earlier op, and it hasn't been delivered yet. */ else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { /* If this op is not the one asking for read, (which means some earlier op has asked), and the read hasn't been delivered. */ if (!curr_op->recv_message && - !stream_state->state_op_done[OP_SUCCEEDED]) + !stream_state->state_callback_received[OP_SUCCEEDED]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; + } } } /* We should see at least one on_write_completed for the trailers that we @@ -625,6 +690,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { OP_SEND_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); /* This OP is the beginning. Reset various states */ + memset(&s->header_array, 0, sizeof(s->header_array)); memset(&stream_state->rs, 0, sizeof(stream_state->rs)); memset(&stream_state->ws, 0, sizeof(stream_state->ws)); memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); @@ -637,6 +703,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); char *url; + s->header_array.headers = NULL; convert_metadata_to_cronet_headers( stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, &s->header_array.headers, &s->header_array.count); @@ -696,6 +763,13 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; + } else if (stream_state->rs.read_stream_closed == true) { + /* No more data will be received */ + CRONET_LOG(GPR_DEBUG, "read stream closed"); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { @@ -808,9 +882,12 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { NULL); oas->state.state_op_done[OP_ON_COMPLETE] = true; oas->done = true; - /* reset any send or receive message state. */ - stream_state->state_callback_received[OP_SEND_MESSAGE] = false; - stream_state->state_op_done[OP_SEND_MESSAGE] = false; + /* reset any send message state, only if this ON_COMPLETE is about a send. + */ + if (stream_op->send_message) { + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + stream_state->state_op_done[OP_SEND_MESSAGE] = false; + } result = ACTION_TAKEN_NO_CALLBACK; /* If this is the on_complete callback being called for a received message - make a note */ @@ -831,9 +908,11 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, const void *server_data) { stream_obj *s = (stream_obj *)gs; memset(&s->storage, 0, sizeof(s->storage)); + s->storage.head = NULL; memset(&s->state, 0, sizeof(s->state)); s->curr_op = NULL; s->cbs = NULL; + memset(&s->header_array, 0, sizeof(s->header_array)); memset(&s->state.rs, 0, sizeof(s->state.rs)); memset(&s->state.ws, 0, sizeof(s->state.ws)); memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); From 17d5c071153b540ee502712a4b0521b911d148bf Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 11 Aug 2016 15:21:43 -0700 Subject: [PATCH 68/97] use sent flag only under mutex and dont fail in send_initial_metadata --- src/ruby/lib/grpc/generic/active_call.rb | 22 ++++++++-------------- src/ruby/lib/grpc/generic/bidi_call.rb | 3 +-- src/ruby/spec/generic/active_call_spec.rb | 4 ++-- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index f9c41f0c0ed..23688dc9244 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -117,10 +117,10 @@ module GRPC end # Sends the initial metadata that has yet to be sent. - # Fails if metadata has already been sent for this call. + # Does nothing if metadata has already been sent for this call. def send_initial_metadata @send_initial_md_mutex.synchronize do - fail('Already send initial metadata') if @metadata_sent + return if @metadata_sent @metadata_tag = ActiveCall.client_invoke(@call, @metadata_to_send) @metadata_sent = true end @@ -201,7 +201,7 @@ module GRPC # @param marshalled [false, true] indicates if the object is already # marshalled. def remote_send(req, marshalled = false) - send_initial_metadata unless @metadata_sent + send_initial_metadata GRPC.logger.debug("sending #{req}, marshalled? #{marshalled}") payload = marshalled ? req : @marshal.call(req) @call.run_batch(SEND_MESSAGE => payload) @@ -217,7 +217,7 @@ module GRPC # list, mulitple metadata for its key are sent def send_status(code = OK, details = '', assert_finished = false, metadata: {}) - send_initial_metadata unless @metadata_sent + send_initial_metadata ops = { SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata) } @@ -318,8 +318,7 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Object] the response received from the server def request_response(req, metadata: {}) - merge_metadata_to_send(metadata) && - send_initial_metadata unless @metadata_sent + merge_metadata_to_send(metadata) && send_initial_metadata remote_send(req) writes_done(false) response = remote_read @@ -343,8 +342,7 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Object] the response received from the server def client_streamer(requests, metadata: {}) - merge_metadata_to_send(metadata) && - send_initial_metadata unless @metadata_sent + merge_metadata_to_send(metadata) && send_initial_metadata requests.each { |r| remote_send(r) } writes_done(false) response = remote_read @@ -370,8 +368,7 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Enumerator|nil] a response Enumerator def server_streamer(req, metadata: {}) - merge_metadata_to_send(metadata) && - send_initial_metadata unless @metadata_sent + merge_metadata_to_send(metadata) && send_initial_metadata remote_send(req) writes_done(false) replies = enum_for(:each_remote_read_then_finish) @@ -410,8 +407,7 @@ module GRPC # a list, multiple metadata for its key are sent # @return [Enumerator, nil] a response Enumerator def bidi_streamer(requests, metadata: {}, &blk) - merge_metadata_to_send(metadata) && - send_initial_metadata unless @metadata_sent + merge_metadata_to_send(metadata) && send_initial_metadata bd = BidiCall.new(@call, @marshal, @unmarshal, @@ -470,9 +466,7 @@ module GRPC # @param metadata [Hash] metadata to be sent to the server. If a value is # a list, multiple metadata for its key are sent def start_call(metadata = {}) - return if @metadata_sent merge_metadata_to_send(metadata) && send_initial_metadata - @metadata_sent = true end def self.view_class(*visible_methods) diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index 0b6ef4918c6..14905b721cc 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -172,8 +172,7 @@ module GRPC payload = @marshal.call(req) # Fails if status already received begin - @req_view.send_initial_metadata unless - @req_view.nil? || @req_view.metadata_sent + @req_view.send_initial_metadata unless @req_view.nil? @call.run_batch(SEND_MESSAGE => payload) rescue GRPC::Core::CallError => e # This is almost definitely caused by a status arriving while still diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index 79f739e8fa7..48bc61e494f 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -221,7 +221,7 @@ describe GRPC::ActiveCall do @client_call.send_initial_metadata end - it 'explicit sending fails if metadata has already been sent' do + it 'explicit sending does nothing if metadata has already been sent' do call = make_test_call @client_call = ActiveCall.new(call, @@ -235,7 +235,7 @@ describe GRPC::ActiveCall do @client_call.send_initial_metadata end - expect { blk.call }.to raise_error + expect { blk.call }.to_not raise_error end end From 91d519532a79e7309e0d63a246c064398eec5288 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Thu, 11 Aug 2016 15:30:52 -0700 Subject: [PATCH 69/97] removed file from commit --- .../CoreCronetEnd2EndTests.m | 403 ------------------ 1 file changed, 403 deletions(-) delete mode 100644 src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m deleted file mode 100644 index d2181120e93..00000000000 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ /dev/null @@ -1,403 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* - * This test file is derived from fixture h2_ssl.c in core end2end test - * (test/core/end2end/fixture/h2_ssl.c). The structure of the fixture is - * preserved as much as possible - * - * This fixture creates a server full stack using chttp2 and a client - * full stack using Cronet. End-to-end tests are run against this - * configuration - * - */ - - -#import -#include "test/core/end2end/end2end_tests.h" - -#include -#include - -#include -#include -#include - -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/security/credentials/credentials.h" -#include "src/core/lib/support/env.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/support/tmpfile.h" -#include "test/core/end2end/data/ssl_test_data.h" -#include "test/core/util/port.h" -#include "test/core/util/test_config.h" - -#include -#import - -typedef struct fullstack_secure_fixture_data { - char *localaddr; -} fullstack_secure_fixture_data; - -static grpc_end2end_test_fixture chttp2_create_fixture_secure_fullstack( - grpc_channel_args *client_args, grpc_channel_args *server_args) { - grpc_end2end_test_fixture f; - int port = grpc_pick_unused_port_or_die(); - fullstack_secure_fixture_data *ffd = - gpr_malloc(sizeof(fullstack_secure_fixture_data)); - memset(&f, 0, sizeof(f)); - - gpr_join_host_port(&ffd->localaddr, "127.0.0.1", port); - - f.fixture_data = ffd; - f.cq = grpc_completion_queue_create(NULL); - - return f; -} - -static void process_auth_failure(void *state, grpc_auth_context *ctx, - const grpc_metadata *md, size_t md_count, - grpc_process_auth_metadata_done_cb cb, - void *user_data) { - GPR_ASSERT(state == NULL); - cb(user_data, NULL, 0, NULL, 0, GRPC_STATUS_UNAUTHENTICATED, NULL); -} - -static void cronet_init_client_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args, - cronet_engine *cronetEngine) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - f->client = - grpc_cronet_secure_channel_create(cronetEngine, ffd->localaddr, client_args, NULL); - GPR_ASSERT(f->client != NULL); -} - -static void chttp2_init_server_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *server_args, - grpc_server_credentials *server_creds) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - if (f->server) { - grpc_server_destroy(f->server); - } - f->server = grpc_server_create(server_args, NULL); - grpc_server_register_completion_queue(f->server, f->cq, NULL); - GPR_ASSERT(grpc_server_add_secure_http2_port(f->server, ffd->localaddr, - server_creds)); - grpc_server_credentials_release(server_creds); - grpc_server_start(f->server); -} - -static void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture *f) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - gpr_free(ffd->localaddr); - gpr_free(ffd); -} - -static void cronet_init_client_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { - grpc_arg ssl_name_override = {GRPC_ARG_STRING, - GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, - {"foo.test.google.fr"}}; - - grpc_channel_args *new_client_args = - grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); - static bool done = false; - // TODO (makdharma): DO NOT CHECK IN THIS HACK!!! - if (!done) { - done = true; - [Cronet setHttp2Enabled:YES]; - NSURL *url = [[[NSFileManager defaultManager] - URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] lastObject]; - NSLog(@"Documents directory: %@", url); - [Cronet start]; - [Cronet startNetLogToFile: @"Documents/cronet_netlog.json" logBytes:YES]; - } - cronet_engine *cronetEngine = [Cronet getGlobalEngine]; - - cronet_init_client_secure_fullstack(f, new_client_args, cronetEngine); - grpc_channel_args_destroy(new_client_args); -} - -static int fail_server_auth_check(grpc_channel_args *server_args) { - size_t i; - if (server_args == NULL) return 0; - for (i = 0; i < server_args->num_args; i++) { - if (strcmp(server_args->args[i].key, FAIL_AUTH_CHECK_SERVER_ARG_NAME) == - 0) { - return 1; - } - } - return 0; -} - -static void chttp2_init_server_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *server_args) { - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; - grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1, 0, NULL); - if (fail_server_auth_check(server_args)) { - grpc_auth_metadata_processor processor = {process_auth_failure, NULL, NULL}; - grpc_server_credentials_set_auth_metadata_processor(ssl_creds, processor); - } - chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); -} - -/* All test configurations */ - -static grpc_end2end_test_config configs[] = { - {"chttp2/simple_ssl_fullstack", - FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION | - FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS, - chttp2_create_fixture_secure_fullstack, - cronet_init_client_simple_ssl_secure_fullstack, - chttp2_init_server_simple_ssl_secure_fullstack, - chttp2_tear_down_secure_fullstack}, -}; - - - -static char *roots_filename; - -@interface CoreCronetEnd2EndTests : XCTestCase - -@end - -@implementation CoreCronetEnd2EndTests - - -// The setUp() function is run before the test cases run and only run once -+ (void)setUp { - [super setUp]; - - FILE *roots_file; - size_t roots_size = strlen(test_root_cert); - - char *argv[] = {"CoreCronetEnd2EndTests"}; - grpc_test_init(1, argv); - grpc_end2end_tests_pre_init(); - - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != NULL); - GPR_ASSERT(roots_file != NULL); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_filename); - - grpc_init(); - -} - -// The tearDown() function is run after all test cases finish running -+ (void)tearDown { - grpc_shutdown(); - - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); - - [super tearDown]; -} - -- (void)testIndividualCase:(char*)test_case { - char *argv[] = {"h2_ssl", test_case}; - - for (int i = 0; i < sizeof(configs) / sizeof(*configs); i++) { - grpc_end2end_tests(sizeof(argv) / sizeof(argv[0]), argv, configs[i]); - } -} - -// TODO(mxyan): Use NSStringFromSelector(_cmd) to acquire test name from the -// test case method name, so that bodies of test cases can stay identical -- (void)testBadHostname { - [self testIndividualCase:"bad_hostname"]; -} - -- (void)testBinaryMetadata { - //[self testIndividualCase:"binary_metadata"]; -} - -- (void)testCallCreds { - [self testIndividualCase:"call_creds"]; -} - -- (void)testCancelAfterAccept { - [self testIndividualCase:"cancel_after_accept"]; -} - -- (void)testCancelAfterClientDone { - [self testIndividualCase:"cancel_after_client_done"]; -} - -- (void)testCancelAfterInvoke { - [self testIndividualCase:"cancel_after_invoke"]; -} - -- (void)testCancelBeforeInvoke { - [self testIndividualCase:"cancel_before_invoke"]; -} - -- (void)testCancelInAVacuum { - [self testIndividualCase:"cancel_in_a_vacuum"]; -} - -- (void)testCancelWithStatus { - [self testIndividualCase:"cancel_with_status"]; -} - -- (void)testCompressedPayload { - [self testIndividualCase:"compressed_payload"]; -} - -- (void)testConnectivity { - [self testIndividualCase:"connectivity"]; -} - -- (void)testDefaultHost { - [self testIndividualCase:"default_host"]; -} - -- (void)testDisappearingServer { - [self testIndividualCase:"disappearing_server"]; -} - -- (void)testEmptyBatch { - [self testIndividualCase:"empty_batch"]; -} - -- (void)testFilterCausesClose { - [self testIndividualCase:"filter_causes_close"]; -} - -- (void)testGracefulServerShutdown { - [self testIndividualCase:"graceful_server_shutdown"]; -} - -- (void)testHighInitialSeqno { - [self testIndividualCase:"high_initial_seqno"]; -} - -- (void)testHpackSize { - [self testIndividualCase:"hpack_size"]; -} - -- (void)testIdempotentRequest { - [self testIndividualCase:"idempotent_request"]; -} - -- (void)testInvokeLargeRequest { - [self testIndividualCase:"invoke_large_request"]; -} - -- (void)testLargeMetadata { - [self testIndividualCase:"large_metadata"]; -} - -- (void)testMaxConcurrentStreams { - [self testIndividualCase:"max_concurrent_streams"]; -} - -- (void)testMaxMessageLength { - [self testIndividualCase:"max_message_length"]; -} - -- (void)testNegativeDeadline { - [self testIndividualCase:"negative_deadline"]; -} - -- (void)testNetworkStatusChange { - [self testIndividualCase:"network_status_change"]; -} - -- (void)testNoOp { - [self testIndividualCase:"no_op"]; -} - -- (void)testPayload { - [self testIndividualCase:"payload"]; -} - -- (void)testPing { - [self testIndividualCase:"ping"]; -} - -- (void)testPingPongStreaming { - [self testIndividualCase:"ping_pong_streaming"]; -} - -- (void)testRegisteredCall { - [self testIndividualCase:"registered_call"]; -} - -- (void)testRequestWithFlags { - [self testIndividualCase:"request_with_flags"]; -} - -- (void)testRequestWithPayload { - [self testIndividualCase:"request_with_payload"]; -} - -- (void)testServerFinishesRequest { - [self testIndividualCase:"server_finishes_request"]; -} - -- (void)testShutdownFinishesCalls { - [self testIndividualCase:"shutdown_finishes_calls"]; -} - -- (void)testShutdownFinishesTags { - [self testIndividualCase:"shutdown_finishes_tags"]; -} - -- (void)testSimpleDelayedRequest { - [self testIndividualCase:"simple_delayed_request"]; -} - -- (void)testSimpleMetadata { - [self testIndividualCase:"simple_metadata"]; -} - -- (void)testSimpleRequest { - [self testIndividualCase:"simple_request"]; -} - -- (void)testStreamingErrorResponse { - [self testIndividualCase:"streaming_error_response"]; -} - -- (void)testTrailingMetadata { - [self testIndividualCase:"trailing_metadata"]; -} - -@end From 5804745aed0e6513706ad0ea0108ed6305003eba Mon Sep 17 00:00:00 2001 From: "David G. Quintas" Date: Thu, 11 Aug 2016 15:40:56 -0700 Subject: [PATCH 70/97] Merge pull request #7688 from dgquintas/cpp_readme fix c++ readme and tutorial --- examples/cpp/README.md | 31 ++-- examples/cpp/cpptutorial.md | 337 ++++++++++++++++++++++++------------ 2 files changed, 243 insertions(+), 125 deletions(-) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 3fa7ad4c784..783935cd53d 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,26 +2,14 @@ ## Installation -To install gRPC on your system, follow the instructions to build from source [here](../../INSTALL.md). This also installs the protocol buffer compiler `protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. +To install gRPC on your system, follow the instructions to build from source +[here](../../INSTALL.md). This also installs the protocol buffer compiler +`protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. ## Hello C++ gRPC! -Here's how to build and run the C++ implementation of the [Hello World](../protos/helloworld.proto) example used in [Getting started](..). - -The example code for this and our other examples lives in the `examples` -directory. Clone this repository to your local machine by running the -following command: - - -```sh -$ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc -``` - -Change your current directory to examples/cpp/helloworld - -```sh -$ cd examples/cpp/helloworld/ -``` +Here's how to build and run the C++ implementation of the [Hello +World](../protos/helloworld.proto) example used in [Getting started](..). ### Client and server implementations @@ -31,18 +19,25 @@ The server implementation is at [greeter_server.cc](helloworld/greeter_server.cc ### Try it! Build client and server: + ```sh $ make ``` + Run the server, which will listen on port 50051: + ```sh $ ./greeter_server ``` + Run the client (in a different terminal): + ```sh $ ./greeter_client ``` -If things go smoothly, you will see the "Greeter received: Hello world" in the client side output. + +If things go smoothly, you will see the "Greeter received: Hello world" in the +client side output. ## Tutorial diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index 80fef07192e..de7e4b26365 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -1,58 +1,77 @@ #gRPC Basics: C++ -This tutorial provides a basic C++ programmer's introduction to working with gRPC. By walking through this example you'll learn how to: +This tutorial provides a basic C++ programmer's introduction to working with +gRPC. By walking through this example you'll learn how to: -- Define a service in a .proto file. +- Define a service in a `.proto` file. - Generate server and client code using the protocol buffer compiler. - Use the C++ gRPC API to write a simple client and server for your service. -It assumes that you have read the [Getting started](..) guide and are familiar with [protocol buffers] (https://developers.google.com/protocol-buffers/docs/overview). Note that the example in this tutorial uses the proto3 version of the protocol buffers language, which is currently in alpha release: you can find out more in the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3) and see the [release notes](https://github.com/google/protobuf/releases) for the new version in the protocol buffers Github repository. - -This isn't a comprehensive guide to using gRPC in C++: more reference documentation is coming soon. +It assumes that you are familiar with +[protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). +Note that the example in this tutorial uses the proto3 version of the protocol +buffers language, which is currently in alpha release: you can find out more in +the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3) +and see the [release notes](https://github.com/google/protobuf/releases) for the +new version in the protocol buffers Github repository. ## Why use gRPC? -Our example is a simple route mapping application that lets clients get information about features on their route, create a summary of their route, and exchange route information such as traffic updates with the server and other clients. +Our example is a simple route mapping application that lets clients get +information about features on their route, create a summary of their route, and +exchange route information such as traffic updates with the server and other +clients. -With gRPC we can define our service once in a .proto file and implement clients and servers in any of gRPC's supported languages, which in turn can be run in environments ranging from servers inside Google to your own tablet - all the complexity of communication between different languages and environments is handled for you by gRPC. We also get all the advantages of working with protocol buffers, including efficient serialization, a simple IDL, and easy interface updating. +With gRPC we can define our service once in a `.proto` file and implement clients +and servers in any of gRPC's supported languages, which in turn can be run in +environments ranging from servers inside Google to your own tablet - all the +complexity of communication between different languages and environments is +handled for you by gRPC. We also get all the advantages of working with protocol +buffers, including efficient serialization, a simple IDL, and easy interface +updating. ## Example code and setup -The example code for our tutorial is in [examples/cpp/route_guide](route_guide). To download the example, clone this repository by running the following command: -```shell -$ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc -``` - -Then change your current directory to `examples/cpp/route_guide`: -```shell -$ cd examples/cpp/route_guide -``` - -You also should have the relevant tools installed to generate the server and client interface code - if you don't already, follow the setup instructions in [gRPC in 3 minutes](README.md). - +The example code for our tutorial is in [examples/cpp/route_guide](route_guide). +You also should have the relevant tools installed to generate the server and +client interface code - if you don't already, follow the setup instructions in +[INSTALL.md](../../INSTALL.md). ## Defining the service -Our first step (as you'll know from [Getting started](..) is to define the gRPC *service* and the method *request* and *response* types using [protocol buffers] (https://developers.google.com/protocol-buffers/docs/overview). You can see the complete .proto file in [`examples/protos/route_guide.proto`](../protos/route_guide.proto). +Our first step is to define the gRPC *service* and the method *request* and +*response* types using +[protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). +You can see the complete `.proto` file in +[`examples/protos/route_guide.proto`](../protos/route_guide.proto). -To define a service, you specify a named `service` in your .proto file: +To define a service, you specify a named `service` in your `.proto` file: -``` +```protobuf service RouteGuide { ... } ``` -Then you define `rpc` methods inside your service definition, specifying their request and response types. gRPC lets you define four kinds of service method, all of which are used in the `RouteGuide` service: +Then you define `rpc` methods inside your service definition, specifying their +request and response types. gRPC lets you define four kinds of service method, +all of which are used in the `RouteGuide` service: -- A *simple RPC* where the client sends a request to the server using the stub and waits for a response to come back, just like a normal function call. -``` +- A *simple RPC* where the client sends a request to the server using the stub + and waits for a response to come back, just like a normal function call. + +```protobuf // Obtains the feature at a given position. rpc GetFeature(Point) returns (Feature) {} ``` -- A *server-side streaming RPC* where the client sends a request to the server and gets a stream to read a sequence of messages back. The client reads from the returned stream until there are no more messages. As you can see in our example, you specify a server-side streaming method by placing the `stream` keyword before the *response* type. -``` +- A *server-side streaming RPC* where the client sends a request to the server + and gets a stream to read a sequence of messages back. The client reads from + the returned stream until there are no more messages. As you can see in our + example, you specify a server-side streaming method by placing the `stream` + keyword before the *response* type. + +```protobuf // Obtains the Features available within the given Rectangle. Results are // streamed rather than returned at once (e.g. in a response message with a // repeated field), as the rectangle may cover a large area and contain a @@ -60,22 +79,38 @@ Then you define `rpc` methods inside your service definition, specifying their r rpc ListFeatures(Rectangle) returns (stream Feature) {} ``` -- A *client-side streaming RPC* where the client writes a sequence of messages and sends them to the server, again using a provided stream. Once the client has finished writing the messages, it waits for the server to read them all and return its response. You specify a client-side streaming method by placing the `stream` keyword before the *request* type. -``` +- A *client-side streaming RPC* where the client writes a sequence of messages + and sends them to the server, again using a provided stream. Once the client + has finished writing the messages, it waits for the server to read them all + and return its response. You specify a client-side streaming method by placing + the `stream` keyword before the *request* type. + +```protobuf // Accepts a stream of Points on a route being traversed, returning a // RouteSummary when traversal is completed. rpc RecordRoute(stream Point) returns (RouteSummary) {} ``` -- A *bidirectional streaming RPC* where both sides send a sequence of messages using a read-write stream. The two streams operate independently, so clients and servers can read and write in whatever order they like: for example, the server could wait to receive all the client messages before writing its responses, or it could alternately read a message then write a message, or some other combination of reads and writes. The order of messages in each stream is preserved. You specify this type of method by placing the `stream` keyword before both the request and the response. -``` +- A *bidirectional streaming RPC* where both sides send a sequence of messages + using a read-write stream. The two streams operate independently, so clients + and servers can read and write in whatever order they like: for example, the + server could wait to receive all the client messages before writing its + responses, or it could alternately read a message then write a message, or + some other combination of reads and writes. The order of messages in each + stream is preserved. You specify this type of method by placing the `stream` + keyword before both the request and the response. + +```protobuf // Accepts a stream of RouteNotes sent while a route is being traversed, // while receiving other RouteNotes (e.g. from other users). rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} ``` -Our .proto file also contains protocol buffer message type definitions for all the request and response types used in our service methods - for example, here's the `Point` message type: -``` +Our `.proto` file also contains protocol buffer message type definitions for all +the request and response types used in our service methods - for example, here's +the `Point` message type: + +```protobuf // Points are represented as latitude-longitude pairs in the E7 representation // (degrees multiplied by 10**7 and rounded to the nearest integer). // Latitudes should be in the range +/- 90 degrees and longitude should be in @@ -86,12 +121,16 @@ message Point { } ``` - ## Generating client and server code -Next we need to generate the gRPC client and server interfaces from our .proto service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. +Next we need to generate the gRPC client and server interfaces from our `.proto` +service definition. We do this using the protocol buffer compiler `protoc` with +a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): +For simplicity, we've provided a [makefile](route_guide/Makefile) that runs +`protoc` for you with the appropriate plugin, input, and output (if you want to +run this yourself, make sure you've installed protoc and followed the gRPC code +[installation instructions](../../INSTALL.md) first): ```shell $ make route_guide.grpc.pb.cc route_guide.pb.cc @@ -107,39 +146,58 @@ $ protoc -I ../../protos --cpp_out=. ../../protos/route_guide.proto Running this command generates the following files in your current directory: - `route_guide.pb.h`, the header which declares your generated message classes - `route_guide.pb.cc`, which contains the implementation of your message classes -- `route_guide.grpc.pb.h`, the header which declares your generated service classes -- `route_guide.grpc.pb.cc`, which contains the implementation of your service classes +- `route_guide.grpc.pb.h`, the header which declares your generated service + classes +- `route_guide.grpc.pb.cc`, which contains the implementation of your service + classes These contain: -- All the protocol buffer code to populate, serialize, and retrieve our request and response message types +- All the protocol buffer code to populate, serialize, and retrieve our request + and response message types - A class called `RouteGuide` that contains - - a remote interface type (or *stub*) for clients to call with the methods defined in the `RouteGuide` service. - - two abstract interfaces for servers to implement, also with the methods defined in the `RouteGuide` service. + - a remote interface type (or *stub*) for clients to call with the methods + defined in the `RouteGuide` service. + - two abstract interfaces for servers to implement, also with the methods + defined in the `RouteGuide` service. ## Creating the server -First let's look at how we create a `RouteGuide` server. If you're only interested in creating gRPC clients, you can skip this section and go straight to [Creating the client](#client) (though you might find it interesting anyway!). +First let's look at how we create a `RouteGuide` server. If you're only +interested in creating gRPC clients, you can skip this section and go straight +to [Creating the client](#client) (though you might find it interesting +anyway!). There are two parts to making our `RouteGuide` service do its job: -- Implementing the service interface generated from our service definition: doing the actual "work" of our service. -- Running a gRPC server to listen for requests from clients and return the service responses. +- Implementing the service interface generated from our service definition: + doing the actual "work" of our service. +- Running a gRPC server to listen for requests from clients and return the + service responses. -You can find our example `RouteGuide` server in [route_guide/route_guide_server.cc](route_guide/route_guide_server.cc). Let's take a closer look at how it works. +You can find our example `RouteGuide` server in +[route_guide/route_guide_server.cc](route_guide/route_guide_server.cc). Let's +take a closer look at how it works. ### Implementing RouteGuide -As you can see, our server has a `RouteGuideImpl` class that implements the generated `RouteGuide::Service` interface: +As you can see, our server has a `RouteGuideImpl` class that implements the +generated `RouteGuide::Service` interface: ```cpp class RouteGuideImpl final : public RouteGuide::Service { ... } ``` -In this case we're implementing the *synchronous* version of `RouteGuide`, which provides our default gRPC server behaviour. It's also possible to implement an asynchronous interface, `RouteGuide::AsyncService`, which allows you to further customize your server's threading behaviour, though we won't look at this in this tutorial. +In this case we're implementing the *synchronous* version of `RouteGuide`, which +provides our default gRPC server behaviour. It's also possible to implement an +asynchronous interface, `RouteGuide::AsyncService`, which allows you to further +customize your server's threading behaviour, though we won't look at this in +this tutorial. -`RouteGuideImpl` implements all our service methods. Let's look at the simplest type first, `GetFeature`, which just gets a `Point` from the client and returns the corresponding feature information from its database in a `Feature`. +`RouteGuideImpl` implements all our service methods. Let's look at the simplest +type first, `GetFeature`, which just gets a `Point` from the client and returns +the corresponding feature information from its database in a `Feature`. ```cpp Status GetFeature(ServerContext* context, const Point* point, @@ -150,34 +208,52 @@ In this case we're implementing the *synchronous* version of `RouteGuide`, which } ``` -The method is passed a context object for the RPC, the client's `Point` protocol buffer request, and a `Feature` protocol buffer to fill in with the response information. In the method we populate the `Feature` with the appropriate information, and then `return` with an `OK` status to tell gRPC that we've finished dealing with the RPC and that the `Feature` can be returned to the client. +The method is passed a context object for the RPC, the client's `Point` protocol +buffer request, and a `Feature` protocol buffer to fill in with the response +information. In the method we populate the `Feature` with the appropriate +information, and then `return` with an `OK` status to tell gRPC that we've +finished dealing with the RPC and that the `Feature` can be returned to the +client. -Now let's look at something a bit more complicated - a streaming RPC. `ListFeatures` is a server-side streaming RPC, so we need to send back multiple `Feature`s to our client. +Now let's look at something a bit more complicated - a streaming RPC. +`ListFeatures` is a server-side streaming RPC, so we need to send back multiple +`Feature`s to our client. ```cpp - Status ListFeatures(ServerContext* context, const Rectangle* rectangle, - ServerWriter* writer) override { - auto lo = rectangle->lo(); - auto hi = rectangle->hi(); - long left = std::min(lo.longitude(), hi.longitude()); - long right = std::max(lo.longitude(), hi.longitude()); - long top = std::max(lo.latitude(), hi.latitude()); - long bottom = std::min(lo.latitude(), hi.latitude()); - for (const Feature& f : feature_list_) { - if (f.location().longitude() >= left && - f.location().longitude() <= right && - f.location().latitude() >= bottom && - f.location().latitude() <= top) { - writer->Write(f); - } +Status ListFeatures(ServerContext* context, const Rectangle* rectangle, + ServerWriter* writer) override { + auto lo = rectangle->lo(); + auto hi = rectangle->hi(); + long left = std::min(lo.longitude(), hi.longitude()); + long right = std::max(lo.longitude(), hi.longitude()); + long top = std::max(lo.latitude(), hi.latitude()); + long bottom = std::min(lo.latitude(), hi.latitude()); + for (const Feature& f : feature_list_) { + if (f.location().longitude() >= left && + f.location().longitude() <= right && + f.location().latitude() >= bottom && + f.location().latitude() <= top) { + writer->Write(f); } - return Status::OK; } + return Status::OK; +} ``` -As you can see, instead of getting simple request and response objects in our method parameters, this time we get a request object (the `Rectangle` in which our client wants to find `Feature`s) and a special `ServerWriter` object. In the method, we populate as many `Feature` objects as we need to return, writing them to the `ServerWriter` using its `Write()` method. Finally, as in our simple RPC, we `return Status::OK` to tell gRPC that we've finished writing responses. +As you can see, instead of getting simple request and response objects in our +method parameters, this time we get a request object (the `Rectangle` in which +our client wants to find `Feature`s) and a special `ServerWriter` object. In the +method, we populate as many `Feature` objects as we need to return, writing them +to the `ServerWriter` using its `Write()` method. Finally, as in our simple RPC, +we `return Status::OK` to tell gRPC that we've finished writing responses. -If you look at the client-side streaming method `RecordRoute` you'll see it's quite similar, except this time we get a `ServerReader` instead of a request object and a single response. We use the `ServerReader`s `Read()` method to repeatedly read in our client's requests to a request object (in this case a `Point`) until there are no more messages: the server needs to check the return value of `Read()` after each call. If `true`, the stream is still good and it can continue reading; if `false` the message stream has ended. +If you look at the client-side streaming method `RecordRoute` you'll see it's +quite similar, except this time we get a `ServerReader` instead of a request +object and a single response. We use the `ServerReader`s `Read()` method to +repeatedly read in our client's requests to a request object (in this case a +`Point`) until there are no more messages: the server needs to check the return +value of `Read()` after each call. If `true`, the stream is still good and it +can continue reading; if `false` the message stream has ended. ```cpp while (stream->Read(&point)) { @@ -205,11 +281,18 @@ Finally, let's look at our bidirectional streaming RPC `RouteChat()`. } ``` -This time we get a `ServerReaderWriter` that can be used to read *and* write messages. The syntax for reading and writing here is exactly the same as for our client-streaming and server-streaming methods. Although each side will always get the other's messages in the order they were written, both the client and server can read and write in any order — the streams operate completely independently. +This time we get a `ServerReaderWriter` that can be used to read *and* write +messages. The syntax for reading and writing here is exactly the same as for our +client-streaming and server-streaming methods. Although each side will always +get the other's messages in the order they were written, both the client and +server can read and write in any order — the streams operate completely +independently. ### Starting the server -Once we've implemented all our methods, we also need to start up a gRPC server so that clients can actually use our service. The following snippet shows how we do this for our `RouteGuide` service: +Once we've implemented all our methods, we also need to start up a gRPC server +so that clients can actually use our service. The following snippet shows how we +do this for our `RouteGuide` service: ```cpp void RunServer(const std::string& db_path) { @@ -227,44 +310,55 @@ void RunServer(const std::string& db_path) { As you can see, we build and start our server using a `ServerBuilder`. To do this, we: 1. Create an instance of our service implementation class `RouteGuideImpl`. -2. Create an instance of the factory `ServerBuilder` class. -3. Specify the address and port we want to use to listen for client requests using the builder's `AddListeningPort()` method. -4. Register our service implementation with the builder. -5. Call `BuildAndStart()` on the builder to create and start an RPC server for our service. -5. Call `Wait()` on the server to do a blocking wait until process is killed or `Shutdown()` is called. +1. Create an instance of the factory `ServerBuilder` class. +1. Specify the address and port we want to use to listen for client requests + using the builder's `AddListeningPort()` method. +1. Register our service implementation with the builder. +1. Call `BuildAndStart()` on the builder to create and start an RPC server for + our service. +1. Call `Wait()` on the server to do a blocking wait until process is killed or + `Shutdown()` is called. ## Creating the client -In this section, we'll look at creating a C++ client for our `RouteGuide` service. You can see our complete example client code in [route_guide/route_guide_client.cc](route_guide/route_guide_client.cc). +In this section, we'll look at creating a C++ client for our `RouteGuide` +service. You can see our complete example client code in +[route_guide/route_guide_client.cc](route_guide/route_guide_client.cc). ### Creating a stub To call service methods, we first need to create a *stub*. -First we need to create a gRPC *channel* for our stub, specifying the server address and port we want to connect to without SSL: +First we need to create a gRPC *channel* for our stub, specifying the server +address and port we want to connect to without SSL: ```cpp grpc::CreateChannel("localhost:50051", grpc::InsecureChannelCredentials()); ``` -Now we can use the channel to create our stub using the `NewStub` method provided in the `RouteGuide` class we generated from our .proto. +Now we can use the channel to create our stub using the `NewStub` method +provided in the `RouteGuide` class we generated from our `.proto`. ```cpp - public: - RouteGuideClient(std::shared_ptr channel, const std::string& db) - : stub_(RouteGuide::NewStub(channel)) { - ... - } +public: + RouteGuideClient(std::shared_ptr channel, const std::string& db) + : stub_(RouteGuide::NewStub(channel)) { + ... + } ``` ### Calling service methods -Now let's look at how we call our service methods. Note that in this tutorial we're calling the *blocking/synchronous* versions of each method: this means that the RPC call waits for the server to respond, and will either return a response or raise an exception. +Now let's look at how we call our service methods. Note that in this tutorial +we're calling the *blocking/synchronous* versions of each method: this means +that the RPC call waits for the server to respond, and will either return a +response or raise an exception. #### Simple RPC -Calling the simple RPC `GetFeature` is nearly as straightforward as calling a local method. +Calling the simple RPC `GetFeature` is nearly as straightforward as calling a +local method. ```cpp Point point; @@ -281,33 +375,53 @@ Calling the simple RPC `GetFeature` is nearly as straightforward as calling a lo } ``` -As you can see, we create and populate a request protocol buffer object (in our case `Point`), and create a response protocol buffer object for the server to fill in. We also create a `ClientContext` object for our call - you can optionally set RPC configuration values on this object, such as deadlines, though for now we'll use the default settings. Note that you cannot reuse this object between calls. Finally, we call the method on the stub, passing it the context, request, and response. If the method returns `OK`, then we can read the response information from the server from our response object. +As you can see, we create and populate a request protocol buffer object (in our +case `Point`), and create a response protocol buffer object for the server to +fill in. We also create a `ClientContext` object for our call - you can +optionally set RPC configuration values on this object, such as deadlines, +though for now we'll use the default settings. Note that you cannot reuse this +object between calls. Finally, we call the method on the stub, passing it the +context, request, and response. If the method returns `OK`, then we can read the +response information from the server from our response object. ```cpp - std::cout << "Found feature called " << feature->name() << " at " - << feature->location().latitude()/kCoordFactor_ << ", " - << feature->location().longitude()/kCoordFactor_ << std::endl; +std::cout << "Found feature called " << feature->name() << " at " + << feature->location().latitude()/kCoordFactor_ << ", " + << feature->location().longitude()/kCoordFactor_ << std::endl; ``` #### Streaming RPCs -Now let's look at our streaming methods. If you've already read [Creating the server](#server) some of this may look very familiar - streaming RPCs are implemented in a similar way on both sides. Here's where we call the server-side streaming method `ListFeatures`, which returns a stream of geographical `Feature`s: +Now let's look at our streaming methods. If you've already read [Creating the +server](#server) some of this may look very familiar - streaming RPCs are +implemented in a similar way on both sides. Here's where we call the server-side +streaming method `ListFeatures`, which returns a stream of geographical +`Feature`s: ```cpp - std::unique_ptr > reader( - stub_->ListFeatures(&context, rect)); - while (reader->Read(&feature)) { - std::cout << "Found feature called " - << feature.name() << " at " - << feature.location().latitude()/kCoordFactor_ << ", " - << feature.location().longitude()/kCoordFactor_ << std::endl; - } - Status status = reader->Finish(); +std::unique_ptr > reader( + stub_->ListFeatures(&context, rect)); +while (reader->Read(&feature)) { + std::cout << "Found feature called " + << feature.name() << " at " + << feature.location().latitude()/kCoordFactor_ << ", " + << feature.location().longitude()/kCoordFactor_ << std::endl; +} +Status status = reader->Finish(); ``` -Instead of passing the method a context, request, and response, we pass it a context and request and get a `ClientReader` object back. The client can use the `ClientReader` to read the server's responses. We use the `ClientReader`s `Read()` method to repeatedly read in the server's responses to a response protocol buffer object (in this case a `Feature`) until there are no more messages: the client needs to check the return value of `Read()` after each call. If `true`, the stream is still good and it can continue reading; if `false` the message stream has ended. Finally, we call `Finish()` on the stream to complete the call and get our RPC status. +Instead of passing the method a context, request, and response, we pass it a +context and request and get a `ClientReader` object back. The client can use the +`ClientReader` to read the server's responses. We use the `ClientReader`s +`Read()` method to repeatedly read in the server's responses to a response +protocol buffer object (in this case a `Feature`) until there are no more +messages: the client needs to check the return value of `Read()` after each +call. If `true`, the stream is still good and it can continue reading; if +`false` the message stream has ended. Finally, we call `Finish()` on the stream +to complete the call and get our RPC status. -The client-side streaming method `RecordRoute` is similar, except there we pass the method a context and response object and get back a `ClientWriter`. +The client-side streaming method `RecordRoute` is similar, except there we pass +the method a context and response object and get back a `ClientWriter`. ```cpp std::unique_ptr > writer( @@ -337,16 +451,26 @@ The client-side streaming method `RecordRoute` is similar, except there we pass } ``` -Once we've finished writing our client's requests to the stream using `Write()`, we need to call `WritesDone()` on the stream to let gRPC know that we've finished writing, then `Finish()` to complete the call and get our RPC status. If the status is `OK`, our response object that we initially passed to `RecordRoute()` will be populated with the server's response. +Once we've finished writing our client's requests to the stream using `Write()`, +we need to call `WritesDone()` on the stream to let gRPC know that we've +finished writing, then `Finish()` to complete the call and get our RPC status. +If the status is `OK`, our response object that we initially passed to +`RecordRoute()` will be populated with the server's response. -Finally, let's look at our bidirectional streaming RPC `RouteChat()`. In this case, we just pass a context to the method and get back a `ClientReaderWriter`, which we can use to both write and read messages. +Finally, let's look at our bidirectional streaming RPC `RouteChat()`. In this +case, we just pass a context to the method and get back a `ClientReaderWriter`, +which we can use to both write and read messages. ```cpp - std::shared_ptr > stream( - stub_->RouteChat(&context)); +std::shared_ptr > stream( + stub_->RouteChat(&context)); ``` -The syntax for reading and writing here is exactly the same as for our client-streaming and server-streaming methods. Although each side will always get the other's messages in the order they were written, both the client and server can read and write in any order — the streams operate completely independently. +The syntax for reading and writing here is exactly the same as for our +client-streaming and server-streaming methods. Although each side will always +get the other's messages in the order they were written, both the client and +server can read and write in any order — the streams operate completely +independently. ## Try it out! @@ -362,4 +486,3 @@ Run the client (in a different terminal): ```shell $ ./route_guide_client ``` - From c61b1d3cd0e14c672741a8d3b27dedb421682abd Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 11 Aug 2016 16:15:41 -0700 Subject: [PATCH 71/97] s/makefile/Makefile --- examples/cpp/cpptutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index de7e4b26365..ae84aba9163 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -127,7 +127,7 @@ Next we need to generate the gRPC client and server interfaces from our `.proto` service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs +For simplicity, we've provided a [Makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): From d161f20a919763a2ee47cd8d4997fba2424fb219 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 11 Aug 2016 17:01:45 -0700 Subject: [PATCH 72/97] Don't get dependencies for Node grpc-health-check package build --- tools/run_tests/build_package_node.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index f20daeaea09..a5636cf87a7 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -50,7 +50,6 @@ cp grpc-*.tgz $artifacts/grpc.tgz mkdir -p bin cd $base/src/node/health_check -npm update npm pack cp grpc-health-check-*.tgz $artifacts/ From f2c074a73edb85ef71616d33f44c6fbda1159b25 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 12 Aug 2016 03:23:10 +0200 Subject: [PATCH 73/97] Updating build package C# scripts. --- src/csharp/build_packages.bat | 8 ++------ templates/src/csharp/build_packages.bat.template | 8 ++------ tools/run_tests/package_targets.py | 9 ++++++++- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index 1d40ef9eb2d..9f544ff085d 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -31,10 +31,7 @@ @rem Current package versions set VERSION=1.0.0-pre2 -set PROTOBUF_VERSION=3.0.0-beta3 - -@rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. -set VERSION_WITH_BETA=%VERSION%-beta +set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe @@ -58,7 +55,6 @@ xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=macos\artifacts\* pr @rem Fetch all dependencies %NUGET% restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error -%NUGET% restore Grpc.sln || goto :error setlocal @@ -73,7 +69,7 @@ endlocal %NUGET% pack Grpc.Auth\Grpc.Auth.nuspec -Symbols -Version %VERSION% || goto :error %NUGET% pack Grpc.Core\Grpc.Core.nuspec -Symbols -Version %VERSION% || goto :error -%NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION_WITH_BETA% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error +%NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error %NUGET% pack Grpc.nuspec -Version %VERSION% || goto :error %NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error diff --git a/templates/src/csharp/build_packages.bat.template b/templates/src/csharp/build_packages.bat.template index ea2acb661ee..5cbd8e3746d 100644 --- a/templates/src/csharp/build_packages.bat.template +++ b/templates/src/csharp/build_packages.bat.template @@ -33,10 +33,7 @@ @rem Current package versions set VERSION=${settings.csharp_version} - set PROTOBUF_VERSION=3.0.0-beta3 - - @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. - set VERSION_WITH_BETA=%VERSION%-beta + set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe @@ -60,7 +57,6 @@ @rem Fetch all dependencies %%NUGET% restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error - %%NUGET% restore Grpc.sln || goto :error setlocal @@ -75,7 +71,7 @@ %%NUGET% pack Grpc.Auth\Grpc.Auth.nuspec -Symbols -Version %VERSION% || goto :error %%NUGET% pack Grpc.Core\Grpc.Core.nuspec -Symbols -Version %VERSION% || goto :error - %%NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION_WITH_BETA% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error + %%NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error %%NUGET% pack Grpc.nuspec -Version %VERSION% || goto :error %%NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 39a11a243d6..5cafa6c296e 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -81,7 +81,14 @@ class CSharpPackage: self.labels += ['windows'] def pre_build_jobspecs(self): - return [] + if self.platform == 'windows': + return [create_jobspec('prebuild_%s' % self.name, + ['tools\\run_tests\\pre_build_csharp.bat'], + shell=True, + flake_retries=5, + timeout_retries=2)] + else: + return [] def build_jobspec(self): if self.use_coreclr: From 6ee23ee614f192ff97b8e0c306f0523375acc045 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 12 Aug 2016 09:46:07 +0200 Subject: [PATCH 74/97] There's no 'platform' in package_targets.py, only labels. --- tools/run_tests/package_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 5cafa6c296e..8217d7698af 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -81,7 +81,7 @@ class CSharpPackage: self.labels += ['windows'] def pre_build_jobspecs(self): - if self.platform == 'windows': + if 'windows' in self.labels: return [create_jobspec('prebuild_%s' % self.name, ['tools\\run_tests\\pre_build_csharp.bat'], shell=True, From 80c704bfa8be8a643e3cefefdacee4b8072f173c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 12 Aug 2016 20:41:57 +0800 Subject: [PATCH 75/97] update distribtest project to Google.Apis.Auth 0.15.0 --- test/distrib/csharp/DistribTest/App.config | 14 ----- .../csharp/DistribTest/DistribTest.csproj | 54 +++++++------------ .../csharp/DistribTest/packages.config | 8 +-- 3 files changed, 21 insertions(+), 55 deletions(-) delete mode 100644 test/distrib/csharp/DistribTest/App.config diff --git a/test/distrib/csharp/DistribTest/App.config b/test/distrib/csharp/DistribTest/App.config deleted file mode 100644 index 30d3e094721..00000000000 --- a/test/distrib/csharp/DistribTest/App.config +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index 1acb34d1b2e..6ca03b2c800 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -1,5 +1,5 @@  - + Debug @@ -41,6 +41,8 @@ prompt MinimumRecommendedRules.ruleset true + 4 + false bin\x64\Release\ @@ -51,39 +53,15 @@ prompt MinimumRecommendedRules.ruleset true + 4 - - ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll - - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.dll - - - ..\packages\Google.Apis.Auth.1.9.3\lib\net40\Google.Apis.Auth.PlatformServices.dll - - - ..\packages\Google.Apis.Core.1.9.3\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll - ..\packages\Grpc.Auth.__GRPC_NUGET_VERSION__\lib\net45\Grpc.Auth.dll ..\packages\Grpc.Core.__GRPC_NUGET_VERSION__\lib\net45\Grpc.Core.dll - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - @@ -91,25 +69,33 @@ - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Extensions.dll - - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Primitives.dll - + + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + + + ..\packages\Google.Apis.Core.1.15.0\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll + - @@ -119,9 +105,7 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - -