From dbfb888e1fa80bab577e262da6691a6dbe8e15d7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Feb 2023 09:18:04 -0800 Subject: [PATCH] Revert "[promises] Server call (#31448)" (#32260) * Revert "[promises] Server call (#31448)" This reverts commit bbeb15006af65d371a933fe432bce5078115b9e3. * clang-fmt --- BUILD | 15 +- CMakeLists.txt | 148 +- Makefile | 8 +- bazel/experiments.bzl | 1 - build_autogenerated.yaml | 259 ++-- config.m4 | 4 +- config.w32 | 4 +- doc/environment_variables.md | 1 - gRPC-C++.podspec | 16 +- gRPC-Core.podspec | 20 +- grpc.gemspec | 12 +- grpc.gyp | 12 +- package.xml | 12 +- src/core/BUILD | 95 +- .../grpclb/client_load_reporting_filter.cc | 21 +- .../filters/http/client/http_client_filter.cc | 44 +- .../message_compress/compression_filter.cc | 225 +-- .../message_compress/compression_filter.h | 21 +- .../filters/http/server/http_server_filter.cc | 47 +- .../stateful_session_filter.cc | 51 +- .../chttp2/transport/chttp2_transport.cc | 32 - .../ext/transport/chttp2/transport/internal.h | 14 +- .../ext/transport/chttp2/transport/parsing.cc | 135 +- .../lib/channel/channel_stack_builder_impl.cc | 15 +- src/core/lib/channel/connected_channel.cc | 1227 +++++------------ src/core/lib/channel/promise_based_filter.cc | 687 +++------ src/core/lib/channel/promise_based_filter.h | 139 +- src/core/lib/event_engine/slice.cc | 2 +- src/core/lib/experiments/experiments.cc | 4 - src/core/lib/experiments/experiments.h | 11 +- src/core/lib/experiments/experiments.yaml | 8 - src/core/lib/gprpp/debug_location.h | 3 + src/core/lib/iomgr/call_combiner.cc | 8 +- src/core/lib/iomgr/closure.cc | 27 - src/core/lib/iomgr/closure.h | 2 - src/core/lib/promise/activity.h | 4 - src/core/lib/promise/for_each.h | 22 - src/core/lib/promise/if.h | 61 - src/core/lib/promise/interceptor_list.h | 309 ----- src/core/lib/promise/intra_activity_waiter.h | 6 - src/core/lib/promise/latch.h | 30 +- src/core/lib/promise/map_pipe.h | 32 +- src/core/lib/promise/{trace.cc => pipe.cc} | 5 +- src/core/lib/promise/pipe.h | 484 +++---- src/core/lib/promise/trace.h | 24 - src/core/lib/promise/try_concurrently.h | 28 - src/core/lib/resource_quota/arena.h | 90 +- src/core/lib/slice/slice.cc | 6 +- src/core/lib/slice/slice.h | 15 +- src/core/lib/slice/slice_buffer.h | 8 - src/core/lib/slice/slice_refcount.cc | 20 - src/core/lib/slice/slice_refcount.h | 26 +- src/core/lib/surface/call.cc | 1077 ++++----------- src/core/lib/surface/call.h | 28 - src/core/lib/surface/call_trace.cc | 28 +- src/core/lib/surface/lame_client.cc | 4 +- src/core/lib/surface/server.cc | 297 +--- src/core/lib/surface/server.h | 3 - src/core/lib/transport/transport.cc | 7 +- src/core/lib/transport/transport.h | 18 +- src/cpp/ext/filters/logging/BUILD | 11 +- src/cpp/ext/filters/logging/logging_filter.cc | 142 +- src/python/grpcio/grpc_core_dependencies.py | 4 +- .../fixtures/h2_sockpair_with_minstack.cc | 15 +- test/core/end2end/generate_tests.bzl | 2 +- .../core/end2end/tests/cancel_after_accept.cc | 3 - test/core/end2end/tests/grpc_authz.cc | 1 - test/core/end2end/tests/max_message_length.cc | 3 - test/core/end2end/tests/no_logging.cc | 2 - test/core/end2end/tests/server_streaming.cc | 11 +- test/core/filters/filter_fuzzer.cc | 28 +- test/core/promise/BUILD | 20 +- test/core/promise/activity_test.cc | 4 +- test/core/promise/for_each_test.cc | 1 - test/core/promise/if_test.cc | 12 - test/core/promise/interceptor_list_test.cc | 144 -- test/core/promise/latch_test.cc | 4 +- test/core/promise/map_pipe_test.cc | 1 - test/core/promise/pipe_test.cc | 262 +--- test/core/resource_quota/arena_test.cc | 45 - test/core/slice/slice_test.cc | 2 +- test/cpp/ext/filters/logging/logging_test.cc | 10 +- tools/doxygen/Doxyfile.c++.internal | 12 +- tools/doxygen/Doxyfile.core.internal | 12 +- tools/run_tests/generated/tests.json | 24 - 85 files changed, 1746 insertions(+), 4991 deletions(-) delete mode 100644 src/core/lib/iomgr/closure.cc delete mode 100644 src/core/lib/promise/interceptor_list.h rename src/core/lib/promise/{trace.cc => pipe.cc} (82%) delete mode 100644 src/core/lib/promise/trace.h delete mode 100644 src/core/lib/slice/slice_refcount.cc delete mode 100644 test/core/promise/interceptor_list_test.cc diff --git a/BUILD b/BUILD index 15930d5149f..a40aed74253 100644 --- a/BUILD +++ b/BUILD @@ -1439,8 +1439,6 @@ grpc_cc_library( "//src/core:arena", "//src/core:arena_promise", "//src/core:atomic_utils", - "//src/core:basic_join", - "//src/core:basic_seq", "//src/core:bitset", "//src/core:channel_args", "//src/core:channel_args_endpoint_config", @@ -1469,7 +1467,7 @@ grpc_cc_library( "//src/core:iomgr_fwd", "//src/core:iomgr_port", "//src/core:json", - "//src/core:map", + "//src/core:latch", "//src/core:match", "//src/core:memory_quota", "//src/core:no_destruct", @@ -1495,8 +1493,6 @@ grpc_cc_library( "//src/core:thread_quota", "//src/core:time", "//src/core:transport_fwd", - "//src/core:try_join", - "//src/core:try_seq", "//src/core:useful", ], ) @@ -3282,9 +3278,9 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_trace", "promise", - "//src/core:activity", "//src/core:arena", "//src/core:arena_promise", + "//src/core:basic_seq", "//src/core:channel_args", "//src/core:channel_fwd", "//src/core:channel_init", @@ -3292,14 +3288,15 @@ grpc_cc_library( "//src/core:context", "//src/core:grpc_message_size_filter", "//src/core:latch", - "//src/core:map", + "//src/core:map_pipe", "//src/core:percent_encoding", "//src/core:pipe", - "//src/core:poll", - "//src/core:race", + "//src/core:promise_like", + "//src/core:seq", "//src/core:slice", "//src/core:slice_buffer", "//src/core:transport_fwd", + "//src/core:try_concurrently", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 58a030b8062..65c76c1dc18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1021,7 +1021,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx init_test) add_dependencies(buildtests_cxx initial_settings_frame_bad_client_test) add_dependencies(buildtests_cxx insecure_security_connector_test) - add_dependencies(buildtests_cxx interceptor_list_test) add_dependencies(buildtests_cxx interop_client) add_dependencies(buildtests_cxx interop_server) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX OR _gRPC_PLATFORM_WINDOWS) @@ -2219,7 +2218,6 @@ add_library(grpc src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -2301,8 +2299,8 @@ add_library(grpc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/matchers/matchers.cc src/core/lib/promise/activity.cc + src/core/lib/promise/pipe.cc src/core/lib/promise/sleep.cc - src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -2380,7 +2378,6 @@ add_library(grpc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -2904,7 +2901,6 @@ add_library(grpc_unsecure src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -2984,8 +2980,8 @@ add_library(grpc_unsecure src/core/lib/load_balancing/lb_policy.cc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/promise/activity.cc + src/core/lib/promise/pipe.cc src/core/lib/promise/sleep.cc - src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -3032,7 +3028,6 @@ add_library(grpc_unsecure src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -4416,7 +4411,6 @@ add_library(grpc_authorization_provider src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -4496,7 +4490,7 @@ add_library(grpc_authorization_provider src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/matchers/matchers.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + src/core/lib/promise/pipe.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -4543,7 +4537,6 @@ add_library(grpc_authorization_provider src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -8304,7 +8297,6 @@ add_executable(chunked_vector_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -8319,7 +8311,6 @@ add_executable(chunked_vector_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/gprpp/chunked_vector_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -10595,7 +10586,6 @@ add_executable(exec_ctx_wakeup_scheduler_test src/core/lib/debug/trace.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -10604,7 +10594,6 @@ add_executable(exec_ctx_wakeup_scheduler_test src/core/lib/promise/activity.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/exec_ctx_wakeup_scheduler_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -11070,7 +11059,6 @@ add_executable(flow_control_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -11084,7 +11072,6 @@ add_executable(flow_control_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/transport/bdp_estimator.cc src/core/lib/transport/pid_controller.cc @@ -11141,14 +11128,13 @@ add_executable(for_each_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + src/core/lib/promise/pipe.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc @@ -11157,7 +11143,6 @@ add_executable(for_each_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/for_each_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -11502,7 +11487,6 @@ add_executable(frame_test src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -11581,7 +11565,7 @@ add_executable(frame_test src/core/lib/load_balancing/lb_policy.cc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + src/core/lib/promise/pipe.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -11605,7 +11589,6 @@ add_executable(frame_test src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -13943,77 +13926,6 @@ target_link_libraries(insecure_security_connector_test ) -endif() -if(gRPC_BUILD_TESTS) - -add_executable(interceptor_list_test - src/core/ext/upb-generated/google/protobuf/any.upb.c - src/core/ext/upb-generated/google/rpc/status.upb.c - src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc - src/core/lib/resource_quota/thread_quota.cc - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/interceptor_list_test.cc - third_party/googletest/googletest/src/gtest-all.cc - third_party/googletest/googlemock/src/gmock-all.cc -) -target_compile_features(interceptor_list_test PUBLIC cxx_std_14) -target_include_directories(interceptor_list_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_RE2_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_XXHASH_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} - third_party/googletest/googletest/include - third_party/googletest/googletest - third_party/googletest/googlemock/include - third_party/googletest/googlemock - ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(interceptor_list_test - ${_gRPC_BASELIB_LIBRARIES} - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ZLIB_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - absl::flat_hash_set - absl::any_invocable - absl::function_ref - absl::hash - absl::type_traits - absl::statusor - absl::utility - gpr - upb -) - - endif() if(gRPC_BUILD_TESTS) @@ -14496,9 +14408,7 @@ endif() if(gRPC_BUILD_TESTS) add_executable(latch_test - src/core/lib/debug/trace.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc test/core/promise/latch_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -14745,14 +14655,13 @@ add_executable(map_pipe_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + src/core/lib/promise/pipe.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc @@ -14761,7 +14670,6 @@ add_executable(map_pipe_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/map_pipe_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -16117,7 +16025,6 @@ add_executable(periodic_update_test src/core/lib/debug/trace.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -16126,7 +16033,6 @@ add_executable(periodic_update_test src/core/lib/resource_quota/periodic_update.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/resource_quota/periodic_update_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -16256,6 +16162,30 @@ endif() if(gRPC_BUILD_TESTS) add_executable(pipe_test + src/core/ext/upb-generated/google/protobuf/any.upb.c + src/core/ext/upb-generated/google/rpc/status.upb.c + src/core/lib/debug/trace.cc + src/core/lib/event_engine/memory_allocator.cc + src/core/lib/experiments/config.cc + src/core/lib/experiments/experiments.cc + src/core/lib/gprpp/status_helper.cc + src/core/lib/gprpp/time.cc + src/core/lib/iomgr/combiner.cc + src/core/lib/iomgr/error.cc + src/core/lib/iomgr/exec_ctx.cc + src/core/lib/iomgr/executor.cc + src/core/lib/iomgr/iomgr_internal.cc + src/core/lib/promise/activity.cc + src/core/lib/promise/pipe.cc + src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/memory_quota.cc + src/core/lib/resource_quota/periodic_update.cc + src/core/lib/resource_quota/resource_quota.cc + src/core/lib/resource_quota/thread_quota.cc + src/core/lib/resource_quota/trace.cc + src/core/lib/slice/percent_encoding.cc + src/core/lib/slice/slice.cc + src/core/lib/slice/slice_string_helpers.cc test/core/promise/pipe_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -16285,7 +16215,15 @@ target_link_libraries(pipe_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ZLIB_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc + absl::flat_hash_set + absl::any_invocable + absl::function_ref + absl::hash + absl::type_traits + absl::statusor + absl::utility + gpr + upb ) @@ -18839,9 +18777,7 @@ endif() if(gRPC_BUILD_TESTS) add_executable(slice_string_helpers_test - src/core/lib/debug/trace.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/slice/slice_string_helpers_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -20235,14 +20171,12 @@ endif() if(gRPC_BUILD_TESTS) add_executable(test_core_event_engine_slice_buffer_test - src/core/lib/debug/trace.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/resolved_address.cc src/core/lib/event_engine/slice.cc src/core/lib/event_engine/slice_buffer.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/event_engine/slice_buffer_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -21590,21 +21524,19 @@ add_executable(try_concurrently_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc - src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + src/core/lib/promise/pipe.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/try_concurrently_test.cc third_party/googletest/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 8c1670e4426..5f6348a3a64 100644 --- a/Makefile +++ b/Makefile @@ -1473,7 +1473,6 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ - src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -1555,8 +1554,8 @@ LIBGRPC_SRC = \ src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/matchers/matchers.cc \ src/core/lib/promise/activity.cc \ + src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ - src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -1634,7 +1633,6 @@ LIBGRPC_SRC = \ src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ - src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ @@ -2011,7 +2009,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ - src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -2091,8 +2088,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/load_balancing/lb_policy.cc \ src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/promise/activity.cc \ + src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ - src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -2139,7 +2136,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ - src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index b86d9688824..bfcbebb19ee 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -25,7 +25,6 @@ EXPERIMENTS = { ], "core_end2end_test": [ "promise_based_client_call", - "promise_based_server_call", ], "endpoint_test": [ "tcp_frame_size_tuning", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 3edb2256a98..a55ea46ec65 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -930,27 +930,25 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h + - src/core/lib/promise/for_each.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h + - src/core/lib/promise/map_pipe.h - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h - - src/core/lib/promise/trace.h - - src/core/lib/promise/try_join.h + - src/core/lib/promise/try_concurrently.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -1610,7 +1608,6 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -1692,8 +1689,8 @@ libs: - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/matchers/matchers.cc - src/core/lib/promise/activity.cc + - src/core/lib/promise/pipe.cc - src/core/lib/promise/sleep.cc - - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -1771,7 +1768,6 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -2262,27 +2258,25 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h + - src/core/lib/promise/for_each.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h + - src/core/lib/promise/map_pipe.h - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h - - src/core/lib/promise/trace.h - - src/core/lib/promise/try_join.h + - src/core/lib/promise/try_concurrently.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -2558,7 +2552,6 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -2638,8 +2631,8 @@ libs: - src/core/lib/load_balancing/lb_policy.cc - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/promise/activity.cc + - src/core/lib/promise/pipe.cc - src/core/lib/promise/sleep.cc - - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -2686,7 +2679,6 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -3711,16 +3703,14 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h + - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - src/core/lib/promise/pipe.h @@ -3728,8 +3718,6 @@ libs: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - - src/core/lib/promise/try_join.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -3890,7 +3878,6 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -3970,7 +3957,7 @@ libs: - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/matchers/matchers.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc + - src/core/lib/promise/pipe.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -4017,7 +4004,6 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -5813,7 +5799,6 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -5828,7 +5813,6 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/gprpp/chunked_vector_test.cc deps: @@ -6813,7 +6797,6 @@ targets: - src/core/lib/debug/trace.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -6822,7 +6805,6 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/exec_ctx_wakeup_scheduler_test.cc deps: @@ -7110,7 +7092,6 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -7124,7 +7105,6 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/pid_controller.cc @@ -7176,8 +7156,6 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h - src/core/lib/promise/loop.h @@ -7186,7 +7164,6 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h - src/core/lib/resource_quota/memory_quota.h @@ -7209,14 +7186,13 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc + - src/core/lib/promise/pipe.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc @@ -7225,7 +7201,6 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/for_each_test.cc deps: @@ -7525,16 +7500,14 @@ targets: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h + - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - src/core/lib/promise/pipe.h @@ -7542,9 +7515,6 @@ targets: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - - src/core/lib/promise/try_join.h - - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h - src/core/lib/resolver/resolver_registry.h @@ -7688,7 +7658,6 @@ targets: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -7767,7 +7736,7 @@ targets: - src/core/lib/load_balancing/lb_policy.cc - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc + - src/core/lib/promise/pipe.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -7791,7 +7760,6 @@ targets: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -8733,7 +8701,6 @@ targets: build: test language: c++ headers: - - src/core/lib/gprpp/construct_destruct.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/if.h @@ -8802,98 +8769,6 @@ targets: - test/core/util/tracer_util.cc deps: - grpc_test_util -- name: interceptor_list_test - gtest: true - build: test - language: c++ - headers: - - src/core/ext/upb-generated/google/protobuf/any.upb.h - - src/core/ext/upb-generated/google/rpc/status.upb.h - - src/core/lib/debug/trace.h - - src/core/lib/experiments/config.h - - src/core/lib/experiments/experiments.h - - src/core/lib/gpr/spinlock.h - - src/core/lib/gprpp/atomic_utils.h - - src/core/lib/gprpp/bitset.h - - src/core/lib/gprpp/cpp_impl_of.h - - src/core/lib/gprpp/manual_constructor.h - - src/core/lib/gprpp/orphanable.h - - src/core/lib/gprpp/ref_counted.h - - src/core/lib/gprpp/ref_counted_ptr.h - - src/core/lib/gprpp/status_helper.h - - src/core/lib/gprpp/time.h - - src/core/lib/iomgr/closure.h - - src/core/lib/iomgr/combiner.h - - src/core/lib/iomgr/error.h - - src/core/lib/iomgr/exec_ctx.h - - src/core/lib/iomgr/executor.h - - src/core/lib/iomgr/iomgr_internal.h - - src/core/lib/promise/activity.h - - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_seq.h - - src/core/lib/promise/detail/promise_factory.h - - src/core/lib/promise/detail/promise_like.h - - src/core/lib/promise/detail/status.h - - src/core/lib/promise/detail/switch.h - - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/interceptor_list.h - - src/core/lib/promise/loop.h - - src/core/lib/promise/map.h - - src/core/lib/promise/poll.h - - src/core/lib/promise/race.h - - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - - src/core/lib/resource_quota/arena.h - - src/core/lib/resource_quota/memory_quota.h - - src/core/lib/resource_quota/periodic_update.h - - src/core/lib/resource_quota/resource_quota.h - - src/core/lib/resource_quota/thread_quota.h - - src/core/lib/resource_quota/trace.h - - src/core/lib/slice/percent_encoding.h - - src/core/lib/slice/slice.h - - src/core/lib/slice/slice_internal.h - - src/core/lib/slice/slice_refcount.h - - src/core/lib/slice/slice_string_helpers.h - - test/core/promise/test_context.h - src: - - src/core/ext/upb-generated/google/protobuf/any.upb.c - - src/core/ext/upb-generated/google/rpc/status.upb.c - - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - - src/core/lib/experiments/config.cc - - src/core/lib/experiments/experiments.cc - - src/core/lib/gprpp/status_helper.cc - - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - - src/core/lib/iomgr/combiner.cc - - src/core/lib/iomgr/error.cc - - src/core/lib/iomgr/exec_ctx.cc - - src/core/lib/iomgr/executor.cc - - src/core/lib/iomgr/iomgr_internal.cc - - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc - - src/core/lib/resource_quota/arena.cc - - src/core/lib/resource_quota/memory_quota.cc - - src/core/lib/resource_quota/periodic_update.cc - - src/core/lib/resource_quota/resource_quota.cc - - src/core/lib/resource_quota/thread_quota.cc - - src/core/lib/resource_quota/trace.cc - - src/core/lib/slice/percent_encoding.cc - - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - - src/core/lib/slice/slice_string_helpers.cc - - test/core/promise/interceptor_list_test.cc - deps: - - absl/container:flat_hash_set - - absl/functional:any_invocable - - absl/functional:function_ref - - absl/hash:hash - - absl/meta:type_traits - - absl/status:statusor - - absl/utility:utility - - gpr - - upb - uses_polling: false - name: interop_client build: test run: false @@ -9098,7 +8973,6 @@ targets: build: test language: c++ headers: - - src/core/lib/debug/trace.h - src/core/lib/gprpp/atomic_utils.h - src/core/lib/gprpp/bitset.h - src/core/lib/gprpp/orphanable.h @@ -9117,12 +8991,9 @@ targets: - src/core/lib/promise/latch.h - src/core/lib/promise/poll.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - test/core/promise/test_wakeup_schedulers.h src: - - src/core/lib/debug/trace.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc - test/core/promise/latch_test.cc deps: - absl/meta:type_traits @@ -9241,8 +9112,6 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h - src/core/lib/promise/loop.h @@ -9252,7 +9121,6 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h - src/core/lib/resource_quota/memory_quota.h @@ -9275,14 +9143,13 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc + - src/core/lib/promise/pipe.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc @@ -9291,7 +9158,6 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/map_pipe_test.cc deps: @@ -9840,7 +9706,6 @@ targets: - src/core/lib/debug/trace.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -9849,7 +9714,6 @@ targets: - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/resource_quota/periodic_update_test.cc deps: @@ -9910,12 +9774,92 @@ targets: build: test language: c++ headers: + - src/core/ext/upb-generated/google/protobuf/any.upb.h + - src/core/ext/upb-generated/google/rpc/status.upb.h + - src/core/lib/debug/trace.h + - src/core/lib/experiments/config.h + - src/core/lib/experiments/experiments.h + - src/core/lib/gpr/spinlock.h + - src/core/lib/gprpp/atomic_utils.h + - src/core/lib/gprpp/bitset.h + - src/core/lib/gprpp/cpp_impl_of.h + - src/core/lib/gprpp/manual_constructor.h + - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/ref_counted.h + - src/core/lib/gprpp/ref_counted_ptr.h + - src/core/lib/gprpp/status_helper.h + - src/core/lib/gprpp/time.h + - src/core/lib/iomgr/closure.h + - src/core/lib/iomgr/combiner.h + - src/core/lib/iomgr/error.h + - src/core/lib/iomgr/exec_ctx.h + - src/core/lib/iomgr/executor.h + - src/core/lib/iomgr/iomgr_internal.h + - src/core/lib/promise/activity.h + - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_join.h + - src/core/lib/promise/detail/basic_seq.h + - src/core/lib/promise/detail/promise_factory.h + - src/core/lib/promise/detail/promise_like.h + - src/core/lib/promise/detail/status.h + - src/core/lib/promise/detail/switch.h + - src/core/lib/promise/exec_ctx_wakeup_scheduler.h + - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h + - src/core/lib/promise/loop.h + - src/core/lib/promise/map.h + - src/core/lib/promise/pipe.h + - src/core/lib/promise/poll.h + - src/core/lib/promise/race.h + - src/core/lib/promise/seq.h + - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/memory_quota.h + - src/core/lib/resource_quota/periodic_update.h + - src/core/lib/resource_quota/resource_quota.h + - src/core/lib/resource_quota/thread_quota.h + - src/core/lib/resource_quota/trace.h + - src/core/lib/slice/percent_encoding.h + - src/core/lib/slice/slice.h + - src/core/lib/slice/slice_internal.h + - src/core/lib/slice/slice_refcount.h + - src/core/lib/slice/slice_string_helpers.h - test/core/promise/test_wakeup_schedulers.h src: + - src/core/ext/upb-generated/google/protobuf/any.upb.c + - src/core/ext/upb-generated/google/rpc/status.upb.c + - src/core/lib/debug/trace.cc + - src/core/lib/event_engine/memory_allocator.cc + - src/core/lib/experiments/config.cc + - src/core/lib/experiments/experiments.cc + - src/core/lib/gprpp/status_helper.cc + - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/combiner.cc + - src/core/lib/iomgr/error.cc + - src/core/lib/iomgr/exec_ctx.cc + - src/core/lib/iomgr/executor.cc + - src/core/lib/iomgr/iomgr_internal.cc + - src/core/lib/promise/activity.cc + - src/core/lib/promise/pipe.cc + - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/memory_quota.cc + - src/core/lib/resource_quota/periodic_update.cc + - src/core/lib/resource_quota/resource_quota.cc + - src/core/lib/resource_quota/thread_quota.cc + - src/core/lib/resource_quota/trace.cc + - src/core/lib/slice/percent_encoding.cc + - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/pipe_test.cc deps: - - grpc + - absl/container:flat_hash_set + - absl/functional:any_invocable + - absl/functional:function_ref + - absl/hash:hash + - absl/meta:type_traits + - absl/status:statusor + - absl/utility:utility + - gpr + - upb uses_polling: false - name: poll_test gtest: true @@ -10988,15 +10932,12 @@ targets: build: test language: c++ headers: - - src/core/lib/debug/trace.h - src/core/lib/slice/slice.h - src/core/lib/slice/slice_internal.h - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h src: - - src/core/lib/debug/trace.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/slice/slice_string_helpers_test.cc deps: @@ -11620,7 +11561,6 @@ targets: build: test language: c++ headers: - - src/core/lib/debug/trace.h - src/core/lib/event_engine/handle_containers.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/iomgr/port.h @@ -11631,14 +11571,12 @@ targets: - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h src: - - src/core/lib/debug/trace.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/resolved_address.cc - src/core/lib/event_engine/slice.cc - src/core/lib/event_engine/slice_buffer.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/event_engine/slice_buffer_test.cc deps: @@ -12246,8 +12184,6 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h - - src/core/lib/promise/if.h - - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h @@ -12256,7 +12192,6 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - - src/core/lib/promise/trace.h - src/core/lib/promise/try_concurrently.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h @@ -12277,21 +12212,19 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/trace.cc + - src/core/lib/promise/pipe.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/try_concurrently_test.cc deps: diff --git a/config.m4 b/config.m4 index 876ef5c9e36..c30182fd782 100644 --- a/config.m4 +++ b/config.m4 @@ -598,7 +598,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ - src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -680,8 +679,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/matchers/matchers.cc \ src/core/lib/promise/activity.cc \ + src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ - src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -759,7 +758,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ - src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ diff --git a/config.w32 b/config.w32 index 616170f76e2..8131eadf4b1 100644 --- a/config.w32 +++ b/config.w32 @@ -564,7 +564,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\iomgr\\buffer_list.cc " + "src\\core\\lib\\iomgr\\call_combiner.cc " + "src\\core\\lib\\iomgr\\cfstream_handle.cc " + - "src\\core\\lib\\iomgr\\closure.cc " + "src\\core\\lib\\iomgr\\combiner.cc " + "src\\core\\lib\\iomgr\\dualstack_socket_posix.cc " + "src\\core\\lib\\iomgr\\endpoint.cc " + @@ -646,8 +645,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\load_balancing\\lb_policy_registry.cc " + "src\\core\\lib\\matchers\\matchers.cc " + "src\\core\\lib\\promise\\activity.cc " + + "src\\core\\lib\\promise\\pipe.cc " + "src\\core\\lib\\promise\\sleep.cc " + - "src\\core\\lib\\promise\\trace.cc " + "src\\core\\lib\\resolver\\resolver.cc " + "src\\core\\lib\\resolver\\resolver_registry.cc " + "src\\core\\lib\\resolver\\server_address.cc " + @@ -725,7 +724,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\slice\\percent_encoding.cc " + "src\\core\\lib\\slice\\slice.cc " + "src\\core\\lib\\slice\\slice_buffer.cc " + - "src\\core\\lib\\slice\\slice_refcount.cc " + "src\\core\\lib\\slice\\slice_string_helpers.cc " + "src\\core\\lib\\surface\\api_trace.cc " + "src\\core\\lib\\surface\\builtins.cc " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index f5f10a285f3..d2d0ba9e798 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -118,7 +118,6 @@ some configuration as environment variables that can be set. - queue_refcount - error_refcount - stream_refcount - - slice_refcount - workqueue_refcount - fd_refcount - cq_refcount diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index cb816108ad5..adf5330fdb1 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -907,27 +907,25 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', - 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/if.h', - 'src/core/lib/promise/interceptor_list.h', + 'src/core/lib/promise/for_each.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/trace.h', - 'src/core/lib/promise/try_join.h', + 'src/core/lib/promise/try_concurrently.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', @@ -1836,27 +1834,25 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', - 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/if.h', - 'src/core/lib/promise/interceptor_list.h', + 'src/core/lib/promise/for_each.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/trace.h', - 'src/core/lib/promise/try_join.h', + 'src/core/lib/promise/try_concurrently.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 6e7f4d423e8..c5b87c8131b 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1321,7 +1321,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/cfstream_handle.cc', 'src/core/lib/iomgr/cfstream_handle.h', - 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/combiner.h', @@ -1477,19 +1476,19 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', - 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/if.h', - 'src/core/lib/promise/interceptor_list.h', + 'src/core/lib/promise/for_each.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/map_pipe.h', + 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', @@ -1497,9 +1496,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.cc', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/trace.cc', - 'src/core/lib/promise/trace.h', - 'src/core/lib/promise/try_join.h', + 'src/core/lib/promise/try_concurrently.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver.h', @@ -1654,7 +1651,6 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_buffer.cc', 'src/core/lib/slice/slice_buffer.h', 'src/core/lib/slice/slice_internal.h', - 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_refcount.h', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/slice/slice_string_helpers.h', @@ -2521,27 +2517,25 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', - 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/if.h', - 'src/core/lib/promise/interceptor_list.h', + 'src/core/lib/promise/for_each.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/trace.h', - 'src/core/lib/promise/try_join.h', + 'src/core/lib/promise/try_concurrently.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', diff --git a/grpc.gemspec b/grpc.gemspec index 5cce81b11c8..4da80f8c12c 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1230,7 +1230,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/call_combiner.h ) s.files += %w( src/core/lib/iomgr/cfstream_handle.cc ) s.files += %w( src/core/lib/iomgr/cfstream_handle.h ) - s.files += %w( src/core/lib/iomgr/closure.cc ) s.files += %w( src/core/lib/iomgr/closure.h ) s.files += %w( src/core/lib/iomgr/combiner.cc ) s.files += %w( src/core/lib/iomgr/combiner.h ) @@ -1386,19 +1385,19 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/activity.h ) s.files += %w( src/core/lib/promise/arena_promise.h ) s.files += %w( src/core/lib/promise/context.h ) - s.files += %w( src/core/lib/promise/detail/basic_join.h ) s.files += %w( src/core/lib/promise/detail/basic_seq.h ) s.files += %w( src/core/lib/promise/detail/promise_factory.h ) s.files += %w( src/core/lib/promise/detail/promise_like.h ) s.files += %w( src/core/lib/promise/detail/status.h ) s.files += %w( src/core/lib/promise/detail/switch.h ) s.files += %w( src/core/lib/promise/exec_ctx_wakeup_scheduler.h ) - s.files += %w( src/core/lib/promise/if.h ) - s.files += %w( src/core/lib/promise/interceptor_list.h ) + s.files += %w( src/core/lib/promise/for_each.h ) s.files += %w( src/core/lib/promise/intra_activity_waiter.h ) s.files += %w( src/core/lib/promise/latch.h ) s.files += %w( src/core/lib/promise/loop.h ) s.files += %w( src/core/lib/promise/map.h ) + s.files += %w( src/core/lib/promise/map_pipe.h ) + s.files += %w( src/core/lib/promise/pipe.cc ) s.files += %w( src/core/lib/promise/pipe.h ) s.files += %w( src/core/lib/promise/poll.h ) s.files += %w( src/core/lib/promise/promise.h ) @@ -1406,9 +1405,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/seq.h ) s.files += %w( src/core/lib/promise/sleep.cc ) s.files += %w( src/core/lib/promise/sleep.h ) - s.files += %w( src/core/lib/promise/trace.cc ) - s.files += %w( src/core/lib/promise/trace.h ) - s.files += %w( src/core/lib/promise/try_join.h ) + s.files += %w( src/core/lib/promise/try_concurrently.h ) s.files += %w( src/core/lib/promise/try_seq.h ) s.files += %w( src/core/lib/resolver/resolver.cc ) s.files += %w( src/core/lib/resolver/resolver.h ) @@ -1563,7 +1560,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/slice/slice_buffer.cc ) s.files += %w( src/core/lib/slice/slice_buffer.h ) s.files += %w( src/core/lib/slice/slice_internal.h ) - s.files += %w( src/core/lib/slice/slice_refcount.cc ) s.files += %w( src/core/lib/slice/slice_refcount.h ) s.files += %w( src/core/lib/slice/slice_string_helpers.cc ) s.files += %w( src/core/lib/slice/slice_string_helpers.h ) diff --git a/grpc.gyp b/grpc.gyp index 42a1ac2eb33..aac09f741a8 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -886,7 +886,6 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -968,8 +967,8 @@ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', + 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', - 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1047,7 +1046,6 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', @@ -1366,7 +1364,6 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -1446,8 +1443,8 @@ 'src/core/lib/load_balancing/lb_policy.cc', 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/promise/activity.cc', + 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', - 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1494,7 +1491,6 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', @@ -1869,7 +1865,6 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -1949,7 +1944,7 @@ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/trace.cc', + 'src/core/lib/promise/pipe.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1996,7 +1991,6 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', diff --git a/package.xml b/package.xml index 8ffbd092a4f..3f97548e386 100644 --- a/package.xml +++ b/package.xml @@ -1212,7 +1212,6 @@ - @@ -1368,19 +1367,19 @@ - - - + + + @@ -1388,9 +1387,7 @@ - - - + @@ -1545,7 +1542,6 @@ - diff --git a/src/core/BUILD b/src/core/BUILD index 0a1706f75ab..6f26534c0e0 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -384,7 +384,6 @@ grpc_cc_library( hdrs = ["lib/promise/try_concurrently.h"], external_deps = [ "absl/status", - "absl/strings", "absl/types:variant", ], language = "c++", @@ -392,7 +391,6 @@ grpc_cc_library( "lib/promise/map_pipe.h", ], deps = [ - "activity", "construct_destruct", "for_each", "map", @@ -401,10 +399,8 @@ grpc_cc_library( "promise_factory", "promise_like", "promise_status", - "promise_trace", "try_seq", "//:gpr", - "//:grpc_trace", ], ) @@ -419,13 +415,9 @@ grpc_cc_library( "for_each", "map", "pipe", - "poll", "promise_factory", - "promise_trace", "try_seq", - "//:gpr", "//:gpr_platform", - "//:grpc_trace", ], ) @@ -536,7 +528,6 @@ grpc_cc_library( language = "c++", public_hdrs = ["lib/promise/if.h"], deps = [ - "construct_destruct", "poll", "promise_factory", "promise_like", @@ -792,18 +783,14 @@ grpc_cc_library( grpc_cc_library( name = "latch", - external_deps = ["absl/strings"], language = "c++", public_hdrs = [ "lib/promise/latch.h", ], deps = [ - "activity", "intra_activity_waiter", "poll", - "promise_trace", "//:gpr", - "//:grpc_trace", ], ) @@ -827,38 +814,15 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "interceptor_list", - hdrs = [ - "lib/promise/interceptor_list.h", - ], - external_deps = [ - "absl/strings", - "absl/strings:str_format", - "absl/types:optional", - "absl/types:variant", - ], - deps = [ - "arena", - "construct_destruct", - "context", - "poll", - "promise_factory", - "promise_trace", - "//:debug_location", - "//:gpr", - "//:grpc_trace", - ], -) - grpc_cc_library( name = "pipe", + srcs = [ + "lib/promise/pipe.cc", + ], hdrs = [ "lib/promise/pipe.h", ], external_deps = [ - "absl/base:core_headers", - "absl/memory", "absl/strings", "absl/types:optional", "absl/types:variant", @@ -868,32 +832,10 @@ grpc_cc_library( "activity", "arena", "context", - "if", - "interceptor_list", "intra_activity_waiter", - "map", "poll", - "promise_trace", - "seq", - "//:debug_location", "//:gpr", "//:grpc_trace", - "//:ref_counted_ptr", - ], -) - -grpc_cc_library( - name = "promise_trace", - srcs = [ - "lib/promise/trace.cc", - ], - hdrs = [ - "lib/promise/trace.h", - ], - language = "c++", - deps = [ - "//:gpr_platform", - "//:grpc_trace", ], ) @@ -918,20 +860,16 @@ grpc_cc_library( name = "for_each", external_deps = [ "absl/status", - "absl/strings", "absl/types:variant", ], language = "c++", public_hdrs = ["lib/promise/for_each.h"], deps = [ - "activity", "construct_destruct", "poll", "promise_factory", - "promise_trace", "//:gpr", "//:gpr_platform", - "//:grpc_trace", ], ) @@ -1189,9 +1127,6 @@ grpc_cc_library( grpc_cc_library( name = "slice_refcount", - srcs = [ - "lib/slice/slice_refcount.cc", - ], hdrs = [ "lib/slice/slice_refcount.h", ], @@ -1199,10 +1134,8 @@ grpc_cc_library( "//:include/grpc/slice.h", ], deps = [ - "//:debug_location", "//:event_engine_base_hdrs", "//:gpr", - "//:grpc_trace", ], ) @@ -1226,7 +1159,6 @@ grpc_cc_library( deps = [ "slice_cast", "slice_refcount", - "//:debug_location", "//:event_engine_base_hdrs", "//:gpr", ], @@ -1241,7 +1173,6 @@ grpc_cc_library( "lib/slice/slice_buffer.h", "//:include/grpc/slice_buffer.h", ], - external_deps = ["absl/memory"], deps = [ "slice", "slice_refcount", @@ -1277,13 +1208,9 @@ grpc_cc_library( grpc_cc_library( name = "closure", - srcs = [ - "lib/iomgr/closure.cc", - ], hdrs = [ "lib/iomgr/closure.h", ], - external_deps = ["absl/strings:str_format"], visibility = ["@grpc:alt_grpc_base_legacy"], deps = [ "error", @@ -3591,6 +3518,7 @@ grpc_cc_library( "ext/filters/stateful_session/stateful_session_service_config_parser.h", ], external_deps = [ + "absl/status", "absl/status:statusor", "absl/strings", "absl/types:optional", @@ -3599,6 +3527,7 @@ grpc_cc_library( deps = [ "arena", "arena_promise", + "basic_seq", "channel_args", "channel_fwd", "context", @@ -3606,12 +3535,12 @@ grpc_cc_library( "json", "json_args", "json_object_loader", - "map", - "pipe", - "poll", + "latch", + "seq", "service_config_parser", "slice", "time", + "try_concurrently", "unique_type_name", "validation_errors", "//:config", @@ -3649,29 +3578,26 @@ grpc_cc_library( ], language = "c++", deps = [ - "arena", "arena_promise", "channel_args", "channel_fwd", "channel_init", "channel_stack_type", "closure", - "context", "error", "gpr_atm", "grpc_sockaddr", "json", "json_args", "json_object_loader", + "latch", "lb_policy", "lb_policy_factory", "lb_policy_registry", - "map", - "pipe", - "poll", "pollset_set", "ref_counted", "resolved_address", + "seq", "slice", "slice_refcount", "status_helper", @@ -3695,6 +3621,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:orphanable", + "//:promise", "//:protobuf_duration_upb", "//:protobuf_timestamp_upb", "//:ref_counted_ptr", diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc index d9e979b7c43..45e875ba2a6 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc @@ -31,11 +31,9 @@ #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/seq.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -63,21 +61,16 @@ ArenaPromise ClientLoadReportingFilter::MakeCallPromise( client_stats.reset(*client_stats_md); } - auto* saw_initial_metadata = GetContext()->New(false); - call_args.server_initial_metadata->InterceptAndMap( - [saw_initial_metadata](ServerMetadataHandle md) { - *saw_initial_metadata = true; - return md; - }); + auto* server_initial_metadata = call_args.server_initial_metadata; - return Map(next_promise_factory(std::move(call_args)), - [saw_initial_metadata, client_stats = std::move(client_stats)]( + return Seq(next_promise_factory(std::move(call_args)), + [server_initial_metadata, client_stats = std::move(client_stats)]( ServerMetadataHandle trailing_metadata) { if (client_stats != nullptr) { client_stats->AddCallFinished( trailing_metadata->get(GrpcStreamNetworkState()) == GrpcStreamNetworkState::kNotSentOnWire, - *saw_initial_metadata); + NowOrNever(server_initial_metadata->Wait()).has_value()); } return trailing_metadata; }); diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index 58f9a709024..7e299e3f2c7 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -40,11 +40,10 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/race.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/try_concurrently.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/transport/status_conversion.h" @@ -119,27 +118,24 @@ ArenaPromise HttpClientFilter::MakeCallPromise( md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); md->Set(UserAgentMetadata(), user_agent_.Ref()); - auto* initial_metadata_err = - GetContext()->New>(); + auto* read_latch = GetContext()->New>(); + auto* write_latch = + std::exchange(call_args.server_initial_metadata, read_latch); - call_args.server_initial_metadata->InterceptAndMap( - [initial_metadata_err]( - ServerMetadataHandle md) -> absl::optional { - auto r = CheckServerMetadata(md.get()); - if (!r.ok()) { - initial_metadata_err->Set(ServerMetadataFromStatus(r)); - return absl::nullopt; - } - return std::move(md); - }); - - return Race(Map(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - auto r = CheckServerMetadata(md.get()); - if (!r.ok()) return ServerMetadataFromStatus(r); - return md; - }), - initial_metadata_err->Wait()); + return TryConcurrently( + Seq(next_promise_factory(std::move(call_args)), + [](ServerMetadataHandle md) -> ServerMetadataHandle { + auto r = CheckServerMetadata(md.get()); + if (!r.ok()) return ServerMetadataFromStatus(r); + return md; + })) + .NecessaryPull(Seq(read_latch->Wait(), + [write_latch](ServerMetadata** md) -> absl::Status { + auto r = *md == nullptr ? absl::OkStatus() + : CheckServerMetadata(*md); + write_latch->Set(*md); + return r; + })); } HttpClientFilter::HttpClientFilter(HttpSchemeMetadata::ValueType scheme, diff --git a/src/core/ext/filters/http/message_compress/compression_filter.cc b/src/core/ext/filters/http/message_compress/compression_filter.cc index 5a84aceefe5..69a7f0a25f7 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.cc +++ b/src/core/ext/filters/http/message_compress/compression_filter.cc @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include "absl/meta/type_traits.h" @@ -43,16 +43,17 @@ #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/map_pipe.h" #include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/race.h" +#include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/try_concurrently.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/call_trace.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -157,19 +158,20 @@ MessageHandle CompressionFilter::CompressMessage( } absl::StatusOr CompressionFilter::DecompressMessage( - MessageHandle message, DecompressArgs args) const { + MessageHandle message, grpc_compression_algorithm algorithm, + absl::optional max_recv_message_length) const { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { gpr_log(GPR_ERROR, "DecompressMessage: len=%" PRIdPTR " max=%d alg=%d", - message->payload()->Length(), - args.max_recv_message_length.value_or(-1), args.algorithm); + message->payload()->Length(), max_recv_message_length.value_or(-1), + algorithm); } // Check max message length. - if (args.max_recv_message_length.has_value() && + if (max_recv_message_length.has_value() && message->payload()->Length() > - static_cast(*args.max_recv_message_length)) { + static_cast(*max_recv_message_length)) { return absl::ResourceExhaustedError(absl::StrFormat( "Received message larger than max (%u vs. %d)", - message->payload()->Length(), *args.max_recv_message_length)); + message->payload()->Length(), *max_recv_message_length)); } // Check if decompression is enabled (if not, we can just pass the message // up). @@ -179,11 +181,11 @@ absl::StatusOr CompressionFilter::DecompressMessage( } // Try to decompress the payload. SliceBuffer decompressed_slices; - if (grpc_msg_decompress(args.algorithm, message->payload()->c_slice_buffer(), + if (grpc_msg_decompress(algorithm, message->payload()->c_slice_buffer(), decompressed_slices.c_slice_buffer()) == 0) { return absl::InternalError( absl::StrCat("Unexpected error decompressing data for algorithm ", - CompressionAlgorithmAsString(args.algorithm))); + CompressionAlgorithmAsString(algorithm))); } // Swap the decompressed slices into the message. message->payload()->Swap(&decompressed_slices); @@ -192,114 +194,123 @@ absl::StatusOr CompressionFilter::DecompressMessage( return std::move(message); } -grpc_compression_algorithm CompressionFilter::HandleOutgoingMetadata( - grpc_metadata_batch& outgoing_metadata) { - const auto algorithm = outgoing_metadata.Take(GrpcInternalEncodingRequest()) - .value_or(default_compression_algorithm()); - // Convey supported compression algorithms. - outgoing_metadata.Set(GrpcAcceptEncodingMetadata(), - enabled_compression_algorithms()); - if (algorithm != GRPC_COMPRESS_NONE) { - outgoing_metadata.Set(GrpcEncodingMetadata(), algorithm); - } - return algorithm; -} +class CompressionFilter::DecompressLoop { + public: + explicit DecompressLoop(CompressionFilter* filter, CallArgs& call_args) + : filter_(filter), + mapper_(PipeMapper::Intercept( + *call_args.incoming_messages)) {} -CompressionFilter::DecompressArgs CompressionFilter::HandleIncomingMetadata( - const grpc_metadata_batch& incoming_metadata) { - // Configure max receive size. - auto max_recv_message_length = max_recv_size_; - const MessageSizeParsedConfig* limits = - MessageSizeParsedConfig::GetFromCallContext( - GetContext(), - message_size_service_config_parser_index_); - if (limits != nullptr && limits->max_recv_size().has_value() && - (!max_recv_message_length.has_value() || - *limits->max_recv_size() < *max_recv_message_length)) { - max_recv_message_length = *limits->max_recv_size(); + // Once we have a compression algorithm we can construct the decompression + // loop. + // Returns a promise that resolves to MessageHandle. + auto TakeAndRun(grpc_compression_algorithm algorithm) { + // Configure max receive size. + auto max_recv_message_length = filter_->max_recv_size_; + const MessageSizeParsedConfig* limits = + MessageSizeParsedConfig::GetFromCallContext( + GetContext(), + filter_->message_size_service_config_parser_index_); + if (limits != nullptr && limits->max_recv_size().has_value() && + (!max_recv_message_length.has_value() || + *limits->max_recv_size() < *max_recv_message_length)) { + max_recv_message_length = *limits->max_recv_size(); + } + // Interject decompression into the message loop. + return mapper_.TakeAndRun([algorithm, max_recv_message_length, + filter = filter_](MessageHandle message) { + return filter->DecompressMessage(std::move(message), algorithm, + max_recv_message_length); + }); } - return DecompressArgs{incoming_metadata.get(GrpcEncodingMetadata()) - .value_or(GRPC_COMPRESS_NONE), - max_recv_message_length}; -} + + private: + CompressionFilter* filter_; + PipeMapper mapper_; +}; + +class CompressionFilter::CompressLoop { + public: + explicit CompressLoop(CompressionFilter* filter, CallArgs& call_args) + : filter_(filter), + mapper_(PipeMapper::Intercept( + *call_args.outgoing_messages)) {} + + // Once we're ready to send initial metadata we can construct the compression + // loop. + // Returns a promise that resolves to MessageHandle. + auto TakeAndRun(grpc_metadata_batch& outgoing_metadata) { + const auto algorithm = + outgoing_metadata.Take(GrpcInternalEncodingRequest()) + .value_or(filter_->default_compression_algorithm()); + // Convey supported compression algorithms. + outgoing_metadata.Set(GrpcAcceptEncodingMetadata(), + filter_->enabled_compression_algorithms()); + if (algorithm != GRPC_COMPRESS_NONE) { + outgoing_metadata.Set(GrpcEncodingMetadata(), algorithm); + } + // Interject compression into the message loop. + return mapper_.TakeAndRun([filter = filter_, algorithm](MessageHandle m) { + return filter->CompressMessage(std::move(m), algorithm); + }); + } + + private: + CompressionFilter* filter_; + PipeMapper mapper_; +}; ArenaPromise ClientCompressionFilter::MakeCallPromise( CallArgs call_args, NextPromiseFactory next_promise_factory) { - auto compression_algorithm = - HandleOutgoingMetadata(*call_args.client_initial_metadata); - call_args.client_to_server_messages->InterceptAndMap( - [compression_algorithm, - this](MessageHandle message) -> absl::optional { - return CompressMessage(std::move(message), compression_algorithm); - }); - auto* decompress_args = GetContext()->New( - DecompressArgs{GRPC_COMPRESS_NONE, absl::nullopt}); - auto* decompress_err = - GetContext()->New>(); - call_args.server_initial_metadata->InterceptAndMap( - [decompress_args, this](ServerMetadataHandle server_initial_metadata) - -> absl::optional { - if (server_initial_metadata == nullptr) return absl::nullopt; - *decompress_args = HandleIncomingMetadata(*server_initial_metadata); - return std::move(server_initial_metadata); - }); - call_args.server_to_client_messages->InterceptAndMap( - [decompress_err, decompress_args, - this](MessageHandle message) -> absl::optional { - auto r = DecompressMessage(std::move(message), *decompress_args); - if (!r.ok()) { - decompress_err->Set(ServerMetadataFromStatus(r.status())); - return absl::nullopt; - } - return std::move(*r); - }); - // Run the next filter, and race it with getting an error from decompression. - return Race(next_promise_factory(std::move(call_args)), - decompress_err->Wait()); + auto compress_loop = CompressLoop(this, call_args) + .TakeAndRun(*call_args.client_initial_metadata); + DecompressLoop decompress_loop(this, call_args); + auto* server_initial_metadata = call_args.server_initial_metadata; + // Concurrently: + // - call the next filter + // - wait for initial metadata from the server and then commence decompression + // - compress outgoing messages + return TryConcurrently(next_promise_factory(std::move(call_args))) + .NecessaryPull(Seq(server_initial_metadata->Wait(), + [decompress_loop = std::move(decompress_loop)]( + ServerMetadata** server_initial_metadata) mutable + -> ArenaPromise { + if (*server_initial_metadata == nullptr) { + return ImmediateOkStatus(); + } + return decompress_loop.TakeAndRun( + (*server_initial_metadata) + ->get(GrpcEncodingMetadata()) + .value_or(GRPC_COMPRESS_NONE)); + })) + .Push(std::move(compress_loop)); } ArenaPromise ServerCompressionFilter::MakeCallPromise( CallArgs call_args, NextPromiseFactory next_promise_factory) { - auto decompress_args = - HandleIncomingMetadata(*call_args.client_initial_metadata); - auto* decompress_err = - GetContext()->New>(); - call_args.client_to_server_messages->InterceptAndMap( - [decompress_err, decompress_args, - this](MessageHandle message) -> absl::optional { - auto r = DecompressMessage(std::move(message), decompress_args); - gpr_log(GPR_DEBUG, "DecompressMessage returned %s", - r.status().ToString().c_str()); - if (!r.ok()) { - decompress_err->Set(ServerMetadataFromStatus(r.status())); - return absl::nullopt; - } - return std::move(*r); - }); - auto* compression_algorithm = - GetContext()->New(); - call_args.server_initial_metadata->InterceptAndMap( - [this, compression_algorithm](ServerMetadataHandle md) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[compression] Write metadata", - Activity::current()->DebugTag().c_str()); - } - // Find the compression algorithm. - *compression_algorithm = HandleOutgoingMetadata(*md); - return md; - }); - call_args.server_to_client_messages->InterceptAndMap( - [compression_algorithm, - this](MessageHandle message) -> absl::optional { - return CompressMessage(std::move(message), *compression_algorithm); - }); + CompressLoop compress_loop(this, call_args); + auto decompress_loop = DecompressLoop(this, call_args) + .TakeAndRun(call_args.client_initial_metadata + ->get(GrpcEncodingMetadata()) + .value_or(GRPC_COMPRESS_NONE)); + auto* read_latch = GetContext()->New>(); + auto* write_latch = + std::exchange(call_args.server_initial_metadata, read_latch); // Concurrently: // - call the next filter // - decompress incoming messages // - wait for initial metadata to be sent, and then commence compression of // outgoing messages - return Race(next_promise_factory(std::move(call_args)), - decompress_err->Wait()); + return TryConcurrently(next_promise_factory(std::move(call_args))) + .Pull(std::move(decompress_loop)) + .Push(Seq(read_latch->Wait(), + [write_latch, compress_loop = std::move(compress_loop)]( + ServerMetadata** md) mutable { + // Find the compression algorithm. + auto loop = compress_loop.TakeAndRun(**md); + write_latch->Set(*md); + return loop; + })); } } // namespace grpc_core diff --git a/src/core/ext/filters/http/message_compress/compression_filter.h b/src/core/ext/filters/http/message_compress/compression_filter.h index caf49c03336..8a17bebe0dc 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.h +++ b/src/core/ext/filters/http/message_compress/compression_filter.h @@ -34,7 +34,6 @@ #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/promise/arena_promise.h" -#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { @@ -63,13 +62,11 @@ namespace grpc_core { class CompressionFilter : public ChannelFilter { protected: - struct DecompressArgs { - grpc_compression_algorithm algorithm; - absl::optional max_recv_message_length; - }; - explicit CompressionFilter(const ChannelArgs& args); + class CompressLoop; + class DecompressLoop; + grpc_compression_algorithm default_compression_algorithm() const { return default_compression_algorithm_; } @@ -78,19 +75,15 @@ class CompressionFilter : public ChannelFilter { return enabled_compression_algorithms_; } - grpc_compression_algorithm HandleOutgoingMetadata( - grpc_metadata_batch& outgoing_metadata); - DecompressArgs HandleIncomingMetadata( - const grpc_metadata_batch& incoming_metadata); - + private: // Compress one message synchronously. MessageHandle CompressMessage(MessageHandle message, grpc_compression_algorithm algorithm) const; // Decompress one message synchronously. - absl::StatusOr DecompressMessage(MessageHandle message, - DecompressArgs args) const; + absl::StatusOr DecompressMessage( + MessageHandle message, grpc_compression_algorithm algorithm, + absl::optional max_recv_message_length) const; - private: // Max receive message length, if set. absl::optional max_recv_size_; size_t message_size_service_config_parser_index_; diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index 8ec0e54df0b..23062d5c007 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -22,7 +22,6 @@ #include #include -#include #include #include "absl/base/attributes.h" @@ -31,19 +30,18 @@ #include "absl/types/optional.h" #include -#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/promise/activity.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/try_concurrently.h" +#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/surface/call_trace.h" #include "src/core/lib/transport/metadata_batch.h" namespace grpc_core { @@ -131,23 +129,24 @@ ArenaPromise HttpServerFilter::MakeCallPromise( md->Remove(UserAgentMetadata()); } - call_args.server_initial_metadata->InterceptAndMap( - [](ServerMetadataHandle md) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[http-server] Write metadata", - Activity::current()->DebugTag().c_str()); - } - FilterOutgoingMetadata(md.get()); - md->Set(HttpStatusMetadata(), 200); - md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); - return md; - }); + auto* read_latch = GetContext()->New>(); + auto* write_latch = + std::exchange(call_args.server_initial_metadata, read_latch); - return Map(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - FilterOutgoingMetadata(md.get()); - return md; - }); + return TryConcurrently( + Seq(next_promise_factory(std::move(call_args)), + [](ServerMetadataHandle md) -> ServerMetadataHandle { + FilterOutgoingMetadata(md.get()); + return md; + })) + .Push(Seq(read_latch->Wait(), [write_latch](ServerMetadata** md) { + FilterOutgoingMetadata(*md); + (*md)->Set(HttpStatusMetadata(), 200); + (*md)->Set(ContentTypeMetadata(), + ContentTypeMetadata::kApplicationGrpc); + write_latch->Set(*md); + return absl::OkStatus(); + })); } absl::StatusOr HttpServerFilter::Create( diff --git a/src/core/ext/filters/stateful_session/stateful_session_filter.cc b/src/core/ext/filters/stateful_session/stateful_session_filter.cc index e64975fd9b4..cefed052921 100644 --- a/src/core/ext/filters/stateful_session/stateful_session_filter.cc +++ b/src/core/ext/filters/stateful_session/stateful_session_filter.cc @@ -27,6 +27,7 @@ #include #include +#include "absl/status/status.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" @@ -46,9 +47,10 @@ #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/try_concurrently.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/slice/slice.h" @@ -163,23 +165,32 @@ ArenaPromise StatefulSessionFilter::MakeCallPromise( *cookie_value); } // Intercept server initial metadata. - call_args.server_initial_metadata->InterceptAndMap( - [cookie_config, cookie_value](ServerMetadataHandle md) { - // Add cookie to server initial metadata if needed. - MaybeUpdateServerInitialMetadata(cookie_config, cookie_value, md.get()); - return md; - }); - return Map(next_promise_factory(std::move(call_args)), - [cookie_config, cookie_value](ServerMetadataHandle md) { - // If we got a Trailers-Only response, then add the - // cookie to the trailing metadata instead of the - // initial metadata. - if (md->get(GrpcTrailersOnly()).value_or(false)) { - MaybeUpdateServerInitialMetadata(cookie_config, cookie_value, - md.get()); - } - return md; - }); + auto* read_latch = GetContext()->New>(); + auto* write_latch = + std::exchange(call_args.server_initial_metadata, read_latch); + return TryConcurrently( + Seq(next_promise_factory(std::move(call_args)), + [cookie_config, cookie_value](ServerMetadataHandle md) { + // If we got a Trailers-Only response, then add the + // cookie to the trailing metadata instead of the + // initial metadata. + if (md->get(GrpcTrailersOnly()).value_or(false)) { + MaybeUpdateServerInitialMetadata(cookie_config, + cookie_value, md.get()); + } + return md; + })) + .NecessaryPull(Seq(read_latch->Wait(), + [write_latch, cookie_config, + cookie_value](ServerMetadata** md) -> absl::Status { + if (*md != nullptr) { + // Add cookie to server initial metadata if needed. + MaybeUpdateServerInitialMetadata( + cookie_config, cookie_value, *md); + } + write_latch->Set(*md); + return absl::OkStatus(); + })); } absl::optional diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 2c54d917d25..ed0b6276575 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -668,10 +668,6 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t, flow_control(&t->flow_control) { if (server_data) { id = static_cast(reinterpret_cast(server_data)); - if (grpc_http_trace.enabled()) { - gpr_log(GPR_DEBUG, "HTTP:%p/%p creating accept stream %d [from %p]", t, - this, id, server_data); - } *t->accepting_stream = this; grpc_chttp2_stream_map_add(&t->stream_map, id, this); post_destructive_reclaimer(t); @@ -1833,12 +1829,6 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, // Lambda is immediately invoked as a big scoped section that can be // exited out of at any point by returning. [&]() { - if (grpc_http_trace.enabled()) { - gpr_log(GPR_DEBUG, - "maybe_complete_recv_message %p final_metadata_requested=%d " - "seen_error=%d", - s, s->final_metadata_requested, s->seen_error); - } if (s->final_metadata_requested && s->seen_error) { grpc_slice_buffer_reset_and_unref(&s->frame_storage); s->recv_message->reset(); @@ -1849,12 +1839,6 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, int64_t min_progress_size; auto r = grpc_deframe_unprocessed_incoming_frames( s, &min_progress_size, &**s->recv_message, s->recv_message_flags); - if (grpc_http_trace.enabled()) { - gpr_log(GPR_DEBUG, "Deframe data frame: %s", - grpc_core::PollToString(r, [](absl::Status r) { - return r.ToString(); - }).c_str()); - } if (absl::holds_alternative(r)) { if (s->read_closed) { grpc_slice_buffer_reset_and_unref(&s->frame_storage); @@ -1905,14 +1889,6 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { grpc_chttp2_maybe_complete_recv_message(t, s); - if (grpc_http_trace.enabled()) { - gpr_log(GPR_DEBUG, - "maybe_complete_recv_trailing_metadata cli=%d s=%p closure=%p " - "read_closed=%d " - "write_closed=%d %" PRIdPTR, - t->is_client, s, s->recv_trailing_metadata_finished, s->read_closed, - s->write_closed, s->frame_storage.length); - } if (s->recv_trailing_metadata_finished != nullptr && s->read_closed && s->write_closed) { if (s->seen_error || !t->is_client) { @@ -2072,14 +2048,6 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t, void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t, grpc_chttp2_stream* s, int close_reads, int close_writes, grpc_error_handle error) { - if (grpc_http_trace.enabled()) { - gpr_log( - GPR_DEBUG, "MARK_STREAM_CLOSED: t=%p s=%p(id=%d) %s [%s]", t, s, s->id, - (close_reads && close_writes) - ? "read+write" - : (close_reads ? "read" : (close_writes ? "write" : "nothing??")), - error.ToString().c_str()); - } if (s->read_closed && s->write_closed) { // already closed, but we should still fake the status if needed. grpc_error_handle overall_error = removal_error(error, s, "Stream removed"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 58b1e02effc..0724e70cfe8 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -389,16 +389,12 @@ struct grpc_chttp2_transport uint32_t incoming_frame_size = 0; uint32_t incoming_stream_id = 0; - grpc_chttp2_stream* incoming_stream = nullptr; // active parser - struct Parser { - const char* name; - grpc_error_handle (*parser)(void* parser_user_data, - grpc_chttp2_transport* t, grpc_chttp2_stream* s, - const grpc_slice& slice, int is_last); - void* user_data = nullptr; - }; - Parser parser; + void* parser_data = nullptr; + grpc_chttp2_stream* incoming_stream = nullptr; + grpc_error_handle (*parser)(void* parser_user_data, grpc_chttp2_transport* t, + grpc_chttp2_stream* s, const grpc_slice& slice, + int is_last); grpc_chttp2_write_cb* write_cb_pool = nullptr; diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index b4d719e5623..2913543e171 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -18,7 +18,7 @@ #include -#include +#include #include #include @@ -28,7 +28,6 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" #include #include @@ -128,65 +127,6 @@ uint32_t grpc_chttp2_min_read_progress_size(grpc_chttp2_transport* t) { GPR_UNREACHABLE_CODE(return 1); } -namespace { -struct KnownFlag { - uint8_t flag; - absl::string_view name; -}; - -std::string MakeFrameTypeString(absl::string_view frame_type, uint8_t flags, - std::initializer_list known_flags) { - std::string result(frame_type); - for (const KnownFlag& known_flag : known_flags) { - if (flags & known_flag.flag) { - absl::StrAppend(&result, ":", known_flag.name); - flags &= ~known_flag.flag; - } - } - if (flags != 0) { - absl::StrAppend(&result, ":UNKNOWN_FLAGS=0x", - absl::Hex(flags, absl::kZeroPad2)); - } - return result; -} - -std::string FrameTypeString(uint8_t frame_type, uint8_t flags) { - switch (frame_type) { - case GRPC_CHTTP2_FRAME_DATA: - return MakeFrameTypeString( - "DATA", flags, {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}}); - case GRPC_CHTTP2_FRAME_HEADER: - return MakeFrameTypeString( - "HEADERS", flags, - {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}, - {GRPC_CHTTP2_DATA_FLAG_END_HEADERS, "END_HEADERS"}, - {GRPC_CHTTP2_FLAG_HAS_PRIORITY, "PRIORITY"}}); - case GRPC_CHTTP2_FRAME_CONTINUATION: - return MakeFrameTypeString( - "HEADERS", flags, - {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}, - {GRPC_CHTTP2_DATA_FLAG_END_HEADERS, "END_HEADERS"}, - {GRPC_CHTTP2_FLAG_HAS_PRIORITY, "PRIORITY"}}); - case GRPC_CHTTP2_FRAME_RST_STREAM: - return MakeFrameTypeString("RST_STREAM", flags, {}); - case GRPC_CHTTP2_FRAME_SETTINGS: - return MakeFrameTypeString("SETTINGS", flags, - {{GRPC_CHTTP2_FLAG_ACK, "ACK"}}); - case GRPC_CHTTP2_FRAME_PING: - return MakeFrameTypeString("PING", flags, - {{GRPC_CHTTP2_FLAG_ACK, "ACK"}}); - case GRPC_CHTTP2_FRAME_GOAWAY: - return MakeFrameTypeString("GOAWAY", flags, {}); - case GRPC_CHTTP2_FRAME_WINDOW_UPDATE: - return MakeFrameTypeString("WINDOW_UPDATE", flags, {}); - default: - return MakeFrameTypeString( - absl::StrCat("UNKNOWN_FRAME_TYPE_", static_cast(frame_type)), - flags, {}); - } -} -} // namespace - grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, const grpc_slice& slice) { const uint8_t* beg = GRPC_SLICE_START_PTR(slice); @@ -310,12 +250,6 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, case GRPC_DTS_FH_8: GPR_DEBUG_ASSERT(cur < end); t->incoming_stream_id |= (static_cast(*cur)); - if (grpc_http_trace.enabled()) { - gpr_log(GPR_INFO, "INCOMING[%p]: %s len:%d id:0x%08x", t, - FrameTypeString(t->incoming_frame_type, t->incoming_frame_flags) - .c_str(), - t->incoming_frame_size, t->incoming_stream_id); - } t->deframe_state = GRPC_DTS_FRAME; err = init_frame_parser(t); if (!err.ok()) { @@ -470,8 +404,8 @@ static HPackParser::LogInfo hpack_parser_log_info( static grpc_error_handle init_header_skip_frame_parser( grpc_chttp2_transport* t, HPackParser::Priority priority_type) { bool is_eoh = t->expect_continuation_stream_id != 0; - t->parser = grpc_chttp2_transport::Parser{ - "header", grpc_chttp2_header_parser_parse, &t->hpack_parser}; + t->parser = grpc_chttp2_header_parser_parse; + t->parser_data = &t->hpack_parser; t->hpack_parser.BeginFrame( nullptr, t->settings[GRPC_ACKED_SETTINGS] @@ -483,17 +417,15 @@ static grpc_error_handle init_header_skip_frame_parser( static grpc_error_handle init_non_header_skip_frame_parser( grpc_chttp2_transport* t) { - t->parser = - grpc_chttp2_transport::Parser{"skip_parser", skip_parser, nullptr}; + t->parser = skip_parser; return absl::OkStatus(); } void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t) { - if (t->parser.parser == grpc_chttp2_header_parser_parse) { + if (t->parser == grpc_chttp2_header_parser_parse) { t->hpack_parser.StopBufferingFrame(); } else { - t->parser = - grpc_chttp2_transport::Parser{"skip_parser", skip_parser, nullptr}; + t->parser = skip_parser; } } @@ -540,8 +472,9 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { error_handler: if (status.ok()) { t->incoming_stream = s; - t->parser = grpc_chttp2_transport::Parser{ - "data", grpc_chttp2_data_parser_parse, nullptr}; + // t->parser = grpc_chttp2_data_parser_parse; + t->parser = grpc_chttp2_data_parser_parse; + t->parser_data = nullptr; t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast(); return absl::OkStatus(); } else if (s != nullptr) { @@ -651,8 +584,8 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t, t->incoming_stream = nullptr; return init_header_skip_frame_parser(t, priority_type); } - t->parser = grpc_chttp2_transport::Parser{ - "header", grpc_chttp2_header_parser_parse, &t->hpack_parser}; + t->parser = grpc_chttp2_header_parser_parse; + t->parser_data = &t->hpack_parser; if (t->header_eof) { s->eos_received = true; } @@ -707,17 +640,12 @@ static grpc_error_handle init_window_update_frame_parser( grpc_chttp2_stream* s = t->incoming_stream = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); if (s == nullptr) { - if (grpc_http_trace.enabled()) { - gpr_log(GPR_ERROR, "Stream %d not found, ignoring WINDOW_UPDATE", - t->incoming_stream_id); - } return init_non_header_skip_frame_parser(t); } s->stats.incoming.framing_bytes += 9; } - t->parser = grpc_chttp2_transport::Parser{ - "window_update", grpc_chttp2_window_update_parser_parse, - &t->simple.window_update}; + t->parser = grpc_chttp2_window_update_parser_parse; + t->parser_data = &t->simple.window_update; return absl::OkStatus(); } @@ -725,8 +653,8 @@ static grpc_error_handle init_ping_parser(grpc_chttp2_transport* t) { grpc_error_handle err = grpc_chttp2_ping_parser_begin_frame( &t->simple.ping, t->incoming_frame_size, t->incoming_frame_flags); if (!err.ok()) return err; - t->parser = grpc_chttp2_transport::Parser{ - "ping", grpc_chttp2_ping_parser_parse, &t->simple.ping}; + t->parser = grpc_chttp2_ping_parser_parse; + t->parser_data = &t->simple.ping; return absl::OkStatus(); } @@ -740,8 +668,8 @@ static grpc_error_handle init_rst_stream_parser(grpc_chttp2_transport* t) { return init_non_header_skip_frame_parser(t); } s->stats.incoming.framing_bytes += 9; - t->parser = grpc_chttp2_transport::Parser{ - "rst_stream", grpc_chttp2_rst_stream_parser_parse, &t->simple.rst_stream}; + t->parser = grpc_chttp2_rst_stream_parser_parse; + t->parser_data = &t->simple.rst_stream; return absl::OkStatus(); } @@ -749,8 +677,8 @@ static grpc_error_handle init_goaway_parser(grpc_chttp2_transport* t) { grpc_error_handle err = grpc_chttp2_goaway_parser_begin_frame( &t->goaway_parser, t->incoming_frame_size, t->incoming_frame_flags); if (!err.ok()) return err; - t->parser = grpc_chttp2_transport::Parser{ - "goaway", grpc_chttp2_goaway_parser_parse, &t->goaway_parser}; + t->parser = grpc_chttp2_goaway_parser_parse; + t->parser_data = &t->goaway_parser; return absl::OkStatus(); } @@ -778,8 +706,8 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { t, nullptr); t->sent_local_settings = false; } - t->parser = grpc_chttp2_transport::Parser{ - "settings", grpc_chttp2_settings_parser_parse, &t->simple.settings}; + t->parser = grpc_chttp2_settings_parser_parse; + t->parser_data = &t->simple.settings; return absl::OkStatus(); } @@ -787,24 +715,15 @@ static grpc_error_handle parse_frame_slice(grpc_chttp2_transport* t, const grpc_slice& slice, int is_last) { grpc_chttp2_stream* s = t->incoming_stream; - if (grpc_http_trace.enabled()) { - gpr_log(GPR_DEBUG, - "INCOMING[%p;%p]: Parse %" PRIdPTR "b %sframe fragment with %s", t, - s, GRPC_SLICE_LENGTH(slice), is_last ? "last " : "", - t->parser.name); - } - grpc_error_handle err = - t->parser.parser(t->parser.user_data, t, s, slice, is_last); + grpc_error_handle err = t->parser(t->parser_data, t, s, slice, is_last); intptr_t unused; if (GPR_LIKELY(err.ok())) { return err; - } - if (grpc_http_trace.enabled()) { - gpr_log(GPR_ERROR, "INCOMING[%p;%p]: Parse failed with %s", t, s, - err.ToString().c_str()); - } - if (grpc_error_get_int(err, grpc_core::StatusIntProperty::kStreamId, - &unused)) { + } else if (grpc_error_get_int(err, grpc_core::StatusIntProperty::kStreamId, + &unused)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { + gpr_log(GPR_ERROR, "%s", grpc_core::StatusToString(err).c_str()); + } grpc_chttp2_parsing_become_skip_parser(t); if (s) { s->forced_close_error = err; diff --git a/src/core/lib/channel/channel_stack_builder_impl.cc b/src/core/lib/channel/channel_stack_builder_impl.cc index 98cb7a0a4c8..c57e12258c4 100644 --- a/src/core/lib/channel/channel_stack_builder_impl.cc +++ b/src/core/lib/channel/channel_stack_builder_impl.cc @@ -37,7 +37,6 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/surface/call_trace.h" -#include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/transport.h" @@ -54,24 +53,12 @@ absl::StatusOr> ChannelStackBuilderImpl::Build() { std::vector stack; const bool is_promising = IsPromising(); - const bool is_client = - grpc_channel_stack_type_is_client(channel_stack_type()); - const bool client_promise_tracing = - is_client && is_promising && grpc_call_trace.enabled(); - const bool server_promise_tracing = - !is_client && is_promising && grpc_call_trace.enabled(); for (const auto* filter : this->stack()) { - if (client_promise_tracing) { + if (is_promising && grpc_call_trace.enabled()) { stack.push_back(PromiseTracingFilterFor(filter)); } stack.push_back(filter); - if (server_promise_tracing) { - stack.push_back(PromiseTracingFilterFor(filter)); - } - } - if (server_promise_tracing) { - stack.pop_back(); // connected_channel must be last => can't be traced } // calculate the size of the channel stack diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index 01e93dfe877..5e74c5d97c5 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -24,15 +24,12 @@ #include #include -#include -#include #include #include #include #include #include "absl/base/thread_annotations.h" -#include "absl/container/inlined_vector.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -40,7 +37,6 @@ #include "absl/types/variant.h" #include -#include #include #include @@ -64,11 +60,10 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" -#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/call_trace.h" @@ -249,12 +244,57 @@ static void connected_channel_get_channel_info( namespace grpc_core { namespace { -#if defined(GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL) || \ - defined(GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL) -class ConnectedChannelStream : public Orphanable { +#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL +class ClientStream : public Orphanable { public: - grpc_transport* transport() { return transport_; } - grpc_closure* stream_destroyed_closure() { return &stream_destroyed_; } + ClientStream(grpc_transport* transport, CallArgs call_args) + : transport_(transport), + stream_(nullptr, StreamDeleter(this)), + server_initial_metadata_latch_(call_args.server_initial_metadata), + client_to_server_messages_(call_args.outgoing_messages), + server_to_client_messages_(call_args.incoming_messages), + client_initial_metadata_(std::move(call_args.client_initial_metadata)) { + call_context_->IncrementRefCount("client_stream"); + GRPC_STREAM_REF_INIT( + &stream_refcount_, 1, + [](void* p, grpc_error_handle) { + static_cast(p)->BeginDestroy(); + }, + this, "client_stream"); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sInitImpl: intitial_metadata=%s", + Activity::current()->DebugTag().c_str(), + client_initial_metadata_->DebugString().c_str()); + } + } + + void Orphan() override { + bool finished; + { + MutexLock lock(&mu_); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sDropStream: %s", + Activity::current()->DebugTag().c_str(), + ActiveOpsString().c_str()); + } + finished = finished_; + } + // If we hadn't already observed the stream to be finished, we need to + // cancel it at the transport. + if (!finished) { + IncrementRefCount("shutdown client stream"); + auto* cancel_op = + GetContext()->New(); + cancel_op->cancel_stream = true; + cancel_op->payload = &batch_payload_; + auto* stream = stream_.get(); + cancel_op->on_complete = NewClosure( + [this](grpc_error_handle) { Unref("shutdown client stream"); }); + batch_payload_.cancel_stream.cancel_error = absl::CancelledError(); + grpc_transport_perform_stream_op(transport_, stream, cancel_op); + } + Unref("orphan client stream"); + } void IncrementRefCount(const char* reason) { #ifndef NDEBUG @@ -274,157 +314,183 @@ class ConnectedChannelStream : public Orphanable { #endif } - void Orphan() final { - bool finished; - { - MutexLock lock(mu()); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] DropStream: %s finished=%s", - Activity::current()->DebugTag().c_str(), - ActiveOpsString().c_str(), finished_ ? "true" : "false"); - } - finished = finished_; + void BeginDestroy() { + if (stream_ != nullptr) { + stream_.reset(); + } else { + StreamDestroyed(); } - // If we hadn't already observed the stream to be finished, we need to - // cancel it at the transport. - if (!finished) { - IncrementRefCount("shutdown client stream"); - auto* cancel_op = - GetContext()->New(); - cancel_op->cancel_stream = true; - cancel_op->payload = batch_payload(); - auto* s = stream(); - cancel_op->on_complete = NewClosure( - [this](grpc_error_handle) { Unref("shutdown client stream"); }); - batch_payload()->cancel_stream.cancel_error = absl::CancelledError(); - grpc_transport_perform_stream_op(transport(), s, cancel_op); - } - Unref("orphan client stream"); } - protected: - explicit ConnectedChannelStream(grpc_transport* transport) - : transport_(transport), stream_(nullptr, StreamDeleter(this)) { - call_context_->IncrementRefCount("connected_channel_stream"); - GRPC_STREAM_REF_INIT( - &stream_refcount_, 1, - [](void* p, grpc_error_handle) { - static_cast(p)->BeginDestroy(); - }, - this, "client_stream"); - } + Poll PollOnce() { + MutexLock lock(&mu_); + GPR_ASSERT(!finished_); - grpc_stream* stream() { return stream_.get(); } - void SetStream(grpc_stream* stream) { stream_.reset(stream); } - grpc_stream_refcount* stream_refcount() { return &stream_refcount_; } - Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; } - grpc_transport_stream_op_batch_payload* batch_payload() { - return &batch_payload_; - } - bool finished() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return finished_; } - void set_finished() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { finished_ = true; } - virtual std::string ActiveOpsString() const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0; - - void SchedulePush(grpc_transport_stream_op_batch* batch) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "%s[connected] Push batch to transport: %s", + gpr_log(GPR_INFO, "%sPollConnectedChannel: %s", Activity::current()->DebugTag().c_str(), - grpc_transport_stream_op_batch_string(batch).c_str()); + ActiveOpsString().c_str()); } - if (push_batches_.empty()) { - IncrementRefCount("push"); - ExecCtx::Run(DEBUG_LOCATION, &push_, absl::OkStatus()); - } - push_batches_.push_back(batch); - } - void PollSendMessage(PipeReceiver* outgoing_messages, - ClientMetadataHandle* client_trailing_metadata) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + auto push_recv_message = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + recv_message_state_ = PendingReceiveMessage{}; + auto& pending_recv_message = + absl::get(recv_message_state_); + memset(&recv_message_, 0, sizeof(recv_message_)); + recv_message_.payload = &batch_payload_; + recv_message_.on_complete = nullptr; + recv_message_.recv_message = true; + batch_payload_.recv_message.recv_message = &pending_recv_message.payload; + batch_payload_.recv_message.flags = &pending_recv_message.flags; + batch_payload_.recv_message.call_failed_before_recv_message = nullptr; + batch_payload_.recv_message.recv_message_ready = + &recv_message_batch_done_; + IncrementRefCount("recv_message"); + recv_message_waker_ = Activity::current()->MakeOwningWaker(); + push_recv_message_ = true; + SchedulePush(); + }; + + if (!std::exchange(requested_metadata_, true)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting metadata", + Activity::current()->DebugTag().c_str()); + } + stream_.reset(static_cast( + GetContext()->Alloc(transport_->vtable->sizeof_stream))); + grpc_transport_init_stream(transport_, stream_.get(), &stream_refcount_, + nullptr, GetContext()); + grpc_transport_set_pops(transport_, stream_.get(), + GetContext()->polling_entity()); + memset(&metadata_, 0, sizeof(metadata_)); + metadata_.send_initial_metadata = true; + metadata_.recv_initial_metadata = true; + metadata_.recv_trailing_metadata = true; + metadata_.payload = &batch_payload_; + metadata_.on_complete = &metadata_batch_done_; + batch_payload_.send_initial_metadata.send_initial_metadata = + client_initial_metadata_.get(); + batch_payload_.send_initial_metadata.peer_string = + GetContext()->peer_string_atm_ptr(); + server_initial_metadata_ = + GetContext()->MakePooled(GetContext()); + batch_payload_.recv_initial_metadata.recv_initial_metadata = + server_initial_metadata_.get(); + batch_payload_.recv_initial_metadata.recv_initial_metadata_ready = + &recv_initial_metadata_ready_; + batch_payload_.recv_initial_metadata.trailing_metadata_available = + nullptr; + batch_payload_.recv_initial_metadata.peer_string = nullptr; + server_trailing_metadata_ = + GetContext()->MakePooled(GetContext()); + batch_payload_.recv_trailing_metadata.recv_trailing_metadata = + server_trailing_metadata_.get(); + batch_payload_.recv_trailing_metadata.collect_stats = + &GetContext()->call_stats()->transport_stream_stats; + batch_payload_.recv_trailing_metadata.recv_trailing_metadata_ready = + &recv_trailing_metadata_ready_; + push_metadata_ = true; + IncrementRefCount("metadata_batch_done"); + IncrementRefCount("initial_metadata_ready"); + IncrementRefCount("trailing_metadata_ready"); + initial_metadata_waker_ = Activity::current()->MakeOwningWaker(); + trailing_metadata_waker_ = Activity::current()->MakeOwningWaker(); + SchedulePush(); + } if (absl::holds_alternative(send_message_state_)) { message_to_send_.reset(); } if (absl::holds_alternative(send_message_state_)) { message_to_send_.reset(); - send_message_state_.emplace>( - outgoing_messages->Next()); + send_message_state_ = client_to_server_messages_->Next(); } - if (auto* next = absl::get_if>( + if (auto* next = absl::get_if::NextType>( &send_message_state_)) { auto r = (*next)(); if (auto* p = absl::get_if>(&r)) { memset(&send_message_, 0, sizeof(send_message_)); - send_message_.payload = batch_payload(); + send_message_.payload = &batch_payload_; send_message_.on_complete = &send_message_batch_done_; // No value => half close from above. if (p->has_value()) { - message_to_send_ = std::move(*p); + message_to_send_ = std::move(**p); send_message_state_ = SendMessageToTransport{}; send_message_.send_message = true; - batch_payload()->send_message.send_message = - (*message_to_send_)->payload(); - batch_payload()->send_message.flags = (*message_to_send_)->flags(); + batch_payload_.send_message.send_message = + message_to_send_->payload(); + batch_payload_.send_message.flags = message_to_send_->flags(); } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: half close", + gpr_log(GPR_INFO, "%sPollConnectedChannel: half close", Activity::current()->DebugTag().c_str()); } GPR_ASSERT(!absl::holds_alternative(send_message_state_)); + client_trailing_metadata_ = + GetContext()->MakePooled( + GetContext()); send_message_state_ = Closed{}; send_message_.send_trailing_metadata = true; - if (client_trailing_metadata != nullptr) { - *client_trailing_metadata = - GetContext()->MakePooled( - GetContext()); - batch_payload()->send_trailing_metadata.send_trailing_metadata = - client_trailing_metadata->get(); - batch_payload()->send_trailing_metadata.sent = nullptr; - } else { - return; // Skip rest of function for server - } + batch_payload_.send_trailing_metadata.send_trailing_metadata = + client_trailing_metadata_.get(); + batch_payload_.send_trailing_metadata.sent = nullptr; } IncrementRefCount("send_message"); send_message_waker_ = Activity::current()->MakeOwningWaker(); - SchedulePush(&send_message_); + push_send_message_ = true; + SchedulePush(); } } - } - - void PollRecvMessage(PipeSender*& incoming_messages) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (auto* pending = absl::get_if(&recv_message_state_)) { if (pending->received) { if (pending->payload.has_value()) { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%s[connected] PollRecvMessage: received payload of " - "%" PRIdPTR " bytes", + "%sRecvMessageBatchDone: received payload of %" PRIdPTR + " bytes", recv_message_waker_.ActivityDebugTag().c_str(), pending->payload->Length()); } - recv_message_state_ = - incoming_messages->Push(GetContext()->MakePooled( + recv_message_state_ = server_to_client_messages_->Push( + GetContext()->MakePooled( std::move(*pending->payload), pending->flags)); } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[connected] PollRecvMessage: received no payload", + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: received no payload", recv_message_waker_.ActivityDebugTag().c_str()); } recv_message_state_ = Closed{}; - std::exchange(incoming_messages, nullptr)->Close(); + std::exchange(server_to_client_messages_, nullptr)->Close(); } } } + if (server_initial_metadata_state_ == + ServerInitialMetadataState::kReceivedButNotSet) { + server_initial_metadata_state_ = ServerInitialMetadataState::kSet; + server_initial_metadata_latch_->Set(server_initial_metadata_.get()); + } if (absl::holds_alternative(recv_message_state_)) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] PollRecvMessage: requesting message", + gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting message", Activity::current()->DebugTag().c_str()); } - PushRecvMessage(); + push_recv_message(); + } + if (server_initial_metadata_state_ == ServerInitialMetadataState::kSet && + !absl::holds_alternative::PushType>( + recv_message_state_) && + !absl::holds_alternative(recv_message_state_) && + std::exchange(queued_trailing_metadata_, false)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sPollConnectedChannel: finished request, returning: {%s}; " + "active_ops: %s", + Activity::current()->DebugTag().c_str(), + server_trailing_metadata_->DebugString().c_str(), + ActiveOpsString().c_str()); + } + finished_ = true; + return ServerMetadataHandle(std::move(server_trailing_metadata_)); } if (auto* push = absl::get_if::PushType>( &recv_message_state_)) { @@ -434,120 +500,57 @@ class ConnectedChannelStream : public Orphanable { if (!finished_) { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%s[connected] PollRecvMessage: pushed message; " - "requesting next", + "%sPollConnectedChannel: pushed message; requesting next", Activity::current()->DebugTag().c_str()); } - PushRecvMessage(); + push_recv_message(); } else { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%s[connected] PollRecvMessage: pushed message " - "and finished; " + "%sPollConnectedChannel: pushed message and finished; " "marking closed", Activity::current()->DebugTag().c_str()); } recv_message_state_ = Closed{}; - std::exchange(incoming_messages, nullptr)->Close(); } } else { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%s[connected] PollRecvMessage: failed to push " - "message; marking " + "%sPollConnectedChannel: failed to push message; marking " "closed", Activity::current()->DebugTag().c_str()); } recv_message_state_ = Closed{}; - std::exchange(incoming_messages, nullptr)->Close(); } } } + return Pending{}; } - std::string SendMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { - return Match( - send_message_state_, [](Idle) -> std::string { return "IDLE"; }, - [](Closed) -> std::string { return "CLOSED"; }, - [](const PipeReceiverNextType&) -> std::string { - return "WAITING"; - }, - [](SendMessageToTransport) -> std::string { return "SENDING"; }); - } - - std::string RecvMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { - return Match( - recv_message_state_, [](Idle) -> std::string { return "IDLE"; }, - [](Closed) -> std::string { return "CLOSED"; }, - [](const PendingReceiveMessage&) -> std::string { return "WAITING"; }, - [](const absl::optional& message) -> std::string { - return absl::StrCat( - "READY:", message.has_value() - ? absl::StrCat((*message)->payload()->Length(), "b") - : "EOS"); - }, - [](const PipeSender::PushType&) -> std::string { - return "PUSHING"; - }); - } - - bool IsPromiseReceiving() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { - return absl::holds_alternative::PushType>( - recv_message_state_) || - absl::holds_alternative(recv_message_state_); - } - - private: - struct SendMessageToTransport {}; - struct Idle {}; - struct Closed {}; - - class StreamDeleter { - public: - explicit StreamDeleter(ConnectedChannelStream* impl) : impl_(impl) {} - void operator()(grpc_stream* stream) const { - if (stream == nullptr) return; - grpc_transport_destroy_stream(impl_->transport(), stream, - impl_->stream_destroyed_closure()); - } - - private: - ConnectedChannelStream* impl_; - }; - using StreamPtr = std::unique_ptr; - - void StreamDestroyed() { - call_context_->RunInContext([this] { - auto* cc = call_context_; - this->~ConnectedChannelStream(); - cc->Unref("child_stream"); - }); - } - - void BeginDestroy() { - if (stream_ != nullptr) { - stream_.reset(); - } else { - StreamDestroyed(); - } - } - - // Called from outside the activity to push work down to the transport. - void Push() { - PushBatches push_batches; + void RecvInitialMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); { MutexLock lock(&mu_); - push_batches.swap(push_batches_); + server_initial_metadata_state_ = + ServerInitialMetadataState::kReceivedButNotSet; + initial_metadata_waker_.Wakeup(); } - for (auto* batch : push_batches) { - if (stream() != nullptr) { - grpc_transport_perform_stream_op(transport(), stream(), batch); - } else { - grpc_transport_stream_op_batch_finish_with_failure_from_transport( - batch, absl::CancelledError()); - } + Unref("initial_metadata_ready"); + } + + void RecvTrailingMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); + { + MutexLock lock(&mu_); + queued_trailing_metadata_ = true; + trailing_metadata_waker_.Wakeup(); } - Unref("push"); + Unref("trailing_metadata_ready"); + } + + void MetadataBatchDone(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); + Unref("metadata_batch_done"); } void SendMessageBatchDone(grpc_error_handle error) { @@ -570,24 +573,21 @@ class ConnectedChannelStream : public Orphanable { void RecvMessageBatchDone(grpc_error_handle error) { { - MutexLock lock(mu()); + MutexLock lock(&mu_); if (error != absl::OkStatus()) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] RecvMessageBatchDone: error=%s", + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: error=%s", recv_message_waker_.ActivityDebugTag().c_str(), StatusToString(error).c_str()); } } else if (absl::holds_alternative(recv_message_state_)) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[connected] RecvMessageBatchDone: already closed, " - "ignoring", + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: already closed, ignoring", recv_message_waker_.ActivityDebugTag().c_str()); } } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[connected] RecvMessageBatchDone: received message", + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: received message", recv_message_waker_.ActivityDebugTag().c_str()); } auto pending = @@ -601,250 +601,95 @@ class ConnectedChannelStream : public Orphanable { Unref("recv_message"); } - void PushRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - recv_message_state_ = PendingReceiveMessage{}; - auto& pending_recv_message = - absl::get(recv_message_state_); - memset(&recv_message_, 0, sizeof(recv_message_)); - recv_message_.payload = batch_payload(); - recv_message_.on_complete = nullptr; - recv_message_.recv_message = true; - batch_payload()->recv_message.recv_message = &pending_recv_message.payload; - batch_payload()->recv_message.flags = &pending_recv_message.flags; - batch_payload()->recv_message.call_failed_before_recv_message = nullptr; - batch_payload()->recv_message.recv_message_ready = - &recv_message_batch_done_; - IncrementRefCount("recv_message"); - recv_message_waker_ = Activity::current()->MakeOwningWaker(); - SchedulePush(&recv_message_); - } - - mutable Mutex mu_; - grpc_transport* const transport_; - CallContext* const call_context_{GetContext()}; - grpc_closure stream_destroyed_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - grpc_stream_refcount stream_refcount_; - StreamPtr stream_; - using PushBatches = absl::InlinedVector; - PushBatches push_batches_ ABSL_GUARDED_BY(mu_); - grpc_closure push_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - - NextResult message_to_send_ ABSL_GUARDED_BY(mu_); - absl::variant, - SendMessageToTransport> - send_message_state_ ABSL_GUARDED_BY(mu_); - grpc_transport_stream_op_batch send_message_; - grpc_closure send_message_batch_done_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - - struct PendingReceiveMessage { - absl::optional payload; - uint32_t flags; - bool received = false; - }; - absl::variant::PushType> - recv_message_state_ ABSL_GUARDED_BY(mu_); - grpc_closure recv_message_batch_done_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - grpc_transport_stream_op_batch recv_message_; - - Waker send_message_waker_ ABSL_GUARDED_BY(mu_); - Waker recv_message_waker_ ABSL_GUARDED_BY(mu_); - bool finished_ ABSL_GUARDED_BY(mu_) = false; - - grpc_transport_stream_op_batch_payload batch_payload_{ - GetContext()}; -}; -#endif - -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL -class ClientStream : public ConnectedChannelStream { - public: - ClientStream(grpc_transport* transport, CallArgs call_args) - : ConnectedChannelStream(transport), - server_initial_metadata_pipe_(call_args.server_initial_metadata), - client_to_server_messages_(call_args.client_to_server_messages), - server_to_client_messages_(call_args.server_to_client_messages), - client_initial_metadata_(std::move(call_args.client_initial_metadata)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] InitImpl: intitial_metadata=%s", - Activity::current()->DebugTag().c_str(), - client_initial_metadata_->DebugString().c_str()); - } - } - - Poll PollOnce() { - MutexLock lock(mu()); - GPR_ASSERT(!finished()); - - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: %s", - Activity::current()->DebugTag().c_str(), - ActiveOpsString().c_str()); - } - - if (!std::exchange(requested_metadata_, true)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[connected] PollConnectedChannel: requesting metadata", - Activity::current()->DebugTag().c_str()); + // Called from outside the activity to push work down to the transport. + void Push() { + auto do_push = [this](grpc_transport_stream_op_batch* batch) { + if (stream_ != nullptr) { + grpc_transport_perform_stream_op(transport_, stream_.get(), batch); + } else { + grpc_transport_stream_op_batch_finish_with_failure_from_transport( + batch, absl::CancelledError()); } - SetStream(static_cast( - GetContext()->Alloc(transport()->vtable->sizeof_stream))); - grpc_transport_init_stream(transport(), stream(), stream_refcount(), - nullptr, GetContext()); - grpc_transport_set_pops(transport(), stream(), - GetContext()->polling_entity()); - memset(&metadata_, 0, sizeof(metadata_)); - metadata_.send_initial_metadata = true; - metadata_.recv_initial_metadata = true; - metadata_.recv_trailing_metadata = true; - metadata_.payload = batch_payload(); - metadata_.on_complete = &metadata_batch_done_; - batch_payload()->send_initial_metadata.send_initial_metadata = - client_initial_metadata_.get(); - batch_payload()->send_initial_metadata.peer_string = - GetContext()->peer_string_atm_ptr(); - server_initial_metadata_ = - GetContext()->MakePooled(GetContext()); - batch_payload()->recv_initial_metadata.recv_initial_metadata = - server_initial_metadata_.get(); - batch_payload()->recv_initial_metadata.recv_initial_metadata_ready = - &recv_initial_metadata_ready_; - batch_payload()->recv_initial_metadata.trailing_metadata_available = - nullptr; - batch_payload()->recv_initial_metadata.peer_string = nullptr; - server_trailing_metadata_ = - GetContext()->MakePooled(GetContext()); - batch_payload()->recv_trailing_metadata.recv_trailing_metadata = - server_trailing_metadata_.get(); - batch_payload()->recv_trailing_metadata.collect_stats = - &GetContext()->call_stats()->transport_stream_stats; - batch_payload()->recv_trailing_metadata.recv_trailing_metadata_ready = - &recv_trailing_metadata_ready_; - IncrementRefCount("metadata_batch_done"); - IncrementRefCount("initial_metadata_ready"); - IncrementRefCount("trailing_metadata_ready"); - initial_metadata_waker_ = Activity::current()->MakeOwningWaker(); - trailing_metadata_waker_ = Activity::current()->MakeOwningWaker(); - SchedulePush(&metadata_); - } - if (server_initial_metadata_state_ == - ServerInitialMetadataState::kReceivedButNotPushed) { - server_initial_metadata_state_ = ServerInitialMetadataState::kPushing; - server_initial_metadata_push_promise_ = - server_initial_metadata_pipe_->Push( - std::move(server_initial_metadata_)); - } - if (server_initial_metadata_state_ == - ServerInitialMetadataState::kPushing) { - auto r = (*server_initial_metadata_push_promise_)(); - if (absl::holds_alternative(r)) { - server_initial_metadata_state_ = ServerInitialMetadataState::kPushed; - server_initial_metadata_push_promise_.reset(); - } - } - PollSendMessage(client_to_server_messages_, &client_trailing_metadata_); - PollRecvMessage(server_to_client_messages_); - if (server_initial_metadata_state_ == ServerInitialMetadataState::kPushed && - !IsPromiseReceiving() && - std::exchange(queued_trailing_metadata_, false)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[connected] PollConnectedChannel: finished request, " - "returning: {%s}; " - "active_ops: %s", - Activity::current()->DebugTag().c_str(), - server_trailing_metadata_->DebugString().c_str(), - ActiveOpsString().c_str()); - } - set_finished(); - return ServerMetadataHandle(std::move(server_trailing_metadata_)); - } - return Pending{}; - } - - void RecvInitialMetadataReady(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); + }; + bool push_metadata; + bool push_send_message; + bool push_recv_message; { - MutexLock lock(mu()); - server_initial_metadata_state_ = - ServerInitialMetadataState::kReceivedButNotPushed; - initial_metadata_waker_.Wakeup(); + MutexLock lock(&mu_); + push_metadata = std::exchange(push_metadata_, false); + push_send_message = std::exchange(push_send_message_, false); + push_recv_message = std::exchange(push_recv_message_, false); + scheduled_push_ = false; } - Unref("initial_metadata_ready"); + if (push_metadata) do_push(&metadata_); + if (push_send_message) do_push(&send_message_); + if (push_recv_message) do_push(&recv_message_); + Unref("push"); } - void RecvTrailingMetadataReady(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); - { - MutexLock lock(mu()); - queued_trailing_metadata_ = true; - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, - "%s[connected] RecvTrailingMetadataReady: " - "queued_trailing_metadata_ " - "set to true; active_ops: %s", - trailing_metadata_waker_.ActivityDebugTag().c_str(), - ActiveOpsString().c_str()); - } - trailing_metadata_waker_.Wakeup(); - } - Unref("trailing_metadata_ready"); - } - - void MetadataBatchDone(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); - Unref("metadata_batch_done"); + void StreamDestroyed() { + call_context_->RunInContext([this] { + auto* cc = call_context_; + this->~ClientStream(); + cc->Unref("child_stream"); + }); } private: + struct Idle {}; + struct Closed {}; + struct SendMessageToTransport {}; + enum class ServerInitialMetadataState : uint8_t { // Initial metadata has not been received from the server. kNotReceived, // Initial metadata has been received from the server via the transport, but - // has not yet been pushed onto the pipe to publish it up the call stack. - kReceivedButNotPushed, + // has not yet been set on the latch to publish it up the call stack. + kReceivedButNotSet, // Initial metadata has been received from the server via the transport and - // has been pushed on the pipe to publish it up the call stack. - // It's still in the pipe and has not been removed by the call at the top - // yet. - kPushing, - // Initial metadata has been received from the server via the transport and - // has been pushed on the pipe to publish it up the call stack AND removed - // by the call at the top. - kPushed, + // has been set on the latch to publish it up the call stack. + kSet, }; - std::string ActiveOpsString() const override - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { + class StreamDeleter { + public: + explicit StreamDeleter(ClientStream* impl) : impl_(impl) {} + void operator()(grpc_stream* stream) const { + if (stream == nullptr) return; + grpc_transport_destroy_stream(impl_->transport_, stream, + &impl_->stream_destroyed_); + } + + private: + ClientStream* impl_; + }; + using StreamPtr = std::unique_ptr; + + void SchedulePush() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (std::exchange(scheduled_push_, true)) return; + IncrementRefCount("push"); + ExecCtx::Run(DEBUG_LOCATION, &push_, absl::OkStatus()); + } + + std::string ActiveOpsString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { std::vector ops; - if (finished()) ops.push_back("FINISHED"); - // Outstanding Operations on Transport - std::vector waiting; - if (initial_metadata_waker_ != Waker()) { - waiting.push_back("initial_metadata"); - } - if (trailing_metadata_waker_ != Waker()) { - waiting.push_back("trailing_metadata"); - } - if (!waiting.empty()) { - ops.push_back(absl::StrCat("waiting:", absl::StrJoin(waiting, ","))); + if (finished_) ops.push_back("FINISHED"); + // Pushes + std::vector pushes; + if (push_metadata_) pushes.push_back("metadata"); + if (push_send_message_) pushes.push_back("send_message"); + if (push_recv_message_) pushes.push_back("recv_message"); + if (!pushes.empty()) { + ops.push_back( + absl::StrCat(scheduled_push_ ? "push:" : "unscheduled-push:", + absl::StrJoin(pushes, ","))); + } else if (scheduled_push_) { + ops.push_back("push:nothing"); } // Results from transport std::vector queued; if (server_initial_metadata_state_ == - ServerInitialMetadataState::kReceivedButNotPushed) { + ServerInitialMetadataState::kReceivedButNotSet) { queued.push_back("initial_metadata"); } if (queued_trailing_metadata_) queued.push_back("trailing_metadata"); @@ -864,31 +709,94 @@ class ClientStream : public ConnectedChannelStream { return absl::StrJoin(ops, " "); } + std::string SendMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return Match( + send_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PipeReceiver::NextType&) -> std::string { + return "WAITING"; + }, + [](SendMessageToTransport) -> std::string { return "SENDING"; }); + } + + std::string RecvMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return Match( + recv_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PendingReceiveMessage&) -> std::string { return "WAITING"; }, + [](const absl::optional& message) -> std::string { + return absl::StrCat( + "READY:", message.has_value() + ? absl::StrCat((*message)->payload()->Length(), "b") + : "EOS"); + }, + [](const PipeSender::PushType&) -> std::string { + return "PUSHING"; + }); + } + + Mutex mu_; bool requested_metadata_ = false; + bool push_metadata_ ABSL_GUARDED_BY(mu_) = false; + bool push_send_message_ ABSL_GUARDED_BY(mu_) = false; + bool push_recv_message_ ABSL_GUARDED_BY(mu_) = false; + bool scheduled_push_ ABSL_GUARDED_BY(mu_) = false; ServerInitialMetadataState server_initial_metadata_state_ - ABSL_GUARDED_BY(mu()) = ServerInitialMetadataState::kNotReceived; - bool queued_trailing_metadata_ ABSL_GUARDED_BY(mu()) = false; - Waker initial_metadata_waker_ ABSL_GUARDED_BY(mu()); - Waker trailing_metadata_waker_ ABSL_GUARDED_BY(mu()); - PipeSender* server_initial_metadata_pipe_; + ABSL_GUARDED_BY(mu_) = ServerInitialMetadataState::kNotReceived; + bool queued_trailing_metadata_ ABSL_GUARDED_BY(mu_) = false; + bool finished_ ABSL_GUARDED_BY(mu_) = false; + CallContext* const call_context_{GetContext()}; + Waker initial_metadata_waker_ ABSL_GUARDED_BY(mu_); + Waker trailing_metadata_waker_ ABSL_GUARDED_BY(mu_); + Waker send_message_waker_ ABSL_GUARDED_BY(mu_); + Waker recv_message_waker_ ABSL_GUARDED_BY(mu_); + grpc_transport* const transport_; + grpc_stream_refcount stream_refcount_; + StreamPtr stream_; + Latch* server_initial_metadata_latch_; PipeReceiver* client_to_server_messages_; PipeSender* server_to_client_messages_; + MessageHandle message_to_send_ ABSL_GUARDED_BY(mu_); + absl::variant::NextType, + SendMessageToTransport> + send_message_state_ ABSL_GUARDED_BY(mu_); + struct PendingReceiveMessage { + absl::optional payload; + uint32_t flags; + bool received = false; + }; + absl::variant::PushType> + recv_message_state_ ABSL_GUARDED_BY(mu_); grpc_closure recv_initial_metadata_ready_ = MakeMemberClosure( this, DEBUG_LOCATION); grpc_closure recv_trailing_metadata_ready_ = MakeMemberClosure( this, DEBUG_LOCATION); + grpc_closure push_ = MakeMemberClosure( + this, DEBUG_LOCATION); ClientMetadataHandle client_initial_metadata_; ClientMetadataHandle client_trailing_metadata_; ServerMetadataHandle server_initial_metadata_; ServerMetadataHandle server_trailing_metadata_; - absl::optional::PushType> - server_initial_metadata_push_promise_; grpc_transport_stream_op_batch metadata_; grpc_closure metadata_batch_done_ = MakeMemberClosure( this, DEBUG_LOCATION); + grpc_transport_stream_op_batch send_message_; + grpc_closure send_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_closure recv_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_transport_stream_op_batch recv_message_; + grpc_transport_stream_op_batch_payload batch_payload_{ + GetContext()}; + grpc_closure stream_destroyed_ = + MakeMemberClosure( + this, DEBUG_LOCATION); }; class ClientConnectedCallPromise { @@ -909,8 +817,7 @@ class ClientConnectedCallPromise { } static ArenaPromise Make(grpc_transport* transport, - CallArgs call_args, - NextPromiseFactory) { + CallArgs call_args) { return ClientConnectedCallPromise(transport, std::move(call_args)); } @@ -921,432 +828,8 @@ class ClientConnectedCallPromise { }; #endif -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL -class ServerStream final : public ConnectedChannelStream { - public: - ServerStream(grpc_transport* transport, - NextPromiseFactory next_promise_factory) - : ConnectedChannelStream(transport) { - SetStream(static_cast( - GetContext()->Alloc(transport->vtable->sizeof_stream))); - grpc_transport_init_stream( - transport, stream(), stream_refcount(), - GetContext()->server_call_context()->server_stream_data(), - GetContext()); - grpc_transport_set_pops(transport, stream(), - GetContext()->polling_entity()); - - // Fetch initial metadata - auto& gim = call_state_.emplace(this); - gim.recv_initial_metadata_ready_waker = - Activity::current()->MakeOwningWaker(); - memset(&gim.recv_initial_metadata, 0, sizeof(gim.recv_initial_metadata)); - gim.recv_initial_metadata.payload = batch_payload(); - gim.recv_initial_metadata.on_complete = nullptr; - gim.recv_initial_metadata.recv_initial_metadata = true; - gim.next_promise_factory = std::move(next_promise_factory); - batch_payload()->recv_initial_metadata.recv_initial_metadata = - gim.client_initial_metadata.get(); - batch_payload()->recv_initial_metadata.recv_initial_metadata_ready = - &gim.recv_initial_metadata_ready; - SchedulePush(&gim.recv_initial_metadata); - - // Fetch trailing metadata (to catch cancellations) - auto& gtm = - client_trailing_metadata_state_.emplace(); - gtm.recv_trailing_metadata_ready = - MakeMemberClosure(this); - memset(>m.recv_trailing_metadata, 0, sizeof(gtm.recv_trailing_metadata)); - gtm.recv_trailing_metadata.payload = batch_payload(); - gtm.recv_trailing_metadata.recv_trailing_metadata = true; - batch_payload()->recv_trailing_metadata.recv_trailing_metadata = - gtm.result.get(); - batch_payload()->recv_trailing_metadata.collect_stats = - &GetContext()->call_stats()->transport_stream_stats; - batch_payload()->recv_trailing_metadata.recv_trailing_metadata_ready = - >m.recv_trailing_metadata_ready; - SchedulePush(>m.recv_trailing_metadata); - gtm.waker = Activity::current()->MakeOwningWaker(); - } - - Poll PollOnce() { - MutexLock lock(mu()); - - auto poll_send_initial_metadata = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED( - mu()) { - if (auto* promise = - absl::get_if>( - &server_initial_metadata_)) { - auto r = (*promise)(); - if (auto* md = absl::get_if>(&r)) { - if (grpc_call_trace.enabled()) { - gpr_log( - GPR_INFO, "%s[connected] got initial metadata %s", - Activity::current()->DebugTag().c_str(), - (md->has_value() ? (**md)->DebugString() : "") - .c_str()); - } - memset(&send_initial_metadata_, 0, sizeof(send_initial_metadata_)); - send_initial_metadata_.send_initial_metadata = true; - send_initial_metadata_.payload = batch_payload(); - send_initial_metadata_.on_complete = &send_initial_metadata_done_; - batch_payload()->send_initial_metadata.send_initial_metadata = - server_initial_metadata_ - .emplace(std::move(**md)) - .get(); - batch_payload()->send_initial_metadata.peer_string = nullptr; - SchedulePush(&send_initial_metadata_); - return true; - } else { - return false; - } - } else { - return true; - } - }; - - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: %s", - Activity::current()->DebugTag().c_str(), - ActiveOpsString().c_str()); - } - - poll_send_initial_metadata(); - - if (auto* p = absl::get_if( - &client_trailing_metadata_state_)) { - pipes_.client_to_server.sender.Close(); - if (!p->result.ok()) { - // client cancelled, we should cancel too - if (absl::holds_alternative(call_state_) || - absl::holds_alternative(call_state_) || - absl::holds_alternative(call_state_)) { - if (!absl::holds_alternative( - server_initial_metadata_)) { - // pretend we've sent initial metadata to stop that op from - // progressing if it's stuck somewhere above us in the stack - server_initial_metadata_.emplace(); - } - // cancel the call - this status will be returned to the server bottom - // promise - call_state_.emplace( - Complete{ServerMetadataFromStatus(p->result)}); - } - } - } - - if (auto* p = absl::get_if(&call_state_)) { - incoming_messages_ = &pipes_.client_to_server.sender; - auto promise = p->next_promise_factory(CallArgs{ - std::move(p->client_initial_metadata), - &pipes_.server_initial_metadata.sender, - &pipes_.client_to_server.receiver, &pipes_.server_to_client.sender}); - call_state_.emplace( - MessageLoop{&pipes_.server_to_client.receiver, std::move(promise)}); - server_initial_metadata_ - .emplace>( - pipes_.server_initial_metadata.receiver.Next()); - } - if (incoming_messages_ != nullptr) { - PollRecvMessage(incoming_messages_); - } - if (auto* p = absl::get_if(&call_state_)) { - if (absl::holds_alternative( - server_initial_metadata_)) { - PollSendMessage(p->outgoing_messages, nullptr); - } - auto poll = p->promise(); - if (auto* r = absl::get_if(&poll)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[connected] got trailing metadata %s; %s", - Activity::current()->DebugTag().c_str(), - (*r)->DebugString().c_str(), ActiveOpsString().c_str()); - } - auto& completing = call_state_.emplace(); - completing.server_trailing_metadata = std::move(*r); - completing.on_complete = - MakeMemberClosure(this); - completing.waker = Activity::current()->MakeOwningWaker(); - auto& op = completing.send_trailing_metadata; - memset(&op, 0, sizeof(op)); - op.payload = batch_payload(); - op.on_complete = &completing.on_complete; - // If we've gotten initial server metadata, we can send trailing - // metadata. - // Otherwise we need to cancel the call. - // There could be an unlucky ordering, so we poll here to make sure. - if (poll_send_initial_metadata()) { - op.send_trailing_metadata = true; - batch_payload()->send_trailing_metadata.send_trailing_metadata = - completing.server_trailing_metadata.get(); - batch_payload()->send_trailing_metadata.sent = &completing.sent; - } else { - op.cancel_stream = true; - const auto status_code = - completing.server_trailing_metadata->get(GrpcStatusMetadata()) - .value_or(GRPC_STATUS_UNKNOWN); - batch_payload()->cancel_stream.cancel_error = grpc_error_set_int( - absl::Status(static_cast(status_code), - completing.server_trailing_metadata - ->GetOrCreatePointer(GrpcMessageMetadata()) - ->as_string_view()), - StatusIntProperty::kRpcStatus, status_code); - } - SchedulePush(&op); - } - } - if (auto* p = absl::get_if(&call_state_)) { - set_finished(); - return std::move(p->result); - } - return Pending{}; - } - - private: - // Call state: we've asked the transport for initial metadata and are - // waiting for it before proceeding. - struct GettingInitialMetadata { - explicit GettingInitialMetadata(ServerStream* stream) - : recv_initial_metadata_ready( - MakeMemberClosure( - stream)) {} - // The batch we're using to get initial metadata. - grpc_transport_stream_op_batch recv_initial_metadata; - // Waker to re-enter the activity once the transport returns. - Waker recv_initial_metadata_ready_waker; - // Initial metadata storage for the transport. - ClientMetadataHandle client_initial_metadata = - GetContext()->MakePooled(GetContext()); - // Closure for the transport to call when it's ready. - grpc_closure recv_initial_metadata_ready; - // Next promise factory to use once we have initial metadata. - NextPromiseFactory next_promise_factory; - }; - - // Call state: transport has returned initial metadata, we're waiting to - // re-enter the activity to process it. - struct GotInitialMetadata { - ClientMetadataHandle client_initial_metadata; - NextPromiseFactory next_promise_factory; - }; - - // Call state: we're sending/receiving messages and processing the filter - // stack. - struct MessageLoop { - PipeReceiver* outgoing_messages; - ArenaPromise promise; - }; - - // Call state: promise stack has returned trailing metadata, we're sending it - // to the transport to communicate. - struct Completing { - ServerMetadataHandle server_trailing_metadata; - grpc_transport_stream_op_batch send_trailing_metadata; - grpc_closure on_complete; - bool sent = false; - Waker waker; - }; - - // Call state: server metadata has been communicated to the transport and sent - // to the client. - // The metadata will be returned down to the server call to tick the - // cancellation bit or not on the originating batch. - struct Complete { - ServerMetadataHandle result; - }; - - // Trailing metadata state: we've asked the transport for trailing metadata - // and are waiting for it before proceeding. - struct WaitingForTrailingMetadata { - ClientMetadataHandle result = - GetContext()->MakePooled(GetContext()); - grpc_transport_stream_op_batch recv_trailing_metadata; - grpc_closure recv_trailing_metadata_ready; - Waker waker; - }; - - // We've received trailing metadata from the transport - which indicates reads - // are closed. - // We convert to an absl::Status here and use that to drive a decision to - // cancel the call (on error) or not. - struct GotClientHalfClose { - absl::Status result; - }; - - void RecvInitialMetadataReady(absl::Status status) { - MutexLock lock(mu()); - auto& getting = absl::get(call_state_); - auto waker = std::move(getting.recv_initial_metadata_ready_waker); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "%sGOT INITIAL METADATA: err=%s %s", - waker.ActivityDebugTag().c_str(), status.ToString().c_str(), - getting.client_initial_metadata->DebugString().c_str()); - } - GotInitialMetadata got{std::move(getting.client_initial_metadata), - std::move(getting.next_promise_factory)}; - call_state_.emplace(std::move(got)); - waker.Wakeup(); - } - - void SendTrailingMetadataDone(absl::Status result) { - MutexLock lock(mu()); - auto& completing = absl::get(call_state_); - auto md = std::move(completing.server_trailing_metadata); - auto waker = std::move(completing.waker); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "%sSEND TRAILING METADATA DONE: err=%s sent=%s %s", - waker.ActivityDebugTag().c_str(), result.ToString().c_str(), - completing.sent ? "true" : "false", md->DebugString().c_str()); - } - md->Set(GrpcStatusFromWire(), completing.sent); - if (!result.ok()) { - md->Clear(); - md->Set(GrpcStatusMetadata(), - static_cast(result.code())); - md->Set(GrpcMessageMetadata(), Slice::FromCopiedString(result.message())); - md->Set(GrpcStatusFromWire(), false); - } - call_state_.emplace(Complete{std::move(md)}); - waker.Wakeup(); - } - - std::string ActiveOpsString() const override - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { - std::vector ops; - ops.push_back(absl::StrCat( - "call_state:", - Match( - call_state_, - [](const absl::monostate&) { return "absl::monostate"; }, - [](const GettingInitialMetadata&) { return "GETTING"; }, - [](const GotInitialMetadata&) { return "GOT"; }, - [](const MessageLoop&) { return "RUNNING"; }, - [](const Completing&) { return "COMPLETING"; }, - [](const Complete&) { return "COMPLETE"; }))); - ops.push_back( - absl::StrCat("client_trailing_metadata_state:", - Match( - client_trailing_metadata_state_, - [](const absl::monostate&) -> std::string { - return "absl::monostate"; - }, - [](const WaitingForTrailingMetadata&) -> std::string { - return "WAITING"; - }, - [](const GotClientHalfClose& got) -> std::string { - return absl::StrCat("GOT:", got.result.ToString()); - }))); - // Send initial metadata - ops.push_back(absl::StrCat( - "server_initial_metadata_state:", - Match( - server_initial_metadata_, - [](const absl::monostate&) { return "absl::monostate"; }, - [](const PipeReceiverNextType&) { - return "WAITING"; - }, - [](const ServerMetadataHandle&) { return "GOT"; }))); - // Send message - std::string send_message_state = SendMessageString(); - if (send_message_state != "WAITING") { - ops.push_back(absl::StrCat("send_message:", send_message_state)); - } - // Receive message - std::string recv_message_state = RecvMessageString(); - if (recv_message_state != "IDLE") { - ops.push_back(absl::StrCat("recv_message:", recv_message_state)); - } - return absl::StrJoin(ops, " "); - } - - void SendInitialMetadataDone() {} - - void RecvTrailingMetadataReady(absl::Status error) { - MutexLock lock(mu()); - auto& state = - absl::get(client_trailing_metadata_state_); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%sRecvTrailingMetadataReady: error:%s metadata:%s state:%s", - state.waker.ActivityDebugTag().c_str(), error.ToString().c_str(), - state.result->DebugString().c_str(), ActiveOpsString().c_str()); - } - auto waker = std::move(state.waker); - ServerMetadataHandle result = std::move(state.result); - if (error.ok()) { - auto* message = result->get_pointer(GrpcMessageMetadata()); - error = absl::Status( - static_cast( - result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN)), - message == nullptr ? "" : message->as_string_view()); - } - client_trailing_metadata_state_.emplace( - GotClientHalfClose{error}); - waker.Wakeup(); - } - - struct Pipes { - Pipe server_to_client; - Pipe client_to_server; - Pipe server_initial_metadata; - }; - - using CallState = - absl::variant; - CallState call_state_ ABSL_GUARDED_BY(mu()) = absl::monostate{}; - using ClientTrailingMetadataState = - absl::variant; - ClientTrailingMetadataState client_trailing_metadata_state_ - ABSL_GUARDED_BY(mu()) = absl::monostate{}; - absl::variant, - ServerMetadataHandle> - ABSL_GUARDED_BY(mu()) server_initial_metadata_ = absl::monostate{}; - PipeSender* incoming_messages_ = nullptr; - grpc_transport_stream_op_batch send_initial_metadata_; - grpc_closure send_initial_metadata_done_ = - MakeMemberClosure( - this); - Pipes pipes_ ABSL_GUARDED_BY(mu()); -}; - -class ServerConnectedCallPromise { - public: - ServerConnectedCallPromise(grpc_transport* transport, - NextPromiseFactory next_promise_factory) - : impl_(GetContext()->New( - transport, std::move(next_promise_factory))) {} - - ServerConnectedCallPromise(const ServerConnectedCallPromise&) = delete; - ServerConnectedCallPromise& operator=(const ServerConnectedCallPromise&) = - delete; - ServerConnectedCallPromise(ServerConnectedCallPromise&& other) noexcept - : impl_(std::exchange(other.impl_, nullptr)) {} - ServerConnectedCallPromise& operator=( - ServerConnectedCallPromise&& other) noexcept { - impl_ = std::move(other.impl_); - return *this; - } - - static ArenaPromise Make(grpc_transport* transport, - CallArgs, - NextPromiseFactory next) { - return ServerConnectedCallPromise(transport, std::move(next)); - } - - Poll operator()() { return impl_->PollOnce(); } - - private: - OrphanablePtr impl_; -}; -#endif - template (*make_call_promise)( - grpc_transport*, CallArgs, NextPromiseFactory)> + grpc_transport*, CallArgs)> grpc_channel_filter MakeConnectedFilter() { // Create a vtable that contains both the legacy call methods (for filter // stack based calls) and the new promise based method for creating promise @@ -1354,15 +837,15 @@ grpc_channel_filter MakeConnectedFilter() { // In this way the filter can be inserted into either kind of channel stack, // and only if all the filters in the stack are promise based will the call // be promise based. - auto make_call_wrapper = +[](grpc_channel_element* elem, CallArgs call_args, - NextPromiseFactory next) { - grpc_transport* transport = - static_cast(elem->channel_data)->transport; - return make_call_promise(transport, std::move(call_args), std::move(next)); - }; + auto make_call_wrapper = + +[](grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory) { + grpc_transport* transport = + static_cast(elem->channel_data)->transport; + return make_call_promise(transport, std::move(call_args)); + }; return { connected_channel_start_transport_stream_op_batch, - make_call_promise != nullptr ? make_call_wrapper : nullptr, + make_call_promise == nullptr ? nullptr : make_call_wrapper, connected_channel_start_transport_op, sizeof(call_data), connected_channel_init_call_elem, @@ -1387,7 +870,7 @@ grpc_channel_filter MakeConnectedFilter() { } ArenaPromise MakeTransportCallPromise( - grpc_transport* transport, CallArgs call_args, NextPromiseFactory) { + grpc_transport* transport, CallArgs call_args) { return transport->vtable->make_call_promise(transport, std::move(call_args)); } @@ -1402,13 +885,7 @@ const grpc_channel_filter kClientEmulatedFilter = MakeConnectedFilter(); #endif -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL -const grpc_channel_filter kServerEmulatedFilter = - MakeConnectedFilter(); -#else -const grpc_channel_filter kServerEmulatedFilter = - MakeConnectedFilter(); -#endif +const grpc_channel_filter kNoPromiseFilter = MakeConnectedFilter(); } // namespace } // namespace grpc_core @@ -1433,8 +910,8 @@ bool grpc_add_connected_filter(grpc_core::ChannelStackBuilder* builder) { builder->AppendFilter(&grpc_core::kClientEmulatedFilter); } else { // Option 3: the transport does not support promise based calls, and we're - // on the server so we use the server filter. - builder->AppendFilter(&grpc_core::kServerEmulatedFilter); + // on the server so we can't construct promise based calls just yet. + builder->AppendFilter(&grpc_core::kNoPromiseFilter); } return true; } diff --git a/src/core/lib/channel/promise_based_filter.cc b/src/core/lib/channel/promise_based_filter.cc index 60eaf3e4422..de348f854df 100644 --- a/src/core/lib/channel/promise_based_filter.cc +++ b/src/core/lib/channel/promise_based_filter.cc @@ -25,7 +25,6 @@ #include "absl/base/attributes.h" #include "absl/functional/function_ref.h" -#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" @@ -39,7 +38,6 @@ #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/slice/slice.h" extern grpc_core::TraceFlag grpc_trace_channel; @@ -83,28 +81,24 @@ absl::Status StatusFromMetadata(const ServerMetadata& md) { /////////////////////////////////////////////////////////////////////////////// // BaseCallData -BaseCallData::BaseCallData( - grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags, - absl::FunctionRef make_send_interceptor, - absl::FunctionRef make_recv_interceptor) +BaseCallData::BaseCallData(grpc_call_element* elem, + const grpc_call_element_args* args, uint8_t flags) : call_stack_(args->call_stack), elem_(elem), arena_(args->arena), call_combiner_(args->call_combiner), deadline_(args->deadline), context_(args->context), - server_initial_metadata_pipe_( + server_initial_metadata_latch_( flags & kFilterExaminesServerInitialMetadata - ? arena_->New>(arena_) - : nullptr), - send_message_( - flags & kFilterExaminesOutboundMessages - ? arena_->New(this, make_send_interceptor()) - : nullptr), - receive_message_( - flags & kFilterExaminesInboundMessages - ? arena_->New(this, make_recv_interceptor()) + ? arena_->New>() : nullptr), + send_message_(flags & kFilterExaminesOutboundMessages + ? arena_->New(this) + : nullptr), + receive_message_(flags & kFilterExaminesInboundMessages + ? arena_->New(this) + : nullptr), event_engine_( static_cast(elem->channel_data) ->hack_until_per_channel_stack_event_engines_land_get_event_engine()) { @@ -118,8 +112,8 @@ BaseCallData::~BaseCallData() { if (receive_message_ != nullptr) { receive_message_->~ReceiveMessage(); } - if (server_initial_metadata_pipe_ != nullptr) { - server_initial_metadata_pipe_->~Pipe(); + if (server_initial_metadata_latch_ != nullptr) { + server_initial_metadata_latch_->~Latch(); } }); } @@ -211,14 +205,7 @@ void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) { auto* batch = std::exchange(batch_, nullptr); GPR_ASSERT(batch != nullptr); uintptr_t& refcnt = *RefCountField(batch); - if (refcnt == 0) { - // refcnt==0 ==> cancelled - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%sRESUME BATCH REQUEST CANCELLED", - Activity::current()->DebugTag().c_str()); - } - return; - } + if (refcnt == 0) return; // refcnt==0 ==> cancelled if (--refcnt == 0) { releaser->Resume(batch); } @@ -241,10 +228,6 @@ void BaseCallData::CapturedBatch::CancelWith(grpc_error_handle error, uintptr_t& refcnt = *RefCountField(batch); if (refcnt == 0) { // refcnt==0 ==> cancelled - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%sCANCEL BATCH REQUEST ALREADY CANCELLED", - Activity::current()->DebugTag().c_str()); - } return; } refcnt = 0; @@ -355,13 +338,12 @@ void BaseCallData::SendMessage::StartOp(CapturedBatch batch) { intercepted_on_complete_ = std::exchange(batch_->on_complete, &on_complete_); } -template -void BaseCallData::SendMessage::GotPipe(T* pipe_end) { +void BaseCallData::SendMessage::GotPipe(PipeReceiver* receiver) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s SendMessage.GotPipe st=%s", base_->LogTag().c_str(), StateString(state_)); } - GPR_ASSERT(pipe_end != nullptr); + GPR_ASSERT(receiver != nullptr); switch (state_) { case State::kInitial: state_ = State::kIdle; @@ -381,7 +363,7 @@ void BaseCallData::SendMessage::GotPipe(T* pipe_end) { case State::kCancelledButNotYetPolled: return; } - interceptor_->GotPipe(pipe_end); + receiver_ = receiver; } bool BaseCallData::SendMessage::IsIdle() const { @@ -425,7 +407,6 @@ void BaseCallData::SendMessage::OnComplete(absl::Status status) { case State::kForwardedBatch: completed_status_ = status; state_ = State::kBatchCompleted; - ScopedContext ctx(base_); base_->WakeInsideCombiner(&flusher); break; } @@ -469,8 +450,7 @@ void BaseCallData::SendMessage::Done(const ServerMetadata& metadata, } } -void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher, - bool allow_push_to_pipe) { +void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s SendMessage.WakeInsideCombiner st=%s%s", base_->LogTag().c_str(), StateString(state_), @@ -482,23 +462,21 @@ void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher, case State::kInitial: case State::kIdle: case State::kGotBatchNoPipe: + case State::kForwardedBatch: case State::kCancelled: break; case State::kCancelledButNotYetPolled: - interceptor()->Push()->Close(); + pipe_.sender.Close(); state_ = State::kCancelled; break; - case State::kGotBatch: - if (allow_push_to_pipe) { - state_ = State::kPushedToPipe; - auto message = GetContext()->MakePooled(); - message->payload()->Swap(batch_->payload->send_message.send_message); - message->mutable_flags() = batch_->payload->send_message.flags; - push_ = interceptor()->Push()->Push(std::move(message)); - next_.emplace(interceptor()->Pull()->Next()); - } else { - break; - } + case State::kGotBatch: { + state_ = State::kPushedToPipe; + auto message = GetContext()->MakePooled(); + message->payload()->Swap(batch_->payload->send_message.send_message); + message->mutable_flags() = batch_->payload->send_message.flags; + push_ = pipe_.sender.Push(std::move(message)); + next_ = receiver_->Next(); + } ABSL_FALLTHROUGH_INTENDED; case State::kPushedToPipe: { GPR_ASSERT(push_.has_value()); @@ -529,25 +507,25 @@ void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher, batch_->payload->send_message.flags = (**p)->flags(); state_ = State::kForwardedBatch; batch_.ResumeWith(flusher); + next_result_ = std::move(*p); next_.reset(); - if (!absl::holds_alternative((*push_)())) push_.reset(); } } break; - case State::kForwardedBatch: - if (push_.has_value() && !absl::holds_alternative((*push_)())) { - push_.reset(); - } - break; case State::kBatchCompleted: - if (push_.has_value() && absl::holds_alternative((*push_)())) { - break; - } + next_result_.reset(); + // We've cleared out the NextResult on the pipe from promise to us, but + // there's also the pipe from us to the promise (so that the promise can + // intercept the sent messages). The push promise here is pushing into the + // latter pipe, and so we need to keep polling it until it's done, which + // depending on what happens inside the promise may take some time. + if (absl::holds_alternative((*push_)())) break; if (completed_status_.ok()) { state_ = State::kIdle; Activity::current()->ForceImmediateRepoll(); } else { state_ = State::kCancelled; } + push_.reset(); flusher->AddClosure(intercepted_on_complete_, completed_status_, "batch_completed"); break; @@ -587,8 +565,6 @@ const char* BaseCallData::ReceiveMessage::StateString(State state) { return "COMPLETED_WHILE_PULLED_FROM_PIPE"; case State::kCompletedWhilePushedToPipe: return "COMPLETED_WHILE_PUSHED_TO_PIPE"; - case State::kCompletedWhileBatchCompleted: - return "COMPLETED_WHILE_BATCH_COMPLETED"; } return "UNKNOWN"; } @@ -611,7 +587,6 @@ void BaseCallData::ReceiveMessage::StartOp(CapturedBatch& batch) { case State::kForwardedBatchNoPipe: case State::kBatchCompleted: case State::kBatchCompletedNoPipe: - case State::kCompletedWhileBatchCompleted: case State::kPushedToPipe: case State::kPulledFromPipe: case State::kCompletedWhilePulledFromPipe: @@ -631,8 +606,7 @@ void BaseCallData::ReceiveMessage::StartOp(CapturedBatch& batch) { batch->payload->recv_message.recv_message_ready, &on_complete_); } -template -void BaseCallData::ReceiveMessage::GotPipe(T* pipe_end) { +void BaseCallData::ReceiveMessage::GotPipe(PipeSender* sender) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s ReceiveMessage.GotPipe st=%s", base_->LogTag().c_str(), StateString(state_)); @@ -655,7 +629,6 @@ void BaseCallData::ReceiveMessage::GotPipe(T* pipe_end) { case State::kPulledFromPipe: case State::kCompletedWhilePulledFromPipe: case State::kCompletedWhilePushedToPipe: - case State::kCompletedWhileBatchCompleted: case State::kCancelledWhilstForwarding: case State::kCancelledWhilstIdle: case State::kBatchCompletedButCancelled: @@ -663,7 +636,7 @@ void BaseCallData::ReceiveMessage::GotPipe(T* pipe_end) { case State::kCancelled: return; } - interceptor()->GotPipe(pipe_end); + sender_ = sender; } void BaseCallData::ReceiveMessage::OnComplete(absl::Status status) { @@ -678,7 +651,6 @@ void BaseCallData::ReceiveMessage::OnComplete(absl::Status status) { case State::kPushedToPipe: case State::kPulledFromPipe: case State::kBatchCompleted: - case State::kCompletedWhileBatchCompleted: case State::kBatchCompletedNoPipe: case State::kCancelled: case State::kBatchCompletedButCancelled: @@ -720,10 +692,6 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, case State::kForwardedBatchNoPipe: state_ = State::kCancelledWhilstForwarding; break; - case State::kCompletedWhileBatchCompleted: - case State::kBatchCompleted: - state_ = State::kCompletedWhileBatchCompleted; - break; case State::kCompletedWhilePulledFromPipe: case State::kCompletedWhilePushedToPipe: case State::kPulledFromPipe: @@ -745,6 +713,7 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, state_ = State::kCancelled; } } break; + case State::kBatchCompleted: case State::kBatchCompletedNoPipe: case State::kBatchCompletedButCancelled: Crash(absl::StrFormat("ILLEGAL STATE: %s", StateString(state_))); @@ -755,15 +724,12 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, } } -void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, - bool allow_push_to_pipe) { +void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, - "%s ReceiveMessage.WakeInsideCombiner st=%s push?=%s next?=%s " - "allow_push_to_pipe=%s", + "%s ReceiveMessage.WakeInsideCombiner st=%s push?=%s next?=%s", base_->LogTag().c_str(), StateString(state_), - push_.has_value() ? "yes" : "no", next_.has_value() ? "yes" : "no", - allow_push_to_pipe ? "yes" : "no"); + push_.has_value() ? "yes" : "no", next_.has_value() ? "yes" : "no"); } switch (state_) { case State::kInitial: @@ -775,38 +741,31 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, case State::kBatchCompletedNoPipe: break; case State::kCancelledWhilstIdle: - interceptor_->Push()->Close(); + sender_->Close(); state_ = State::kCancelled; break; case State::kBatchCompletedButCancelled: - case State::kCompletedWhileBatchCompleted: - interceptor()->Push()->Close(); + sender_->Close(); state_ = State::kCancelled; flusher->AddClosure(std::exchange(intercepted_on_complete_, nullptr), completed_status_, "recv_message"); break; case State::kBatchCompleted: if (completed_status_.ok() && intercepted_slice_buffer_->has_value()) { - if (!allow_push_to_pipe) break; - if (state_ == State::kBatchCompleted) { - state_ = State::kPushedToPipe; - } else { - state_ = State::kCompletedWhilePushedToPipe; - } + state_ = State::kPushedToPipe; auto message = GetContext()->MakePooled(); message->payload()->Swap(&**intercepted_slice_buffer_); message->mutable_flags() = *intercepted_flags_; - push_ = interceptor()->Push()->Push(std::move(message)); - next_.emplace(interceptor()->Pull()->Next()); + push_ = sender_->Push(std::move(message)); + next_ = pipe_.receiver.Next(); } else { - interceptor()->Push()->Close(); + sender_->Close(); state_ = State::kCancelled; flusher->AddClosure(std::exchange(intercepted_on_complete_, nullptr), completed_status_, "recv_message"); break; } - GPR_ASSERT(state_ == State::kPushedToPipe || - state_ == State::kCompletedWhilePushedToPipe); + GPR_ASSERT(state_ == State::kPushedToPipe); ABSL_FALLTHROUGH_INTENDED; case State::kCompletedWhilePushedToPipe: case State::kPushedToPipe: { @@ -826,6 +785,12 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, GPR_ASSERT(next_.has_value()); auto r_next = (*next_)(); if (auto* p = absl::get_if>(&r_next)) { + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, + "%s ReceiveMessage.WakeInsideCombiner next complete: %s", + base_->LogTag().c_str(), + p->has_value() ? "got message" : "end of stream"); + } next_.reset(); if (p->has_value()) { *intercepted_slice_buffer_ = std::move(*(**p)->payload()); @@ -839,18 +804,6 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, *intercepted_slice_buffer_ = absl::nullopt; *intercepted_flags_ = 0; state_ = State::kCancelled; - flusher->AddClosure( - std::exchange(intercepted_on_complete_, nullptr), - p->cancelled() ? absl::CancelledError() : absl::OkStatus(), - "recv_message"); - } - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, - "%s ReceiveMessage.WakeInsideCombiner next complete: %s " - "new_state=%s", - base_->LogTag().c_str(), - p->has_value() ? "got message" : "end of stream", - StateString(state_)); } } } @@ -869,7 +822,7 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, base_->LogTag().c_str()); } if (state_ == State::kCompletedWhilePulledFromPipe) { - interceptor()->Push()->Close(); + sender_->Close(); state_ = State::kCancelled; } else { state_ = State::kIdle; @@ -891,76 +844,56 @@ struct ClientCallData::RecvInitialMetadata final { // Initial state; no op seen kInitial, // No op seen, but we have a latch that would like to modify it when we do - kGotPipe, + kGotLatch, // Responded to trailing metadata prior to getting a recv_initial_metadata kRespondedToTrailingMetadataPriorToHook, // Hooked, no latch yet - kHookedWaitingForPipe, + kHookedWaitingForLatch, // Hooked, latch seen - kHookedAndGotPipe, + kHookedAndGotLatch, // Got the callback, haven't set latch yet - kCompleteWaitingForPipe, + kCompleteWaitingForLatch, // Got the callback and got the latch - kCompleteAndGotPipe, + kCompleteAndGotLatch, // Got the callback and set the latch - kCompleteAndPushedToPipe, + kCompleteAndSetLatch, // Called the original callback kResponded, // Called the original callback with an error: still need to set the latch - kRespondedButNeedToClosePipe, + kRespondedButNeedToSetLatch, }; State state = kInitial; grpc_closure* original_on_ready = nullptr; grpc_closure on_ready; grpc_metadata_batch* metadata = nullptr; - PipeSender* server_initial_metadata_publisher = nullptr; - absl::optional::PushType> metadata_push_; - absl::optional> metadata_next_; + Latch* server_initial_metadata_publisher = nullptr; static const char* StateString(State state) { switch (state) { case kInitial: return "INITIAL"; - case kGotPipe: - return "GOT_PIPE"; + case kGotLatch: + return "GOT_LATCH"; case kRespondedToTrailingMetadataPriorToHook: return "RESPONDED_TO_TRAILING_METADATA_PRIOR_TO_HOOK"; - case kHookedWaitingForPipe: - return "HOOKED_WAITING_FOR_PIPE"; - case kHookedAndGotPipe: - return "HOOKED_AND_GOT_PIPE"; - case kCompleteWaitingForPipe: - return "COMPLETE_WAITING_FOR_PIPE"; - case kCompleteAndGotPipe: - return "COMPLETE_AND_GOT_PIPE"; - case kCompleteAndPushedToPipe: - return "COMPLETE_AND_PUSHED_TO_PIPE"; + case kHookedWaitingForLatch: + return "HOOKED_WAITING_FOR_LATCH"; + case kHookedAndGotLatch: + return "HOOKED_AND_GOT_LATCH"; + case kCompleteWaitingForLatch: + return "COMPLETE_WAITING_FOR_LATCH"; + case kCompleteAndGotLatch: + return "COMPLETE_AND_GOT_LATCH"; + case kCompleteAndSetLatch: + return "COMPLETE_AND_SET_LATCH"; case kResponded: return "RESPONDED"; - case kRespondedButNeedToClosePipe: - return "RESPONDED_BUT_NEED_TO_CLOSE_PIPE"; + case kRespondedButNeedToSetLatch: + return "RESPONDED_BUT_NEED_TO_SET_LATCH"; } return "UNKNOWN"; } - - bool AllowRecvMessage() const { - switch (state) { - case kInitial: - case kGotPipe: - case kHookedWaitingForPipe: - case kHookedAndGotPipe: - case kCompleteWaitingForPipe: - case kCompleteAndGotPipe: - case kCompleteAndPushedToPipe: - case kRespondedToTrailingMetadataPriorToHook: - return false; - case kResponded: - case kRespondedButNeedToClosePipe: - return true; - } - GPR_UNREACHABLE_CODE(return false); - } }; class ClientCallData::PollContext { @@ -978,7 +911,6 @@ class ClientCallData::PollContext { PollContext& operator=(const PollContext&) = delete; void Run() { - GPR_DEBUG_ASSERT(HasContext()); if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s ClientCallData.PollContext.Run %s", self_->LogTag().c_str(), self_->DebugString().c_str()); @@ -986,69 +918,43 @@ class ClientCallData::PollContext { GPR_ASSERT(have_scoped_activity_); repoll_ = false; if (self_->send_message() != nullptr) { - self_->send_message()->WakeInsideCombiner(flusher_, true); + self_->send_message()->WakeInsideCombiner(flusher_); } if (self_->receive_message() != nullptr) { - self_->receive_message()->WakeInsideCombiner( - flusher_, self_->recv_initial_metadata_ == nullptr - ? true - : self_->recv_initial_metadata_->AllowRecvMessage()); + self_->receive_message()->WakeInsideCombiner(flusher_); } - if (self_->server_initial_metadata_pipe() != nullptr) { - if (self_->recv_initial_metadata_->metadata_push_.has_value()) { - if (!absl::holds_alternative( - (*self_->recv_initial_metadata_->metadata_push_)())) { - self_->recv_initial_metadata_->metadata_push_.reset(); - } - } + if (self_->server_initial_metadata_latch() != nullptr) { switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: - case RecvInitialMetadata::kHookedWaitingForPipe: - case RecvInitialMetadata::kHookedAndGotPipe: - case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kCompleteWaitingForLatch: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: break; - case RecvInitialMetadata::kRespondedButNeedToClosePipe: - self_->recv_initial_metadata_->server_initial_metadata_publisher - ->Close(); + case RecvInitialMetadata::kRespondedButNeedToSetLatch: + self_->recv_initial_metadata_->server_initial_metadata_publisher->Set( + nullptr); self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; break; - case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndGotLatch: self_->recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndPushedToPipe; - GPR_ASSERT( - !self_->recv_initial_metadata_->metadata_push_.has_value()); - GPR_ASSERT( - !self_->recv_initial_metadata_->metadata_next_.has_value()); - self_->recv_initial_metadata_->metadata_push_.emplace( - self_->recv_initial_metadata_->server_initial_metadata_publisher - ->Push(ServerMetadataHandle( - self_->recv_initial_metadata_->metadata, - Arena::PooledDeleter(nullptr)))); - repoll_ = true; // ensure Push() gets polled. - self_->recv_initial_metadata_->metadata_next_.emplace( - self_->server_initial_metadata_pipe()->receiver.Next()); + RecvInitialMetadata::kCompleteAndSetLatch; + self_->recv_initial_metadata_->server_initial_metadata_publisher->Set( + self_->recv_initial_metadata_->metadata); ABSL_FALLTHROUGH_INTENDED; - case RecvInitialMetadata::kCompleteAndPushedToPipe: { - GPR_ASSERT(self_->recv_initial_metadata_->metadata_next_.has_value()); - Poll> p = - (*self_->recv_initial_metadata_->metadata_next_)(); - if (NextResult* nr = - absl::get_if(&p)) { - if (nr->has_value()) { - ServerMetadataHandle md = std::move(nr->value()); - if (self_->recv_initial_metadata_->metadata != md.get()) { - *self_->recv_initial_metadata_->metadata = std::move(*md); - } - } else { - self_->recv_initial_metadata_->metadata->Clear(); + case RecvInitialMetadata::kCompleteAndSetLatch: { + Poll p = + self_->server_initial_metadata_latch()->Wait()(); + if (ServerMetadata*** ppp = absl::get_if(&p)) { + ServerMetadata* md = **ppp; + if (self_->recv_initial_metadata_->metadata != md) { + *self_->recv_initial_metadata_->metadata = std::move(*md); } self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; - repoll_ = true; flusher_->AddClosure( std::exchange(self_->recv_initial_metadata_->original_on_ready, nullptr), @@ -1094,25 +1000,25 @@ class ClientCallData::PollContext { if (self_->recv_initial_metadata_ != nullptr) { switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kGotLatch: self_->recv_initial_metadata_->state = RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook; break; case RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat("ILLEGAL STATE: %s", RecvInitialMetadata::StateString( self_->recv_initial_metadata_ ->state))); // not reachable break; - case RecvInitialMetadata::kHookedWaitingForPipe: - case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedAndGotLatch: case RecvInitialMetadata::kResponded: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: break; - case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteWaitingForLatch: self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; flusher_->AddClosure( @@ -1129,25 +1035,25 @@ class ClientCallData::PollContext { if (self_->recv_initial_metadata_ != nullptr) { switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kGotLatch: self_->recv_initial_metadata_->state = RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook; break; - case RecvInitialMetadata::kHookedWaitingForPipe: - case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedAndGotLatch: case RecvInitialMetadata::kResponded: break; case RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat("ILLEGAL STATE: %s", RecvInitialMetadata::StateString( self_->recv_initial_metadata_ ->state))); // not reachable break; - case RecvInitialMetadata::kCompleteWaitingForPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; flusher_->AddClosure( @@ -1212,7 +1118,6 @@ class ClientCallData::PollContext { auto run = [](void* p, grpc_error_handle) { auto* next_poll = static_cast(p); { - ScopedContext ctx(next_poll->call_data); Flusher flusher(next_poll->call_data); next_poll->call_data->WakeInsideCombiner(&flusher); } @@ -1247,16 +1152,11 @@ class ClientCallData::PollContext { ClientCallData::ClientCallData(grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags) - : BaseCallData( - elem, args, flags, - [args]() { - return args->arena->New(args->arena); - }, - [args]() { return args->arena->New(args->arena); }) { + : BaseCallData(elem, args, flags) { GRPC_CLOSURE_INIT(&recv_trailing_metadata_ready_, RecvTrailingMetadataReadyCallback, this, grpc_schedule_on_exec_ctx); - if (server_initial_metadata_pipe() != nullptr) { + if (server_initial_metadata_latch() != nullptr) { recv_initial_metadata_ = arena()->New(); } } @@ -1319,7 +1219,7 @@ std::string ClientCallData::DebugString() const { " sent_initial_state=", StateString(send_initial_state_), " recv_trailing_state=", StateString(recv_trailing_state_), " captured={", absl::StrJoin(captured, ","), "}", - server_initial_metadata_pipe() == nullptr + server_initial_metadata_latch() == nullptr ? "" : absl::StrCat(" recv_initial_metadata=", RecvInitialMetadata::StateString( @@ -1361,21 +1261,21 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) { switch (recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: recv_initial_metadata_->state = - RecvInitialMetadata::kHookedWaitingForPipe; + RecvInitialMetadata::kHookedWaitingForLatch; break; - case RecvInitialMetadata::kGotPipe: - recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotPipe; + case RecvInitialMetadata::kGotLatch: + recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch; break; case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: hook = false; break; - case RecvInitialMetadata::kHookedWaitingForPipe: - case RecvInitialMetadata::kHookedAndGotPipe: - case RecvInitialMetadata::kCompleteWaitingForPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: case RecvInitialMetadata::kResponded: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1479,9 +1379,9 @@ void ClientCallData::Cancel(grpc_error_handle error, Flusher* flusher) { } if (recv_initial_metadata_ != nullptr) { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kCompleteWaitingForPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: recv_initial_metadata_->state = RecvInitialMetadata::kResponded; GRPC_CALL_COMBINER_START( call_combiner(), @@ -1489,13 +1389,13 @@ void ClientCallData::Cancel(grpc_error_handle error, Flusher* flusher) { error, "propagate cancellation"); break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kGotLatch: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kHookedWaitingForPipe: - case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedAndGotLatch: case RecvInitialMetadata::kResponded: break; - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString(recv_initial_metadata_->state))); @@ -1521,15 +1421,8 @@ void ClientCallData::StartPromise(Flusher* flusher) { promise_ = filter->MakeCallPromise( CallArgs{WrapMetadata(send_initial_metadata_batch_->payload ->send_initial_metadata.send_initial_metadata), - server_initial_metadata_pipe() == nullptr - ? nullptr - : &server_initial_metadata_pipe()->sender, - send_message() == nullptr - ? nullptr - : send_message()->interceptor()->original_receiver(), - receive_message() == nullptr - ? nullptr - : receive_message()->interceptor()->original_sender()}, + server_initial_metadata_latch(), outgoing_messages_pipe(), + incoming_messages_pipe()}, [this](CallArgs call_args) { return MakeNextPromise(std::move(call_args)); }); @@ -1538,30 +1431,28 @@ void ClientCallData::StartPromise(Flusher* flusher) { void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) { if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, - "%s ClientCallData.RecvInitialMetadataReady %s error:%s md:%s", - LogTag().c_str(), DebugString().c_str(), error.ToString().c_str(), - recv_initial_metadata_->metadata->DebugString().c_str()); + gpr_log(GPR_INFO, "%s ClientCallData.RecvInitialMetadataReady %s", + LogTag().c_str(), DebugString().c_str()); } ScopedContext context(this); Flusher flusher(this); if (!error.ok()) { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: recv_initial_metadata_->state = RecvInitialMetadata::kResponded; break; - case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kHookedAndGotLatch: recv_initial_metadata_->state = - RecvInitialMetadata::kRespondedButNeedToClosePipe; + RecvInitialMetadata::kRespondedButNeedToSetLatch; break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: - case RecvInitialMetadata::kCompleteWaitingForPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1578,22 +1469,22 @@ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) { cancelled_error_, "propagate cancellation"); } else { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedWaitingForLatch: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteWaitingForPipe; + RecvInitialMetadata::kCompleteWaitingForLatch; break; - case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kHookedAndGotLatch: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndGotPipe; + RecvInitialMetadata::kCompleteAndGotLatch; break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotPipe: - case RecvInitialMetadata::kCompleteWaitingForPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1640,24 +1531,24 @@ ArenaPromise ClientCallData::MakeNextPromise( call_args.server_initial_metadata; switch (recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - recv_initial_metadata_->state = RecvInitialMetadata::kGotPipe; + recv_initial_metadata_->state = RecvInitialMetadata::kGotLatch; break; - case RecvInitialMetadata::kHookedWaitingForPipe: - recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotPipe; + case RecvInitialMetadata::kHookedWaitingForLatch: + recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch; poll_ctx_->Repoll(); break; - case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteWaitingForLatch: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndGotPipe; + RecvInitialMetadata::kCompleteAndGotLatch; poll_ctx_->Repoll(); break; - case RecvInitialMetadata::kGotPipe: - case RecvInitialMetadata::kHookedAndGotPipe: - case RecvInitialMetadata::kCompleteAndGotPipe: - case RecvInitialMetadata::kCompleteAndPushedToPipe: + case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndSetLatch: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToClosePipe: + case RecvInitialMetadata::kRespondedButNeedToSetLatch: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1667,14 +1558,14 @@ ArenaPromise ClientCallData::MakeNextPromise( GPR_ASSERT(call_args.server_initial_metadata == nullptr); } if (send_message() != nullptr) { - send_message()->GotPipe(call_args.client_to_server_messages); + send_message()->GotPipe(call_args.outgoing_messages); } else { - GPR_ASSERT(call_args.client_to_server_messages == nullptr); + GPR_ASSERT(call_args.outgoing_messages == nullptr); } if (receive_message() != nullptr) { - receive_message()->GotPipe(call_args.server_to_client_messages); + receive_message()->GotPipe(call_args.incoming_messages); } else { - GPR_ASSERT(call_args.server_to_client_messages == nullptr); + GPR_ASSERT(call_args.incoming_messages == nullptr); } return ArenaPromise( [this]() { return PollTrailingMetadata(); }); @@ -1805,31 +1696,29 @@ void ClientCallData::OnWakeup() { struct ServerCallData::SendInitialMetadata { enum State { kInitial, - kGotPipe, - kQueuedWaitingForPipe, - kQueuedAndGotPipe, - kQueuedAndPushedToPipe, + kGotLatch, + kQueuedWaitingForLatch, + kQueuedAndGotLatch, + kQueuedAndSetLatch, kForwarded, kCancelled, }; State state = kInitial; CapturedBatch batch; - PipeSender* server_initial_metadata_publisher = nullptr; - absl::optional::PushType> metadata_push_; - absl::optional> metadata_next_; + Latch* server_initial_metadata_publisher = nullptr; static const char* StateString(State state) { switch (state) { case kInitial: return "INITIAL"; - case kGotPipe: - return "GOT_PIPE"; - case kQueuedWaitingForPipe: - return "QUEUED_WAITING_FOR_PIPE"; - case kQueuedAndGotPipe: - return "QUEUED_AND_GOT_PIPE"; - case kQueuedAndPushedToPipe: - return "QUEUED_AND_PUSHED_TO_PIPE"; + case kGotLatch: + return "GOT_LATCH"; + case kQueuedWaitingForLatch: + return "QUEUED_WAITING_FOR_LATCH"; + case kQueuedAndGotLatch: + return "QUEUED_AND_GOT_LATCH"; + case kQueuedAndSetLatch: + return "QUEUED_AND_SET_LATCH"; case kForwarded: return "FORWARDED"; case kCancelled: @@ -1864,7 +1753,6 @@ class ServerCallData::PollContext { auto* next_poll = static_cast(p); { Flusher flusher(next_poll->call_data); - ScopedContext context(next_poll->call_data); next_poll->call_data->WakeInsideCombiner(&flusher); } GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll"); @@ -1912,8 +1800,6 @@ const char* ServerCallData::StateString(SendTrailingState state) { return "FORWARDED"; case SendTrailingState::kQueuedBehindSendMessage: return "QUEUED_BEHIND_SEND_MESSAGE"; - case SendTrailingState::kQueuedButHaventClosedSends: - return "QUEUED_BUT_HAVENT_CLOSED_SENDS"; case SendTrailingState::kQueued: return "QUEUED"; case SendTrailingState::kCancelled: @@ -1925,13 +1811,8 @@ const char* ServerCallData::StateString(SendTrailingState state) { ServerCallData::ServerCallData(grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags) - : BaseCallData( - elem, args, flags, - [args]() { return args->arena->New(args->arena); }, - [args]() { - return args->arena->New(args->arena); - }) { - if (server_initial_metadata_pipe() != nullptr) { + : BaseCallData(elem, args, flags) { + if (server_initial_metadata_latch() != nullptr) { send_initial_metadata_ = arena()->New(); } GRPC_CLOSURE_INIT(&recv_initial_metadata_ready_, @@ -2019,19 +1900,19 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: send_initial_metadata_->state = - SendInitialMetadata::kQueuedWaitingForPipe; + SendInitialMetadata::kQueuedWaitingForLatch; break; - case SendInitialMetadata::kGotPipe: - send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotPipe; + case SendInitialMetadata::kGotLatch: + send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch; break; case SendInitialMetadata::kCancelled: batch.CancelWith( cancelled_error_.ok() ? absl::CancelledError() : cancelled_error_, &flusher); break; - case SendInitialMetadata::kQueuedAndGotPipe: - case SendInitialMetadata::kQueuedWaitingForPipe: - case SendInitialMetadata::kQueuedAndPushedToPipe: + case SendInitialMetadata::kQueuedAndGotLatch: + case SendInitialMetadata::kQueuedWaitingForLatch: + case SendInitialMetadata::kQueuedAndSetLatch: case SendInitialMetadata::kForwarded: Crash(absl::StrFormat( "ILLEGAL STATE: %s", @@ -2064,9 +1945,6 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { } if (send_message() != nullptr && !send_message()->IsIdle()) { send_trailing_state_ = SendTrailingState::kQueuedBehindSendMessage; - } else if (send_message() != nullptr) { - send_trailing_state_ = SendTrailingState::kQueuedButHaventClosedSends; - wake = true; } else { send_trailing_state_ = SendTrailingState::kQueued; wake = true; @@ -2074,7 +1952,6 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { break; case SendTrailingState::kQueued: case SendTrailingState::kQueuedBehindSendMessage: - case SendTrailingState::kQueuedButHaventClosedSends: case SendTrailingState::kForwarded: Crash( absl::StrFormat("ILLEGAL STATE: %s", @@ -2094,55 +1971,26 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { // Handle cancellation. void ServerCallData::Completed(grpc_error_handle error, Flusher* flusher) { - if (grpc_trace_channel.enabled()) { - gpr_log( - GPR_DEBUG, - "%sServerCallData::Completed: send_trailing_state=%s " - "send_initial_state=%s error=%s", - LogTag().c_str(), StateString(send_trailing_state_), - send_initial_metadata_ == nullptr - ? "null" - : SendInitialMetadata::StateString(send_initial_metadata_->state), - error.ToString().c_str()); - } // Track the latest reason for cancellation. cancelled_error_ = error; // Stop running the promise. promise_ = ArenaPromise(); - switch (send_trailing_state_) { - case SendTrailingState::kInitial: - case SendTrailingState::kForwarded: - send_trailing_state_ = SendTrailingState::kCancelled; - if (!error.ok()) { - auto* batch = grpc_make_transport_stream_op( - NewClosure([call_combiner = call_combiner()](absl::Status) { - GRPC_CALL_COMBINER_STOP(call_combiner, "done-cancel"); - })); - batch->cancel_stream = true; - batch->payload->cancel_stream.cancel_error = error; - flusher->Resume(batch); - } - break; - case SendTrailingState::kQueued: - send_trailing_state_ = SendTrailingState::kCancelled; - send_trailing_metadata_batch_.CancelWith(error, flusher); - break; - case SendTrailingState::kQueuedBehindSendMessage: - case SendTrailingState::kQueuedButHaventClosedSends: - case SendTrailingState::kCancelled: - send_trailing_state_ = SendTrailingState::kCancelled; - break; + if (send_trailing_state_ == SendTrailingState::kQueued) { + send_trailing_state_ = SendTrailingState::kCancelled; + send_trailing_metadata_batch_.CancelWith(error, flusher); + } else { + send_trailing_state_ = SendTrailingState::kCancelled; } if (send_initial_metadata_ != nullptr) { switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: - case SendInitialMetadata::kGotPipe: + case SendInitialMetadata::kGotLatch: case SendInitialMetadata::kForwarded: case SendInitialMetadata::kCancelled: break; - case SendInitialMetadata::kQueuedWaitingForPipe: - case SendInitialMetadata::kQueuedAndGotPipe: - case SendInitialMetadata::kQueuedAndPushedToPipe: + case SendInitialMetadata::kQueuedWaitingForLatch: + case SendInitialMetadata::kQueuedAndGotLatch: + case SendInitialMetadata::kQueuedAndSetLatch: send_initial_metadata_->batch.CancelWith(error, flusher); break; } @@ -2179,19 +2027,19 @@ ArenaPromise ServerCallData::MakeNextPromise( call_args.server_initial_metadata; switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: - send_initial_metadata_->state = SendInitialMetadata::kGotPipe; + send_initial_metadata_->state = SendInitialMetadata::kGotLatch; break; - case SendInitialMetadata::kGotPipe: - case SendInitialMetadata::kQueuedAndGotPipe: - case SendInitialMetadata::kQueuedAndPushedToPipe: + case SendInitialMetadata::kGotLatch: + case SendInitialMetadata::kQueuedAndGotLatch: + case SendInitialMetadata::kQueuedAndSetLatch: case SendInitialMetadata::kForwarded: Crash(absl::StrFormat( "ILLEGAL STATE: %s", SendInitialMetadata::StateString( send_initial_metadata_->state))); // not reachable break; - case SendInitialMetadata::kQueuedWaitingForPipe: - send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotPipe; + case SendInitialMetadata::kQueuedWaitingForLatch: + send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch; break; case SendInitialMetadata::kCancelled: break; @@ -2200,14 +2048,14 @@ ArenaPromise ServerCallData::MakeNextPromise( GPR_ASSERT(call_args.server_initial_metadata == nullptr); } if (send_message() != nullptr) { - send_message()->GotPipe(call_args.server_to_client_messages); + send_message()->GotPipe(call_args.outgoing_messages); } else { - GPR_ASSERT(call_args.server_to_client_messages == nullptr); + GPR_ASSERT(call_args.outgoing_messages == nullptr); } if (receive_message() != nullptr) { - receive_message()->GotPipe(call_args.client_to_server_messages); + receive_message()->GotPipe(call_args.incoming_messages); } else { - GPR_ASSERT(call_args.client_to_server_messages == nullptr); + GPR_ASSERT(call_args.incoming_messages == nullptr); } return ArenaPromise( [this]() { return PollTrailingMetadata(); }); @@ -2217,14 +2065,9 @@ ArenaPromise ServerCallData::MakeNextPromise( // All polls: await sending the trailing metadata, then foward it down the // stack. Poll ServerCallData::PollTrailingMetadata() { - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%s PollTrailingMetadata: %s", LogTag().c_str(), - StateString(send_trailing_state_)); - } switch (send_trailing_state_) { case SendTrailingState::kInitial: case SendTrailingState::kQueuedBehindSendMessage: - case SendTrailingState::kQueuedButHaventClosedSends: return Pending{}; case SendTrailingState::kQueued: return WrapMetadata(send_trailing_metadata_batch_->payload @@ -2290,15 +2133,8 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) { FakeActivity(this).Run([this, filter] { promise_ = filter->MakeCallPromise( CallArgs{WrapMetadata(recv_initial_metadata_), - server_initial_metadata_pipe() == nullptr - ? nullptr - : &server_initial_metadata_pipe()->sender, - receive_message() == nullptr - ? nullptr - : receive_message()->interceptor()->original_receiver(), - send_message() == nullptr - ? nullptr - : send_message()->interceptor()->original_sender()}, + server_initial_metadata_latch(), outgoing_messages_pipe(), + incoming_messages_pipe()}, [this](CallArgs call_args) { return MakeNextPromise(std::move(call_args)); }); @@ -2335,110 +2171,46 @@ void ServerCallData::WakeInsideCombiner(Flusher* flusher) { gpr_log(GPR_INFO, "%s: WakeInsideCombiner %s", LogTag().c_str(), DebugString().c_str()); } - poll_ctx.ClearRepoll(); - if (send_initial_metadata_ != nullptr) { - if (send_initial_metadata_->state == - SendInitialMetadata::kQueuedAndGotPipe) { - send_initial_metadata_->state = - SendInitialMetadata::kQueuedAndPushedToPipe; - GPR_ASSERT(!send_initial_metadata_->metadata_push_.has_value()); - GPR_ASSERT(!send_initial_metadata_->metadata_next_.has_value()); - send_initial_metadata_->metadata_push_.emplace( - send_initial_metadata_->server_initial_metadata_publisher->Push( - ServerMetadataHandle( - send_initial_metadata_->batch->payload->send_initial_metadata - .send_initial_metadata, - Arena::PooledDeleter(nullptr)))); - send_initial_metadata_->metadata_next_.emplace( - server_initial_metadata_pipe()->receiver.Next()); - } - if (send_initial_metadata_->metadata_push_.has_value()) { - if (!absl::holds_alternative( - (*send_initial_metadata_->metadata_push_)())) { - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%s: WakeInsideCombiner: metadata_push done", - LogTag().c_str()); - } - send_initial_metadata_->metadata_push_.reset(); - } else if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%s: WakeInsideCombiner: metadata_push pending", - LogTag().c_str()); - } - } + if (send_initial_metadata_ != nullptr && + send_initial_metadata_->state == + SendInitialMetadata::kQueuedAndGotLatch) { + send_initial_metadata_->state = SendInitialMetadata::kQueuedAndSetLatch; + send_initial_metadata_->server_initial_metadata_publisher->Set( + send_initial_metadata_->batch->payload->send_initial_metadata + .send_initial_metadata); } + poll_ctx.ClearRepoll(); if (send_message() != nullptr) { - if (send_trailing_state_ == - SendTrailingState::kQueuedButHaventClosedSends) { - send_trailing_state_ = SendTrailingState::kQueued; - send_message()->Done(*send_trailing_metadata_batch_->payload - ->send_trailing_metadata.send_trailing_metadata, - flusher); - } - send_message()->WakeInsideCombiner( - flusher, - send_initial_metadata_ == nullptr || - send_initial_metadata_->state == SendInitialMetadata::kForwarded); - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_DEBUG, - "%s: After send_message WakeInsideCombiner %s is_idle=%s " - "is_forwarded=%s", - LogTag().c_str(), DebugString().c_str(), - send_message()->IsIdle() ? "true" : "false", - send_message()->IsForwarded() ? "true" : "false"); - } + send_message()->WakeInsideCombiner(flusher); if (send_trailing_state_ == SendTrailingState::kQueuedBehindSendMessage && - (send_message()->IsIdle() || - (send_trailing_metadata_batch_->send_message && - send_message()->IsForwarded()))) { + send_message()->IsIdle()) { send_trailing_state_ = SendTrailingState::kQueued; - send_message()->Done(*send_trailing_metadata_batch_->payload - ->send_trailing_metadata.send_trailing_metadata, - flusher); } } if (receive_message() != nullptr) { - receive_message()->WakeInsideCombiner(flusher, true); + receive_message()->WakeInsideCombiner(flusher); } if (promise_.has_value()) { Poll poll; poll = promise_(); if (grpc_trace_channel.enabled()) { - gpr_log( - GPR_INFO, - "%s: WakeInsideCombiner poll=%s; send_initial_metadata=%s " - "send_trailing_metadata=%s", - LogTag().c_str(), - PollToString( - poll, - [](const ServerMetadataHandle& h) { return h->DebugString(); }) - .c_str(), - send_initial_metadata_ == nullptr - ? "null" - : SendInitialMetadata::StateString(send_initial_metadata_->state), - StateString(send_trailing_state_)); + gpr_log(GPR_INFO, "%s: WakeInsideCombiner poll=%s", LogTag().c_str(), + PollToString(poll, [](const ServerMetadataHandle& h) { + return h->DebugString(); + }).c_str()); } if (send_initial_metadata_ != nullptr && send_initial_metadata_->state == - SendInitialMetadata::kQueuedAndPushedToPipe) { - GPR_ASSERT(send_initial_metadata_->metadata_next_.has_value()); - auto p = (*send_initial_metadata_->metadata_next_)(); - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, - "%s: WakeInsideCombiner send_initial_metadata poll=%s", - LogTag().c_str(), - PollToString(p, [](const NextResult& h) { - return (*h)->DebugString(); - }).c_str()); - } - if (auto* nr = absl::get_if>(&p)) { - ServerMetadataHandle md = std::move(nr->value()); + SendInitialMetadata::kQueuedAndSetLatch) { + Poll p = server_initial_metadata_latch()->Wait()(); + if (ServerMetadata*** ppp = absl::get_if(&p)) { + ServerMetadata* md = **ppp; if (send_initial_metadata_->batch->payload->send_initial_metadata - .send_initial_metadata != md.get()) { + .send_initial_metadata != md) { *send_initial_metadata_->batch->payload->send_initial_metadata .send_initial_metadata = std::move(*md); } send_initial_metadata_->state = SendInitialMetadata::kForwarded; - poll_ctx.Repoll(); send_initial_metadata_->batch.ResumeWith(flusher); } } @@ -2454,7 +2226,6 @@ void ServerCallData::WakeInsideCombiner(Flusher* flusher) { } switch (send_trailing_state_) { case SendTrailingState::kQueuedBehindSendMessage: - case SendTrailingState::kQueuedButHaventClosedSends: case SendTrailingState::kQueued: { if (send_trailing_metadata_batch_->payload->send_trailing_metadata .send_trailing_metadata != md) { diff --git a/src/core/lib/channel/promise_based_filter.h b/src/core/lib/channel/promise_based_filter.h index 0c604390b7b..57fa2608a87 100644 --- a/src/core/lib/channel/promise_based_filter.h +++ b/src/core/lib/channel/promise_based_filter.h @@ -19,13 +19,15 @@ // promise-style. Most of this will be removed once the promises conversion is // completed. +// TODO(ctiller): When removing this file, also reduce the number of *'s on the +// server initial metadata latch. + #include #include #include #include -#include #include #include #include @@ -33,7 +35,6 @@ #include #include "absl/container/inlined_vector.h" -#include "absl/functional/function_ref.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" @@ -59,6 +60,7 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" @@ -150,27 +152,9 @@ class InvalidChannelFilter : public ChannelFilter { // Call data shared between all implementations of promise-based filters. class BaseCallData : public Activity, private Wakeable { - protected: - // Hook to allow interception of messages on the send/receive path by - // PipeSender and PipeReceiver, as appropriate according to whether we're - // client or server. - class Interceptor { - public: - virtual PipeSender* Push() = 0; - virtual PipeReceiver* Pull() = 0; - virtual PipeReceiver* original_receiver() = 0; - virtual PipeSender* original_sender() = 0; - virtual void GotPipe(PipeSender*) = 0; - virtual void GotPipe(PipeReceiver*) = 0; - virtual ~Interceptor() = default; - }; - - BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args, - uint8_t flags, - absl::FunctionRef make_send_interceptor, - absl::FunctionRef make_recv_interceptor); - public: + BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args, + uint8_t flags); ~BaseCallData() override; void set_pollent(grpc_polling_entity* pollent) { @@ -286,54 +270,6 @@ class BaseCallData : public Activity, private Wakeable { return p.release(); } - class ReceiveInterceptor final : public Interceptor { - public: - explicit ReceiveInterceptor(Arena* arena) : pipe_{arena} {} - - PipeReceiver* original_receiver() override { - return &pipe_.receiver; - } - PipeSender* original_sender() override { abort(); } - - void GotPipe(PipeReceiver* receiver) override { - GPR_ASSERT(receiver_ == nullptr); - receiver_ = receiver; - } - - void GotPipe(PipeSender*) override { abort(); } - - PipeSender* Push() override { return &pipe_.sender; } - PipeReceiver* Pull() override { return receiver_; } - - private: - Pipe pipe_; - PipeReceiver* receiver_ = nullptr; - }; - - class SendInterceptor final : public Interceptor { - public: - explicit SendInterceptor(Arena* arena) : pipe_{arena} {} - - PipeReceiver* original_receiver() override { abort(); } - PipeSender* original_sender() override { - return &pipe_.sender; - } - - void GotPipe(PipeReceiver*) override { abort(); } - - void GotPipe(PipeSender* sender) override { - GPR_ASSERT(sender_ == nullptr); - sender_ = sender; - } - - PipeSender* Push() override { return sender_; } - PipeReceiver* Pull() override { return &pipe_.receiver; } - - private: - Pipe pipe_; - PipeSender* sender_ = nullptr; - }; - // State machine for sending messages: handles intercepting send_message ops // and forwarding them through pipes to the promise, then getting the result // down the stack. @@ -341,11 +277,9 @@ class BaseCallData : public Activity, private Wakeable { // these members for filters that don't need to intercept sent messages. class SendMessage { public: - SendMessage(BaseCallData* base, Interceptor* interceptor) - : base_(base), interceptor_(interceptor) {} - ~SendMessage() { interceptor_->~Interceptor(); } - - Interceptor* interceptor() { return interceptor_; } + explicit SendMessage(BaseCallData* base) + : base_(base), pipe_(base->arena()) {} + PipeReceiver* outgoing_pipe() { return &pipe_.receiver; } // Start a send_message op. void StartOp(CapturedBatch batch); @@ -353,19 +287,16 @@ class BaseCallData : public Activity, private Wakeable { // This happens when the promise requests to call the next filter: until // this occurs messages can't be sent as we don't know the pipe that the // promise expects to send on. - template - void GotPipe(T* pipe); + void GotPipe(PipeReceiver* receiver); // Called from client/server polling to do the send message part of the // work. - void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe); + void WakeInsideCombiner(Flusher* flusher); // Call is completed, we have trailing metadata. Close things out. void Done(const ServerMetadata& metadata, Flusher* flusher); // Return true if we have a batch captured (for debug logs) bool HaveCapturedBatch() const { return batch_.is_captured(); } // Return true if we're not actively sending a message. bool IsIdle() const; - // Return true if we've released the message for forwarding down the stack. - bool IsForwarded() const { return state_ == State::kForwardedBatch; } private: enum class State : uint8_t { @@ -400,9 +331,11 @@ class BaseCallData : public Activity, private Wakeable { BaseCallData* const base_; State state_ = State::kInitial; - Interceptor* const interceptor_; + Pipe pipe_; + PipeReceiver* receiver_ = nullptr; absl::optional::PushType> push_; - absl::optional> next_; + absl::optional::NextType> next_; + absl::optional> next_result_; CapturedBatch batch_; grpc_closure* intercepted_on_complete_; grpc_closure on_complete_ = @@ -418,11 +351,9 @@ class BaseCallData : public Activity, private Wakeable { // these members for filters that don't need to intercept sent messages. class ReceiveMessage { public: - ReceiveMessage(BaseCallData* base, Interceptor* interceptor) - : base_(base), interceptor_(interceptor) {} - ~ReceiveMessage() { interceptor_->~Interceptor(); } - - Interceptor* interceptor() { return interceptor_; } + explicit ReceiveMessage(BaseCallData* base) + : base_(base), pipe_(base->arena()) {} + PipeSender* incoming_pipe() { return &pipe_.sender; } // Start a recv_message op. void StartOp(CapturedBatch& batch); @@ -430,11 +361,10 @@ class BaseCallData : public Activity, private Wakeable { // This happens when the promise requests to call the next filter: until // this occurs messages can't be received as we don't know the pipe that the // promise expects to forward them with. - template - void GotPipe(T* pipe); + void GotPipe(PipeSender* sender); // Called from client/server polling to do the receive message part of the // work. - void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe); + void WakeInsideCombiner(Flusher* flusher); // Call is completed, we have trailing metadata. Close things out. void Done(const ServerMetadata& metadata, Flusher* flusher); @@ -478,28 +408,23 @@ class BaseCallData : public Activity, private Wakeable { // On the next poll we'll close things out and forward on completions, // then transition to cancelled. kBatchCompletedButCancelled, - // Completed successfully while we're processing a recv message - see - // kPushedToPipe. + // Completed successfully while we're processing a recv message. kCompletedWhilePushedToPipe, - // Completed successfully while we're processing a recv message - see - // kPulledFromPipe. kCompletedWhilePulledFromPipe, - // Completed successfully while we were waiting to process - // kBatchCompleted. - kCompletedWhileBatchCompleted, }; static const char* StateString(State); void OnComplete(absl::Status status); BaseCallData* const base_; - Interceptor* const interceptor_; + Pipe pipe_; + PipeSender* sender_; State state_ = State::kInitial; uint32_t scratch_flags_; absl::optional* intercepted_slice_buffer_; uint32_t* intercepted_flags_; absl::optional::PushType> push_; - absl::optional> next_; + absl::optional::NextType> next_; absl::Status completed_status_; grpc_closure* intercepted_on_complete_; grpc_closure on_complete_ = @@ -511,8 +436,15 @@ class BaseCallData : public Activity, private Wakeable { CallCombiner* call_combiner() const { return call_combiner_; } Timestamp deadline() const { return deadline_; } grpc_call_stack* call_stack() const { return call_stack_; } - Pipe* server_initial_metadata_pipe() const { - return server_initial_metadata_pipe_; + Latch* server_initial_metadata_latch() const { + return server_initial_metadata_latch_; + } + PipeReceiver* outgoing_messages_pipe() const { + return send_message_ == nullptr ? nullptr : send_message_->outgoing_pipe(); + } + PipeSender* incoming_messages_pipe() const { + return receive_message_ == nullptr ? nullptr + : receive_message_->incoming_pipe(); } SendMessage* send_message() const { return send_message_; } ReceiveMessage* receive_message() const { return receive_message_; } @@ -542,7 +474,7 @@ class BaseCallData : public Activity, private Wakeable { CallFinalization finalization_; grpc_call_context_element* const context_; std::atomic pollent_{nullptr}; - Pipe* const server_initial_metadata_pipe_; + Latch* const server_initial_metadata_latch_; SendMessage* const send_message_; ReceiveMessage* const receive_message_; grpc_event_engine::experimental::EventEngine* event_engine_; @@ -694,9 +626,6 @@ class ServerCallData : public BaseCallData { // metadata. kQueuedBehindSendMessage, // We saw the op, and are waiting for the promise to complete - // to forward it. First however we need to close sends. - kQueuedButHaventClosedSends, - // We saw the op, and are waiting for the promise to complete // to forward it. kQueued, // We've forwarded the op to the next filter. diff --git a/src/core/lib/event_engine/slice.cc b/src/core/lib/event_engine/slice.cc index b9266714bb2..bcb09c50345 100644 --- a/src/core/lib/event_engine/slice.cc +++ b/src/core/lib/event_engine/slice.cc @@ -93,7 +93,7 @@ Slice Slice::FromRefcountAndBytes(grpc_slice_refcount* r, const uint8_t* begin, const uint8_t* end) { grpc_slice out; out.refcount = r; - if (r != grpc_slice_refcount::NoopRefcount()) r->Ref({}); + if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(); out.data.refcounted.bytes = const_cast(begin); out.data.refcounted.length = end - begin; return Slice(out); diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index a0fe3ee013f..521e0786fd4 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -49,9 +49,6 @@ const char* const description_promise_based_client_call = "(ie when all filters in a stack are promise based)"; const char* const description_free_large_allocator = "If set, return all free bytes from a \042big\042 allocator"; -const char* const description_promise_based_server_call = - "If set, use the new gRPC promise based call code when it's appropriate " - "(ie when all filters in a stack are promise based)"; const char* const description_transport_supplies_client_latency = "If set, use the transport represented value for client latency in " "opencensus"; @@ -75,7 +72,6 @@ const ExperimentMetadata g_experiment_metadata[] = { {"monitoring_experiment", description_monitoring_experiment, true}, {"promise_based_client_call", description_promise_based_client_call, false}, {"free_large_allocator", description_free_large_allocator, false}, - {"promise_based_server_call", description_promise_based_server_call, false}, {"transport_supplies_client_latency", description_transport_supplies_client_latency, false}, {"event_engine_listener", description_event_engine_listener, false}, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 8cb206f5f13..2a3e4cbb27e 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -71,7 +71,6 @@ inline bool IsEventEngineClientEnabled() { return false; } inline bool IsMonitoringExperimentEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsFreeLargeAllocatorEnabled() { return false; } -inline bool IsPromiseBasedServerCallEnabled() { return false; } inline bool IsTransportSuppliesClientLatencyEnabled() { return false; } inline bool IsEventEngineListenerEnabled() { return false; } #else @@ -101,18 +100,14 @@ inline bool IsMonitoringExperimentEnabled() { return IsExperimentEnabled(8); } inline bool IsPromiseBasedClientCallEnabled() { return IsExperimentEnabled(9); } #define GRPC_EXPERIMENT_IS_INCLUDED_FREE_LARGE_ALLOCATOR inline bool IsFreeLargeAllocatorEnabled() { return IsExperimentEnabled(10); } -#define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL -inline bool IsPromiseBasedServerCallEnabled() { - return IsExperimentEnabled(11); -} #define GRPC_EXPERIMENT_IS_INCLUDED_TRANSPORT_SUPPLIES_CLIENT_LATENCY inline bool IsTransportSuppliesClientLatencyEnabled() { - return IsExperimentEnabled(12); + return IsExperimentEnabled(11); } #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER -inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(13); } +inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(12); } -constexpr const size_t kNumExperiments = 14; +constexpr const size_t kNumExperiments = 13; extern const ExperimentMetadata g_experiment_metadata[kNumExperiments]; #endif diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 3bd0827f7dc..8f917b5439e 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -119,14 +119,6 @@ expiry: 2023/04/01 owner: alishananda@google.com test_tags: [resource_quota_test] -- name: promise_based_server_call - description: - If set, use the new gRPC promise based call code when it's appropriate - (ie when all filters in a stack are promise based) - default: false - expiry: 2023/06/01 - owner: ctiller@google.com - test_tags: ["core_end2end_test"] - name: transport_supplies_client_latency description: If set, use the transport represented value for client latency in opencensus diff --git a/src/core/lib/gprpp/debug_location.h b/src/core/lib/gprpp/debug_location.h index 1a29f495383..1b75086f4ec 100644 --- a/src/core/lib/gprpp/debug_location.h +++ b/src/core/lib/gprpp/debug_location.h @@ -56,6 +56,9 @@ class SourceLocation { // No-op for non-debug builds. // Callers can use the DEBUG_LOCATION macro in either case. #ifndef NDEBUG +// TODO(roth): See if there's a way to automatically populate this, +// similarly to how absl::SourceLocation::current() works, so that +// callers don't need to explicitly pass DEBUG_LOCATION anywhere. class DebugLocation { public: DebugLocation(const char* file = GRPC_DEFAULT_FILE, diff --git a/src/core/lib/iomgr/call_combiner.cc b/src/core/lib/iomgr/call_combiner.cc index c48766f781a..2b2996c2c4f 100644 --- a/src/core/lib/iomgr/call_combiner.cc +++ b/src/core/lib/iomgr/call_combiner.cc @@ -116,9 +116,9 @@ void CallCombiner::Start(grpc_closure* closure, grpc_error_handle error, DEBUG_ARGS const char* reason) { if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) { gpr_log(GPR_INFO, - "==> CallCombiner::Start() [%p] closure=%s [" DEBUG_FMT_STR + "==> CallCombiner::Start() [%p] closure=%p [" DEBUG_FMT_STR "%s] error=%s", - this, closure->DebugString().c_str() DEBUG_FMT_ARGS, reason, + this, closure DEBUG_FMT_ARGS, reason, StatusToString(error).c_str()); } size_t prev_size = @@ -176,8 +176,8 @@ void CallCombiner::Stop(DEBUG_ARGS const char* reason) { internal::StatusMoveFromHeapPtr(closure->error_data.error); closure->error_data.error = 0; if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) { - gpr_log(GPR_INFO, " EXECUTING FROM QUEUE: closure=%s error=%s", - closure->DebugString().c_str(), StatusToString(error).c_str()); + gpr_log(GPR_INFO, " EXECUTING FROM QUEUE: closure=%p error=%s", + closure, StatusToString(error).c_str()); } ScheduleClosure(closure, error); break; diff --git a/src/core/lib/iomgr/closure.cc b/src/core/lib/iomgr/closure.cc deleted file mode 100644 index cf1a184032c..00000000000 --- a/src/core/lib/iomgr/closure.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "src/core/lib/iomgr/closure.h" - -#include "absl/strings/str_format.h" - -std::string grpc_closure::DebugString() const { -#ifdef NDEBUG - return absl::StrFormat("%p", this); -#else - return absl::StrFormat("%p|created=%s:%d", this, file_created, line_created); -#endif -} diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index adf8ebf2b74..9996f163bcd 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -86,8 +86,6 @@ struct grpc_closure { const char* file_initiated; int line_initiated; #endif - - std::string DebugString() const; }; #ifndef NDEBUG diff --git a/src/core/lib/promise/activity.h b/src/core/lib/promise/activity.h index 73e9bfd3574..771dcd39c23 100644 --- a/src/core/lib/promise/activity.h +++ b/src/core/lib/promise/activity.h @@ -102,10 +102,6 @@ class Waker { return wakeable_ == other.wakeable_; } - bool operator!=(const Waker& other) const noexcept { - return wakeable_ != other.wakeable_; - } - std::string ActivityDebugTag() { return wakeable_ == nullptr ? "" : wakeable_->ActivityDebugTag(); } diff --git a/src/core/lib/promise/for_each.h b/src/core/lib/promise/for_each.h index cee3a730b30..8b3f2ff82b2 100644 --- a/src/core/lib/promise/for_each.h +++ b/src/core/lib/promise/for_each.h @@ -17,24 +17,17 @@ #include -#include - -#include #include #include #include "absl/status/status.h" -#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include -#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/construct_destruct.h" -#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -108,21 +101,9 @@ class ForEach { ReaderResult result; }; - std::string DebugTag() { - return absl::StrCat(Activity::current()->DebugTag(), " FOR_EACH[0x", - reinterpret_cast(this), "]: "); - } - Poll PollReaderNext() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%s PollReaderNext", DebugTag().c_str()); - } auto r = reader_next_(); if (auto* p = absl::get_if(&r)) { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%s PollReaderNext: got has_value=%s", - DebugTag().c_str(), p->has_value() ? "true" : "false"); - } if (p->has_value()) { Destruct(&reader_next_); auto action = action_factory_.Make(std::move(**p)); @@ -137,9 +118,6 @@ class ForEach { } Poll PollAction() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%s PollAction", DebugTag().c_str()); - } auto r = in_action_.promise(); if (auto* p = absl::get_if(&r)) { if (p->ok()) { diff --git a/src/core/lib/promise/if.h b/src/core/lib/promise/if.h index 20b83685315..09a62a8aea0 100644 --- a/src/core/lib/promise/if.h +++ b/src/core/lib/promise/if.h @@ -22,7 +22,6 @@ #include "absl/status/statusor.h" #include "absl/types/variant.h" -#include "src/core/lib/gprpp/construct_destruct.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/poll.h" @@ -117,66 +116,6 @@ class If { }; }; -template -class If { - private: - using TrueFactory = promise_detail::OncePromiseFactory; - using FalseFactory = promise_detail::OncePromiseFactory; - using TruePromise = typename TrueFactory::Promise; - using FalsePromise = typename FalseFactory::Promise; - using Result = - typename PollTraits()())>::Type; - - public: - If(bool condition, T if_true, F if_false) : condition_(condition) { - TrueFactory true_factory(std::move(if_true)); - FalseFactory false_factory(std::move(if_false)); - if (condition_) { - Construct(&if_true_, true_factory.Make()); - } else { - Construct(&if_false_, false_factory.Make()); - } - } - ~If() { - if (condition_) { - Destruct(&if_true_); - } else { - Destruct(&if_false_); - } - } - - If(const If&) = delete; - If& operator=(const If&) = delete; - If(If&& other) noexcept : condition_(other.condition_) { - if (condition_) { - Construct(&if_true_, std::move(other.if_true_)); - } else { - Construct(&if_false_, std::move(other.if_false_)); - } - } - If& operator=(If&& other) noexcept { - if (&other == this) return *this; - Destruct(this); - Construct(this, std::move(other)); - return *this; - } - - Poll operator()() { - if (condition_) { - return if_true_(); - } else { - return if_false_(); - } - } - - private: - bool condition_; - union { - TruePromise if_true_; - FalsePromise if_false_; - }; -}; - } // namespace promise_detail // If promise combinator. diff --git a/src/core/lib/promise/interceptor_list.h b/src/core/lib/promise/interceptor_list.h deleted file mode 100644 index 2e09f574ae5..00000000000 --- a/src/core/lib/promise/interceptor_list.h +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H -#define GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H - -#include - -#include - -#include -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/types/optional.h" -#include "absl/types/variant.h" - -#include - -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/construct_destruct.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/promise_factory.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/trace.h" -#include "src/core/lib/resource_quota/arena.h" - -namespace grpc_core { - -// Tracks a list of maps of T -> optional via promises. -// When Run, runs each transformation in order, and resolves to the ulimate -// result. -// If a map resolves to nullopt, the chain is terminated and the result is -// nullopt. -// Maps can also have synchronous cleanup functions, which are guaranteed to be -// called at the termination of each run through the chain. -template -class InterceptorList { - private: - // A map of T -> T via promises. - class Map { - public: - explicit Map(DebugLocation from) : from_(from) {} - // Construct a promise to transform x into some other value at memory. - virtual void MakePromise(T x, void* memory) = 0; - // Destroy a promise constructed at memory. - virtual void Destroy(void* memory) = 0; - // Poll a promise constructed at memory. - // Resolves to an optional -- if nullopt it means terminate the chain and - // resolve. - virtual Poll> PollOnce(void* memory) = 0; - virtual ~Map() = default; - - // Update the next pointer stored with this map. - // This is only valid to call once, and only before the map is used. - void SetNext(Map* next) { - GPR_DEBUG_ASSERT(next_ == nullptr); - next_ = next; - } - - // Access the creation location for this map (for debug tracing). - DebugLocation from() const { return from_; } - - // Access the next map in the chain (or nullptr if this is the last map). - Map* next() const { return next_; } - - private: - GPR_NO_UNIQUE_ADDRESS const DebugLocation from_; - Map* next_ = nullptr; - }; - - public: - // The result of Run: a promise that will execute the entire chain. - class RunPromise { - public: - RunPromise(size_t memory_required, Map* factory, absl::optional value) { - if (!value.has_value() || factory == nullptr) { - is_immediately_resolved_ = true; - Construct(&result_, std::move(value)); - } else { - is_immediately_resolved_ = false; - Construct(&async_resolution_, memory_required); - factory->MakePromise(std::move(*value), async_resolution_.space.get()); - async_resolution_.current_factory = factory; - } - } - - ~RunPromise() { - if (is_immediately_resolved_) { - Destruct(&result_); - } else { - if (async_resolution_.current_factory != nullptr) { - async_resolution_.current_factory->Destroy( - async_resolution_.space.get()); - } - Destruct(&async_resolution_); - } - } - - RunPromise(const RunPromise&) = delete; - RunPromise& operator=(const RunPromise&) = delete; - - RunPromise(RunPromise&& other) noexcept - : is_immediately_resolved_(other.is_immediately_resolved_) { - if (is_immediately_resolved_) { - Construct(&result_, std::move(other.result_)); - } else { - Construct(&async_resolution_, std::move(other.async_resolution_)); - } - } - - RunPromise& operator=(RunPromise&& other) noexcept = delete; - - Poll> operator()() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "InterceptorList::RunPromise: %s", - DebugString().c_str()); - } - if (is_immediately_resolved_) return std::move(result_); - while (true) { - auto r = async_resolution_.current_factory->PollOnce( - async_resolution_.space.get()); - if (auto* p = absl::get_if(&r)) { - async_resolution_.current_factory->Destroy( - async_resolution_.space.get()); - async_resolution_.current_factory = - async_resolution_.current_factory->next(); - if (async_resolution_.current_factory == nullptr || !p->has_value()) { - return std::move(*p); - } - async_resolution_.current_factory->MakePromise( - std::move(**p), async_resolution_.space.get()); - continue; - } - return Pending{}; - } - } - - private: - std::string DebugString() const { - if (is_immediately_resolved_) { - return absl::StrFormat("Result:has_value:%d", result_.has_value()); - } else { - return absl::StrCat( - "Running:", - async_resolution_.current_factory == nullptr - ? "END" - : ([p = async_resolution_.current_factory->from()]() { - return absl::StrCat(p.file(), ":", p.line()); - })() - .c_str()); - } - } - struct AsyncResolution { - explicit AsyncResolution(size_t max_size) - : space(GetContext()->MakePooledArray(max_size)) {} - AsyncResolution(const AsyncResolution&) = delete; - AsyncResolution& operator=(const AsyncResolution&) = delete; - AsyncResolution(AsyncResolution&& other) noexcept - : current_factory(std::exchange(other.current_factory, nullptr)), - space(std::move(other.space)) {} - Map* current_factory; - Arena::PoolPtr space; - }; - union { - AsyncResolution async_resolution_; - absl::optional result_; - }; - // If true, the result_ union is valid, otherwise async_resolution_ is. - // Indicates whether the promise resolved immediately at construction or if - // additional steps were needed. - bool is_immediately_resolved_; - }; - - InterceptorList() = default; - InterceptorList(const InterceptorList&) = delete; - InterceptorList& operator=(const InterceptorList&) = delete; - ~InterceptorList() { DeleteFactories(); } - - RunPromise Run(absl::optional initial_value) { - return RunPromise(promise_memory_required_, first_map_, - std::move(initial_value)); - } - - // Append a new map to the end of the chain. - template - void AppendMap(Fn fn, DebugLocation from) { - Append(MakeMapToAdd( - std::move(fn), [] {}, from)); - } - - // Prepend a new map to the beginning of the chain. - template - void PrependMap(Fn fn, DebugLocation from) { - Prepend(MakeMapToAdd( - std::move(fn), [] {}, from)); - } - - // Append a new map to the end of the chain, with a cleanup function to be - // called at the end of run promise execution. - template - void AppendMapWithCleanup(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { - Append(MakeMapToAdd(std::move(fn), std::move(cleanup_fn), from)); - } - - // Prepend a new map to the beginning of the chain, with a cleanup function to - // be called at the end of run promise execution. - template - void PrependMapWithCleanup(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { - Prepend(MakeMapToAdd(std::move(fn), std::move(cleanup_fn), from)); - } - - protected: - // Clear the interceptor list - void ResetInterceptorList() { - DeleteFactories(); - first_map_ = nullptr; - last_map_ = nullptr; - promise_memory_required_ = 0; - } - - private: - template - class MapImpl final : public Map { - public: - using PromiseFactory = promise_detail::RepeatedPromiseFactory; - using Promise = typename PromiseFactory::Promise; - - explicit MapImpl(Fn fn, CleanupFn cleanup_fn, DebugLocation from) - : Map(from), fn_(std::move(fn)), cleanup_fn_(std::move(cleanup_fn)) {} - ~MapImpl() override { cleanup_fn_(); } - void MakePromise(T x, void* memory) override { - new (memory) Promise(fn_.Make(std::move(x))); - } - void Destroy(void* memory) override { - static_cast(memory)->~Promise(); - } - Poll> PollOnce(void* memory) override { - return poll_cast>((*static_cast(memory))()); - } - - private: - GPR_NO_UNIQUE_ADDRESS PromiseFactory fn_; - GPR_NO_UNIQUE_ADDRESS CleanupFn cleanup_fn_; - }; - - template - Map* MakeMapToAdd(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { - using FactoryType = MapImpl; - promise_memory_required_ = std::max(promise_memory_required_, - sizeof(typename FactoryType::Promise)); - return GetContext()->New(std::move(fn), - std::move(cleanup_fn), from); - } - - void Append(Map* f) { - if (first_map_ == nullptr) { - first_map_ = f; - last_map_ = f; - } else { - last_map_->SetNext(f); - last_map_ = f; - } - } - - void Prepend(Map* f) { - if (first_map_ == nullptr) { - first_map_ = f; - last_map_ = f; - } else { - f->SetNext(first_map_); - first_map_ = f; - } - } - - void DeleteFactories() { - for (auto* f = first_map_; f != nullptr;) { - auto* next = f->next(); - f->~Map(); - f = next; - } - } - - // The first map in the chain. - Map* first_map_ = nullptr; - // The last map in the chain. - Map* last_map_ = nullptr; - // The amount of memory required to store the largest promise in the chain. - size_t promise_memory_required_ = 0; -}; - -} // namespace grpc_core - -#endif // GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H diff --git a/src/core/lib/promise/intra_activity_waiter.h b/src/core/lib/promise/intra_activity_waiter.h index 736ec04ae7d..73bc93f4100 100644 --- a/src/core/lib/promise/intra_activity_waiter.h +++ b/src/core/lib/promise/intra_activity_waiter.h @@ -17,8 +17,6 @@ #include -#include - #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/poll.h" @@ -42,10 +40,6 @@ class IntraActivityWaiter { } } - std::string DebugString() const { - return waiting_ ? "WAITING" : "NOT_WAITING"; - } - private: bool waiting_ = false; }; diff --git a/src/core/lib/promise/latch.h b/src/core/lib/promise/latch.h index c1dca66efc1..d66b461f46c 100644 --- a/src/core/lib/promise/latch.h +++ b/src/core/lib/promise/latch.h @@ -17,21 +17,12 @@ #include -#include - -#include #include -#include - -#include "absl/strings/str_cat.h" #include -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/intra_activity_waiter.h" #include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -65,13 +56,9 @@ class Latch { #ifndef NDEBUG has_had_waiters_ = true; #endif - return [this]() -> Poll { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_INFO, "%sPollWait %s", DebugTag().c_str(), - StateString().c_str()); - } + return [this]() -> Poll { if (has_value_) { - return std::move(value_); + return &value_; } else { return waiter_.pending(); } @@ -80,9 +67,6 @@ class Latch { // Set the value of the latch. Can only be called once. void Set(T value) { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str()); - } GPR_DEBUG_ASSERT(!has_value_); value_ = std::move(value); has_value_ = true; @@ -90,16 +74,6 @@ class Latch { } private: - std::string DebugTag() { - return absl::StrCat(Activity::current()->DebugTag(), " LATCH[0x", - reinterpret_cast(this), "]: "); - } - - std::string StateString() { - return absl::StrCat("has_value:", has_value_ ? "true" : "false", - " waiter:", waiter_.DebugString()); - } - // The value stored (if has_value_ is true), otherwise some random value, we // don't care. // Why not absl::optional<>? Writing things this way lets us compress diff --git a/src/core/lib/promise/map_pipe.h b/src/core/lib/promise/map_pipe.h index 6f7312da047..45df76e32d4 100644 --- a/src/core/lib/promise/map_pipe.h +++ b/src/core/lib/promise/map_pipe.h @@ -19,15 +19,10 @@ #include "absl/status/status.h" -#include - -#include "src/core/lib/debug/trace.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/for_each.h" #include "src/core/lib/promise/map.h" #include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/trace.h" #include "src/core/lib/promise/try_seq.h" namespace grpc_core { @@ -45,25 +40,14 @@ auto MapPipe(PipeReceiver src, PipeSender dst, Filter filter_factory) { [filter_factory = promise_detail::RepeatedPromiseFactory( std::move(filter_factory)), dst = std::move(dst)](T t) mutable { - return TrySeq( - [] { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "MapPipe: start map"); - } - return Empty{}; - }, - filter_factory.Make(std::move(t)), - [&dst](T t) { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "MapPipe: start push"); - } - return Map(dst.Push(std::move(t)), [](bool successful_push) { - if (successful_push) { - return absl::OkStatus(); - } - return absl::CancelledError(); - }); - }); + return TrySeq(filter_factory.Make(std::move(t)), [&dst](T t) { + return Map(dst.Push(std::move(t)), [](bool successful_push) { + if (successful_push) { + return absl::OkStatus(); + } + return absl::CancelledError(); + }); + }); }); } diff --git a/src/core/lib/promise/trace.cc b/src/core/lib/promise/pipe.cc similarity index 82% rename from src/core/lib/promise/trace.cc rename to src/core/lib/promise/pipe.cc index 21b430526df..cbb408bea19 100644 --- a/src/core/lib/promise/trace.cc +++ b/src/core/lib/promise/pipe.cc @@ -14,7 +14,6 @@ #include -#include "src/core/lib/promise/trace.h" +#include "src/core/lib/promise/pipe.h" -grpc_core::DebugOnlyTraceFlag grpc_trace_promise_primitives( - false, "promise_primitives"); +grpc_core::DebugOnlyTraceFlag grpc_trace_promise_pipe(false, "promise_pipe"); diff --git a/src/core/lib/promise/pipe.h b/src/core/lib/promise/pipe.h index 8033201fac8..a1f4ed42b56 100644 --- a/src/core/lib/promise/pipe.h +++ b/src/core/lib/promise/pipe.h @@ -18,15 +18,10 @@ #include #include -#include -#include #include -#include #include -#include "absl/base/attributes.h" -#include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/types/optional.h" #include "absl/types/variant.h" @@ -34,19 +29,14 @@ #include #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/if.h" -#include "src/core/lib/promise/interceptor_list.h" #include "src/core/lib/promise/intra_activity_waiter.h" -#include "src/core/lib/promise/map.h" #include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/trace.h" #include "src/core/lib/resource_quota/arena.h" +extern grpc_core::DebugOnlyTraceFlag grpc_trace_promise_pipe; + namespace grpc_core { namespace pipe_detail { @@ -66,24 +56,21 @@ struct Pipe; template class NextResult final { public: - NextResult() : center_(nullptr) {} - explicit NextResult(RefCountedPtr> center) - : center_(std::move(center)) { - GPR_ASSERT(center_ != nullptr); - } - explicit NextResult(bool cancelled) - : center_(nullptr), cancelled_(cancelled) {} + explicit NextResult(pipe_detail::Center* center) : center_(center) {} ~NextResult(); NextResult(const NextResult&) = delete; NextResult& operator=(const NextResult&) = delete; - NextResult(NextResult&& other) noexcept = default; - NextResult& operator=(NextResult&& other) noexcept = default; + NextResult(NextResult&& other) noexcept + : center_(std::exchange(other.center_, nullptr)) {} + NextResult& operator=(NextResult&& other) noexcept { + center_ = std::exchange(other.center_, nullptr); + return *this; + } using value_type = T; void reset(); bool has_value() const; - // Only valid if has_value() const T& value() const { GPR_ASSERT(has_value()); return **this; @@ -94,12 +81,9 @@ class NextResult final { } const T& operator*() const; T& operator*(); - // Only valid if !has_value() - bool cancelled() { return cancelled_; } private: - RefCountedPtr> center_; - bool cancelled_; + pipe_detail::Center* center_; }; namespace pipe_detail { @@ -112,39 +96,71 @@ class Next; // Center sits between a sender and a receiver to provide a one-deep buffer of // Ts template -class Center : public InterceptorList { +class Center { public: // Initialize with one send ref (held by PipeSender) and one recv ref (held by // PipeReceiver) Center() { - refs_ = 2; + send_refs_ = 1; + recv_refs_ = 1; value_state_ = ValueState::kEmpty; } - // Add one ref to this object, and return this. - void IncrementRefCount() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%s", DebugOpString("IncrementRefCount").c_str()); + // Add one ref to the send side of this object, and return this. + Center* RefSend() { + if (grpc_trace_promise_pipe.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("RefSend").c_str()); } - refs_++; - GPR_DEBUG_ASSERT(refs_ != 0); + send_refs_++; + GPR_ASSERT(send_refs_ != 0); + return this; } - RefCountedPtr
Ref() { - IncrementRefCount(); - return RefCountedPtr
(this); + // Add one ref to the recv side of this object, and return this. + Center* RefRecv() { + if (grpc_trace_promise_pipe.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("RefRecv").c_str()); + } + recv_refs_++; + GPR_ASSERT(recv_refs_ != 0); + return this; } - // Drop a ref + // Drop a send side ref + // If no send refs remain, wake due to send closure // If no refs remain, destroy this object - void Unref() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%s", DebugOpString("Unref").c_str()); + void UnrefSend() { + if (grpc_trace_promise_pipe.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("UnrefSend").c_str()); } - GPR_DEBUG_ASSERT(refs_ > 0); - refs_--; - if (0 == refs_) { - this->~Center(); + GPR_DEBUG_ASSERT(send_refs_ > 0); + send_refs_--; + if (0 == send_refs_) { + on_full_.Wake(); + on_empty_.Wake(); + if (0 == recv_refs_) { + this->~Center(); + } + } + } + + // Drop a recv side ref + // If no recv refs remain, wake due to recv closure + // If no refs remain, destroy this object + void UnrefRecv() { + if (grpc_trace_promise_pipe.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("UnrefRecv").c_str()); + } + GPR_DEBUG_ASSERT(recv_refs_ > 0); + recv_refs_--; + if (0 == recv_refs_) { + on_full_.Wake(); + on_empty_.Wake(); + if (0 == send_refs_) { + this->~Center(); + } else if (value_state_ == ValueState::kReady) { + ResetValue(); + } } } @@ -153,45 +169,26 @@ class Center : public InterceptorList { // Return true if the value was pushed. // Return false if the recv end is closed. Poll Push(T* value) { - if (grpc_trace_promise_primitives.enabled()) { + if (grpc_trace_promise_pipe.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("Push").c_str()); } - GPR_DEBUG_ASSERT(refs_ != 0); - switch (value_state_) { - case ValueState::kClosed: - case ValueState::kReadyClosed: - case ValueState::kCancelled: - return false; - case ValueState::kReady: - case ValueState::kAcked: - return on_empty_.pending(); - case ValueState::kEmpty: - value_state_ = ValueState::kReady; - value_ = std::move(*value); - on_full_.Wake(); - return true; - } - GPR_UNREACHABLE_CODE(return false); + GPR_DEBUG_ASSERT(send_refs_ != 0); + if (recv_refs_ == 0) return false; + if (value_state_ != ValueState::kEmpty) return on_empty_.pending(); + value_state_ = ValueState::kReady; + value_ = std::move(*value); + on_full_.Wake(); + return true; } Poll PollAck() { - if (grpc_trace_promise_primitives.enabled()) { + if (grpc_trace_promise_pipe.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("PollAck").c_str()); } - GPR_DEBUG_ASSERT(refs_ != 0); - switch (value_state_) { - case ValueState::kClosed: - case ValueState::kReadyClosed: - case ValueState::kCancelled: - return false; - case ValueState::kReady: - case ValueState::kEmpty: - return on_empty_.pending(); - case ValueState::kAcked: - value_state_ = ValueState::kEmpty; - on_empty_.Wake(); - return true; - } + GPR_DEBUG_ASSERT(send_refs_ != 0); + if (recv_refs_ == 0) return value_state_ == ValueState::kAcked; + if (value_state_ != ValueState::kAcked) return on_empty_.pending(); + value_state_ = ValueState::kEmpty; return true; } @@ -199,107 +196,47 @@ class Center : public InterceptorList { // Return Pending if there is no value. // Return the value if one was retrieved. // Return nullopt if the send end is closed and no value had been pushed. - Poll> Next() { - if (grpc_trace_promise_primitives.enabled()) { + Poll> Next() { + if (grpc_trace_promise_pipe.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("Next").c_str()); } - GPR_DEBUG_ASSERT(refs_ != 0); - switch (value_state_) { - case ValueState::kEmpty: - case ValueState::kAcked: - return on_full_.pending(); - case ValueState::kReadyClosed: - this->ResetInterceptorList(); - value_state_ = ValueState::kClosed; - ABSL_FALLTHROUGH_INTENDED; - case ValueState::kReady: - return std::move(value_); - case ValueState::kClosed: - case ValueState::kCancelled: - return absl::nullopt; + GPR_DEBUG_ASSERT(recv_refs_ != 0); + if (value_state_ != ValueState::kReady) { + if (send_refs_ == 0) return NextResult(nullptr); + return on_full_.pending(); } - GPR_UNREACHABLE_CODE(return absl::nullopt); + return NextResult(RefRecv()); } void AckNext() { - if (grpc_trace_promise_primitives.enabled()) { + if (grpc_trace_promise_pipe.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("AckNext").c_str()); } - switch (value_state_) { - case ValueState::kReady: - value_state_ = ValueState::kAcked; - on_empty_.Wake(); - break; - case ValueState::kReadyClosed: - this->ResetInterceptorList(); - value_state_ = ValueState::kClosed; - break; - case ValueState::kClosed: - case ValueState::kCancelled: - break; - case ValueState::kEmpty: - case ValueState::kAcked: - abort(); - } + GPR_DEBUG_ASSERT(value_state_ == ValueState::kReady); + value_state_ = ValueState::kAcked; + on_empty_.Wake(); + UnrefRecv(); } - void MarkClosed() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("MarkClosed").c_str()); - } - switch (value_state_) { - case ValueState::kEmpty: - case ValueState::kAcked: - this->ResetInterceptorList(); - value_state_ = ValueState::kClosed; - on_full_.Wake(); - break; - case ValueState::kReady: - value_state_ = ValueState::kReadyClosed; - break; - case ValueState::kReadyClosed: - case ValueState::kClosed: - case ValueState::kCancelled: - break; - } - } - - void MarkCancelled() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("MarkCancelled").c_str()); - } - switch (value_state_) { - case ValueState::kEmpty: - case ValueState::kAcked: - case ValueState::kReady: - case ValueState::kReadyClosed: - this->ResetInterceptorList(); - value_state_ = ValueState::kCancelled; - on_full_.Wake(); - break; - case ValueState::kClosed: - value_state_ = ValueState::kCancelled; - break; - case ValueState::kCancelled: - break; - } - } - - bool cancelled() { return value_state_ == ValueState::kCancelled; } - T& value() { return value_; } const T& value() const { return value_; } - std::string DebugTag() { - if (auto* activity = Activity::current()) { - return absl::StrCat(activity->DebugTag(), " PIPE[0x", - reinterpret_cast(this), "]: "); - } else { - return absl::StrCat("PIPE[0x", reinterpret_cast(this), "]: "); - } - } - private: + std::string DebugTag() { + return absl::StrCat(Activity::current()->DebugTag(), "PIPE[0x", + reinterpret_cast(this), "]: "); + } + std::string DebugOpString(std::string op) { + return absl::StrCat(DebugTag(), op, " send_refs=", send_refs_, + " recv_refs=", recv_refs_, + " value_state=", ValueStateName(value_state_)); + } + void ResetValue() { + // Fancy dance to move out of value in the off chance that we reclaim some + // memory earlier. + [](T) {}(std::move(value_)); + value_state_ = ValueState::kEmpty; + } // State of value_. enum class ValueState : uint8_t { // No value is set, it's possible to send. @@ -309,52 +246,31 @@ class Center : public InterceptorList { // Value has been received and acked, we can unblock senders and transition // to empty. kAcked, - // Pipe is closed successfully, no more values can be sent - kClosed, - // Pipe is closed successfully, no more values can be sent - // (but one value is queued and ready to be received) - kReadyClosed, - // Pipe is closed unsuccessfully, no more values can be sent - kCancelled, }; - - std::string DebugOpString(std::string op) { - return absl::StrCat(DebugTag(), op, " refs=", refs_, - " value_state=", ValueStateName(value_state_), - " on_empty=", on_empty_.DebugString().c_str(), - " on_full=", on_full_.DebugString().c_str()); - } - static const char* ValueStateName(ValueState state) { switch (state) { case ValueState::kEmpty: - return "Empty"; + return "kEmpty"; case ValueState::kReady: - return "Ready"; + return "kReady"; case ValueState::kAcked: - return "Acked"; - case ValueState::kClosed: - return "Closed"; - case ValueState::kReadyClosed: - return "ReadyClosed"; - case ValueState::kCancelled: - return "Cancelled"; + return "kAcked"; } GPR_UNREACHABLE_CODE(return "unknown"); } - T value_; - // Number of refs - uint8_t refs_; + // Number of sending objects. + // 0 => send is closed. + // 1 ref each for PipeSender and Push. + uint8_t send_refs_ : 2; + // Number of receiving objects. + // 0 => recv is closed. + // 1 ref each for PipeReceiver, Next, and NextResult. + uint8_t recv_refs_ : 2; // Current state of the value. - ValueState value_state_; + ValueState value_state_ : 2; IntraActivityWaiter on_empty_; IntraActivityWaiter on_full_; - - // Make failure to destruct show up in ASAN builds. -#ifndef NDEBUG - std::unique_ptr asan_canary_ = absl::make_unique(0); -#endif }; } // namespace pipe_detail @@ -367,18 +283,23 @@ class PipeSender { PipeSender(const PipeSender&) = delete; PipeSender& operator=(const PipeSender&) = delete; - PipeSender(PipeSender&& other) noexcept = default; - PipeSender& operator=(PipeSender&& other) noexcept = default; + + PipeSender(PipeSender&& other) noexcept : center_(other.center_) { + other.center_ = nullptr; + } + PipeSender& operator=(PipeSender&& other) noexcept { + if (center_ != nullptr) center_->UnrefSend(); + center_ = other.center_; + other.center_ = nullptr; + return *this; + } ~PipeSender() { - if (center_ != nullptr) center_->MarkClosed(); + if (center_ != nullptr) center_->UnrefSend(); } void Close() { - if (center_ != nullptr) { - center_->MarkClosed(); - center_.reset(); - } + if (auto* center = std::exchange(center_, nullptr)) center->UnrefSend(); } void Swap(PipeSender* other) { std::swap(center_, other->center_); } @@ -389,37 +310,32 @@ class PipeSender { // receiver is either closed or able to receive another message. PushType Push(T value); - template - void InterceptAndMap(Fn f, DebugLocation from = {}) { - center_->PrependMap(std::move(f), from); - } - - template - void InterceptAndMap(Fn f, OnHalfClose cleanup_fn, DebugLocation from = {}) { - center_->PrependMapWithCleanup(std::move(f), std::move(cleanup_fn), from); - } - private: friend struct Pipe; explicit PipeSender(pipe_detail::Center* center) : center_(center) {} - RefCountedPtr> center_; - - // Make failure to destruct show up in ASAN builds. -#ifndef NDEBUG - std::unique_ptr asan_canary_ = absl::make_unique(0); -#endif + pipe_detail::Center* center_; }; // Receive end of a Pipe. template class PipeReceiver { public: + using NextType = pipe_detail::Next; + PipeReceiver(const PipeReceiver&) = delete; PipeReceiver& operator=(const PipeReceiver&) = delete; - PipeReceiver(PipeReceiver&& other) noexcept = default; - PipeReceiver& operator=(PipeReceiver&& other) noexcept = default; + + PipeReceiver(PipeReceiver&& other) noexcept : center_(other.center_) { + other.center_ = nullptr; + } + PipeReceiver& operator=(PipeReceiver&& other) noexcept { + if (center_ != nullptr) center_->UnrefRecv(); + center_ = other.center_; + other.center_ = nullptr; + return *this; + } ~PipeReceiver() { - if (center_ != nullptr) center_->MarkClosed(); + if (center_ != nullptr) center_->UnrefRecv(); } void Swap(PipeReceiver* other) { std::swap(center_, other->center_); } @@ -429,28 +345,12 @@ class PipeReceiver { // message was received, or no value if the other end of the pipe was closed. // Blocks the promise until the receiver is either closed or a message is // available. - auto Next(); - - template - void InterceptAndMap(Fn f, DebugLocation from = {}) { - center_->AppendMap(std::move(f), from); - } - - template - void InterceptAndMapWithHalfClose(Fn f, OnHalfClose cleanup_fn, - DebugLocation from = {}) { - center_->AppendMapWithCleanup(std::move(f), std::move(cleanup_fn), from); - } + NextType Next(); private: friend struct Pipe; explicit PipeReceiver(pipe_detail::Center* center) : center_(center) {} - RefCountedPtr> center_; - - // Make failure to destruct show up in ASAN builds. -#ifndef NDEBUG - std::unique_ptr asan_canary_ = absl::make_unique(0); -#endif + pipe_detail::Center* center_; }; namespace pipe_detail { @@ -461,33 +361,42 @@ class Push { public: Push(const Push&) = delete; Push& operator=(const Push&) = delete; - Push(Push&& other) noexcept = default; - Push& operator=(Push&& other) noexcept = default; + Push(Push&& other) noexcept + : center_(other.center_), push_(std::move(other.push_)) { + other.center_ = nullptr; + } + Push& operator=(Push&& other) noexcept { + if (center_ != nullptr) center_->UnrefSend(); + center_ = other.center_; + other.center_ = nullptr; + push_ = std::move(other.push_); + return *this; + } + + ~Push() { + if (center_ != nullptr) center_->UnrefSend(); + } Poll operator()() { - if (center_ == nullptr) return false; - if (auto* p = absl::get_if(&state_)) { - auto r = center_->Push(p); + if (push_.has_value()) { + auto r = center_->Push(&*push_); if (auto* ok = absl::get_if(&r)) { - state_.template emplace(); + push_.reset(); if (!*ok) return false; } else { return Pending{}; } } - GPR_DEBUG_ASSERT(absl::holds_alternative(state_)); + GPR_DEBUG_ASSERT(!push_.has_value()); return center_->PollAck(); } private: - struct AwaitingAck {}; - friend class PipeSender; - explicit Push(RefCountedPtr> center, T push) - : center_(std::move(center)), state_(std::move(push)) {} - - RefCountedPtr> center_; - absl::variant state_; + explicit Push(pipe_detail::Center* center, T push) + : center_(center), push_(std::move(push)) {} + Center* center_; + absl::optional push_; }; // Implementation of PipeReceiver::Next promise. @@ -496,56 +405,46 @@ class Next { public: Next(const Next&) = delete; Next& operator=(const Next&) = delete; - Next(Next&& other) noexcept = default; - Next& operator=(Next&& other) noexcept = default; + Next(Next&& other) noexcept : center_(other.center_) { + other.center_ = nullptr; + } + Next& operator=(Next&& other) noexcept { + if (center_ != nullptr) center_->UnrefRecv(); + center_ = other.center_; + other.center_ = nullptr; + return *this; + } - Poll> operator()() { return center_->Next(); } + ~Next() { + if (center_ != nullptr) center_->UnrefRecv(); + } + + Poll> operator()() { + auto r = center_->Next(); + if (!absl::holds_alternative(r)) { + std::exchange(center_, nullptr)->UnrefRecv(); + } + return r; + } private: friend class PipeReceiver; - explicit Next(RefCountedPtr> center) : center_(std::move(center)) {} - - RefCountedPtr> center_; + explicit Next(pipe_detail::Center* center) : center_(center) {} + Center* center_; }; } // namespace pipe_detail template pipe_detail::Push PipeSender::Push(T value) { - return pipe_detail::Push(center_ == nullptr ? nullptr : center_->Ref(), - std::move(value)); + return pipe_detail::Push(center_->RefSend(), std::move(value)); } template -auto PipeReceiver::Next() { - return Seq( - pipe_detail::Next(center_->Ref()), - [center = center_->Ref()](absl::optional value) { - bool open = value.has_value(); - bool cancelled = center->cancelled(); - return If( - open, - [center = std::move(center), value = std::move(value)]() mutable { - auto run_interceptors = center->Run(std::move(value)); - return Map(std::move(run_interceptors), - [center = std::move(center)]( - absl::optional value) mutable { - if (value.has_value()) { - center->value() = std::move(*value); - return NextResult(std::move(center)); - } else { - center->MarkCancelled(); - return NextResult(true); - } - }); - }, - [cancelled]() { return NextResult(cancelled); }); - }); +pipe_detail::Next PipeReceiver::Next() { + return pipe_detail::Next(center_->RefRecv()); } -template -using PipeReceiverNextType = decltype(std::declval>().Next()); - template bool NextResult::has_value() const { return center_ != nullptr; @@ -568,10 +467,7 @@ NextResult::~NextResult() { template void NextResult::reset() { - if (center_ != nullptr) { - center_->AckNext(); - center_.reset(); - } + if (auto* p = std::exchange(center_, nullptr)) p->AckNext(); } // A Pipe is an intra-Activity communications channel that transmits T's from diff --git a/src/core/lib/promise/trace.h b/src/core/lib/promise/trace.h deleted file mode 100644 index de66138fec6..00000000000 --- a/src/core/lib/promise/trace.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GRPC_SRC_CORE_LIB_PROMISE_TRACE_H -#define GRPC_SRC_CORE_LIB_PROMISE_TRACE_H - -#include - -#include "src/core/lib/debug/trace.h" - -extern grpc_core::DebugOnlyTraceFlag grpc_trace_promise_primitives; - -#endif // GRPC_SRC_CORE_LIB_PROMISE_TRACE_H diff --git a/src/core/lib/promise/try_concurrently.h b/src/core/lib/promise/try_concurrently.h index 06ca3739bed..b05c3089b8e 100644 --- a/src/core/lib/promise/try_concurrently.h +++ b/src/core/lib/promise/try_concurrently.h @@ -20,22 +20,17 @@ #include #include -#include #include #include -#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include -#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/construct_destruct.h" -#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/detail/status.h" #include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -190,18 +185,9 @@ class TryConcurrently { typename PollTraits>()())>::Type; Poll operator()() { - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%sBEGIN POLL: done_bits=%x necessary_bits=%x", - DebugTag().c_str(), done_bits_, NecessaryBits()); - } auto r = pre_main_.template Run(done_bits_); if (auto* status = absl::get_if(&r)) { GPR_DEBUG_ASSERT(!IsStatusOk(*status)); - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, - "%sFAIL POLL PRE-MAIN: done_bits=%x necessary_bits=%x", - DebugTag().c_str(), done_bits_, NecessaryBits()); - } return std::move(*status); } if ((done_bits_ & 1) == 0) { @@ -215,17 +201,8 @@ class TryConcurrently { r = post_main_.template Run(done_bits_); if (auto* status = absl::get_if(&r)) { GPR_DEBUG_ASSERT(!IsStatusOk(*status)); - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, - "%sFAIL POLL POST-MAIN: done_bits=%x necessary_bits=%x", - DebugTag().c_str(), done_bits_, NecessaryBits()); - } return std::move(*status); } - if (grpc_trace_promise_primitives.enabled()) { - gpr_log(GPR_DEBUG, "%sEND POLL: done_bits=%x necessary_bits=%x", - DebugTag().c_str(), done_bits_, NecessaryBits()); - } if ((done_bits_ & NecessaryBits()) == NecessaryBits()) { return std::move(result_); } @@ -257,11 +234,6 @@ class TryConcurrently { // that is all promises that are not the main one. constexpr uint8_t HelperBits() { return AllBits() ^ 1; } - std::string DebugTag() { - return absl::StrCat(Activity::current()->DebugTag(), " TRY_CONCURRENTLY[0x", - reinterpret_cast(this), "]: "); - } - // done_bits signifies which operations have completed. // Bit 0 is set if main_ has completed. // The next higher bits correspond one per pre-main promise. diff --git a/src/core/lib/resource_quota/arena.h b/src/core/lib/resource_quota/arena.h index 09d559664b0..9b67d1ffba9 100644 --- a/src/core/lib/resource_quota/arena.h +++ b/src/core/lib/resource_quota/arena.h @@ -30,7 +30,6 @@ #include #include -#include #include #include #include @@ -49,11 +48,6 @@ namespace grpc_core { namespace arena_detail { -struct PoolAndSize { - size_t alloc_size; - size_t pool_index; -}; - template struct PoolIndexForSize; @@ -87,37 +81,10 @@ constexpr size_t AllocationSizeFromObjectSize( return PoolIndexForSize::kSize; } -template -struct ChoosePoolForAllocationSizeImpl; - -template -struct ChoosePoolForAllocationSizeImpl { - static PoolAndSize Fn(size_t n) { - if (n <= kBucketSize) return {kBucketSize, kIndex}; - return ChoosePoolForAllocationSizeImpl::Fn(n); - } -}; - -template -struct ChoosePoolForAllocationSizeImpl { - static PoolAndSize Fn(size_t n) { - return PoolAndSize{n, std::numeric_limits::max()}; - } -}; - -template -PoolAndSize ChoosePoolForAllocationSize( - size_t n, absl::integer_sequence) { - return ChoosePoolForAllocationSizeImpl<0, kBucketSizes...>::Fn(n); -} - } // namespace arena_detail class Arena { using PoolSizes = absl::integer_sequence; - struct FreePoolNode { - FreePoolNode* next; - }; public: // Create an arena, with \a initial_size bytes in the first allocated buffer. @@ -166,8 +133,7 @@ class Arena { class PooledDeleter { public: - explicit PooledDeleter(std::atomic* free_list) - : free_list_(free_list) {} + explicit PooledDeleter(Arena* arena) : arena_(arena) {} PooledDeleter() = default; template void operator()(T* p) { @@ -176,57 +142,24 @@ class Arena { // by setting the arena to nullptr. // This is a transitional hack and should be removed once promise based // filter is removed. - if (free_list_ != nullptr) { - p->~T(); - FreePooled(p, free_list_); - } + if (arena_ != nullptr) arena_->DeletePooled(p); } - bool has_freelist() const { return free_list_ != nullptr; } - private: - std::atomic* free_list_; + Arena* arena_; }; template using PoolPtr = std::unique_ptr; - // Make a unique_ptr to T that is allocated from the arena. - // When the pointer is released, the memory may be reused for other - // MakePooled(.*) calls. - // CAUTION: The amount of memory allocated is rounded up to the nearest - // value in Arena::PoolSizes, and so this may pessimize total - // arena size. template PoolPtr MakePooled(Args&&... args) { - auto* free_list = - &pools_[arena_detail::PoolFromObjectSize(PoolSizes())]; return PoolPtr( new (AllocPooled( arena_detail::AllocationSizeFromObjectSize(PoolSizes()), - free_list)) T(std::forward(args)...), - PooledDeleter(free_list)); - } - - // Make a unique_ptr to an array of T that is allocated from the arena. - // When the pointer is released, the memory may be reused for other - // MakePooled(.*) calls. - // One can use MakePooledArray to allocate a buffer of bytes. - // CAUTION: The amount of memory allocated is rounded up to the nearest - // value in Arena::PoolSizes, and so this may pessimize total - // arena size. - template - PoolPtr MakePooledArray(size_t n) { - auto where = - arena_detail::ChoosePoolForAllocationSize(n * sizeof(T), PoolSizes()); - if (where.pool_index == std::numeric_limits::max()) { - return PoolPtr(new (Alloc(where.alloc_size)) T[n], - PooledDeleter(nullptr)); - } else { - return PoolPtr( - new (AllocPooled(where.alloc_size, &pools_[where.pool_index])) T[n], - PooledDeleter(&pools_[where.pool_index])); - } + &pools_[arena_detail::PoolFromObjectSize(PoolSizes())])) + T(std::forward(args)...), + PooledDeleter(this)); } private: @@ -269,6 +202,17 @@ class Arena { void* AllocZone(size_t size); + template + void DeletePooled(T* p) { + p->~T(); + FreePooled( + p, &pools_[arena_detail::PoolFromObjectSize(PoolSizes())]); + } + + struct FreePoolNode { + FreePoolNode* next; + }; + void* AllocPooled(size_t alloc_size, std::atomic* head); static void FreePooled(void* p, std::atomic* head); diff --git a/src/core/lib/slice/slice.cc b/src/core/lib/slice/slice.cc index 51ee3a83644..177ca456ec5 100644 --- a/src/core/lib/slice/slice.cc +++ b/src/core/lib/slice/slice.cc @@ -283,7 +283,7 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { subset = grpc_slice_sub_no_ref(source, begin, end); // Bump the refcount if (subset.refcount != grpc_slice_refcount::NoopRefcount()) { - subset.refcount->Ref({}); + subset.refcount->Ref(); } } return subset; @@ -332,7 +332,7 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice* source, size_t split, tail.refcount = source->refcount; // Bump the refcount if (tail.refcount != grpc_slice_refcount::NoopRefcount()) { - tail.refcount->Ref({}); + tail.refcount->Ref(); } break; } @@ -378,7 +378,7 @@ grpc_slice grpc_slice_split_head(grpc_slice* source, size_t split) { head.refcount = source->refcount; // Bump the refcount if (head.refcount != grpc_slice_refcount::NoopRefcount()) { - head.refcount->Ref({}); + head.refcount->Ref(); } // Point into the source array head.data.refcounted.bytes = source->data.refcounted.bytes; diff --git a/src/core/lib/slice/slice.h b/src/core/lib/slice/slice.h index 2640ee80644..17047008eca 100644 --- a/src/core/lib/slice/slice.h +++ b/src/core/lib/slice/slice.h @@ -31,7 +31,6 @@ #include #include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" @@ -55,17 +54,16 @@ namespace grpc_core { -inline const grpc_slice& CSliceRef(const grpc_slice& slice, - DebugLocation loc = {}) { +inline const grpc_slice& CSliceRef(const grpc_slice& slice) { if (reinterpret_cast(slice.refcount) > 1) { - slice.refcount->Ref(loc); + slice.refcount->Ref(); } return slice; } -inline void CSliceUnref(const grpc_slice& slice, DebugLocation loc = {}) { +inline void CSliceUnref(const grpc_slice& slice) { if (reinterpret_cast(slice.refcount) > 1) { - slice.refcount->Unref(loc); + slice.refcount->Unref(); } } @@ -393,11 +391,10 @@ class GPR_MSVC_EMPTY_BASE_CLASS_WORKAROUND Slice Slice Copy() const { return Slice(grpc_slice_copy(c_slice())); } static Slice FromRefcountAndBytes(grpc_slice_refcount* r, - const uint8_t* begin, const uint8_t* end, - DebugLocation location = {}) { + const uint8_t* begin, const uint8_t* end) { grpc_slice out; out.refcount = r; - if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(location); + if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(); out.data.refcounted.bytes = const_cast(begin); out.data.refcounted.length = end - begin; return Slice(out); diff --git a/src/core/lib/slice/slice_buffer.h b/src/core/lib/slice/slice_buffer.h index 45d41031380..b87a91e1306 100644 --- a/src/core/lib/slice/slice_buffer.h +++ b/src/core/lib/slice/slice_buffer.h @@ -20,11 +20,8 @@ #include #include -#include #include -#include "absl/memory/memory.h" - #include #include @@ -154,11 +151,6 @@ class SliceBuffer { private: /// The backing raw slice buffer. grpc_slice_buffer slice_buffer_; - -// Make failure to destruct show up in ASAN builds. -#ifndef NDEBUG - std::unique_ptr asan_canary_ = absl::make_unique(0); -#endif }; } // namespace grpc_core diff --git a/src/core/lib/slice/slice_refcount.cc b/src/core/lib/slice/slice_refcount.cc deleted file mode 100644 index 0a05f0a333a..00000000000 --- a/src/core/lib/slice/slice_refcount.cc +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "src/core/lib/slice/slice_refcount.h" - -grpc_core::DebugOnlyTraceFlag grpc_slice_refcount_trace(false, - "slice_refcount"); diff --git a/src/core/lib/slice/slice_refcount.h b/src/core/lib/slice/slice_refcount.h index 50fdb6efd9a..5a862542d45 100644 --- a/src/core/lib/slice/slice_refcount.h +++ b/src/core/lib/slice/slice_refcount.h @@ -17,18 +17,10 @@ #include -#include #include #include -#include - -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/debug_location.h" - -extern grpc_core::DebugOnlyTraceFlag grpc_slice_refcount_trace; - // grpc_slice_refcount : A reference count for grpc_slice. struct grpc_slice_refcount { public: @@ -48,21 +40,9 @@ struct grpc_slice_refcount { explicit grpc_slice_refcount(DestroyerFn destroyer_fn) : destroyer_fn_(destroyer_fn) {} - void Ref(grpc_core::DebugLocation location) { - auto prev_refs = ref_.fetch_add(1, std::memory_order_relaxed); - if (grpc_slice_refcount_trace.enabled()) { - gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_INFO, - "REF %p %" PRIdPTR "->%" PRIdPTR, this, prev_refs, prev_refs + 1); - } - } - void Unref(grpc_core::DebugLocation location) { - auto prev_refs = ref_.fetch_sub(1, std::memory_order_acq_rel); - if (grpc_slice_refcount_trace.enabled()) { - gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_INFO, - "UNREF %p %" PRIdPTR "->%" PRIdPTR, this, prev_refs, - prev_refs - 1); - } - if (prev_refs == 1) { + void Ref() { ref_.fetch_add(1, std::memory_order_relaxed); } + void Unref() { + if (ref_.fetch_sub(1, std::memory_order_acq_rel) == 1) { destroyer_fn_(this); } } diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 647f2a46875..d610e3d72fc 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -84,7 +84,7 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" @@ -103,7 +103,7 @@ grpc_core::TraceFlag grpc_call_error_trace(false, "call_error"); grpc_core::TraceFlag grpc_compression_trace(false, "compression"); grpc_core::TraceFlag grpc_call_trace(false, "call"); -grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount"); +grpc_core::TraceFlag grpc_call_refcount_trace(false, "call_refcount"); namespace grpc_core { @@ -452,31 +452,6 @@ class FilterStackCall final : public Call { static constexpr gpr_atm kRecvNone = 0; static constexpr gpr_atm kRecvInitialMetadataFirst = 1; - enum class PendingOp { - kRecvMessage, - kRecvInitialMetadata, - kRecvTrailingMetadata, - kSends - }; - static intptr_t PendingOpMask(PendingOp op) { - return static_cast(1) << static_cast(op); - } - static std::string PendingOpString(intptr_t pending_ops) { - std::vector pending_op_strings; - if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) { - pending_op_strings.push_back("kRecvMessage"); - } - if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) { - pending_op_strings.push_back("kRecvInitialMetadata"); - } - if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) { - pending_op_strings.push_back("kRecvTrailingMetadata"); - } - if (pending_ops & PendingOpMask(PendingOp::kSends)) { - pending_op_strings.push_back("kSends"); - } - return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}"); - } struct BatchControl { FilterStackCall* call_ = nullptr; grpc_transport_stream_op_batch op_; @@ -501,26 +476,17 @@ class FilterStackCall final : public Call { } completion_data_; grpc_closure start_batch_; grpc_closure finish_batch_; - std::atomic ops_pending_{0}; + std::atomic steps_to_complete_{0}; AtomicError batch_error_; - void set_pending_ops(uintptr_t ops) { - ops_pending_.store(ops, std::memory_order_release); + void set_num_steps_to_complete(uintptr_t steps) { + steps_to_complete_.store(steps, std::memory_order_release); } - bool completed_batch_step(PendingOp op) { - auto mask = PendingOpMask(op); - auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this, - PendingOpString(mask).c_str(), - PendingOpString(r & ~mask).c_str(), - completion_data_.notify_tag.tag); - } - GPR_ASSERT((r & mask) != 0); - return r == mask; + bool completed_batch_step() { + return steps_to_complete_.fetch_sub(1, std::memory_order_acq_rel) == 1; } void PostCompletion(); - void FinishStep(PendingOp op); + void FinishStep(); void ProcessDataAfterMetadata(); void ReceivingStreamReady(grpc_error_handle error); void ValidateFilteredMetadata(); @@ -1139,10 +1105,6 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl( void FilterStackCall::BatchControl::PostCompletion() { FilterStackCall* call = call_; grpc_error_handle error = batch_error_.get(); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "tag:%p batch_error=%s", completion_data_.notify_tag.tag, - error.ToString().c_str()); - } if (op_.send_initial_metadata) { call->send_initial_metadata_.Clear(); @@ -1172,12 +1134,14 @@ void FilterStackCall::BatchControl::PostCompletion() { batch_error_.set(absl::OkStatus()); if (completion_data_.notify_tag.is_closure) { + // unrefs error call_ = nullptr; Closure::Run(DEBUG_LOCATION, static_cast(completion_data_.notify_tag.tag), error); call->InternalUnref("completion"); } else { + // unrefs error grpc_cq_end_op( call->cq_, completion_data_.notify_tag.tag, error, [](void* user_data, grpc_cq_completion* /*storage*/) { @@ -1190,8 +1154,8 @@ void FilterStackCall::BatchControl::PostCompletion() { } } -void FilterStackCall::BatchControl::FinishStep(PendingOp op) { - if (GPR_UNLIKELY(completed_batch_step(op))) { +void FilterStackCall::BatchControl::FinishStep() { + if (GPR_UNLIKELY(completed_batch_step())) { PostCompletion(); } } @@ -1201,7 +1165,7 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { if (!call->receiving_slice_buffer_.has_value()) { *call->receiving_buffer_ = nullptr; call->receiving_message_ = false; - FinishStep(PendingOp::kRecvMessage); + FinishStep(); } else { call->test_only_last_message_flags_ = call->receiving_stream_flags_; if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) && @@ -1216,20 +1180,12 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { &(*call->receiving_buffer_)->data.raw.slice_buffer); call->receiving_message_ = false; call->receiving_slice_buffer_.reset(); - FinishStep(PendingOp::kRecvMessage); + FinishStep(); } } void FilterStackCall::BatchControl::ReceivingStreamReady( grpc_error_handle error) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, - "tag:%p ReceivingStreamReady error=%s " - "receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR, - completion_data_.notify_tag.tag, error.ToString().c_str(), - call_->receiving_slice_buffer_.has_value(), - gpr_atm_no_barrier_load(&call_->recv_state_)); - } FilterStackCall* call = call_; if (!error.ok()) { call->receiving_slice_buffer_.reset(); @@ -1347,7 +1303,7 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady( Closure::Run(DEBUG_LOCATION, saved_rsr_closure, error); } - FinishStep(PendingOp::kRecvInitialMetadata); + FinishStep(); } void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady( @@ -1356,7 +1312,7 @@ void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady( "recv_trailing_metadata_ready"); grpc_metadata_batch* md = &call_->recv_trailing_metadata_; call_->RecvTrailingFilter(md, error); - FinishStep(PendingOp::kRecvTrailingMetadata); + FinishStep(); } void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) { @@ -1367,7 +1323,7 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) { if (!error.ok()) { call_->CancelWithError(error); } - FinishStep(PendingOp::kSends); + FinishStep(); } namespace { @@ -1394,11 +1350,12 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, size_t i; const grpc_op* op; BatchControl* bctl; + bool has_send_ops = false; + int num_recv_ops = 0; grpc_call_error error = GRPC_CALL_OK; grpc_transport_stream_op_batch* stream_op; grpc_transport_stream_op_batch_payload* stream_op_payload; uint32_t seen_ops = 0; - intptr_t pending_ops = 0; for (i = 0; i < nops; i++) { if (seen_ops & (1u << ops[i].op)) { @@ -1515,7 +1472,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->send_initial_metadata.peer_string = peer_string_atm_ptr(); } - pending_ops |= PendingOpMask(PendingOp::kSends); + has_send_ops = true; break; } case GRPC_OP_SEND_MESSAGE: { @@ -1547,7 +1504,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, send_slice_buffer_.c_slice_buffer()); stream_op_payload->send_message.flags = flags; stream_op_payload->send_message.send_message = &send_slice_buffer_; - pending_ops |= PendingOpMask(PendingOp::kSends); + has_send_ops = true; break; } case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { @@ -1568,7 +1525,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, sent_final_op_ = true; stream_op_payload->send_trailing_metadata.send_trailing_metadata = &send_trailing_metadata_; - pending_ops |= PendingOpMask(PendingOp::kSends); + has_send_ops = true; break; } case GRPC_OP_SEND_STATUS_FROM_SERVER: { @@ -1632,7 +1589,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, &send_trailing_metadata_; stream_op_payload->send_trailing_metadata.sent = &sent_server_trailing_metadata_; - pending_ops |= PendingOpMask(PendingOp::kSends); + has_send_ops = true; break; } case GRPC_OP_RECV_INITIAL_METADATA: { @@ -1667,7 +1624,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->recv_initial_metadata.peer_string = peer_string_atm_ptr(); } - pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata); + ++num_recv_ops; break; } case GRPC_OP_RECV_MESSAGE: { @@ -1703,7 +1660,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_message.recv_message_ready = &receiving_stream_ready_; - pending_ops |= PendingOpMask(PendingOp::kRecvMessage); + ++num_recv_ops; break; } case GRPC_OP_RECV_STATUS_ON_CLIENT: { @@ -1742,7 +1699,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready = &receiving_trailing_metadata_ready_; - pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata); + ++num_recv_ops; break; } case GRPC_OP_RECV_CLOSE_ON_SERVER: { @@ -1775,7 +1732,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready = &receiving_trailing_metadata_ready_; - pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata); + ++num_recv_ops; break; } } @@ -1785,9 +1742,9 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, if (!is_notify_tag_closure) { GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag)); } - bctl->set_pending_ops(pending_ops); + bctl->set_num_steps_to_complete((has_send_ops ? 1 : 0) + num_recv_ops); - if (pending_ops & PendingOpMask(PendingOp::kSends)) { + if (has_send_ops) { GRPC_CLOSURE_INIT( &bctl->finish_batch_, [](void* bctl, grpc_error_handle error) { @@ -1797,13 +1754,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op->on_complete = &bctl->finish_batch_; } - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl, - PendingOpString(pending_ops).c_str(), - grpc_transport_stream_op_batch_string(stream_op).c_str(), - bctl->completion_data_.notify_tag.tag); - } - ExecuteBatch(stream_op, &bctl->start_batch_); done: @@ -1877,52 +1827,21 @@ class PromiseBasedCall : public Call, public grpc_event_engine::experimental::EventEngine:: Closure /* for deadlines */ { public: - PromiseBasedCall(Arena* arena, uint32_t initial_external_refs, - const grpc_call_create_args& args); + PromiseBasedCall(Arena* arena, const grpc_call_create_args& args); void ContextSet(grpc_context_index elem, void* value, void (*destroy)(void* value)) override; void* ContextGet(grpc_context_index elem) const override; void SetCompletionQueue(grpc_completion_queue* cq) override; - void SetCompletionQueueLocked(grpc_completion_queue* cq) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) { - MutexLock lock(&mu_); - CancelWithErrorLocked(std::move(error)); - } - virtual void CancelWithErrorLocked(absl::Status error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0; - bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) { - MutexLock lock(&mu_); - return completed_; - } - - void Orphan() final { - MutexLock lock(&mu_); - if (!completed_) { - CancelWithErrorLocked(absl::CancelledError("Call orphaned")); - } - } // Implementation of call refcounting: move this to DualRefCounted once we // don't need to maintain FilterStackCall compatibility void ExternalRef() final { - const uint64_t prev_ref_pair = - refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed); - if (grpc_call_refcount_trace.enabled()) { - gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(), - GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), - GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair)); - } + refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed); } void ExternalUnref() final { const uint64_t prev_ref_pair = refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel); - if (grpc_call_refcount_trace.enabled()) { - gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(), - GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), - GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1); - } const uint32_t strong_refs = GetStrongRefs(prev_ref_pair); if (GPR_UNLIKELY(strong_refs == 1)) { Orphan(); @@ -1930,22 +1849,12 @@ class PromiseBasedCall : public Call, // Now drop the weak ref. InternalUnref("external_ref"); } - void InternalRef(const char* reason) final { - uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed); - if (grpc_call_refcount_trace.enabled()) { - gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason, - GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n), - GetWeakRefs(n) + 1); - } + void InternalRef(const char*) final { + refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed); } - void InternalUnref(const char* reason) final { + void InternalUnref(const char*) final { const uint64_t prev_ref_pair = refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel); - if (grpc_call_refcount_trace.enabled()) { - gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(), - reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), - GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1); - } if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) { DeleteThis(); } @@ -2045,12 +1954,9 @@ class PromiseBasedCall : public Call, void UpdateDeadline(Timestamp deadline); void ResetDeadline(); - // Implementation of EventEngine::Closure, called when deadline expires void Run() override; - virtual ServerCallContext* server_call_context() { return nullptr; } - protected: class ScopedContext : public ScopedActivity, @@ -2088,6 +1994,11 @@ class PromiseBasedCall : public Call, uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); } bool has_value() const { return index_ != kNullIndex; } + std::string ToString() const { + return index_ == kNullIndex ? "null" + : std::to_string(static_cast(index_)); + } + private: enum : uint8_t { kNullIndex = 0xff }; uint8_t index_; @@ -2105,27 +2016,22 @@ class PromiseBasedCall : public Call, // The following correspond with the batch operations from above kReceiveInitialMetadata, kReceiveStatusOnClient, - kReceiveCloseOnServer = kReceiveStatusOnClient, kSendMessage, kReceiveMessage, - kSendStatusFromServer, - kSendCloseFromClient = kSendStatusFromServer, }; - const char* PendingOpString(PendingOp reason) const { + static constexpr const char* PendingOpString(PendingOp reason) { switch (reason) { case PendingOp::kStartingBatch: return "StartingBatch"; case PendingOp::kReceiveInitialMetadata: return "ReceiveInitialMetadata"; case PendingOp::kReceiveStatusOnClient: - return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer"; + return "ReceiveStatusOnClient"; case PendingOp::kSendMessage: return "SendMessage"; case PendingOp::kReceiveMessage: return "ReceiveMessage"; - case PendingOp::kSendStatusFromServer: - return is_client() ? "SendCloseFromClient" : "SendStatusFromServer"; } return "Unknown"; } @@ -2135,6 +2041,7 @@ class PromiseBasedCall : public Call, } Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; } + // Begin work on a completion, recording the tag/closure to notify. // Use the op selected in \a ops to determine the index to allocate into. // Starts the "StartingBatch" PendingOp immediately. @@ -2144,21 +2051,12 @@ class PromiseBasedCall : public Call, // Add one pending op to the completion, and return it. Completion AddOpToCompletion(const Completion& completion, PendingOp reason) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Stringify a completion - std::string CompletionString(const Completion& completion) const { - return completion.has_value() - ? absl::StrFormat( - "%d:tag=%p", static_cast(completion.index()), - completion_info_[completion.index()].pending.tag) - : "no-completion"; - } // Finish one op on the completion. Must have been previously been added. // The completion as a whole finishes when all pending ops finish. void FinishOpOnCompletion(Completion* completion, PendingOp reason) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Mark the completion as failed. Does not finish it. - void FailCompletion(const Completion& completion, - SourceLocation source_location = {}); + void FailCompletion(const Completion& completion); // Run the promise polling loop until it stalls. void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Update the promise state once. @@ -2189,56 +2087,6 @@ class PromiseBasedCall : public Call, finalization_.Run(&final_info); } - std::string PresentAndCompletionText(const char* caption, bool has, - const Completion& completion) const { - if (has) { - if (completion.has_value()) { - return absl::StrCat(caption, ":", CompletionString(completion), " "); - } else { - return absl::StrCat(caption, - ":!!BUG:operation is present, no completion!! "); - } - } else { - if (!completion.has_value()) { - return ""; - } else { - return absl::StrCat(caption, ":no-op:", CompletionString(completion), - " "); - } - } - } - - std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return absl::StrCat(PresentAndCompletionText("outstanding_send", - outstanding_send_.has_value(), - send_message_completion_) - .c_str(), - PresentAndCompletionText("outstanding_recv", - outstanding_recv_.has_value(), - recv_message_completion_) - .c_str()); - } - - void StartRecvMessage(const grpc_op& op, const Completion& completion, - PipeReceiver* receiver) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void PollRecvMessage(grpc_compression_algorithm compression_algorithm) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void StartSendMessage(const grpc_op& op, const Completion& completion, - PipeSender* sender) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - - bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return completed_; - } - void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; } - bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return outstanding_send_.has_value(); - } - private: union CompletionInfo { struct Pending { @@ -2338,7 +2186,7 @@ class PromiseBasedCall : public Call, } mutable Mutex mu_; - std::atomic refs_; + std::atomic refs_{MakeRefPair(1, 0)}; CallContext call_context_{this}; bool keep_polling_ ABSL_GUARDED_BY(mu()) = false; @@ -2352,14 +2200,6 @@ class PromiseBasedCall : public Call, // Current deadline. Timestamp deadline_ = Timestamp::InfFuture(); grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_; - absl::optional::PushType> outstanding_send_ - ABSL_GUARDED_BY(mu_); - absl::optional> outstanding_recv_ - ABSL_GUARDED_BY(mu_); - grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr; - Completion send_message_completion_ ABSL_GUARDED_BY(mu_); - Completion recv_message_completion_ ABSL_GUARDED_BY(mu_); - bool completed_ ABSL_GUARDED_BY(mu_) = false; }; template @@ -2375,11 +2215,10 @@ grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args, return absl::OkStatus(); } -PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs, +PromiseBasedCall::PromiseBasedCall(Arena* arena, const grpc_call_create_args& args) : Call(arena, args.server_transport_data == nullptr, args.send_deadline, args.channel->Ref()), - refs_(MakeRefPair(initial_external_refs, 0)), cq_(args.cq) { if (args.cq != nullptr) { GPR_ASSERT(args.pollset_set_alternative == nullptr && @@ -2439,8 +2278,8 @@ PromiseBasedCall::Completion PromiseBasedCall::StartCompletion( void* tag, bool is_closure, const grpc_op* ops) { Completion c(BatchSlotForOp(ops[0].op)); if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(), - CompletionString(c).c_str(), tag); + gpr_log(GPR_INFO, "%sStartCompletion %s tag=%p", DebugTag().c_str(), + c.ToString().c_str(), tag); } if (!is_closure) { grpc_cq_begin_op(cq(), tag); @@ -2453,10 +2292,9 @@ PromiseBasedCall::Completion PromiseBasedCall::StartCompletion( PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion( const Completion& completion, PendingOp reason) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(), - CompletionString(completion).c_str(), PendingOpString(reason)); + gpr_log(GPR_INFO, "%sAddOpToCompletion %s %s", DebugTag().c_str(), + completion.ToString().c_str(), PendingOpString(reason)); } - GPR_ASSERT(completion.has_value()); auto& pending_op_bits = completion_info_[completion.index()].pending.pending_op_bits; GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0); @@ -2464,12 +2302,10 @@ PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion( return Completion(completion.index()); } -void PromiseBasedCall::FailCompletion(const Completion& completion, - SourceLocation location) { +void PromiseBasedCall::FailCompletion(const Completion& completion) { if (grpc_call_trace.enabled()) { - gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR, - "%s[call] FailCompletion %s", DebugTag().c_str(), - CompletionString(completion).c_str()); + gpr_log(GPR_INFO, "%sFailCompletion %s", DebugTag().c_str(), + completion.ToString().c_str()); } completion_info_[completion.index()].pending.success = false; } @@ -2488,9 +2324,8 @@ void PromiseBasedCall::FinishOpOnCompletion(Completion* completion, } } gpr_log( - GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s", - DebugTag().c_str(), completion_info_[completion->index()].pending.tag, - CompletionString(*completion).c_str(), PendingOpString(reason), + GPR_INFO, "%sFinishOpOnCompletion %s %s %s", DebugTag().c_str(), + completion->ToString().c_str(), PendingOpString(reason), (pending.empty() ? (success ? std::string("done") : std::string("failed")) : absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ","))) @@ -2525,10 +2360,6 @@ void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; } void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) { MutexLock lock(&mu_); - SetCompletionQueueLocked(cq); -} - -void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) { cq_ = cq; GRPC_CQ_INTERNAL_REF(cq, "bind"); call_context_.pollent_ = @@ -2561,121 +2392,6 @@ void PromiseBasedCall::Run() { InternalUnref("deadline"); } -void PromiseBasedCall::StartSendMessage(const grpc_op& op, - const Completion& completion, - PipeSender* sender) { - GPR_ASSERT(!outstanding_send_.has_value()); - if (!completed_) { - send_message_completion_ = - AddOpToCompletion(completion, PendingOp::kSendMessage); - SliceBuffer send; - grpc_slice_buffer_swap( - &op.data.send_message.send_message->data.raw.slice_buffer, - send.c_slice_buffer()); - outstanding_send_.emplace(sender->Push( - GetContext()->MakePooled(std::move(send), op.flags))); - } else { - FailCompletion(completion); - } -} - -bool PromiseBasedCall::PollSendMessage() { - if (!outstanding_send_.has_value()) return true; - Poll r = (*outstanding_send_)(); - if (const bool* result = absl::get_if(&r)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(), - *result ? "successfully" : "with failure"); - } - if (!*result) { - FailCompletion(send_message_completion_); - return false; - } - FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); - outstanding_send_.reset(); - } - return true; -} - -void PromiseBasedCall::CancelSendMessage() { - if (!outstanding_send_.has_value()) return; - FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); - outstanding_send_.reset(); -} - -void PromiseBasedCall::StartRecvMessage(const grpc_op& op, - const Completion& completion, - PipeReceiver* receiver) { - GPR_ASSERT(!outstanding_recv_.has_value()); - recv_message_ = op.data.recv_message.recv_message; - recv_message_completion_ = - AddOpToCompletion(completion, PendingOp::kReceiveMessage); - outstanding_recv_.emplace(receiver->Next()); -} - -void PromiseBasedCall::PollRecvMessage( - grpc_compression_algorithm incoming_compression_algorithm) { - if (!outstanding_recv_.has_value()) return; - Poll> r = (*outstanding_recv_)(); - if (auto* result = absl::get_if>(&r)) { - outstanding_recv_.reset(); - if (result->has_value()) { - MessageHandle& message = **result; - if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && - (incoming_compression_algorithm != GRPC_COMPRESS_NONE)) { - *recv_message_ = grpc_raw_compressed_byte_buffer_create( - nullptr, 0, incoming_compression_algorithm); - } else { - *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0); - } - grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(), - &(*recv_message_)->data.raw.slice_buffer); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[call] PollRecvMessage: outstanding_recv finishes: received " - "%" PRIdPTR " byte message", - DebugTag().c_str(), - (*recv_message_)->data.raw.slice_buffer.length); - } - } else if (result->cancelled()) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[call] PollRecvMessage: outstanding_recv finishes: received " - "end-of-stream with error", - DebugTag().c_str()); - } - FailCompletion(recv_message_completion_); - *recv_message_ = nullptr; - } else { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[call] PollRecvMessage: outstanding_recv finishes: received " - "end-of-stream", - DebugTag().c_str()); - } - *recv_message_ = nullptr; - } - FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); - } else if (completed_) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%s[call] UpdateOnce: outstanding_recv finishes: promise has " - "completed without queuing a message, forcing end-of-stream", - DebugTag().c_str()); - } - outstanding_recv_.reset(); - *recv_message_ = nullptr; - FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); - } -} - -void PromiseBasedCall::CancelRecvMessage() { - if (!outstanding_recv_.has_value()) return; - *recv_message_ = nullptr; - outstanding_recv_.reset(); - FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); -} - /////////////////////////////////////////////////////////////////////////////// // CallContext @@ -2689,35 +2405,10 @@ void CallContext::IncrementRefCount(const char* reason) { void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); } -gpr_atm* CallContext::peer_string_atm_ptr() { - return call_->peer_string_atm_ptr(); -} - void CallContext::UpdateDeadline(Timestamp deadline) { call_->UpdateDeadline(deadline); } -ServerCallContext* CallContext::server_call_context() { - return call_->server_call_context(); -} - -/////////////////////////////////////////////////////////////////////////////// -// PublishMetadataArray - -namespace { -void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) { - const auto md_count = md->count(); - if (md_count > array->capacity) { - array->capacity = - std::max(array->capacity + md->count(), array->capacity * 3 / 2); - array->metadata = static_cast( - gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity)); - } - PublishToAppEncoder encoder(array); - md->Encode(&encoder); -} -} // namespace - /////////////////////////////////////////////////////////////////////////////// // ClientPromiseBasedCall @@ -2725,7 +2416,7 @@ void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) { class ClientPromiseBasedCall final : public PromiseBasedCall { public: ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args) - : PromiseBasedCall(arena, 1, *args) { + : PromiseBasedCall(arena, *args) { global_stats().IncrementClientCallsCreated(); ScopedContext context(this); send_initial_metadata_ = @@ -2753,12 +2444,16 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { // before context. auto c2s = std::move(client_to_server_messages_); auto s2c = std::move(server_to_client_messages_); - auto sim = std::move(server_initial_metadata_); } absl::string_view GetServerAuthority() const override { abort(); } - void CancelWithErrorLocked(grpc_error_handle error) override - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + void CancelWithError(grpc_error_handle error) override; + bool Completed() override; + void Orphan() override { + MutexLock lock(mu()); + ScopedContext ctx(this); + if (!completed_) Finish(ServerMetadataFromStatus(absl::CancelledError())); + } bool is_trailers_only() const override { MutexLock lock(mu()); return is_trailers_only_; @@ -2788,6 +2483,9 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { // Start the underlying promise. void StartPromise(ClientMetadataHandle client_initial_metadata) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Publish some metadata out to the application. + static void PublishMetadataArray(grpc_metadata_array* array, + ServerMetadata* md); // Publish status out to the application. void PublishStatus( grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args, @@ -2798,23 +2496,29 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); ArenaPromise promise_ ABSL_GUARDED_BY(mu()); - Pipe server_initial_metadata_ ABSL_GUARDED_BY(mu()){ - arena()}; + Latch server_initial_metadata_ ABSL_GUARDED_BY(mu()); Pipe client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()}; Pipe server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()}; ClientMetadataHandle send_initial_metadata_; grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr; + grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu()) = nullptr; absl::variant recv_status_on_client_ ABSL_GUARDED_BY(mu()); - absl::optional> + absl::optional::PushType> outstanding_send_ + ABSL_GUARDED_BY(mu()); + absl::optional::NextType> outstanding_recv_ + ABSL_GUARDED_BY(mu()); + absl::optional> server_initial_metadata_ready_; absl::optional incoming_compression_algorithm_; Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu()); Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu()); - Completion close_send_completion_ ABSL_GUARDED_BY(mu()); + Completion send_message_completion_ ABSL_GUARDED_BY(mu()); + Completion recv_message_completion_ ABSL_GUARDED_BY(mu()); + bool completed_ ABSL_GUARDED_BY(mu()) = false; bool is_trailers_only_ ABSL_GUARDED_BY(mu()); }; @@ -2823,13 +2527,14 @@ void ClientPromiseBasedCall::StartPromise( GPR_ASSERT(!promise_.has_value()); promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{ std::move(client_initial_metadata), - &server_initial_metadata_.sender, + &server_initial_metadata_, &client_to_server_messages_.receiver, &server_to_client_messages_.sender, }); } -void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) { +void ClientPromiseBasedCall::CancelWithError(grpc_error_handle error) { + MutexLock lock(mu()); ScopedContext context(this); Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error))); } @@ -2879,7 +2584,7 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, // compression not implemented GPR_ASSERT( !op.data.send_initial_metadata.maybe_compression_level.is_set); - if (!completed()) { + if (!completed_) { CToMetadata(op.data.send_initial_metadata.metadata, op.data.send_initial_metadata.count, send_initial_metadata_.get()); @@ -2889,8 +2594,7 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, case GRPC_OP_RECV_INITIAL_METADATA: { recv_initial_metadata_ = op.data.recv_initial_metadata.recv_initial_metadata; - server_initial_metadata_ready_.emplace( - server_initial_metadata_.receiver.Next()); + server_initial_metadata_ready_.emplace(server_initial_metadata_.Wait()); recv_initial_metadata_completion_ = AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata); } break; @@ -2905,17 +2609,32 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, recv_status_on_client_ = op.data.recv_status_on_client; } } break; - case GRPC_OP_SEND_MESSAGE: - StartSendMessage(op, completion, &client_to_server_messages_.sender); - break; - case GRPC_OP_RECV_MESSAGE: - StartRecvMessage(op, completion, &server_to_client_messages_.receiver); - break; - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: - close_send_completion_ = - AddOpToCompletion(completion, PendingOp::kSendCloseFromClient); - GPR_ASSERT(close_send_completion_.has_value()); - break; + case GRPC_OP_SEND_MESSAGE: { + GPR_ASSERT(!outstanding_send_.has_value()); + if (!completed_) { + send_message_completion_ = + AddOpToCompletion(completion, PendingOp::kSendMessage); + SliceBuffer send; + grpc_slice_buffer_swap( + &op.data.send_message.send_message->data.raw.slice_buffer, + send.c_slice_buffer()); + outstanding_send_.emplace(client_to_server_messages_.sender.Push( + GetContext()->MakePooled(std::move(send), + op.flags))); + } else { + FailCompletion(completion); + } + } break; + case GRPC_OP_RECV_MESSAGE: { + GPR_ASSERT(!outstanding_recv_.has_value()); + recv_message_ = op.data.recv_message.recv_message; + recv_message_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveMessage); + outstanding_recv_.emplace(server_to_client_messages_.receiver.Next()); + } break; + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { + client_to_server_messages_.sender.Close(); + } break; case GRPC_OP_SEND_STATUS_FROM_SERVER: case GRPC_OP_RECV_CLOSE_ON_SERVER: abort(); // unreachable @@ -2950,48 +2669,78 @@ void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) { metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE); server_initial_metadata_ready_.reset(); GPR_ASSERT(recv_initial_metadata_ != nullptr); - PublishMetadataArray(metadata, - std::exchange(recv_initial_metadata_, nullptr)); + PublishMetadataArray(std::exchange(recv_initial_metadata_, nullptr), + metadata); FinishOpOnCompletion(&recv_initial_metadata_completion_, PendingOp::kReceiveInitialMetadata); } void ClientPromiseBasedCall::UpdateOnce() { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s", - DebugTag().c_str(), - PresentAndCompletionText("server_initial_metadata_ready", - server_initial_metadata_ready_.has_value(), - recv_initial_metadata_completion_) - .c_str(), - PollStateDebugString().c_str(), - promise_.has_value() ? "true" : "false"); + auto present_and_completion_text = + [](const char* caption, bool has, + const Completion& completion) -> std::string { + if (has) { + if (completion.has_value()) { + return absl::StrCat(caption, ":", + static_cast(completion.index()), " "); + } else { + return absl::StrCat(caption, + ":!!BUG:operation is present, no completion!! "); + } + } else { + if (!completion.has_value()) { + return ""; + } else { + return absl::StrCat( + caption, ":no-op:", static_cast(completion.index()), " "); + } + } + }; + gpr_log( + GPR_INFO, "%sUpdateOnce: %s%s%shas_promise=%s", DebugTag().c_str(), + present_and_completion_text("server_initial_metadata_ready", + server_initial_metadata_ready_.has_value(), + recv_initial_metadata_completion_) + .c_str(), + present_and_completion_text("outstanding_send", + outstanding_send_.has_value(), + send_message_completion_) + .c_str(), + present_and_completion_text("outstanding_recv", + outstanding_recv_.has_value(), + recv_message_completion_) + .c_str(), + promise_.has_value() ? "true" : "false"); + } + if (send_message_completion_.has_value()) { + FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); } if (server_initial_metadata_ready_.has_value()) { - Poll> r = - (*server_initial_metadata_ready_)(); - if (auto* server_initial_metadata = - absl::get_if>(&r)) { - PublishInitialMetadata(server_initial_metadata->value().get()); - } else if (completed()) { + Poll r = (*server_initial_metadata_ready_)(); + if (ServerMetadata*** server_initial_metadata = + absl::get_if(&r)) { + PublishInitialMetadata(**server_initial_metadata); + } else if (completed_) { ServerMetadata no_metadata{GetContext()}; PublishInitialMetadata(&no_metadata); } } - if (!PollSendMessage()) { - Finish(ServerMetadataFromStatus(absl::Status( - absl::StatusCode::kInternal, "Failed to send message to server"))); - } - if (!is_sending() && close_send_completion_.has_value()) { - client_to_server_messages_.sender.Close(); - FinishOpOnCompletion(&close_send_completion_, - PendingOp::kSendCloseFromClient); + if (outstanding_send_.has_value()) { + Poll r = (*outstanding_send_)(); + if (const bool* result = absl::get_if(&r)) { + outstanding_send_.reset(); + if (!*result) { + FailCompletion(send_message_completion_); + Finish(ServerMetadataFromStatus(absl::Status( + absl::StatusCode::kInternal, "Failed to send message to server"))); + } + } } if (promise_.has_value()) { Poll r = promise_(); if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s", - DebugTag().c_str(), + gpr_log(GPR_INFO, "%sUpdateOnce: promise returns %s", DebugTag().c_str(), PollToString(r, [](const ServerMetadataHandle& h) { return h->DebugString(); }).c_str()); @@ -3001,33 +2750,72 @@ void ClientPromiseBasedCall::UpdateOnce() { Finish(std::move(*result)); } } - if (incoming_compression_algorithm_.has_value()) { - PollRecvMessage(*incoming_compression_algorithm_); + if (incoming_compression_algorithm_.has_value() && + outstanding_recv_.has_value()) { + Poll> r = (*outstanding_recv_)(); + if (auto* result = absl::get_if>(&r)) { + outstanding_recv_.reset(); + if (result->has_value()) { + MessageHandle& message = **result; + if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && + (incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { + *recv_message_ = grpc_raw_compressed_byte_buffer_create( + nullptr, 0, *incoming_compression_algorithm_); + } else { + *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0); + } + grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(), + &(*recv_message_)->data.raw.slice_buffer); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: received %" PRIdPTR + " byte message", + DebugTag().c_str(), + (*recv_message_)->data.raw.slice_buffer.length); + } + } else { + if (grpc_call_trace.enabled()) { + gpr_log( + GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: received end-of-stream", + DebugTag().c_str()); + } + *recv_message_ = nullptr; + } + FinishOpOnCompletion(&recv_message_completion_, + PendingOp::kReceiveMessage); + } else if (completed_) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: promise has " + "completed without queuing a message, forcing end-of-stream", + DebugTag().c_str()); + } + outstanding_recv_.reset(); + *recv_message_ = nullptr; + FinishOpOnCompletion(&recv_message_completion_, + PendingOp::kReceiveMessage); + } } } void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(), + gpr_log(GPR_INFO, "%sFinish: %s", DebugTag().c_str(), trailing_metadata->DebugString().c_str()); } promise_ = ArenaPromise(); ResetDeadline(); - set_completed(); + completed_ = true; if (recv_initial_metadata_ != nullptr) { ForceImmediateRepoll(); } const bool pending_initial_metadata = server_initial_metadata_ready_.has_value(); - if (!pending_initial_metadata) { - server_initial_metadata_ready_.emplace( - server_initial_metadata_.receiver.Next()); - } - Poll> r = - (*server_initial_metadata_ready_)(); server_initial_metadata_ready_.reset(); - if (auto* result = absl::get_if>(&r)) { - if (pending_initial_metadata) PublishInitialMetadata(result->value().get()); + Poll r = server_initial_metadata_.Wait()(); + if (auto* result = absl::get_if(&r)) { + if (pending_initial_metadata) PublishInitialMetadata(**result); is_trailers_only_ = false; } else { if (pending_initial_metadata) { @@ -3099,426 +2887,34 @@ void ClientPromiseBasedCall::PublishStatus( *op_args.error_string = gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str()); } - PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata); + PublishMetadataArray(op_args.trailing_metadata, trailing_metadata.get()); // Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding // (so we don't call through twice) recv_status_on_client_ = absl::monostate(); FinishOpOnCompletion(&recv_status_on_client_completion_, PendingOp::kReceiveStatusOnClient); } -#endif -/////////////////////////////////////////////////////////////////////////////// -// ServerPromiseBasedCall - -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL - -class ServerPromiseBasedCall final : public PromiseBasedCall { - public: - ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args); - - void CancelWithErrorLocked(grpc_error_handle) override - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); - grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, - bool is_notify_tag_closure) override; - bool failed_before_recv_message() const override { abort(); } - bool is_trailers_only() const override { abort(); } - absl::string_view GetServerAuthority() const override { return ""; } - - // Polling order for the server promise stack: - // - // │ ┌───────────────────────────────────────┐ - // │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management, - // │ ├───────────────────────────────────────┤ signal call end to app - // │ │ ConnectedChannel ├─┐ - // │ ├───────────────────────────────────────┤ └► Interactions with the - // │ │ ... closest to transport filter │ transport - send/recv msgs - // │ ├───────────────────────────────────────┤ and metadata, call phase - // │ │ ... │ ordering - // │ ├───────────────────────────────────────┤ - // │ │ ... closest to app filter │ ┌► Request matching, initial - // │ ├───────────────────────────────────────┤ │ setup, publishing call to - // │ │ Server::ChannelData::MakeCallPromise ├─┘ application - // │ ├───────────────────────────────────────┤ - // │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions, - // ▼ └───────────────────────────────────────┘ forwarding messages, - // Polling & sending trailing metadata - // instantiation - // order - - void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override; - Poll PollTopOfCall() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); - - std::string DebugTag() const override { - return absl::StrFormat("SERVER_CALL[%p]: ", this); +void ClientPromiseBasedCall::PublishMetadataArray(grpc_metadata_array* array, + ServerMetadata* md) { + const auto md_count = md->count(); + if (md_count > array->capacity) { + array->capacity = + std::max(array->capacity + md->count(), array->capacity * 3 / 2); + array->metadata = static_cast( + gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity)); } + PublishToAppEncoder encoder(array); + md->Encode(&encoder); +} - ServerCallContext* server_call_context() override { return &call_context_; } - - private: - class RecvCloseOpCancelState { - public: - // Request that receiver be filled in per grpc_op_recv_close_on_server. - // Returns true if the request can be fulfilled immediately. - // Returns false if the request will be fulfilled later. - bool ReceiveCloseOnServerOpStarted(int* receiver) { - switch (state_) { - case kUnset: - state_ = reinterpret_cast(receiver); - return false; - case kFinishedWithFailure: - *receiver = 1; - return true; - case kFinishedWithSuccess: - *receiver = 0; - return true; - default: - abort(); // unreachable - } - } - - // Mark the call as having completed. - // Returns true if this finishes a previous RequestReceiveCloseOnServer. - bool CompleteCall(bool success) { - switch (state_) { - case kUnset: - state_ = success ? kFinishedWithSuccess : kFinishedWithFailure; - return false; - case kFinishedWithFailure: - case kFinishedWithSuccess: - abort(); // unreachable - default: - *reinterpret_cast(state_) = success ? 0 : 1; - state_ = success ? kFinishedWithSuccess : kFinishedWithFailure; - return true; - } - } - - std::string ToString() const { - switch (state_) { - case kUnset: - return "Unset"; - case kFinishedWithFailure: - return "FinishedWithFailure"; - case kFinishedWithSuccess: - return "FinishedWithSuccess"; - default: - return absl::StrFormat("WaitingForReceiver(%p)", - reinterpret_cast(state_)); - } - } - - private: - static constexpr uintptr_t kUnset = 0; - static constexpr uintptr_t kFinishedWithFailure = 1; - static constexpr uintptr_t kFinishedWithSuccess = 2; - // Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess - // OR an int* that wants to receive the final status. - uintptr_t state_ = kUnset; - }; - - grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const; - void CommitBatch(const grpc_op* ops, size_t nops, - const Completion& completion) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); - - friend class ServerCallContext; - ServerCallContext call_context_; - Server* const server_; - ArenaPromise promise_ ABSL_GUARDED_BY(mu()); - PipeSender* server_to_client_messages_ ABSL_GUARDED_BY(mu()) = - nullptr; - PipeReceiver* client_to_server_messages_ - ABSL_GUARDED_BY(mu()) = nullptr; - using SendInitialMetadataState = - absl::variant*, - typename PipeSender::PushType>; - SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) = - absl::monostate{}; - ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu()); - grpc_compression_algorithm incoming_compression_algorithm_ - ABSL_GUARDED_BY(mu()); - RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu()); - Completion recv_close_completion_ ABSL_GUARDED_BY(mu()); - bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false; - Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu()); - ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu()); -}; - -ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena, - grpc_call_create_args* args) - : PromiseBasedCall(arena, 0, *args), - call_context_(this, args->server_transport_data), - server_(args->server) { - global_stats().IncrementServerCallsCreated(); - channelz::ServerNode* channelz_node = server_->channelz_node(); - if (channelz_node != nullptr) { - channelz_node->RecordCallStarted(); - } +bool ClientPromiseBasedCall::Completed() { MutexLock lock(mu()); - ScopedContext activity_context(this); - promise_ = channel()->channel_stack()->MakeServerCallPromise( - CallArgs{nullptr, nullptr, nullptr, nullptr}); + return completed_; } -Poll ServerPromiseBasedCall::PollTopOfCall() { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(), - cancel_send_and_receive_ ? "force-" : "", - send_trailing_metadata_ != nullptr - ? absl::StrCat("send-metadata:", - send_trailing_metadata_->DebugString()) - .c_str() - : " ", - PollStateDebugString().c_str()); - } - - if (cancel_send_and_receive_) { - CancelSendMessage(); - CancelRecvMessage(); - } - - PollSendMessage(); - PollRecvMessage(incoming_compression_algorithm_); - - if (!is_sending() && send_trailing_metadata_ != nullptr) { - server_to_client_messages_->Close(); - return std::move(send_trailing_metadata_); - } - - return Pending{}; -} - -void ServerPromiseBasedCall::UpdateOnce() { - if (grpc_call_trace.enabled()) { - gpr_log( - GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s", - DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(), - recv_close_completion_.has_value() - ? absl::StrCat(":", CompletionString(recv_close_completion_)) - .c_str() - : "", - send_status_from_server_completion_.has_value() - ? absl::StrCat( - "send_status:", - CompletionString(send_status_from_server_completion_), " ") - .c_str() - : "", - PollStateDebugString().c_str(), - promise_.has_value() ? "true" : "false"); - } - if (auto* p = - absl::get_if::PushType>( - &send_initial_metadata_state_)) { - if (!absl::holds_alternative((*p)())) { - send_initial_metadata_state_ = absl::monostate{}; - } - } - if (promise_.has_value()) { - auto r = promise_(); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s", - DebugTag().c_str(), - PollToString(r, [](const ServerMetadataHandle& h) { - return h->DebugString(); - }).c_str()); - } - if (auto* result = absl::get_if(&r)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s", - DebugTag().c_str(), - recv_close_op_cancel_state_.ToString().c_str(), - (*result)->DebugString().c_str()); - } - if (recv_close_op_cancel_state_.CompleteCall( - (*result)->get(GrpcStatusFromWire()).value_or(false))) { - FinishOpOnCompletion(&recv_close_completion_, - PendingOp::kReceiveCloseOnServer); - } - channelz::ServerNode* channelz_node = server_->channelz_node(); - if (channelz_node != nullptr) { - if ((*result) - ->get(GrpcStatusMetadata()) - .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) { - channelz_node->RecordCallSucceeded(); - } else { - channelz_node->RecordCallFailed(); - } - } - if (send_status_from_server_completion_.has_value()) { - FinishOpOnCompletion(&send_status_from_server_completion_, - PendingOp::kSendStatusFromServer); - } - CancelSendMessage(); - CancelRecvMessage(); - set_completed(); - promise_ = ArenaPromise(); - } - } -} - -grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops, - size_t nops) const { - BitSet<8> got_ops; - for (size_t op_idx = 0; op_idx < nops; op_idx++) { - const grpc_op& op = ops[op_idx]; - switch (op.op) { - case GRPC_OP_SEND_INITIAL_METADATA: - if (!AreInitialMetadataFlagsValid(op.flags)) { - return GRPC_CALL_ERROR_INVALID_FLAGS; - } - if (!ValidateMetadata(op.data.send_initial_metadata.count, - op.data.send_initial_metadata.metadata)) { - return GRPC_CALL_ERROR_INVALID_METADATA; - } - break; - case GRPC_OP_SEND_MESSAGE: - if (!AreWriteFlagsValid(op.flags)) { - return GRPC_CALL_ERROR_INVALID_FLAGS; - } - break; - case GRPC_OP_RECV_MESSAGE: - case GRPC_OP_RECV_CLOSE_ON_SERVER: - case GRPC_OP_SEND_STATUS_FROM_SERVER: - if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS; - break; - case GRPC_OP_RECV_INITIAL_METADATA: - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: - case GRPC_OP_RECV_STATUS_ON_CLIENT: - return GRPC_CALL_ERROR_NOT_ON_SERVER; - } - if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; - got_ops.set(op.op); - } - return GRPC_CALL_OK; -} - -void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, - const Completion& completion) { - for (size_t op_idx = 0; op_idx < nops; op_idx++) { - const grpc_op& op = ops[op_idx]; - switch (op.op) { - case GRPC_OP_SEND_INITIAL_METADATA: { - // compression not implemented - GPR_ASSERT( - !op.data.send_initial_metadata.maybe_compression_level.is_set); - if (!completed()) { - auto metadata = arena()->MakePooled(arena()); - CToMetadata(op.data.send_initial_metadata.metadata, - op.data.send_initial_metadata.count, metadata.get()); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] Send initial metadata", - DebugTag().c_str()); - } - auto* pipe = absl::get*>( - send_initial_metadata_state_); - send_initial_metadata_state_ = pipe->Push(std::move(metadata)); - } - } break; - case GRPC_OP_SEND_MESSAGE: - StartSendMessage(op, completion, server_to_client_messages_); - break; - case GRPC_OP_RECV_MESSAGE: - StartRecvMessage(op, completion, client_to_server_messages_); - break; - case GRPC_OP_SEND_STATUS_FROM_SERVER: - send_trailing_metadata_ = arena()->MakePooled(arena()); - CToMetadata(op.data.send_status_from_server.trailing_metadata, - op.data.send_status_from_server.trailing_metadata_count, - send_trailing_metadata_.get()); - send_trailing_metadata_->Set(GrpcStatusMetadata(), - op.data.send_status_from_server.status); - if (auto* details = op.data.send_status_from_server.status_details) { - send_trailing_metadata_->Set(GrpcMessageMetadata(), - Slice(CSliceRef(*details))); - } - send_status_from_server_completion_ = - AddOpToCompletion(completion, PendingOp::kSendStatusFromServer); - break; - case GRPC_OP_RECV_CLOSE_ON_SERVER: - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s", - DebugTag().c_str(), - recv_close_op_cancel_state_.ToString().c_str()); - } - if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted( - op.data.recv_close_on_server.cancelled)) { - recv_close_completion_ = - AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer); - } - break; - case GRPC_OP_RECV_STATUS_ON_CLIENT: - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: - case GRPC_OP_RECV_INITIAL_METADATA: - abort(); // unreachable - } - } -} - -grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops, - size_t nops, - void* notify_tag, - bool is_notify_tag_closure) { - MutexLock lock(mu()); - ScopedContext activity_context(this); - if (nops == 0) { - EndOpImmediately(cq(), notify_tag, is_notify_tag_closure); - return GRPC_CALL_OK; - } - const grpc_call_error validation_result = ValidateBatch(ops, nops); - if (validation_result != GRPC_CALL_OK) { - return validation_result; - } - Completion completion = - StartCompletion(notify_tag, is_notify_tag_closure, ops); - CommitBatch(ops, nops, completion); - Update(); - FinishOpOnCompletion(&completion, PendingOp::kStartingBatch); - return GRPC_CALL_OK; -} - -void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) { - if (!promise_.has_value()) return; - cancel_send_and_receive_ = true; - send_trailing_metadata_ = ServerMetadataFromStatus(error, arena()); - ForceWakeup(); -} - -#endif - -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL -ArenaPromise -ServerCallContext::MakeTopOfServerCallPromise( - CallArgs call_args, grpc_completion_queue* cq, - grpc_metadata_array* publish_initial_metadata, - absl::FunctionRef publish) { - call_->mu()->AssertHeld(); - call_->SetCompletionQueueLocked(cq); - call_->server_to_client_messages_ = call_args.server_to_client_messages; - call_->client_to_server_messages_ = call_args.client_to_server_messages; - call_->send_initial_metadata_state_ = call_args.server_initial_metadata; - call_->incoming_compression_algorithm_ = - call_args.client_initial_metadata->get(GrpcEncodingMetadata()) - .value_or(GRPC_COMPRESS_NONE); - call_->client_initial_metadata_ = - std::move(call_args.client_initial_metadata); - PublishMetadataArray(call_->client_initial_metadata_.get(), - publish_initial_metadata); - call_->ExternalRef(); - publish(call_->c_ptr()); - return [this]() { - call_->mu()->AssertHeld(); - return call_->PollTopOfCall(); - }; -} -#else -ArenaPromise -ServerCallContext::MakeTopOfServerCallPromise( - CallArgs, grpc_completion_queue*, grpc_metadata_array*, - absl::FunctionRef) { - (void)call_; - Crash("Promise-based server call is not enabled"); +gpr_atm* CallContext::peer_string_atm_ptr() { + return call_->peer_string_atm_ptr(); } #endif @@ -3540,16 +2936,11 @@ grpc_error_handle grpc_call_create(grpc_call_create_args* args, grpc_call** out_call) { #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL if (grpc_core::IsPromiseBasedClientCallEnabled() && - args->server_transport_data == nullptr && args->channel->is_promising()) { - return grpc_core::MakePromiseBasedCall( - args, out_call); - } -#endif -#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL - if (grpc_core::IsPromiseBasedServerCallEnabled() && - args->server_transport_data != nullptr && args->channel->is_promising()) { - return grpc_core::MakePromiseBasedCall( - args, out_call); + args->channel->is_promising()) { + if (args->server_transport_data == nullptr) { + return grpc_core::MakePromiseBasedCall( + args, out_call); + } } #endif return grpc_core::FilterStackCall::Create(args, out_call); diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 0a77709946d..92974700960 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -25,7 +25,6 @@ #include #include "absl/functional/any_invocable.h" -#include "absl/functional/function_ref.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -44,14 +43,12 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/iomgr/polling_entity.h" -#include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/server.h" -#include "src/core/lib/transport/transport.h" typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success, void* user_data); @@ -77,28 +74,6 @@ typedef struct grpc_call_create_args { namespace grpc_core { class PromiseBasedCall; -class ServerPromiseBasedCall; - -class ServerCallContext { - public: - ServerCallContext(ServerPromiseBasedCall* call, - const void* server_stream_data) - : call_(call), server_stream_data_(server_stream_data) {} - ArenaPromise MakeTopOfServerCallPromise( - CallArgs call_args, grpc_completion_queue* cq, - grpc_metadata_array* publish_initial_metadata, - absl::FunctionRef publish); - - // Server stream data as supplied by the transport (so we can link the - // transport stream up with the call again). - // TODO(ctiller): legacy API - once we move transports to promises we'll - // create the promise directly and not need to pass around this token. - const void* server_stream_data() { return server_stream_data_; } - - private: - ServerPromiseBasedCall* const call_; - const void* const server_stream_data_; -}; // TODO(ctiller): move more call things into this type class CallContext { @@ -123,8 +98,6 @@ class CallContext { gpr_atm* peer_string_atm_ptr(); grpc_polling_entity* polling_entity() { return &pollent_; } - ServerCallContext* server_call_context(); - private: friend class PromiseBasedCall; // Call final info. @@ -139,7 +112,6 @@ class CallContext { template <> struct ContextType {}; - } // namespace grpc_core // Create a new call based on \a args. diff --git a/src/core/lib/surface/call_trace.cc b/src/core/lib/surface/call_trace.cc index 7f5a07e1b52..f2798aefbc0 100644 --- a/src/core/lib/surface/call_trace.cc +++ b/src/core/lib/surface/call_trace.cc @@ -25,7 +25,6 @@ #include "absl/container/flat_hash_map.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" -#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include @@ -46,8 +45,7 @@ const grpc_channel_filter* PromiseTracingFilterFor( struct DerivedFilter : public grpc_channel_filter { explicit DerivedFilter(const grpc_channel_filter* filter) : grpc_channel_filter{ - // start_transport_stream_op_batch: - grpc_call_next_op, + /* start_transport_stream_op_batch: */ grpc_call_next_op, // make_call_promise: [](grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory next_promise_factory) @@ -56,22 +54,22 @@ const grpc_channel_filter* PromiseTracingFilterFor( static_cast(elem->filter)->filter; gpr_log( GPR_DEBUG, - "%s[%s] CreateCallPromise: client_initial_metadata=%s", + "%sCreateCallPromise[%s]: client_initial_metadata=%s", Activity::current()->DebugTag().c_str(), source_filter->name, call_args.client_initial_metadata->DebugString().c_str()); return [source_filter, child = next_promise_factory( std::move(call_args))]() mutable { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: begin", + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: begin", Activity::current()->DebugTag().c_str(), source_filter->name); auto r = child(); if (auto* p = absl::get_if(&r)) { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: done: %s", + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: done: %s", Activity::current()->DebugTag().c_str(), source_filter->name, (*p)->DebugString().c_str()); } else { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: <>", + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: <", Activity::current()->DebugTag().c_str(), source_filter->name); } @@ -87,24 +85,16 @@ const grpc_channel_filter* PromiseTracingFilterFor( // destroy_call_elem: [](grpc_call_element*, const grpc_call_final_info*, grpc_closure*) {}, - // sizeof_channel_data: - 0, - // init_channel_elem: + /* sizeof_channel_data: */ 0, // init_channel_elem: [](grpc_channel_element*, grpc_channel_element_args*) { return absl::OkStatus(); }, // post_init_channel_elem: [](grpc_channel_stack*, grpc_channel_element*) {}, - // destroy_channel_elem: - [](grpc_channel_element*) {}, grpc_channel_next_get_info, - // name: - nullptr}, - filter(filter), - name_str(absl::StrCat(filter->name, ".trace")) { - this->name = name_str.c_str(); - } + /* destroy_channel_elem: */ [](grpc_channel_element*) {}, + grpc_channel_next_get_info, filter->name}, + filter(filter) {} const grpc_channel_filter* const filter; - const std::string name_str; }; struct Globals { Mutex mu; diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index ecbc9eed098..181beb9b4b1 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -76,9 +76,7 @@ ArenaPromise LameClientFilter::MakeCallPromise( CallArgs args, NextPromiseFactory) { // TODO(ctiller): remove if check once promise_based_filter is removed (Close // is still needed) - if (args.server_to_client_messages != nullptr) { - args.server_to_client_messages->Close(); - } + if (args.incoming_messages != nullptr) args.incoming_messages->Close(); return Immediate(ServerMetadataFromStatus(error_)); } diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 319039accaf..985342265f0 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -24,19 +24,14 @@ #include #include -#include #include #include #include -#include -#include #include #include -#include "absl/cleanup/cleanup.h" #include "absl/status/status.h" #include "absl/types/optional.h" -#include "absl/types/variant.h" #include #include @@ -49,26 +44,13 @@ #include "src/core/lib/channel/channel_trace.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/match.h" #include "src/core/lib/gprpp/mpscq.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/pollset_set.h" -#include "src/core/lib/promise/activity.h" -#include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_join.h" -#include "src/core/lib/promise/detail/basic_seq.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/promise/promise.h" -#include "src/core/lib/promise/try_join.h" -#include "src/core/lib/promise/try_seq.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" @@ -195,20 +177,6 @@ class Server::RequestMatcherInterface { virtual void RequestCallWithPossiblePublish(size_t request_queue_index, RequestedCall* call) = 0; - struct MatchResult { - size_t cq_idx; - RequestedCall* requested_call; - }; - - // This function is invoked on an incoming promise based RPC. - // The RequestMatcher will try to match it against an application-requested - // RPC if possible or will place it in the pending queue otherwise. To enable - // some measure of fairness between server CQs, the match is done starting at - // the start_request_queue_index parameter in a cyclic order rather than - // always starting at 0. - virtual ArenaPromise> MatchRequest( - size_t start_request_queue_index) = 0; - // This function is invoked on an incoming RPC, represented by the calld // object. The RequestMatcher will try to match it against an // application-requested RPC if possible or will place it in the pending queue @@ -240,15 +208,9 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { void ZombifyPending() override { while (!pending_.empty()) { - Match( - pending_.front(), - [](CallData* calld) { - calld->SetState(CallData::CallState::ZOMBIED); - calld->KillZombie(); - }, - [](const std::shared_ptr& w) { - w->Finish(absl::InternalError("Server closed")); - }); + CallData* calld = pending_.front(); + calld->SetState(CallData::CallState::ZOMBIED); + calld->KillZombie(); pending_.pop(); } } @@ -272,19 +234,19 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { if (requests_per_cq_[request_queue_index].Push(&call->mpscq_node)) { // this was the first queued request: we need to lock and start // matching calls - struct NextPendingCall { + struct PendingCall { RequestedCall* rc = nullptr; - PendingCall pending; + CallData* calld; }; auto pop_next_pending = [this, request_queue_index] { - NextPendingCall pending_call; + PendingCall pending_call; { MutexLock lock(&server_->mu_call_); if (!pending_.empty()) { pending_call.rc = reinterpret_cast( requests_per_cq_[request_queue_index].Pop()); if (pending_call.rc != nullptr) { - pending_call.pending = std::move(pending_.front()); + pending_call.calld = pending_.front(); pending_.pop(); } } @@ -292,20 +254,14 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { return pending_call; }; while (true) { - NextPendingCall next_pending = pop_next_pending(); + PendingCall next_pending = pop_next_pending(); if (next_pending.rc == nullptr) break; - auto mr = MatchResult{request_queue_index, next_pending.rc}; - Match( - next_pending.pending, - [mr](CallData* calld) { - if (!calld->MaybeActivate()) { - // Zombied Call - calld->KillZombie(); - } else { - calld->Publish(mr.cq_idx, mr.requested_call); - } - }, - [mr](const std::shared_ptr& w) { w->Finish(mr); }); + if (!next_pending.calld->MaybeActivate()) { + // Zombied Call + next_pending.calld->KillZombie(); + } else { + next_pending.calld->Publish(request_queue_index, next_pending.rc); + } } } } @@ -350,66 +306,11 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { calld->Publish(cq_idx, rc); } - ArenaPromise> MatchRequest( - size_t start_request_queue_index) override { - for (size_t i = 0; i < requests_per_cq_.size(); i++) { - size_t cq_idx = (start_request_queue_index + i) % requests_per_cq_.size(); - RequestedCall* rc = - reinterpret_cast(requests_per_cq_[cq_idx].TryPop()); - if (rc != nullptr) { - return Immediate(MatchResult{cq_idx, rc}); - } - } - // No cq to take the request found; queue it on the slow list. - // We need to ensure that all the queues are empty. We do this under - // the server mu_call_ lock to ensure that if something is added to - // an empty request queue, it will block until the call is actually - // added to the pending list. - RequestedCall* rc = nullptr; - size_t cq_idx = 0; - size_t loop_count; - { - MutexLock lock(&server_->mu_call_); - for (loop_count = 0; loop_count < requests_per_cq_.size(); loop_count++) { - cq_idx = - (start_request_queue_index + loop_count) % requests_per_cq_.size(); - rc = reinterpret_cast(requests_per_cq_[cq_idx].Pop()); - if (rc != nullptr) { - break; - } - } - if (rc == nullptr) { - auto w = std::make_shared( - Activity::current()->MakeNonOwningWaker()); - pending_.push(w); - return [w]() -> Poll> { - std::unique_ptr> r( - w->result.exchange(nullptr, std::memory_order_acq_rel)); - if (r == nullptr) return Pending{}; - return std::move(*r); - }; - } - } - return Immediate(MatchResult{cq_idx, rc}); - } - Server* server() const override { return server_; } private: Server* const server_; - struct ActivityWaiter { - explicit ActivityWaiter(Waker waker) : waker(std::move(waker)) {} - ~ActivityWaiter() { delete result.load(std::memory_order_acquire); } - void Finish(absl::StatusOr r) { - result.store(new absl::StatusOr(std::move(r)), - std::memory_order_release); - waker.Wakeup(); - } - Waker waker; - std::atomic*> result{nullptr}; - }; - using PendingCall = absl::variant>; - std::queue pending_; + std::queue pending_; std::vector requests_per_cq_; }; @@ -470,10 +371,7 @@ class Server::AllocatingRequestMatcherBatch void MatchOrQueue(size_t /*start_request_queue_index*/, CallData* calld) override { - const bool still_running = server()->ShutdownRefOnRequest(); - auto cleanup_ref = - absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); - if (still_running) { + if (server()->ShutdownRefOnRequest()) { BatchCallAllocation call_info = allocator_(); GPR_ASSERT(server()->ValidateServerRequest( cq(), static_cast(call_info.tag), nullptr, @@ -486,25 +384,7 @@ class Server::AllocatingRequestMatcherBatch } else { calld->FailCallCreation(); } - } - - ArenaPromise> MatchRequest( - size_t /*start_request_queue_index*/) override { - const bool still_running = server()->ShutdownRefOnRequest(); - auto cleanup_ref = - absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); - if (still_running) { - BatchCallAllocation call_info = allocator_(); - GPR_ASSERT(server()->ValidateServerRequest( - cq(), static_cast(call_info.tag), nullptr, - nullptr) == GRPC_CALL_OK); - RequestedCall* rc = new RequestedCall( - static_cast(call_info.tag), call_info.cq, call_info.call, - call_info.initial_metadata, call_info.details); - return Immediate(MatchResult{cq_idx(), rc}); - } else { - return Immediate(absl::InternalError("Server shutdown")); - } + server()->ShutdownUnrefOnRequest(); } private: @@ -524,8 +404,6 @@ class Server::AllocatingRequestMatcherRegistered void MatchOrQueue(size_t /*start_request_queue_index*/, CallData* calld) override { - auto cleanup_ref = - absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); if (server()->ShutdownRefOnRequest()) { RegisteredCallAllocation call_info = allocator_(); GPR_ASSERT(server()->ValidateServerRequest( @@ -540,26 +418,7 @@ class Server::AllocatingRequestMatcherRegistered } else { calld->FailCallCreation(); } - } - - ArenaPromise> MatchRequest( - size_t /*start_request_queue_index*/) override { - const bool still_running = server()->ShutdownRefOnRequest(); - auto cleanup_ref = - absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); - if (still_running) { - RegisteredCallAllocation call_info = allocator_(); - GPR_ASSERT(server()->ValidateServerRequest( - cq(), call_info.tag, call_info.optional_payload, - registered_method_) == GRPC_CALL_OK); - RequestedCall* rc = - new RequestedCall(call_info.tag, call_info.cq, call_info.call, - call_info.initial_metadata, registered_method_, - call_info.deadline, call_info.optional_payload); - return Immediate(MatchResult{cq_idx(), rc}); - } else { - return Immediate(absl::InternalError("Server shutdown")); - } + server()->ShutdownUnrefOnRequest(); } private: @@ -634,7 +493,7 @@ class ChannelBroadcaster { const grpc_channel_filter Server::kServerTopFilter = { Server::CallData::StartTransportStreamOpBatch, - Server::ChannelData::MakeCallPromise, + nullptr, grpc_channel_next_op, sizeof(Server::CallData), Server::CallData::InitCallElement, @@ -1241,119 +1100,14 @@ void Server::ChannelData::AcceptStream(void* arg, grpc_transport* /*transport*/, args.send_deadline = Timestamp::InfFuture(); grpc_call* call; grpc_error_handle error = grpc_call_create(&args, &call); - grpc_call_stack* call_stack = grpc_call_get_call_stack(call); - if (call_stack == nullptr) { // Promise based calls do not have a call stack - GPR_ASSERT(error.ok()); - GPR_ASSERT(IsPromiseBasedServerCallEnabled()); + grpc_call_element* elem = + grpc_call_stack_element(grpc_call_get_call_stack(call), 0); + auto* calld = static_cast(elem->call_data); + if (!error.ok()) { + calld->FailCallCreation(); return; - } else { - grpc_call_element* elem = grpc_call_stack_element(call_stack, 0); - auto* calld = static_cast(elem->call_data); - if (!error.ok()) { - calld->FailCallCreation(); - return; - } - calld->Start(elem); } -} - -ArenaPromise Server::ChannelData::MakeCallPromise( - grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory) { - auto* chand = static_cast(elem->channel_data); - auto* server = chand->server_.get(); - if (server->ShutdownCalled()) { - return [] { - return ServerMetadataFromStatus(absl::InternalError("Server shutdown")); - }; - } - absl::optional path = - call_args.client_initial_metadata->Take(HttpPathMetadata()); - if (!path.has_value()) { - return [] { - return ServerMetadataFromStatus( - absl::InternalError("Missing :path header")); - }; - } - auto host_ptr = - call_args.client_initial_metadata->get_pointer(HttpAuthorityMetadata()); - if (host_ptr == nullptr) { - return [] { - return ServerMetadataFromStatus( - absl::InternalError("Missing :authority header")); - }; - } - // TODO(ctiller): deadline handling - Timestamp deadline = Timestamp::InfFuture(); - // Find request matcher. - RequestMatcherInterface* matcher; - ChannelRegisteredMethod* rm = - chand->GetRegisteredMethod(host_ptr->c_slice(), path->c_slice()); - ArenaPromise>> - maybe_read_first_message([] { return NextResult(); }); - if (rm != nullptr) { - matcher = rm->server_registered_method->matcher.get(); - switch (rm->server_registered_method->payload_handling) { - case GRPC_SRM_PAYLOAD_NONE: - break; - case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: - maybe_read_first_message = - Map(call_args.client_to_server_messages->Next(), - [](NextResult msg) - -> absl::StatusOr> { - return std::move(msg); - }); - } - } else { - matcher = server->unregistered_request_matcher_.get(); - } - return TrySeq( - TryJoin(matcher->MatchRequest(chand->cq_idx()), - std::move(maybe_read_first_message)), - [path = std::move(*path), host = std::move(*host_ptr), deadline, server, - call_args = std::move(call_args)]( - std::tuple> - match_result_and_payload) mutable { - auto& mr = std::get<0>(match_result_and_payload); - auto& payload = std::get<1>(match_result_and_payload); - auto* rc = mr.requested_call; - auto* cq_for_new_request = server->cqs_[mr.cq_idx]; - switch (rc->type) { - case RequestedCall::Type::BATCH_CALL: - GPR_ASSERT(!payload.has_value()); - rc->data.batch.details->host = CSliceRef(host.c_slice()); - rc->data.batch.details->method = CSliceRef(path.c_slice()); - rc->data.batch.details->deadline = - deadline.as_timespec(GPR_CLOCK_MONOTONIC); - break; - case RequestedCall::Type::REGISTERED_CALL: - *rc->data.registered.deadline = - deadline.as_timespec(GPR_CLOCK_MONOTONIC); - if (rc->data.registered.optional_payload != nullptr) { - if (payload.has_value()) { - auto* sb = payload.value()->payload()->c_slice_buffer(); - *rc->data.registered.optional_payload = - grpc_raw_byte_buffer_create(sb->slices, sb->count); - } else { - *rc->data.registered.optional_payload = nullptr; - } - } - break; - default: - GPR_UNREACHABLE_CODE(abort()); - } - return GetContext() - ->server_call_context() - ->MakeTopOfServerCallPromise( - std::move(call_args), rc->cq_bound_to_call, - rc->initial_metadata, - [rc, cq_for_new_request](grpc_call* call) { - *rc->call = call; - grpc_cq_end_op(cq_for_new_request, rc->tag, absl::OkStatus(), - Server::DoneRequestEvent, rc, &rc->completion, - true); - }); - }); + calld->Start(elem); } void Server::ChannelData::FinishDestroy(void* arg, @@ -1519,7 +1273,6 @@ void Server::CallData::KillZombie() { ExecCtx::Run(DEBUG_LOCATION, &kill_zombie_closure_, absl::OkStatus()); } -// If this changes, change MakeCallPromise too. void Server::CallData::StartNewRpc(grpc_call_element* elem) { auto* chand = static_cast(elem->channel_data); if (server_->ShutdownCalled()) { diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index d635da585b7..35f5cc5a27f 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -56,7 +56,6 @@ #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" -#include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" @@ -237,8 +236,6 @@ class Server : public InternallyRefCounted, static grpc_error_handle InitChannelElement( grpc_channel_element* elem, grpc_channel_element_args* args); static void DestroyChannelElement(grpc_channel_element* elem); - static ArenaPromise MakeCallPromise( - grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory); private: class ConnectivityWatcher; diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index 6e2e48ac24c..beff91dd21d 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -32,6 +32,7 @@ #include "src/core/lib/event_engine/default_event_engine.h" #include "src/core/lib/gpr/alloc.h" #include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/promise/context.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/transport/transport_impl.h" @@ -268,9 +269,9 @@ grpc_transport_stream_op_batch* grpc_make_transport_stream_op( namespace grpc_core { -ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status, - Arena* arena) { - auto hdl = arena->MakePooled(arena); +ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status) { + auto hdl = + GetContext()->MakePooled(GetContext()); hdl->Set(GrpcStatusMetadata(), static_cast(status.code())); if (!status.ok()) { hdl->Set(GrpcMessageMetadata(), Slice::FromCopiedString(status.message())); diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index bcd151cc6af..3be543a59ff 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -52,8 +52,8 @@ #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/promise/arena_promise.h" -#include "src/core/lib/promise/context.h" #include "src/core/lib/promise/detail/status.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" @@ -120,8 +120,7 @@ inline bool IsStatusOk(const ServerMetadataHandle& m) { GRPC_STATUS_OK; } -ServerMetadataHandle ServerMetadataFromStatus( - const absl::Status& status, Arena* arena = GetContext()); +ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status); template <> struct StatusCastImpl { @@ -137,13 +136,6 @@ struct StatusCastImpl { } }; -template <> -struct StatusCastImpl { - static ServerMetadataHandle Cast(const absl::Status& m) { - return ServerMetadataFromStatus(m); - } -}; - struct CallArgs { // Initial metadata from the client to the server. // During promise setup this can be manipulated by filters (and then @@ -153,11 +145,11 @@ struct CallArgs { // Set once when it's available. // During promise setup filters can substitute their own latch for this // and consequently intercept the sent value and mutate/observe it. - PipeSender* server_initial_metadata; + Latch* server_initial_metadata; // Messages travelling from the application to the transport. - PipeReceiver* client_to_server_messages; + PipeReceiver* outgoing_messages; // Messages travelling from the transport to the application. - PipeSender* server_to_client_messages; + PipeSender* incoming_messages; }; using NextPromiseFactory = diff --git a/src/cpp/ext/filters/logging/BUILD b/src/cpp/ext/filters/logging/BUILD index 8d7b3440224..dff7ba34dbc 100644 --- a/src/cpp/ext/filters/logging/BUILD +++ b/src/cpp/ext/filters/logging/BUILD @@ -55,6 +55,7 @@ grpc_cc_library( ], external_deps = [ "absl/random", + "absl/status", "absl/status:statusor", "absl/strings", "absl/types:optional", @@ -73,18 +74,22 @@ grpc_cc_library( "//:grpc_base", "//:grpc_client_channel", "//:grpc_resolver", + "//:promise", "//:uri_parser", "//src/core:arena", "//src/core:arena_promise", + "//src/core:basic_seq", "//src/core:cancel_callback", "//src/core:channel_args", "//src/core:channel_fwd", "//src/core:channel_stack_type", "//src/core:context", - "//src/core:map", - "//src/core:pipe", - "//src/core:poll", + "//src/core:for_each", + "//src/core:latch", + "//src/core:promise_like", + "//src/core:seq", "//src/core:slice", "//src/core:slice_buffer", + "//src/core:try_concurrently", ], ) diff --git a/src/cpp/ext/filters/logging/logging_filter.cc b/src/cpp/ext/filters/logging/logging_filter.cc index 9725d80d38a..2e664e92d18 100644 --- a/src/cpp/ext/filters/logging/logging_filter.cc +++ b/src/cpp/ext/filters/logging/logging_filter.cc @@ -34,6 +34,7 @@ #include #include "absl/random/random.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -57,9 +58,14 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/cancel_callback.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/map.h" -#include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/detail/promise_like.h" +#include "src/core/lib/promise/for_each.h" +#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/map_pipe.h" +#include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/try_concurrently.h" #include "src/core/lib/resolver/resolver_registry.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" @@ -359,28 +365,52 @@ class ClientLoggingFilter final : public grpc_core::ChannelFilter { } calld->LogClientHeader(/*is_client=*/true, call_args.client_initial_metadata); - call_args.server_initial_metadata->InterceptAndMap( - [calld](grpc_core::ServerMetadataHandle metadata) { - calld->LogServerHeader(/*is_client=*/true, metadata.get()); - return metadata; - }); - call_args.client_to_server_messages->InterceptAndMapWithHalfClose( - [calld](grpc_core::MessageHandle message) { - calld->LogClientMessage(/*is_client=*/true, message->payload()); - return message; - }, - [calld] { calld->LogClientHalfClose(/*is_client=*/true); }); - call_args.server_to_client_messages->InterceptAndMap( - [calld](grpc_core::MessageHandle message) { - calld->LogServerMessage(/*is_client=*/true, message->payload()); - return message; - }); + auto* server_initial_metadata = call_args.server_initial_metadata; + auto incoming_mapper = + grpc_core::PipeMapper::Intercept( + *call_args.incoming_messages); + grpc_core::PipeMapper outgoing_mapper = + grpc_core::PipeMapper::Intercept( + *call_args.outgoing_messages); return grpc_core::OnCancel( - Map(next_promise_factory(std::move(call_args)), - [calld](grpc_core::ServerMetadataHandle md) { - calld->LogServerTrailer(/*is_client=*/true, md.get()); - return md; - }), + grpc_core::TryConcurrently( + grpc_core::Seq( + next_promise_factory(std::move(call_args)), + [calld](grpc_core::ServerMetadataHandle metadata) mutable + -> grpc_core::ServerMetadataHandle { + calld->LogServerTrailer(/*is_client=*/true, metadata.get()); + return metadata; + })) + .NecessaryPull(grpc_core::Seq( + server_initial_metadata->Wait(), + [calld]( + grpc_core::ServerMetadata** server_initial_metadata) mutable + -> grpc_core::ArenaPromise { + if (server_initial_metadata != nullptr) { + calld->LogServerHeader(/*is_client=*/true, + *server_initial_metadata); + } + return grpc_core::ImmediateOkStatus(); + })) + .NecessaryPull(incoming_mapper.TakeAndRun( + [calld](grpc_core::MessageHandle message) + -> absl::StatusOr { + calld->LogServerMessage(/*is_client=*/true, + message->payload()); + return message; + })) + .NecessaryPush(grpc_core::Seq( + outgoing_mapper.TakeAndRun( + [calld](grpc_core::MessageHandle message) + -> absl::StatusOr { + calld->LogClientMessage(/*is_client=*/true, + message->payload()); + return message; + }), + [calld]() mutable -> grpc_core::ArenaPromise { + calld->LogClientHalfClose(/*is_client=*/true); + return grpc_core::ImmediateOkStatus(); + })), [calld]() { calld->LogCancel(/*is_client=*/true); }); } @@ -419,28 +449,50 @@ class ServerLoggingFilter final : public grpc_core::ChannelFilter { } calld->LogClientHeader(/*is_client=*/false, call_args.client_initial_metadata); - call_args.server_initial_metadata->InterceptAndMap( - [calld](grpc_core::ServerMetadataHandle metadata) { - calld->LogServerHeader(/*is_client=*/false, metadata.get()); - return metadata; - }); - call_args.client_to_server_messages->InterceptAndMapWithHalfClose( - [calld](grpc_core::MessageHandle message) { - calld->LogClientMessage(/*is_client=*/false, message->payload()); - return message; - }, - [calld] { calld->LogClientHalfClose(/*is_client=*/false); }); - call_args.server_to_client_messages->InterceptAndMap( - [calld](grpc_core::MessageHandle message) { - calld->LogServerMessage(/*is_client=*/false, message->payload()); - return message; - }); + auto* server_initial_metadata = call_args.server_initial_metadata; + auto incoming_mapper = + grpc_core::PipeMapper::Intercept( + *call_args.incoming_messages); + grpc_core::PipeMapper outgoing_mapper = + grpc_core::PipeMapper::Intercept( + *call_args.outgoing_messages); return grpc_core::OnCancel( - Map(next_promise_factory(std::move(call_args)), - [calld](grpc_core::ServerMetadataHandle md) { - calld->LogServerTrailer(/*is_client=*/false, md.get()); - return md; - }), + grpc_core::TryConcurrently( + grpc_core::Seq( + next_promise_factory(std::move(call_args)), + [calld](grpc_core::ServerMetadataHandle metadata) mutable + -> grpc_core::ServerMetadataHandle { + calld->LogServerTrailer(/*is_client=*/false, metadata.get()); + return metadata; + })) + .Push(grpc_core::Seq( + server_initial_metadata->Wait(), + [calld]( + grpc_core::ServerMetadata** server_initial_metadata) mutable + -> grpc_core::ArenaPromise { + calld->LogServerHeader(/*is_client=*/false, + *server_initial_metadata); + return grpc_core::ImmediateOkStatus(); + })) + .Push(outgoing_mapper.TakeAndRun( + [calld](grpc_core::MessageHandle message) + -> absl::StatusOr { + calld->LogServerMessage(/*is_client=*/false, + message->payload()); + return message; + })) + .NecessaryPull(grpc_core::Seq( + incoming_mapper.TakeAndRun( + [calld](grpc_core::MessageHandle message) + -> absl::StatusOr { + calld->LogClientMessage(/*is_client=*/false, + message->payload()); + return message; + }), + [calld]() mutable -> grpc_core::ArenaPromise { + calld->LogClientHalfClose(/*is_client=*/false); + return grpc_core::ImmediateOkStatus(); + })), [calld]() { calld->LogCancel(/*is_client=*/false); }); } }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 6eb36ade19b..80fd72a77a0 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -573,7 +573,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -655,8 +654,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', + 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', - 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -734,7 +733,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', diff --git a/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc index 5eb130e7474..84c93922c5b 100644 --- a/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc +++ b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc @@ -130,16 +130,15 @@ static void chttp2_init_server_socketpair( auto* fixture_data = static_cast(f->fixture_data); grpc_transport* transport; GPR_ASSERT(!f->server); - f->server = grpc_server_create(grpc_core::ChannelArgs::FromC(server_args) - .Set(GRPC_ARG_MINIMAL_STACK, true) - .ToC() - .get(), - nullptr); + f->server = grpc_server_create(server_args, nullptr); grpc_server_register_completion_queue(f->server, f->cq, nullptr); grpc_server_start(f->server); - transport = grpc_create_chttp2_transport( - grpc_core::Server::FromC(f->server)->channel_args(), - fixture_data->ep.server, false); + auto final_server_args = grpc_core::CoreConfiguration::Get() + .channel_args_preconditioning() + .PreconditionChannelArgs(server_args) + .Set(GRPC_ARG_MINIMAL_STACK, true); + transport = grpc_create_chttp2_transport(final_server_args, + fixture_data->ep.server, false); server_setup_transport(f, transport); } diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index f833fbc33bf..a901ae39fdf 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -261,7 +261,7 @@ END2END_TESTS = { exclude_inproc = True, exclude_minstack = True, ), - "max_connection_age": _test_options(exclude_minstack = True, exclude_inproc = True), + "max_connection_age": _test_options(exclude_inproc = True), "max_connection_idle": _test_options(needs_fullstack = True, proxyable = False), "max_message_length": _test_options(exclude_minstack = True), "negative_deadline": _test_options(exclude_minstack = True), diff --git a/test/core/end2end/tests/cancel_after_accept.cc b/test/core/end2end/tests/cancel_after_accept.cc index c440c154d0e..f70b599e3a8 100644 --- a/test/core/end2end/tests/cancel_after_accept.cc +++ b/test/core/end2end/tests/cancel_after_accept.cc @@ -19,8 +19,6 @@ #include #include -#include - #include #include #include @@ -46,7 +44,6 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args* client_args, grpc_channel_args* server_args) { grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "%s", std::string(80, '*').c_str()); gpr_log(GPR_INFO, "Running test: %s/%s/%s/%s", test_name, config.name, mode.name, use_service_config ? "service_config" : "client_api"); f = config.create_fixture(client_args, server_args); diff --git a/test/core/end2end/tests/grpc_authz.cc b/test/core/end2end/tests/grpc_authz.cc index 90479dec716..4536faa2332 100644 --- a/test/core/end2end/tests/grpc_authz.cc +++ b/test/core/end2end/tests/grpc_authz.cc @@ -48,7 +48,6 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args* client_args, grpc_channel_args* server_args) { grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "%s", std::string(80, '*').c_str()); gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); diff --git a/test/core/end2end/tests/max_message_length.cc b/test/core/end2end/tests/max_message_length.cc index b778ae42b47..d8f039d0bee 100644 --- a/test/core/end2end/tests/max_message_length.cc +++ b/test/core/end2end/tests/max_message_length.cc @@ -596,9 +596,6 @@ static void test_max_receive_message_length_on_compressed_request( nullptr); GPR_ASSERT(GRPC_CALL_OK == error); - // WARNING!! - // It's believed the following line (and the associated batch) is the only - // test we have for failing a receive operation in a batch. cqv.Expect(tag(102), false); cqv.Expect(tag(103), true); cqv.Expect(tag(1), true); diff --git a/test/core/end2end/tests/no_logging.cc b/test/core/end2end/tests/no_logging.cc index cce5786cc86..771e04d748f 100644 --- a/test/core/end2end/tests/no_logging.cc +++ b/test/core/end2end/tests/no_logging.cc @@ -33,7 +33,6 @@ #include #include -#include "src/core/lib/gprpp/env.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/util/test_config.h" @@ -285,7 +284,6 @@ static void test_no_logging_in_one_request(grpc_end2end_test_config config) { } void no_logging(grpc_end2end_test_config config) { - grpc_core::SetEnv("GRPC_TRACE", ""); gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); grpc_tracer_set_enabled("all", 0); gpr_set_log_function(log_dispatcher_func); diff --git a/test/core/end2end/tests/server_streaming.cc b/test/core/end2end/tests/server_streaming.cc index 91f899749ff..f5b4e942918 100644 --- a/test/core/end2end/tests/server_streaming.cc +++ b/test/core/end2end/tests/server_streaming.cc @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -39,11 +38,9 @@ static void* tag(intptr_t t) { return reinterpret_cast(t); } static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, const char* test_name, grpc_channel_args* client_args, - grpc_channel_args* server_args, - int num_messages) { + grpc_channel_args* server_args) { grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "%s\nRunning test: %s/%s/%d", std::string(100, '*').c_str(), - test_name, config.name, num_messages); + gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); config.init_client(&f, client_args); @@ -97,8 +94,8 @@ static void end_test(grpc_end2end_test_fixture* f) { // writing, and expects to get the status after the messages. static void test_server_streaming(grpc_end2end_test_config config, int num_messages) { - grpc_end2end_test_fixture f = begin_test(config, "test_server_streaming", - nullptr, nullptr, num_messages); + grpc_end2end_test_fixture f = + begin_test(config, "test_server_streaming", nullptr, nullptr); grpc_call* c; grpc_call* s; auto cqv = std::make_unique(f.cq); diff --git a/test/core/filters/filter_fuzzer.cc b/test/core/filters/filter_fuzzer.cc index 72b2435fd83..dd8b38b0337 100644 --- a/test/core/filters/filter_fuzzer.cc +++ b/test/core/filters/filter_fuzzer.cc @@ -66,7 +66,7 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" @@ -436,12 +436,10 @@ class MainLoop { CallArgs call_args, NextPromiseFactory) override { Call* call = static_cast(Activity::current()); if (call->server_initial_metadata_) { - call->server_initial_metadata_push_promise_.emplace( - call_args.server_initial_metadata->Push( - ServerMetadataHandle(call->server_initial_metadata_.get(), - Arena::PooledDeleter(nullptr)))); + call_args.server_initial_metadata->Set( + call->server_initial_metadata_.get()); } else { - call->unpushed_incoming_server_initial_metadata_pipe_ = + call->unset_incoming_server_initial_metadata_latch_ = call_args.server_initial_metadata; } return [call]() -> Poll { @@ -474,10 +472,10 @@ class MainLoop { const filter_fuzzer::Metadata& client_initial_metadata, bool is_client) : main_loop_(main_loop), id_(id) { ScopedContext context(this); - auto* server_initial_metadata = arena_->New>(); + auto* server_initial_metadata = arena_->New>(); CallArgs call_args{std::move(*LoadMetadata(client_initial_metadata, &client_initial_metadata_)), - &server_initial_metadata->sender, nullptr, nullptr}; + server_initial_metadata, nullptr, nullptr}; if (is_client) { promise_ = main_loop_->channel_stack_->MakeClientCallPromise( std::move(call_args)); @@ -534,12 +532,10 @@ class MainLoop { void RecvInitialMetadata(const filter_fuzzer::Metadata& metadata) { if (server_initial_metadata_ == nullptr) { LoadMetadata(metadata, &server_initial_metadata_); - if (auto* pipe = std::exchange( - unpushed_incoming_server_initial_metadata_pipe_, nullptr)) { + if (auto* latch = std::exchange( + unset_incoming_server_initial_metadata_latch_, nullptr)) { ScopedContext context(this); - server_initial_metadata_push_promise_.emplace( - pipe->Push(ServerMetadataHandle(server_initial_metadata_.get(), - Arena::PooledDeleter(nullptr)))); + latch->Set(server_initial_metadata_.get()); } } } @@ -629,10 +625,8 @@ class MainLoop { std::unique_ptr final_info_; std::unique_ptr client_initial_metadata_; std::unique_ptr server_initial_metadata_; - PipeSender* - unpushed_incoming_server_initial_metadata_pipe_ = nullptr; - absl::optional::PushType> - server_initial_metadata_push_promise_; + Latch* unset_incoming_server_initial_metadata_latch_ = + nullptr; std::unique_ptr server_trailing_metadata_; Waker server_trailing_metadata_waker_; CallFinalization finalization_; diff --git a/test/core/promise/BUILD b/test/core/promise/BUILD index 00d5fa85c7f..6546e89525c 100644 --- a/test/core/promise/BUILD +++ b/test/core/promise/BUILD @@ -45,24 +45,6 @@ grpc_cc_test( deps = ["//src/core:poll"], ) -grpc_cc_test( - name = "interceptor_list_test", - srcs = ["interceptor_list_test.cc"], - external_deps = ["gtest"], - language = "c++", - uses_event_engine = False, - uses_polling = False, - deps = [ - "test_context", - "//:ref_counted_ptr", - "//src/core:arena", - "//src/core:event_engine_memory_allocator", - "//src/core:interceptor_list", - "//src/core:memory_quota", - "//src/core:resource_quota", - ], -) - grpc_cc_test( name = "context_test", srcs = ["context_test.cc"], @@ -396,7 +378,7 @@ grpc_cc_test( uses_polling = False, deps = [ "test_wakeup_schedulers", - "//:grpc", + "//:gpr", "//:ref_counted_ptr", "//src/core:activity", "//src/core:basic_join", diff --git a/test/core/promise/activity_test.cc b/test/core/promise/activity_test.cc index 5ba0770caa1..279ebbe9de6 100644 --- a/test/core/promise/activity_test.cc +++ b/test/core/promise/activity_test.cc @@ -128,12 +128,12 @@ TEST(ActivityTest, DropImmediately) { } template -class BarrierTest : public ::testing::Test { +class BarrierTest : public testing::Test { public: using Type = B; }; -using BarrierTestTypes = ::testing::Types; +using BarrierTestTypes = testing::Types; TYPED_TEST_SUITE(BarrierTest, BarrierTestTypes); TYPED_TEST(BarrierTest, Barrier) { diff --git a/test/core/promise/for_each_test.cc b/test/core/promise/for_each_test.cc index 71ac9b35238..73db7d537ef 100644 --- a/test/core/promise/for_each_test.cc +++ b/test/core/promise/for_each_test.cc @@ -16,7 +16,6 @@ #include -#include #include #include "gmock/gmock.h" diff --git a/test/core/promise/if_test.cc b/test/core/promise/if_test.cc index 0a2ccc5cbe1..5df68acdac1 100644 --- a/test/core/promise/if_test.cc +++ b/test/core/promise/if_test.cc @@ -52,18 +52,6 @@ TEST(IfTest, ChooseFailure) { Poll>(absl::StatusOr())); } -TEST(IfTest, ImmediateChooseTrue) { - EXPECT_EQ(If( - true, []() { return 1; }, []() { return 2; })(), - Poll(1)); -} - -TEST(IfTest, ImmediateChooseFalse) { - EXPECT_EQ(If( - false, []() { return 1; }, []() { return 2; })(), - Poll(2)); -} - } // namespace grpc_core int main(int argc, char** argv) { diff --git a/test/core/promise/interceptor_list_test.cc b/test/core/promise/interceptor_list_test.cc deleted file mode 100644 index 22f0aef7803..00000000000 --- a/test/core/promise/interceptor_list_test.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/core/lib/promise/interceptor_list.h" - -#include -#include - -#include "gtest/gtest.h" - -#include - -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/resource_quota/arena.h" -#include "src/core/lib/resource_quota/memory_quota.h" -#include "src/core/lib/resource_quota/resource_quota.h" -#include "test/core/promise/test_context.h" - -namespace grpc_core { -namespace { - -class InterceptorListTest : public ::testing::Test { - protected: - MemoryAllocator memory_allocator_ = MemoryAllocator( - ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator("test")); - ScopedArenaPtr arena_ = MakeScopedArena(1024, &memory_allocator_); - TestContext arena_ctx_{arena_.get()}; -}; - -TEST_F(InterceptorListTest, NoOp) { InterceptorList(); } - -TEST_F(InterceptorListTest, CanRunOne) { - InterceptorList list; - list.AppendMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); - EXPECT_EQ(list.Run("hello")(), Poll>("helloa")); -} - -TEST_F(InterceptorListTest, CanRunTwo) { - InterceptorList list; - list.AppendMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); - list.AppendMap([](std::string s) { return s + "b"; }, DEBUG_LOCATION); - EXPECT_EQ(list.Run("hello")(), Poll>("helloab")); -} - -TEST_F(InterceptorListTest, CanRunTwoTwice) { - InterceptorList list; - list.AppendMap([](std::string s) { return s + s; }, DEBUG_LOCATION); - list.AppendMap([](std::string s) { return s + s + s; }, DEBUG_LOCATION); - EXPECT_EQ(absl::get(list.Run(std::string(10, 'a'))()).value(), - std::string(60, 'a')); - EXPECT_EQ(absl::get(list.Run(std::string(100, 'b'))()).value(), - std::string(600, 'b')); -} - -TEST_F(InterceptorListTest, CanRunManyWithCaptures) { - InterceptorList list; - for (size_t i = 0; i < 26 * 1000; i++) { - list.AppendMap( - [i = std::make_shared(i)](std::string s) { - return s + static_cast((*i % 26) + 'a'); - }, - DEBUG_LOCATION); - } - std::string expected; - for (size_t i = 0; i < 1000; i++) { - expected += "abcdefghijklmnopqrstuvwxyz"; - } - EXPECT_EQ(absl::get(list.Run("")()).value(), expected); -} - -TEST_F(InterceptorListTest, CanRunOnePrepended) { - InterceptorList list; - list.PrependMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); - EXPECT_EQ(list.Run("hello")(), Poll>("helloa")); -} - -TEST_F(InterceptorListTest, CanRunTwoPrepended) { - InterceptorList list; - list.PrependMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); - list.PrependMap([](std::string s) { return s + "b"; }, DEBUG_LOCATION); - EXPECT_EQ(list.Run("hello")(), Poll>("helloba")); -} - -TEST_F(InterceptorListTest, CanRunManyWithCapturesPrepended) { - InterceptorList list; - for (size_t i = 0; i < 26 * 1000; i++) { - list.PrependMap( - [i = std::make_shared(i)](std::string s) { - return s + static_cast((*i % 26) + 'a'); - }, - DEBUG_LOCATION); - } - std::string expected; - for (size_t i = 0; i < 1000; i++) { - expected += "zyxwvutsrqponmlkjihgfedcba"; - } - EXPECT_EQ(absl::get(list.Run("")()).value(), expected); -} - -TEST_F(InterceptorListTest, CanRunManyWithCapturesThatDelay) { - InterceptorList list; - for (size_t i = 0; i < 26 * 1000; i++) { - list.AppendMap( - [i = std::make_shared(i)](std::string s) { - return - [x = false, i, s]() mutable -> Poll> { - if (!x) { - x = true; - return Pending{}; - } - return s + static_cast((*i % 26) + 'a'); - }; - }, - DEBUG_LOCATION); - } - auto promise = list.Run(""); - for (size_t i = 0; i < 26 * 1000; i++) { - EXPECT_TRUE(absl::holds_alternative(promise())) << i; - } - std::string expected; - for (size_t i = 0; i < 1000; i++) { - expected += "abcdefghijklmnopqrstuvwxyz"; - } - EXPECT_EQ(absl::get(promise()).value(), expected); -} - -} // namespace -} // namespace grpc_core - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/core/promise/latch_test.cc b/test/core/promise/latch_test.cc index d7258e71fa2..a3b4bc96a8a 100644 --- a/test/core/promise/latch_test.cc +++ b/test/core/promise/latch_test.cc @@ -43,8 +43,8 @@ TEST(LatchTest, Works) { latch.Set(42); return true; }), - [](std::tuple result) { - EXPECT_EQ(std::get<0>(result), 42); + [](std::tuple result) { + EXPECT_EQ(*std::get<0>(result), 42); return absl::OkStatus(); }); }, diff --git a/test/core/promise/map_pipe_test.cc b/test/core/promise/map_pipe_test.cc index 4fe4654b456..4ad63e66967 100644 --- a/test/core/promise/map_pipe_test.cc +++ b/test/core/promise/map_pipe_test.cc @@ -16,7 +16,6 @@ #include -#include #include #include diff --git a/test/core/promise/pipe_test.cc b/test/core/promise/pipe_test.cc index e78b4acfe1f..bcf1adbe276 100644 --- a/test/core/promise/pipe_test.cc +++ b/test/core/promise/pipe_test.cc @@ -14,9 +14,9 @@ #include "src/core/lib/promise/pipe.h" -#include #include #include +#include #include #include "absl/status/status.h" @@ -24,7 +24,7 @@ #include "gtest/gtest.h" #include -#include +#include #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/promise/activity.h" @@ -52,11 +52,11 @@ TEST_F(PipeTest, CanSendAndReceive) { EXPECT_CALL(on_done, Call(absl::OkStatus())); MakeActivity( [] { - auto* pipe = GetContext()->ManagedNew>(); + Pipe pipe; return Seq( // Concurrently: send 42 into the pipe, and receive from the pipe. - Join(pipe->sender.Push(42), - Map(pipe->receiver.Next(), + Join(pipe.sender.Push(42), + Map(pipe.receiver.Next(), [](NextResult r) { return r.value(); })), // Once complete, verify successful sending and the received value // is 42. @@ -71,103 +71,17 @@ TEST_F(PipeTest, CanSendAndReceive) { MakeScopedArena(1024, &memory_allocator_)); } -TEST_F(PipeTest, CanInterceptAndMapAtSender) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - pipe->sender.InterceptAndMap([](int value) { return value / 2; }); - return Seq( - // Concurrently: send 42 into the pipe, and receive from the pipe. - Join(pipe->sender.Push(42), - Map(pipe->receiver.Next(), - [](NextResult r) { return r.value(); })), - // Once complete, verify successful sending and the received value - // is 21. - [](std::tuple result) { - EXPECT_TRUE(std::get<0>(result)); - EXPECT_EQ(21, std::get<1>(result)); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - -TEST_F(PipeTest, CanInterceptAndMapAtReceiver) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - pipe->receiver.InterceptAndMap([](int value) { return value / 2; }); - return Seq( - // Concurrently: send 42 into the pipe, and receive from the pipe. - Join(pipe->sender.Push(42), - Map(pipe->receiver.Next(), - [](NextResult r) { return r.value(); })), - // Once complete, verify successful sending and the received value - // is 21. - [](std::tuple result) { - EXPECT_TRUE(std::get<0>(result)); - EXPECT_EQ(21, std::get<1>(result)); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - -TEST_F(PipeTest, InterceptionOrderingIsCorrect) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - auto appender = [](char c) { - return [c](std::string value) { - value += c; - return value; - }; - }; - // Interception get added outwardly from the center, and run from sender - // to receiver, so the following should result in append "abcd". - pipe->receiver.InterceptAndMap(appender('c')); - pipe->sender.InterceptAndMap(appender('b')); - pipe->receiver.InterceptAndMap(appender('d')); - pipe->sender.InterceptAndMap(appender('a')); - return Seq( - // Concurrently: send "" into the pipe, and receive from the pipe. - Join(pipe->sender.Push(""), - Map(pipe->receiver.Next(), - [](NextResult r) { return r.value(); })), - // Once complete, verify successful sending and the received value - // is 21. - [](std::tuple result) { - EXPECT_TRUE(std::get<0>(result)); - EXPECT_EQ("abcd", std::get<1>(result)); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - TEST_F(PipeTest, CanReceiveAndSend) { StrictMock> on_done; EXPECT_CALL(on_done, Call(absl::OkStatus())); MakeActivity( [] { - auto* pipe = GetContext()->ManagedNew>(); + Pipe pipe; return Seq( // Concurrently: receive from the pipe, and send 42 into the pipe. - Join(Map(pipe->receiver.Next(), + Join(Map(pipe.receiver.Next(), [](NextResult r) { return r.value(); }), - pipe->sender.Push(42)), + pipe.sender.Push(42)), // Once complete, verify the received value is 42 and successful // sending. [](std::tuple result) { @@ -243,95 +157,6 @@ TEST_F(PipeTest, CanSeeClosedOnReceive) { MakeScopedArena(1024, &memory_allocator_)); } -TEST_F(PipeTest, CanCloseSend) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - return Seq( - // Concurrently: - // - wait for a received value (will stall forever since we push - // nothing into the queue) - // - close the sender, which will signal the receiver to return an - // end-of-stream. - Join(pipe->receiver.Next(), - [pipe]() mutable { - pipe->sender.Close(); - return absl::OkStatus(); - }), - // Verify we received end-of-stream and closed the sender. - [](std::tuple, absl::Status> result) { - EXPECT_FALSE(std::get<0>(result).has_value()); - EXPECT_FALSE(std::get<0>(result).cancelled()); - EXPECT_EQ(std::get<1>(result), absl::OkStatus()); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - -TEST_F(PipeTest, CanCloseSendWithInterceptor) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - pipe->sender.InterceptAndMap([](int value) { return value + 1; }); - return Seq( - // Concurrently: - // - wait for a received value (will stall forever since we push - // nothing into the queue) - // - close the sender, which will signal the receiver to return an - // end-of-stream. - Join(pipe->receiver.Next(), - [pipe]() mutable { - pipe->sender.Close(); - return absl::OkStatus(); - }), - // Verify we received end-of-stream and closed the sender. - [](std::tuple, absl::Status> result) { - EXPECT_FALSE(std::get<0>(result).has_value()); - EXPECT_FALSE(std::get<0>(result).cancelled()); - EXPECT_EQ(std::get<1>(result), absl::OkStatus()); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - -TEST_F(PipeTest, CanCancelSendWithInterceptor) { - StrictMock> on_done; - EXPECT_CALL(on_done, Call(absl::OkStatus())); - MakeActivity( - [] { - auto* pipe = GetContext()->ManagedNew>(); - pipe->sender.InterceptAndMap([](int) { return absl::nullopt; }); - return Seq( - // Concurrently: - // - wait for a received value (will stall forever since we push - // nothing into the queue) - // - close the sender, which will signal the receiver to return an - // end-of-stream. - Join(pipe->receiver.Next(), pipe->sender.Push(3)), - // Verify we received end-of-stream with cancellation and sent - // successfully. - [](std::tuple, bool> result) { - EXPECT_FALSE(std::get<0>(result).has_value()); - EXPECT_TRUE(std::get<0>(result).cancelled()); - EXPECT_FALSE(std::get<1>(result)); - return absl::OkStatus(); - }); - }, - NoWakeupScheduler(), - [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, - MakeScopedArena(1024, &memory_allocator_)); -} - TEST_F(PipeTest, CanFlowControlThroughManyStages) { StrictMock> on_done; EXPECT_CALL(on_done, Call(absl::OkStatus())); @@ -342,38 +167,39 @@ TEST_F(PipeTest, CanFlowControlThroughManyStages) { // completes. MakeActivity( [done] { - auto* pipe1 = GetContext()->ManagedNew>(); - auto* pipe2 = GetContext()->ManagedNew>(); - auto* pipe3 = GetContext()->ManagedNew>(); - auto* sender1 = &pipe1->sender; - auto* receiver1 = &pipe1->receiver; - auto* sender2 = &pipe2->sender; - auto* receiver2 = &pipe2->receiver; - auto* sender3 = &pipe3->sender; - auto* receiver3 = &pipe3->receiver; - return Seq(Join(Seq(sender1->Push(1), - [done] { - *done = true; - return 1; - }), - Seq(receiver1->Next(), - [sender2](NextResult r) mutable { - return sender2->Push(r.value()); - }), - Seq(receiver2->Next(), - [sender3](NextResult r) mutable { - return sender3->Push(r.value()); - }), - Seq(receiver3->Next(), - [done](NextResult r) { - EXPECT_EQ(r.value(), 1); - EXPECT_FALSE(*done); - return 2; - })), - [](std::tuple result) { - EXPECT_EQ(result, std::make_tuple(1, true, true, 2)); - return absl::OkStatus(); - }); + Pipe pipe1; + Pipe pipe2; + Pipe pipe3; + auto sender1 = std::move(pipe1.sender); + auto receiver1 = std::move(pipe1.receiver); + auto sender2 = std::move(pipe2.sender); + auto receiver2 = std::move(pipe2.receiver); + auto sender3 = std::move(pipe3.sender); + auto receiver3 = std::move(pipe3.receiver); + return Seq( + Join(Seq(sender1.Push(1), + [done] { + *done = true; + return 1; + }), + Seq(receiver1.Next(), + [sender2 = std::move(sender2)](NextResult r) mutable { + return sender2.Push(r.value()); + }), + Seq(receiver2.Next(), + [sender3 = std::move(sender3)](NextResult r) mutable { + return sender3.Push(r.value()); + }), + Seq(receiver3.Next(), + [done](NextResult r) { + EXPECT_EQ(r.value(), 1); + EXPECT_FALSE(*done); + return 2; + })), + [](std::tuple result) { + EXPECT_EQ(result, std::make_tuple(1, true, true, 2)); + return absl::OkStatus(); + }); }, NoWakeupScheduler(), [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, @@ -384,9 +210,7 @@ TEST_F(PipeTest, CanFlowControlThroughManyStages) { } // namespace grpc_core int main(int argc, char** argv) { + gpr_log_verbosity_init(); ::testing::InitGoogleTest(&argc, argv); - grpc_init(); - int r = RUN_ALL_TESTS(); - grpc_shutdown(); - return r; + return RUN_ALL_TESTS(); } diff --git a/test/core/resource_quota/arena_test.cc b/test/core/resource_quota/arena_test.cc index 577deb3aad4..be23aaad935 100644 --- a/test/core/resource_quota/arena_test.cc +++ b/test/core/resource_quota/arena_test.cc @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -175,21 +174,6 @@ TEST_F(ArenaTest, ConcurrentManagedNew) { args.arena->Destroy(); } -template -void Scribble(Int* ints, int n, int offset) { - for (int i = 0; i < n; i++) { - ints[i] = static_cast(i + offset); - } -} - -template -bool IsScribbled(Int* ints, int n, int offset) { - for (int i = 0; i < n; i++) { - if (ints[i] != static_cast(i + offset)) return false; - } - return true; -} - TEST_F(ArenaTest, PooledObjectsArePooled) { struct TestObj { char a[100]; @@ -197,15 +181,10 @@ TEST_F(ArenaTest, PooledObjectsArePooled) { auto arena = MakeScopedArena(1024, &memory_allocator_); auto obj = arena->MakePooled(); - Scribble(obj->a, 100, 1); - EXPECT_TRUE(IsScribbled(obj->a, 100, 1)); void* p = obj.get(); obj.reset(); obj = arena->MakePooled(); - EXPECT_FALSE(IsScribbled(obj->a, 100, 1)); EXPECT_EQ(p, obj.get()); - Scribble(obj->a, 100, 2); - EXPECT_TRUE(IsScribbled(obj->a, 100, 2)); } TEST_F(ArenaTest, CreateManyObjects) { @@ -217,31 +196,7 @@ TEST_F(ArenaTest, CreateManyObjects) { objs.reserve(1000); for (int i = 0; i < 1000; i++) { objs.emplace_back(arena->MakePooled()); - Scribble(objs.back()->a, 100, i); } - for (int i = 0; i < 1000; i++) { - EXPECT_TRUE(IsScribbled(objs[i]->a, 100, i)); - } -} - -TEST_F(ArenaTest, CreateManyObjectsWithDestructors) { - using TestObj = std::unique_ptr; - auto arena = MakeScopedArena(1024, &memory_allocator_); - std::vector> objs; - objs.reserve(1000); - for (int i = 0; i < 1000; i++) { - objs.emplace_back(arena->MakePooled(new int(i))); - } -} - -TEST_F(ArenaTest, CreatePoolArray) { - auto arena = MakeScopedArena(1024, &memory_allocator_); - auto p = arena->MakePooledArray(1024); - EXPECT_FALSE(p.get_deleter().has_freelist()); - p = arena->MakePooledArray(5); - EXPECT_TRUE(p.get_deleter().has_freelist()); - Scribble(p.get(), 5, 1); - EXPECT_TRUE(IsScribbled(p.get(), 5, 1)); } } // namespace grpc_core diff --git a/test/core/slice/slice_test.cc b/test/core/slice/slice_test.cc index a292c7cd324..864ac2f01e4 100644 --- a/test/core/slice/slice_test.cc +++ b/test/core/slice/slice_test.cc @@ -347,7 +347,7 @@ INSTANTIATE_TEST_SUITE_P(SliceSizedTest, SliceSizedTest, } return out; }()), - [](const ::testing::TestParamInfo& info) { + [](const testing::TestParamInfo& info) { return std::to_string(info.param); }); diff --git a/test/cpp/ext/filters/logging/logging_test.cc b/test/cpp/ext/filters/logging/logging_test.cc index e42ed471e8a..926d6cb2519 100644 --- a/test/cpp/ext/filters/logging/logging_test.cc +++ b/test/cpp/ext/filters/logging/logging_test.cc @@ -147,7 +147,7 @@ class LoggingTest : public ::testing::Test { std::unique_ptr stub_; }; -TEST_F(LoggingTest, SimpleRpc) { +TEST_F(LoggingTest, DISABLED_SimpleRpc) { g_test_logging_sink->SetConfig( grpc::internal::LoggingSink::Config(4096, 4096)); EchoRequest request; @@ -304,7 +304,7 @@ TEST_F(LoggingTest, SimpleRpc) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, LoggingDisabled) { +TEST_F(LoggingTest, DISABLED_LoggingDisabled) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config()); EchoRequest request; request.set_message("foo"); @@ -316,7 +316,7 @@ TEST_F(LoggingTest, LoggingDisabled) { EXPECT_TRUE(g_test_logging_sink->entries().empty()); } -TEST_F(LoggingTest, MetadataTruncated) { +TEST_F(LoggingTest, DISABLED_MetadataTruncated) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config( 40 /* expect truncated metadata*/, 4096)); EchoRequest request; @@ -474,7 +474,7 @@ TEST_F(LoggingTest, MetadataTruncated) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, PayloadTruncated) { +TEST_F(LoggingTest, DISABLED_PayloadTruncated) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config(4096, 10)); EchoRequest request; // The following message should get truncated @@ -635,7 +635,7 @@ TEST_F(LoggingTest, PayloadTruncated) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, CancelledRpc) { +TEST_F(LoggingTest, DISABLED_CancelledRpc) { g_test_logging_sink->SetConfig( grpc::internal::LoggingSink::Config(4096, 4096)); EchoRequest request; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 4bfac6921fe..9bd60293755 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2224,7 +2224,6 @@ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/cfstream_handle.cc \ src/core/lib/iomgr/cfstream_handle.h \ -src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/combiner.h \ @@ -2380,19 +2379,19 @@ src/core/lib/promise/activity.cc \ src/core/lib/promise/activity.h \ src/core/lib/promise/arena_promise.h \ src/core/lib/promise/context.h \ -src/core/lib/promise/detail/basic_join.h \ src/core/lib/promise/detail/basic_seq.h \ src/core/lib/promise/detail/promise_factory.h \ src/core/lib/promise/detail/promise_like.h \ src/core/lib/promise/detail/status.h \ src/core/lib/promise/detail/switch.h \ src/core/lib/promise/exec_ctx_wakeup_scheduler.h \ -src/core/lib/promise/if.h \ -src/core/lib/promise/interceptor_list.h \ +src/core/lib/promise/for_each.h \ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ +src/core/lib/promise/map_pipe.h \ +src/core/lib/promise/pipe.cc \ src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ @@ -2400,9 +2399,7 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ -src/core/lib/promise/trace.cc \ -src/core/lib/promise/trace.h \ -src/core/lib/promise/try_join.h \ +src/core/lib/promise/try_concurrently.h \ src/core/lib/promise/try_seq.h \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver.h \ @@ -2557,7 +2554,6 @@ src/core/lib/slice/slice.h \ src/core/lib/slice/slice_buffer.cc \ src/core/lib/slice/slice_buffer.h \ src/core/lib/slice/slice_internal.h \ -src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_refcount.h \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/slice/slice_string_helpers.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 4ad1acbea33..6740bde46d1 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2006,7 +2006,6 @@ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/cfstream_handle.cc \ src/core/lib/iomgr/cfstream_handle.h \ -src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/combiner.h \ @@ -2162,19 +2161,19 @@ src/core/lib/promise/activity.cc \ src/core/lib/promise/activity.h \ src/core/lib/promise/arena_promise.h \ src/core/lib/promise/context.h \ -src/core/lib/promise/detail/basic_join.h \ src/core/lib/promise/detail/basic_seq.h \ src/core/lib/promise/detail/promise_factory.h \ src/core/lib/promise/detail/promise_like.h \ src/core/lib/promise/detail/status.h \ src/core/lib/promise/detail/switch.h \ src/core/lib/promise/exec_ctx_wakeup_scheduler.h \ -src/core/lib/promise/if.h \ -src/core/lib/promise/interceptor_list.h \ +src/core/lib/promise/for_each.h \ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ +src/core/lib/promise/map_pipe.h \ +src/core/lib/promise/pipe.cc \ src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ @@ -2182,9 +2181,7 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ -src/core/lib/promise/trace.cc \ -src/core/lib/promise/trace.h \ -src/core/lib/promise/try_join.h \ +src/core/lib/promise/try_concurrently.h \ src/core/lib/promise/try_seq.h \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver.h \ @@ -2339,7 +2336,6 @@ src/core/lib/slice/slice.h \ src/core/lib/slice/slice_buffer.cc \ src/core/lib/slice/slice_buffer.h \ src/core/lib/slice/slice_internal.h \ -src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_refcount.h \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/slice/slice_string_helpers.h \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index bcabd59a646..475d568443b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4193,30 +4193,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "interceptor_list_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": false - }, { "args": [], "benchmark": false,