natas_hw
/
grpc
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[metadata] Expose metadata limits and annotation info to filter stack (#35970) 

<!--

If you know who should review your pull request, please assign it to that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the appropriate
lang label.

-->

Closes #35970

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35970 from ananda1066:client_metadata b62a9ae146
PiperOrigin-RevId: 610830228
This commit is contained in:
Alisha Nanda 2024-02-27 12:04:19 -08:00 committed by Copybara-Service
parent 338fc05c21
commit fa2bae473a
21 changed files with 190 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BUILD
View File

@ -4247,6 +4247,7 @@ grpc_cc_library(
"//src/core:hpack_constants",
"//src/core:match",
"//src/core:metadata_batch",
"//src/core:metadata_info",
"//src/core:parsed_metadata",
"//src/core:random_early_detection",
"//src/core:slice",
@ -4424,6 +4425,7 @@ grpc_cc_library(
"//src/core:max_concurrent_streams_policy",
"//src/core:memory_quota",
"//src/core:metadata_batch",
"//src/core:metadata_info",
"//src/core:ping_abuse_policy",
"//src/core:ping_callbacks",
"//src/core:ping_rate_policy",

2
CMakeLists.txt generated
View File

@ -2489,6 +2489,7 @@ add_library(grpc
src/core/lib/transport/message.cc
src/core/lib/transport/metadata.cc
src/core/lib/transport/metadata_batch.cc
src/core/lib/transport/metadata_info.cc
src/core/lib/transport/parsed_metadata.cc
src/core/lib/transport/status_conversion.cc
src/core/lib/transport/tcp_connect_handshaker.cc
@ -3218,6 +3219,7 @@ add_library(grpc_unsecure
src/core/lib/transport/message.cc
src/core/lib/transport/metadata.cc
src/core/lib/transport/metadata_batch.cc
src/core/lib/transport/metadata_info.cc
src/core/lib/transport/parsed_metadata.cc
src/core/lib/transport/status_conversion.cc
src/core/lib/transport/tcp_connect_handshaker.cc

2
Makefile generated
View File

@ -1670,6 +1670,7 @@ LIBGRPC_SRC = \
src/core/lib/transport/message.cc \
src/core/lib/transport/metadata.cc \
src/core/lib/transport/metadata_batch.cc \
src/core/lib/transport/metadata_info.cc \
src/core/lib/transport/parsed_metadata.cc \
src/core/lib/transport/status_conversion.cc \
src/core/lib/transport/tcp_connect_handshaker.cc \
@ -2233,6 +2234,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/lib/transport/message.cc \
src/core/lib/transport/metadata.cc \
src/core/lib/transport/metadata_batch.cc \
src/core/lib/transport/metadata_info.cc \
src/core/lib/transport/parsed_metadata.cc \
src/core/lib/transport/status_conversion.cc \
src/core/lib/transport/tcp_connect_handshaker.cc \

2
Package.swift generated
View File

@ -1831,6 +1831,8 @@ let package = Package(
"src/core/lib/transport/metadata_batch.cc",
"src/core/lib/transport/metadata_batch.h",
"src/core/lib/transport/metadata_compression_traits.h",
"src/core/lib/transport/metadata_info.cc",
"src/core/lib/transport/metadata_info.h",
"src/core/lib/transport/parsed_metadata.cc",
"src/core/lib/transport/parsed_metadata.h",
"src/core/lib/transport/simple_slice_based_metadata.h",

4
build_autogenerated.yaml generated
View File

@ -1146,6 +1146,7 @@ libs:
- src/core/lib/transport/metadata.h
- src/core/lib/transport/metadata_batch.h
- src/core/lib/transport/metadata_compression_traits.h
- src/core/lib/transport/metadata_info.h
- src/core/lib/transport/parsed_metadata.h
- src/core/lib/transport/simple_slice_based_metadata.h
- src/core/lib/transport/status_conversion.h
@ -1946,6 +1947,7 @@ libs:
- src/core/lib/transport/message.cc
- src/core/lib/transport/metadata.cc
- src/core/lib/transport/metadata_batch.cc
- src/core/lib/transport/metadata_info.cc
- src/core/lib/transport/parsed_metadata.cc
- src/core/lib/transport/status_conversion.cc
- src/core/lib/transport/tcp_connect_handshaker.cc
@ -2609,6 +2611,7 @@ libs:
- src/core/lib/transport/metadata.h
- src/core/lib/transport/metadata_batch.h
- src/core/lib/transport/metadata_compression_traits.h
- src/core/lib/transport/metadata_info.h
- src/core/lib/transport/parsed_metadata.h
- src/core/lib/transport/simple_slice_based_metadata.h
- src/core/lib/transport/status_conversion.h
@ -3029,6 +3032,7 @@ libs:
- src/core/lib/transport/message.cc
- src/core/lib/transport/metadata.cc
- src/core/lib/transport/metadata_batch.cc
- src/core/lib/transport/metadata_info.cc
- src/core/lib/transport/parsed_metadata.cc
- src/core/lib/transport/status_conversion.cc
- src/core/lib/transport/tcp_connect_handshaker.cc

1
config.m4 generated
View File

@ -798,6 +798,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/lib/transport/message.cc \
src/core/lib/transport/metadata.cc \
src/core/lib/transport/metadata_batch.cc \
src/core/lib/transport/metadata_info.cc \
src/core/lib/transport/parsed_metadata.cc \
src/core/lib/transport/status_conversion.cc \
src/core/lib/transport/tcp_connect_handshaker.cc \

1
config.w32 generated
View File

@ -763,6 +763,7 @@ if (PHP_GRPC != "no") {
"src\\core\\lib\\transport\\message.cc " +
"src\\core\\lib\\transport\\metadata.cc " +
"src\\core\\lib\\transport\\metadata_batch.cc " +
"src\\core\\lib\\transport\\metadata_info.cc " +
"src\\core\\lib\\transport\\parsed_metadata.cc " +
"src\\core\\lib\\transport\\status_conversion.cc " +
"src\\core\\lib\\transport\\tcp_connect_handshaker.cc " +

2
gRPC-C++.podspec generated
View File

@ -1251,6 +1251,7 @@ Pod::Spec.new do |s|
'src/core/lib/transport/metadata.h',
'src/core/lib/transport/metadata_batch.h',
'src/core/lib/transport/metadata_compression_traits.h',
'src/core/lib/transport/metadata_info.h',
'src/core/lib/transport/parsed_metadata.h',
'src/core/lib/transport/simple_slice_based_metadata.h',
'src/core/lib/transport/status_conversion.h',
@ -2509,6 +2510,7 @@ Pod::Spec.new do |s|
'src/core/lib/transport/metadata.h',
'src/core/lib/transport/metadata_batch.h',
'src/core/lib/transport/metadata_compression_traits.h',
'src/core/lib/transport/metadata_info.h',
'src/core/lib/transport/parsed_metadata.h',
'src/core/lib/transport/simple_slice_based_metadata.h',
'src/core/lib/transport/status_conversion.h',

3
gRPC-Core.podspec generated
View File

@ -1940,6 +1940,8 @@ Pod::Spec.new do |s|
'src/core/lib/transport/metadata_batch.cc',
'src/core/lib/transport/metadata_batch.h',
'src/core/lib/transport/metadata_compression_traits.h',
'src/core/lib/transport/metadata_info.cc',
'src/core/lib/transport/metadata_info.h',
'src/core/lib/transport/parsed_metadata.cc',
'src/core/lib/transport/parsed_metadata.h',
'src/core/lib/transport/simple_slice_based_metadata.h',
@ -3289,6 +3291,7 @@ Pod::Spec.new do |s|
'src/core/lib/transport/metadata.h',
'src/core/lib/transport/metadata_batch.h',
'src/core/lib/transport/metadata_compression_traits.h',
'src/core/lib/transport/metadata_info.h',
'src/core/lib/transport/parsed_metadata.h',
'src/core/lib/transport/simple_slice_based_metadata.h',
'src/core/lib/transport/status_conversion.h',

2
grpc.gemspec generated
View File

@ -1833,6 +1833,8 @@ Gem::Specification.new do |s|
s.files += %w( src/core/lib/transport/metadata_batch.cc )
s.files += %w( src/core/lib/transport/metadata_batch.h )
s.files += %w( src/core/lib/transport/metadata_compression_traits.h )
s.files += %w( src/core/lib/transport/metadata_info.cc )
s.files += %w( src/core/lib/transport/metadata_info.h )
s.files += %w( src/core/lib/transport/parsed_metadata.cc )
s.files += %w( src/core/lib/transport/parsed_metadata.h )
s.files += %w( src/core/lib/transport/simple_slice_based_metadata.h )

2
grpc.gyp generated
View File

@ -984,6 +984,7 @@
'src/core/lib/transport/message.cc',
'src/core/lib/transport/metadata.cc',
'src/core/lib/transport/metadata_batch.cc',
'src/core/lib/transport/metadata_info.cc',
'src/core/lib/transport/parsed_metadata.cc',
'src/core/lib/transport/status_conversion.cc',
'src/core/lib/transport/tcp_connect_handshaker.cc',
@ -1487,6 +1488,7 @@
'src/core/lib/transport/message.cc',
'src/core/lib/transport/metadata.cc',
'src/core/lib/transport/metadata_batch.cc',
'src/core/lib/transport/metadata_info.cc',
'src/core/lib/transport/parsed_metadata.cc',
'src/core/lib/transport/status_conversion.cc',
'src/core/lib/transport/tcp_connect_handshaker.cc',

2
package.xml generated
View File

@ -1815,6 +1815,8 @@
<file baseinstalldir="/" name="src/core/lib/transport/metadata_batch.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/metadata_batch.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/metadata_compression_traits.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/metadata_info.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/metadata_info.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/parsed_metadata.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/parsed_metadata.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/simple_slice_based_metadata.h" role="src" />

14
src/core/BUILD
View File

@ -128,6 +128,20 @@ grpc_cc_library(
deps = ["//:gpr_platform"],
)
grpc_cc_library(
name = "metadata_info",
srcs = ["lib/transport/metadata_info.cc"],
hdrs = ["lib/transport/metadata_info.h"],
deps = [
"channel_args",
"hpack_constants",
"metadata_batch",
"slice",
"//:gpr_platform",
"//:grpc_base",
],
)
grpc_cc_library(
name = "experiments",
srcs = [

38
src/core/ext/transport/chttp2/transport/chttp2_transport.cc
View File

@ -106,6 +106,7 @@
#include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/http2_errors.h"
#include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/metadata_info.h"
#include "src/core/lib/transport/status_conversion.h"
#include "src/core/lib/transport/transport.h"
@ -116,8 +117,6 @@
#define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
#define MAX_WINDOW 0x7fffffffu
#define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT (8 * 1024)
#define KEEPALIVE_TIME_BACKOFF_MULTIPLIER 2
@ -525,21 +524,8 @@ static void read_channel_args(grpc_chttp2_transport* t,
.GetDurationFromIntMillis(GRPC_ARG_HTTP_TARPIT_MAX_DURATION_MS)
.value_or(grpc_core::Duration::Seconds(1))
.millis();
const int soft_limit =
channel_args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
if (soft_limit < 0) {
// Set soft limit to 0.8 * hard limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT` and
// `GRPC_ARG_MAX_METADATA_SIZE` is not set.
t->max_header_list_size_soft_limit = std::max(
DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT,
static_cast<int>(
0.8 * channel_args.GetInt(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE)
.value_or(-1)));
} else {
t->max_header_list_size_soft_limit = soft_limit;
}
t->max_header_list_size_soft_limit =
grpc_core::GetSoftLimitFromChannelArgs(channel_args);
int value;
if (!is_client) {
@ -557,22 +543,8 @@ static void read_channel_args(grpc_chttp2_transport* t,
if (value >= 0) {
t->settings.mutable_local().SetHeaderTableSize(value);
}
value = channel_args.GetInt(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE).value_or(-1);
if (value >= 0) {
t->settings.mutable_local().SetMaxHeaderListSize(value);
} else {
// Set value to 1.25 * soft limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE` and
// `GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE` is not set.
const int soft_limit =
channel_args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
const int value = (soft_limit >= 0 && soft_limit < (INT_MAX / 1.25))
? static_cast<int>(soft_limit * 1.25)
: soft_limit;
if (value > DEFAULT_MAX_HEADER_LIST_SIZE) {
t->settings.mutable_local().SetMaxHeaderListSize(value);
}
}
t->settings.mutable_local().SetMaxHeaderListSize(
grpc_core::GetHardLimitFromChannelArgs(channel_args));
value = channel_args.GetInt(GRPC_ARG_HTTP2_MAX_FRAME_SIZE).value_or(-1);
if (value >= 0) {
t->settings.mutable_local().SetMaxFrameSize(value);

50
src/core/ext/transport/chttp2/transport/hpack_parser.cc
View File

@ -52,6 +52,7 @@
#include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/surface/validate_metadata.h"
#include "src/core/lib/transport/metadata_info.h"
#include "src/core/lib/transport/parsed_metadata.h"
// IWYU pragma: no_include <type_traits>
@ -1076,55 +1077,6 @@ Slice HPackParser::String::Take() {
GPR_UNREACHABLE_CODE(return Slice());
}
class HPackParser::MetadataSizeEncoder {
public:
explicit MetadataSizeEncoder(std::string& summary) : summary_(summary) {}
void Encode(const Slice& key, const Slice& value) {
AddToSummary(key.as_string_view(), value.size());
}
template <typename Key, typename Value>
void Encode(Key, const Value& value) {
AddToSummary(Key::key(), EncodedSizeOfKey(Key(), value));
}
private:
void AddToSummary(absl::string_view key,
size_t value_length) GPR_ATTRIBUTE_NOINLINE {
absl::StrAppend(&summary_, key, ":",
hpack_constants::SizeForEntry(key.size(), value_length),
",");
}
std::string& summary_;
};
class HPackParser::MetadataSizesAnnotation
: public CallTracerAnnotationInterface::Annotation {
public:
MetadataSizesAnnotation(grpc_metadata_batch* metadata_buffer,
uint64_t soft_limit, uint64_t hard_limit)
: CallTracerAnnotationInterface::Annotation(
CallTracerAnnotationInterface::AnnotationType::kMetadataSizes),
metadata_buffer_(metadata_buffer),
soft_limit_(soft_limit),
hard_limit_(hard_limit) {}
std::string ToString() const override {
std::string metadata_annotation =
absl::StrCat("gRPC metadata soft_limit:", soft_limit_,
",hard_limit:", hard_limit_, ",");
MetadataSizeEncoder encoder(metadata_annotation);
metadata_buffer_->Encode(&encoder);
return metadata_annotation;
}
private:
grpc_metadata_batch* metadata_buffer_;
uint64_t soft_limit_;
uint64_t hard_limit_;
};
// PUBLIC INTERFACE
HPackParser::HPackParser() = default;

2
src/core/ext/transport/chttp2/transport/hpack_parser.h
View File

@ -123,8 +123,6 @@ class HPackParser {
// Helper classes: see implementation
class Parser;
class Input;
class MetadataSizeEncoder;
class MetadataSizesAnnotation;
// Helper to parse a string and turn it into a slice with appropriate memory
// management characteristics

55
src/core/lib/transport/metadata_info.cc Normal file
View File

@ -0,0 +1,55 @@
// Copyright 2023 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <grpc/support/port_platform.h>
#include "src/core/lib/transport/metadata_info.h"
#include "src/core/lib/slice/slice.h"
namespace grpc_core {
class MetadataSizesAnnotation::MetadataSizeEncoder {
public:
explicit MetadataSizeEncoder(std::string& summary) : summary_(summary) {}
void Encode(const Slice& key, const Slice& value) {
AddToSummary(key.as_string_view(), value.size());
}
template <typename Key, typename Value>
void Encode(Key, const Value& value) {
AddToSummary(Key::key(), EncodedSizeOfKey(Key(), value));
}
private:
void AddToSummary(absl::string_view key,
size_t value_length) GPR_ATTRIBUTE_NOINLINE {
absl::StrAppend(&summary_, key, ":",
hpack_constants::SizeForEntry(key.size(), value_length),
",");
}
std::string& summary_;
};
std::string MetadataSizesAnnotation::ToString() const {
std::string metadata_annotation =
absl::StrCat("gRPC metadata soft_limit:", soft_limit_,
",hard_limit:", hard_limit_, ",");
MetadataSizeEncoder encoder(metadata_annotation);
metadata_buffer_->Encode(&encoder);
return metadata_annotation;
}
} // namespace grpc_core

85
src/core/lib/transport/metadata_info.h Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2023 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_METADATA_INFO_H
#define GRPC_SRC_CORE_LIB_TRANSPORT_METADATA_INFO_H
#include <grpc/support/port_platform.h>
#include "src/core/ext/transport/chttp2/transport/hpack_constants.h"
#include "src/core/lib/channel/call_tracer.h"
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/transport/metadata_batch.h"
namespace grpc_core {
#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT (8 * 1024)
inline uint32_t GetSoftLimitFromChannelArgs(const ChannelArgs& args) {
const int soft_limit = args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
if (soft_limit < 0) {
// Set soft limit to 0.8 * hard limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT` and
// `GRPC_ARG_MAX_METADATA_SIZE` is not set.
return std::max(
DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT,
static_cast<int>(
0.8 *
args.GetInt(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE).value_or(-1)));
} else {
return soft_limit;
}
}
inline uint32_t GetHardLimitFromChannelArgs(const ChannelArgs& args) {
int hard_limit =
args.GetInt(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE).value_or(-1);
if (hard_limit >= 0) {
return hard_limit;
} else {
// Set value to 1.25 * soft limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE` and
// `GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE` is not set.
const int soft_limit = args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
const int value = (soft_limit >= 0 && soft_limit < (INT_MAX / 1.25))
? static_cast<int>(soft_limit * 1.25)
: soft_limit;
return std::max(value, DEFAULT_MAX_HEADER_LIST_SIZE);
}
}
class MetadataSizesAnnotation
: public CallTracerAnnotationInterface::Annotation {
public:
MetadataSizesAnnotation(grpc_metadata_batch* metadata_buffer,
uint64_t soft_limit, uint64_t hard_limit)
: CallTracerAnnotationInterface::Annotation(
CallTracerAnnotationInterface::AnnotationType::kMetadataSizes),
metadata_buffer_(metadata_buffer),
soft_limit_(soft_limit),
hard_limit_(hard_limit) {}
std::string ToString() const override;
private:
class MetadataSizeEncoder;
grpc_metadata_batch* metadata_buffer_;
uint64_t soft_limit_;
uint64_t hard_limit_;
};
} // namespace grpc_core
#endif // GRPC_SRC_CORE_LIB_TRANSPORT_METADATA_INFO_H

1
src/python/grpcio/grpc_core_dependencies.py generated
View File

@ -772,6 +772,7 @@ CORE_SOURCE_FILES = [
'src/core/lib/transport/message.cc',
'src/core/lib/transport/metadata.cc',
'src/core/lib/transport/metadata_batch.cc',
'src/core/lib/transport/metadata_info.cc',
'src/core/lib/transport/parsed_metadata.cc',
'src/core/lib/transport/status_conversion.cc',
'src/core/lib/transport/tcp_connect_handshaker.cc',

2
tools/doxygen/Doxyfile.c++.internal generated
View File

@ -2832,6 +2832,8 @@ src/core/lib/transport/metadata.h \
src/core/lib/transport/metadata_batch.cc \
src/core/lib/transport/metadata_batch.h \
src/core/lib/transport/metadata_compression_traits.h \
src/core/lib/transport/metadata_info.cc \
src/core/lib/transport/metadata_info.h \
src/core/lib/transport/parsed_metadata.cc \
src/core/lib/transport/parsed_metadata.h \
src/core/lib/transport/simple_slice_based_metadata.h \

2
tools/doxygen/Doxyfile.core.internal generated
View File

@ -2609,6 +2609,8 @@ src/core/lib/transport/metadata.h \
src/core/lib/transport/metadata_batch.cc \
src/core/lib/transport/metadata_batch.h \
src/core/lib/transport/metadata_compression_traits.h \
src/core/lib/transport/metadata_info.cc \
src/core/lib/transport/metadata_info.h \
src/core/lib/transport/parsed_metadata.cc \
src/core/lib/transport/parsed_metadata.h \
src/core/lib/transport/simple_slice_based_metadata.h \