natas_hw
/
grpc
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

xds end2end tests: change tests to check all RPC failure statuses (#29974) 

* WIP

* fix from merge

* fix ring_hash tests

* ring hash: fix picker propagation bug in xds_cluster_manager policy

* fix build

* clang-format (ish)

* fix build

* more conversion

* remove old CheckRpcSendFailure

* fix SendRpcsAndCountFailuresWithMessage

* fix WaitForBackend

* clang-format

* revert xds resolver change, that will be done in a separate PR

* Automated change: Fix sanity tests

Co-authored-by: markdroth <markdroth@users.noreply.github.com>
This commit is contained in:
Mark D. Roth 2022-06-09 18:00:38 -07:00 committed by GitHub
parent 475623c489
commit 8459824e5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 672 additions and 699 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
test/cpp/end2end/xds/xds_cluster_end2end_test.cc
View File

@ -39,6 +39,7 @@ using ClientStats = LrsServiceImpl::ClientStats;
constexpr char kLbDropType[] = "lb";
constexpr char kThrottleDropType[] = "throttle";
constexpr char kStatusMessageDropPrefix[] = "EDS-configured drop: ";
//
// CDS tests
@ -239,11 +240,14 @@ TEST_P(CdsTest, ClusterRemoved) {
// Unset CDS resource.
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
// Wait for RPCs to start failing.
do {
} while (SendRpc(RpcOptions(), nullptr).ok());
// Make sure RPCs are still failing.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(1000));
SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
EXPECT_EQ(absl::StrCat("CDS resource \"", kDefaultClusterName,
"\" does not exist"),
result.status.error_message());
return false;
});
// Make sure we ACK'ed the update.
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
@ -349,23 +353,7 @@ TEST_P(CdsTest, ClusterChangeAfterAdsCallFails) {
cluster.mutable_eds_cluster_config()->set_service_name(kNewEdsResourceName);
balancer_->ads_service()->SetCdsResource(cluster);
// Make sure client sees the change.
// TODO(roth): This should not be allowing errors. The errors are
// being caused by a bug that triggers in the following situation:
//
// 1. xDS call fails.
// 2. When xDS call is restarted, the server sends the updated CDS
// resource that points to the new EDS resource name.
// 3. When the client receives the CDS update, it does two things:
// - Sends the update to the CDS LB policy, which creates a new
// xds_cluster_resolver policy using the new EDS service name.
// - Notices that the CDS update no longer refers to the old EDS
// service name, so removes that resource, notifying the old
// xds_cluster_resolver policy that the resource no longer exists.
//
// Need to figure out a way to fix this bug, and then change this to
// not allow failures.
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 1);
}
//
@ -455,13 +443,21 @@ TEST_P(EdsTest, InitiallyEmptyServerlist) {
EdsResourceArgs args({std::move(empty_locality)});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// RPCs should fail.
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Send non-empty serverlist.
args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends()}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// RPCs should eventually succeed.
WaitForAllBackends(DEBUG_LOCATION, 0, 1,
WaitForBackendOptions().set_allow_failures(true));
WaitForAllBackends(DEBUG_LOCATION, 0, 1, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
// Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if
@ -493,17 +489,16 @@ TEST_P(EdsTest, BackendsRestart) {
WaitForAllBackends(DEBUG_LOCATION);
// Stop backends. RPCs should fail.
ShutdownAllBackends();
// Sending multiple failed requests instead of just one to ensure that the
// client notices that all backends are down before we restart them. If we
// didn't do this, then a single RPC could fail here due to the race
// condition between the LB pick and the GOAWAY from the chosen backend
// being shut down, which would not actually prove that the client noticed
// that all of the backends are down. Then, when we send another request
// below (which we expect to succeed), if the callbacks happen in the wrong
// order, the same race condition could happen again due to the client not
// yet having noticed that the backends were all down.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(backends_.size()));
// Wait for channel to report TRANSIENT_FAILURE.
EXPECT_TRUE(channel_->WaitForStateChange(
GRPC_CHANNEL_READY, grpc_timeout_seconds_to_deadline(5)));
EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false));
// RPCs should fail.
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
// Restart all backends. RPCs should start succeeding again.
StartAllBackends();
CheckRpcSendOk(DEBUG_LOCATION, 1,
@ -672,7 +667,7 @@ TEST_P(EdsTest, ManyLocalitiesStressTest) {
}
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Wait until backend 0 is ready.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
@ -837,7 +832,8 @@ TEST_P(EdsTest, Drops) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
@ -858,7 +854,8 @@ TEST_P(EdsTest, DropPerHundred) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate,
@ -879,7 +876,8 @@ TEST_P(EdsTest, DropPerTenThousand) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate,
@ -907,7 +905,8 @@ TEST_P(EdsTest, DropConfigUpdate) {
// Send kNumRpcsLbOnly RPCs and count the drops.
gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcsLbOnly, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcsLbOnly, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
// The drop rate should be roughly equal to the expectation.
double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsLbOnly;
@ -924,24 +923,27 @@ TEST_P(EdsTest, DropConfigUpdate) {
const double kDropRateThreshold =
(kDropRateForLb + kDropRateForLbAndThrottle) / 2;
size_t num_rpcs = kNumRpcsBoth;
while (seen_drop_rate < kDropRateThreshold) {
EchoResponse response;
const Status status = SendRpc(RpcOptions(), &response);
++num_rpcs;
if (!status.ok() &&
absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
++num_drops;
} else {
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
EXPECT_EQ(response.message(), kRequestMessage);
}
seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
}
SendRpcsUntil(
DEBUG_LOCATION,
[&](const RpcResult& result) {
++num_rpcs;
if (result.status.ok()) {
EXPECT_EQ(result.response.message(), kRequestMessage);
} else {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_THAT(result.status.error_message(),
::testing::StartsWith(kStatusMessageDropPrefix));
++num_drops;
}
seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
return seen_drop_rate < kDropRateThreshold;
},
/*timeout_ms=*/20000);
// Send kNumRpcsBoth RPCs and count the drops.
gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
num_drops = SendRpcsAndCountFailuresWithMessage(DEBUG_LOCATION, kNumRpcsBoth,
"EDS-configured drop: ");
StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
// The new drop rate should be roughly equal to the expectation.
seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsBoth;
@ -962,7 +964,8 @@ TEST_P(EdsTest, DropAll) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and all of them are dropped.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
EXPECT_EQ(num_drops, kNumRpcs);
}
@ -997,7 +1000,7 @@ TEST_P(FailoverTest, ChooseHighestPriority) {
0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
for (size_t i = 0; i < 3; ++i) {
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
@ -1017,7 +1020,7 @@ TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) {
{"locality3", {}, kDefaultLocalityWeight, 0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
for (size_t i = 1; i < 3; ++i) {
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
@ -1049,7 +1052,7 @@ TEST_P(FailoverTest, Failover) {
{"locality3", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
}
@ -1071,16 +1074,10 @@ TEST_P(FailoverTest, SwitchBackToHigherPriority) {
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3);
ShutdownBackend(3);
backends_[3]->StopListeningAndSendGoaways();
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
ShutdownBackend(0);
WaitForBackend(
DEBUG_LOCATION, 1,
WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
true));
for (size_t i = 0; i < backends_.size(); ++i) {
if (i == 1) continue;
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
}
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
@ -1096,7 +1093,11 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) {
{"locality1", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 1},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
args = EdsResourceArgs({
{"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
0},
@ -1104,8 +1105,12 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) {
1},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
// Tests that after the localities' priorities are updated, we still choose
@ -1124,7 +1129,7 @@ TEST_P(FailoverTest, UpdatePriority) {
0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0U, backends_[0]->backend_service()->request_count());
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
@ -1161,7 +1166,7 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) {
// When we get the first update, all backends in priority 0 are down,
// so we will create priority 1. Backends 0 and 1 should have traffic,
// but backend 2 should not.
WaitForAllBackends(DEBUG_LOCATION, 0, 2,
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count());
// Second update:
@ -1227,7 +1232,7 @@ TEST_P(FailoverTest, PriorityChildNameChurn) {
2},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
// P2 should not have gotten any traffic in this change.
EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count());
@ -1255,14 +1260,15 @@ TEST_P(ClientLoadReportingTest, Vanilla) {
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Wait until all backends are ready.
size_t num_warmup_rpcs = WaitForAllBackends(
DEBUG_LOCATION, 0, 4, WaitForBackendOptions().set_reset_counters(false));
size_t num_warmup_rpcs =
WaitForAllBackends(DEBUG_LOCATION, 0, 4, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
// Send kNumRpcsPerAddress RPCs per server.
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size());
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(kNumFailuresPerAddress * backends_.size())
.set_rpc_options(RpcOptions().set_server_fail(true)));
for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "",
RpcOptions().set_server_fail(true));
}
const size_t total_successful_rpcs_sent =
(kNumRpcsPerAddress * backends_.size()) + num_warmup_rpcs;
const size_t total_failed_rpcs_sent =
@ -1321,10 +1327,10 @@ TEST_P(ClientLoadReportingTest, SendAllClusters) {
size_t num_warmup_rpcs = WaitForAllBackends(DEBUG_LOCATION);
// Send kNumRpcsPerAddress RPCs per server.
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size());
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(kNumFailuresPerAddress * backends_.size())
.set_rpc_options(RpcOptions().set_server_fail(true)));
for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "",
RpcOptions().set_server_fail(true));
}
// Check that each backend got the right number of requests.
for (size_t i = 0; i < backends_.size(); ++i) {
EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
@ -1552,7 +1558,6 @@ TEST_P(ClientLoadReportingTest, DropStats) {
kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
const size_t kNumRpcs =
ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
const char kStatusMessageDropPrefix[] = "EDS-configured drop: ";
// The ADS response contains two drop categories.
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
@ -1560,7 +1565,8 @@ TEST_P(ClientLoadReportingTest, DropStats) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, kStatusMessageDropPrefix);
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,

46
test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc
View File

@ -286,7 +286,7 @@ using AggregateClusterTest = ClusterTypeTest;
INSTANTIATE_TEST_SUITE_P(XdsTest, AggregateClusterTest,
::testing::Values(XdsTestType()), &XdsTestType::Name);
TEST_P(AggregateClusterTest, ) {
TEST_P(AggregateClusterTest, Basic) {
CreateAndStartBackends(2);
const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1";
@ -326,13 +326,13 @@ TEST_P(AggregateClusterTest, ) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
// Bring backend 0 back and ensure all traffic goes back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -386,13 +386,13 @@ TEST_P(AggregateClusterTest, DiamondDependency) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -522,8 +522,8 @@ TEST_P(AggregateClusterTest, FallBackWithConnectivityChurn) {
connection_attempt_injector.Start();
// Wait for P0 backend.
// Increase timeout to account for subchannel connection delays.
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(),
RpcOptions().set_timeout_ms(2000));
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), RpcOptions().set_timeout_ms(2000));
// Send GOAWAY from the P0 backend.
// We don't actually shut it down here to avoid flakiness caused by
// failing an RPC after the client has already sent it but before the
@ -583,13 +583,13 @@ TEST_P(AggregateClusterTest, EdsToLogicalDns) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -644,13 +644,13 @@ TEST_P(AggregateClusterTest, LogicalDnsToEds) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -713,7 +713,11 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) {
std::move(result));
}
// When an RPC fails, we know the channel has seen the update.
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Figure out how to get some sort of resolution note
// included here as part of https://github.com/grpc/grpc/issues/22883.
"empty address list: ";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Send an EDS update that moves locality1 to priority 0.
args1 = EdsResourceArgs({
{"locality1", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
@ -723,8 +727,12 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) {
});
balancer_->ads_service()->SetEdsResource(
BuildEdsResource(args1, kNewEdsService1Name));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
TEST_P(AggregateClusterTest, MultipleClustersWithSameLocalities) {

131
test/cpp/end2end/xds/xds_core_end2end_test.cc
View File

@ -198,9 +198,10 @@ TEST_P(XdsClientTest, MultipleBadCdsResources) {
{"cluster", kClusterName2},
};
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_metadata(std::move(metadata_cluster_2))));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
"cluster_name_2: UNAVAILABLE: invalid resource: INVALID_ARGUMENT:.*"
"errors parsing CDS resource.*DiscoveryType is not valid.*",
RpcOptions().set_metadata(std::move(metadata_cluster_2)));
}
TEST_P(XdsClientTest, XdsStreamErrorPropagation) {
@ -394,16 +395,20 @@ INSTANTIATE_TEST_SUITE_P(
TEST_P(TimeoutTest, LdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kLdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, LdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
@ -417,8 +422,8 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000));
// Create second channel for a new server name.
// This should fail because there is no LDS resource for this server name.
auto channel2 =
CreateChannel(/*failover_timeout_ms=*/0, "new-server.example.com");
const char* kNewServerName = "new-server.example.com";
auto channel2 = CreateChannel(/*failover_timeout_ms=*/0, kNewServerName);
auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
ClientContext context;
EchoRequest request;
@ -428,21 +433,29 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
auto status =
SendRpcMethod(stub2.get(), rpc_options, &context, request, &response);
EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
EXPECT_THAT(status.error_message(),
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
}
TEST_P(TimeoutTest, RdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kRdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, RdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kRdsTypeUrl,
kDefaultRouteConfigurationName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
@ -456,12 +469,13 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000));
// Add listener for 2nd channel, but no RDS resource.
const char* kNewServerName = "new-server.example.com";
const char* kNewRouteConfigName = "rds_resource_does_not_exist";
Listener listener = default_listener_;
listener.set_name(kNewServerName);
HttpConnectionManager http_connection_manager =
ClientHcmAccessor().Unpack(listener);
auto* rds = http_connection_manager.mutable_rds();
rds->set_route_config_name("rds_resource_does_not_exist");
rds->set_route_config_name(kNewRouteConfigName);
rds->mutable_config_source()->mutable_self();
ClientHcmAccessor().Pack(http_connection_manager, &listener);
balancer_->ads_service()->SetLdsResource(listener);
@ -478,20 +492,26 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
auto status =
SendRpcMethod(stub2.get(), rpc_options, &context, request, &response);
EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
EXPECT_EQ(status.error_message(),
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
}
TEST_P(TimeoutTest, CdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kCdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, CdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) {
@ -509,31 +529,38 @@ TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) {
balancer_->ads_service()->SetRdsResource(route_config);
// New cluster times out.
// May need to wait a bit for the change to propagate to the client.
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
bool error_seen = false;
do {
auto status = SendRpc();
if (status.error_code() == StatusCode::UNAVAILABLE) {
error_seen = true;
break;
}
} while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0);
EXPECT_TRUE(error_seen);
SendRpcsUntil(
DEBUG_LOCATION,
[&](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
EXPECT_EQ(absl::StrCat("CDS resource \"", kNewClusterName,
"\" does not exist"),
result.status.error_message());
return false;
},
/*timeout_ms=*/30000, RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kEdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsResourceNotPresentInRequest) {
// No need to remove EDS resource, since the test suite does not add it
// by default.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) {
@ -557,16 +584,20 @@ TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) {
balancer_->ads_service()->SetRdsResource(route_config);
// New EDS resource times out.
// May need to wait a bit for the RDS change to propagate to the client.
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
bool error_seen = false;
do {
auto status = SendRpc(RpcOptions().set_rpc_method(METHOD_ECHO1));
if (status.error_code() == StatusCode::UNAVAILABLE) {
error_seen = true;
break;
}
} while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0);
EXPECT_TRUE(error_seen);
SendRpcsUntil(
DEBUG_LOCATION,
[](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
EXPECT_EQ(
result.status.error_message(),
"weighted_target: all children report state TRANSIENT_FAILURE");
return false;
},
/*timeout_ms=*/30000,
RpcOptions().set_rpc_method(METHOD_ECHO1).set_timeout_ms(4000));
}
TEST_P(TimeoutTest, ServerDoesNotResendAfterAdsStreamRestart) {

48
test/cpp/end2end/xds/xds_csds_end2end_test.cc
View File

@ -560,11 +560,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) {
TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) {
int kTimeoutMillisecond = 1000;
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -591,11 +589,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) {
routes2->mutable_route()->set_cluster(kClusterName2);
SetRouteConfiguration(balancer_.get(), route_config);
// Try to get the configs plumb through
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::AllOf(
@ -639,10 +635,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -656,10 +652,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) {
balancer_->ads_service()->UnsetResource(kRdsTypeUrl,
kDefaultRouteConfigurationName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(
csds_response.config(0).generic_xds_configs(),
@ -672,10 +668,9 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -687,10 +682,11 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kEdsTypeUrl, kDefaultEdsServiceName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(
csds_response.config(0).generic_xds_configs(),

40
test/cpp/end2end/xds/xds_end2end_test.cc
View File

@ -369,8 +369,16 @@ class XdsSecurityTest : public XdsEnd2endTest {
continue;
}
} else {
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state "
"TRANSIENT_FAILURE");
}
});
Status status = SendRpc();
if (!status.ok()) {
gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
@ -768,11 +776,15 @@ TEST_P(XdsSecurityTest, TestTlsConfigurationInCombinedValidationContext) {
->set_instance_name("fake_plugin1");
transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
balancer_->ads_service()->SetCdsResource(cluster);
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
Status status = SendRpc();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
}
});
}
// TODO(yashykt): Remove this test once we stop supporting old fields
@ -789,11 +801,15 @@ TEST_P(XdsSecurityTest,
->set_instance_name("fake_plugin1");
transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
balancer_->ads_service()->SetCdsResource(cluster);
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
Status status = SendRpc();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
}
});
}
TEST_P(XdsSecurityTest, TestMtlsConfigurationWithNoSanMatchers) {

122
test/cpp/end2end/xds/xds_end2end_test_lib.cc
View File

@ -834,18 +834,39 @@ Status XdsEnd2endTest::SendRpc(const RpcOptions& rpc_options,
return status;
}
void XdsEnd2endTest::SendRpcsUntil(
const grpc_core::DebugLocation& debug_location,
std::function<bool(const RpcResult&)> continue_predicate, int timeout_ms,
const RpcOptions& rpc_options) {
absl::Time deadline = absl::InfiniteFuture();
if (timeout_ms != 0) {
deadline = absl::Now() +
(absl::Milliseconds(timeout_ms) * grpc_test_slowdown_factor());
}
while (true) {
RpcResult result;
result.status = SendRpc(rpc_options, &result.response);
if (!continue_predicate(result)) return;
EXPECT_LE(absl::Now(), deadline)
<< debug_location.file() << ":" << debug_location.line();
if (absl::Now() >= deadline) break;
}
}
void XdsEnd2endTest::CheckRpcSendOk(
const grpc_core::DebugLocation& debug_location, const size_t times,
const RpcOptions& rpc_options) {
for (size_t i = 0; i < times; ++i) {
EchoResponse response;
const Status status = SendRpc(rpc_options, &response);
EXPECT_TRUE(status.ok())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
EXPECT_EQ(response.message(), kRequestMessage);
}
SendRpcsUntil(
debug_location,
[debug_location, times, n = size_t(0)](const RpcResult& result) mutable {
EXPECT_TRUE(result.status.ok())
<< "code=" << result.status.error_code()
<< " message=" << result.status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
EXPECT_EQ(result.response.message(), kRequestMessage);
return ++n < times;
},
/*timeout_ms=*/0, rpc_options);
}
void XdsEnd2endTest::CheckRpcSendFailure(
@ -857,41 +878,29 @@ void XdsEnd2endTest::CheckRpcSendFailure(
EXPECT_EQ(expected_status, status.error_code())
<< debug_location.file() << ":" << debug_location.line();
EXPECT_THAT(status.error_message(),
::testing::ContainsRegex(expected_message_regex))
::testing::MatchesRegex(expected_message_regex))
<< debug_location.file() << ":" << debug_location.line();
}
void XdsEnd2endTest::CheckRpcSendFailure(
const grpc_core::DebugLocation& debug_location,
const CheckRpcSendFailureOptions& options) {
for (size_t i = 0; options.continue_predicate(i); ++i) {
const Status status = SendRpc(options.rpc_options);
EXPECT_FALSE(status.ok())
<< " at " << debug_location.file() << ":" << debug_location.line();
if (options.expected_error_code != StatusCode::OK) {
EXPECT_EQ(options.expected_error_code, status.error_code())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
}
}
}
size_t XdsEnd2endTest::SendRpcsAndCountFailuresWithMessage(
const grpc_core::DebugLocation& debug_location, size_t num_rpcs,
const char* drop_error_message_prefix, const RpcOptions& rpc_options) {
StatusCode expected_status, absl::string_view expected_message_prefix,
const RpcOptions& rpc_options) {
size_t num_failed = 0;
for (size_t i = 0; i < num_rpcs; ++i) {
Status status = SendRpc(rpc_options);
if (!status.ok()) {
EXPECT_THAT(status.error_message(),
::testing::StartsWith(drop_error_message_prefix))
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
++num_failed;
}
}
SendRpcsUntil(
debug_location,
[&, n = size_t(0)](const RpcResult& result) mutable {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), expected_status)
<< debug_location.file() << ":" << debug_location.line();
EXPECT_THAT(result.status.error_message(),
::testing::StartsWith(expected_message_prefix))
<< debug_location.file() << ":" << debug_location.line();
++num_failed;
}
return ++n < num_rpcs;
},
/*timeout_ms=*/0, rpc_options);
return num_failed;
}
@ -954,28 +963,29 @@ std::vector<XdsEnd2endTest::ConcurrentRpc> XdsEnd2endTest::SendConcurrentRpcs(
size_t XdsEnd2endTest::WaitForAllBackends(
const grpc_core::DebugLocation& debug_location, size_t start_index,
size_t stop_index, const WaitForBackendOptions& wait_options,
const RpcOptions& rpc_options) {
size_t num_rpcs = 0;
auto deadline = absl::Now() + (absl::Milliseconds(wait_options.timeout_ms) *
grpc_test_slowdown_factor());
size_t stop_index, std::function<void(const RpcResult&)> check_status,
const WaitForBackendOptions& wait_options, const RpcOptions& rpc_options) {
if (check_status == nullptr) {
check_status = [&](const RpcResult& result) {
EXPECT_TRUE(result.status.ok())
<< "code=" << result.status.error_code()
<< " message=" << result.status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
};
}
gpr_log(GPR_INFO,
"========= WAITING FOR BACKENDS [%" PRIuPTR ", %" PRIuPTR
") ==========",
start_index, stop_index);
while (!SeenAllBackends(start_index, stop_index, rpc_options.service)) {
Status status = SendRpc(rpc_options);
if (!wait_options.allow_failures) {
EXPECT_TRUE(status.ok())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
}
EXPECT_LE(absl::Now(), deadline)
<< " at " << debug_location.file() << ":" << debug_location.line();
if (absl::Now() >= deadline) break;
++num_rpcs;
}
size_t num_rpcs = 0;
SendRpcsUntil(
debug_location,
[&](const RpcResult& result) {
++num_rpcs;
check_status(result);
return !SeenAllBackends(start_index, stop_index, rpc_options.service);
},
wait_options.timeout_ms, rpc_options);
if (wait_options.reset_counters) ResetBackendCounters();
gpr_log(GPR_INFO, "Backends up; sent %" PRIuPTR " warm up requests",
num_rpcs);

67
test/cpp/end2end/xds/xds_end2end_test_lib.h
View File

@ -801,6 +801,17 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
GPR_UNREACHABLE_CODE(return grpc::Status::OK);
}
// Send RPCs in a loop until either continue_predicate() returns false
// or timeout_ms elapses.
struct RpcResult {
Status status;
EchoResponse response;
};
void SendRpcsUntil(const grpc_core::DebugLocation& debug_location,
std::function<bool(const RpcResult&)> continue_predicate,
int timeout_ms = 5000,
const RpcOptions& rpc_options = RpcOptions());
// Sends the specified number of RPCs and fails if the RPC fails.
void CheckRpcSendOk(const grpc_core::DebugLocation& debug_location,
const size_t times = 1,
@ -813,49 +824,12 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
absl::string_view expected_message_regex,
const RpcOptions& rpc_options = RpcOptions());
// DEPRECATED -- USE THE ABOVE VARIANT INSTEAD.
// TODO(roth): Change all existing callers to use the above variant
// instead and then remove this.
struct CheckRpcSendFailureOptions {
std::function<bool(size_t)> continue_predicate = [](size_t i) {
return i < 1;
};
RpcOptions rpc_options;
StatusCode expected_error_code = StatusCode::OK;
CheckRpcSendFailureOptions() {}
CheckRpcSendFailureOptions& set_times(size_t times) {
continue_predicate = [times](size_t i) { return i < times; };
return *this;
}
CheckRpcSendFailureOptions& set_continue_predicate(
std::function<bool(size_t)> pred) {
continue_predicate = std::move(pred);
return *this;
}
CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) {
rpc_options = options;
return *this;
}
CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) {
expected_error_code = code;
return *this;
}
};
void CheckRpcSendFailure(
const grpc_core::DebugLocation& debug_location,
const CheckRpcSendFailureOptions& options = CheckRpcSendFailureOptions());
// Sends num_rpcs RPCs, counting how many of them fail with a message
// matching the specfied drop_error_message_prefix.
// Any failure with a non-matching message is a test failure.
// matching the specfied expected_message_prefix.
// Any failure with a non-matching status or message is a test failure.
size_t SendRpcsAndCountFailuresWithMessage(
const grpc_core::DebugLocation& debug_location, size_t num_rpcs,
const char* drop_error_message_prefix,
StatusCode expected_status, absl::string_view expected_message_prefix,
const RpcOptions& rpc_options = RpcOptions());
// A class for running a long-running RPC in its own thread.
@ -901,8 +875,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
struct WaitForBackendOptions {
// If true, resets the backend counters before returning.
bool reset_counters = true;
// If true, RPC failures will not cause the test to fail.
bool allow_failures = false;
// How long to wait for the backend(s) to see requests.
int timeout_ms = 5000;
@ -913,11 +885,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
return *this;
}
WaitForBackendOptions& set_allow_failures(bool enable) {
allow_failures = enable;
return *this;
}
WaitForBackendOptions& set_timeout_ms(int ms) {
timeout_ms = ms;
return *this;
@ -925,20 +892,24 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
};
// Sends RPCs until all of the backends in the specified range see requests.
// The check_status callback will be invoked to check the status of
// every RPC; if null, the default is to check that the RPC succeeded.
// Returns the total number of RPCs sent.
size_t WaitForAllBackends(
const grpc_core::DebugLocation& debug_location, size_t start_index = 0,
size_t stop_index = 0,
std::function<void(const RpcResult&)> check_status = nullptr,
const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
const RpcOptions& rpc_options = RpcOptions());
// Sends RPCs until the backend at index backend_idx sees requests.
void WaitForBackend(
const grpc_core::DebugLocation& debug_location, size_t backend_idx,
std::function<void(const RpcResult&)> check_status = nullptr,
const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
const RpcOptions& rpc_options = RpcOptions()) {
WaitForAllBackends(debug_location, backend_idx, backend_idx + 1,
wait_options, rpc_options);
check_status, wait_options, rpc_options);
}
//

14
test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
View File

@ -116,12 +116,10 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) {
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Fire several RPCs, and expect all of them to be aborted.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(5)
.set_rpc_options(RpcOptions().set_wait_for_ready(true))
.set_expected_error_code(StatusCode::ABORTED));
for (size_t i = 0; i < 5; ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::ABORTED, "Fault injected",
RpcOptions().set_wait_for_ready(true));
}
}
// Without the listener config, the fault injection won't be enabled.
@ -166,7 +164,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) {
SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the aborts.
size_t num_aborted = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "Fault injected");
DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected");
// The abort rate should be roughly equal to the expectation.
const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
EXPECT_THAT(seen_abort_rate,
@ -196,7 +194,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
{"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)},
};
size_t num_aborted = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "Fault injected",
DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected",
RpcOptions().set_metadata(metadata));
// The abort rate should be roughly equal to the expectation.
const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;

366
test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
View File

@ -99,17 +99,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEjectionAndUnejection) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend is ejected, rpc destinated to it are now hashed to the other
@ -177,28 +175,24 @@ TEST_P(OutlierDetectionTest, SuccessRateMaxPercent) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options2);
CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options3);
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
@ -280,17 +274,15 @@ TEST_P(OutlierDetectionTest, SuccessRateStdevFactor) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the stdev_factor is high, no
@ -351,17 +343,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEnforcementPercentage) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the enforcement percentage is 0, no
@ -421,17 +411,15 @@ TEST_P(OutlierDetectionTest, SuccessRateMinimumHosts) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -492,17 +480,15 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -567,18 +553,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEjectionAndUnejection) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend is ejected all traffic going to the ejected backend should now
@ -646,28 +630,24 @@ TEST_P(OutlierDetectionTest, FailurePercentageMaxPercentage) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend should be ejected, trafficed picked up by another backend.
@ -743,18 +723,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageThreshold) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced 1 failure, but since the threshold is 50 % no
@ -815,18 +793,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEnforcementPercentage) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the enforcement percentage is 0, no
@ -886,18 +862,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageMinimumHosts) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -959,18 +933,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageRequestVolume) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -1050,10 +1022,14 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1
// outlier detection interval to pass. The 2 errors to the 1 backend will make
// exactly 1 outlier from the success rate algorithm; all 4 errors will make 3
@ -1063,33 +1039,21 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
// eject another backend because of failure percentage we will stop as we have
// reached our 50% limit.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
@ -1149,17 +1113,15 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentageBothDisabled) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since there is no counting there is no
@ -1235,42 +1197,34 @@ TEST_P(OutlierDetectionTest,
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1
// outlier detection interval to pass. The errors should have caused 2
// ejctionss but since the policy is disabled we are not ejecting any and
// traffic flow as usual and RPCs reach destinated backends.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);

47
test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
View File

@ -272,10 +272,14 @@ TEST_P(RingHashTest, HeaderHashing) {
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options1);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options2);
@ -352,7 +356,8 @@ TEST_P(RingHashTest, NoHashPolicy) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -418,7 +423,8 @@ TEST_P(RingHashTest, HashOnHeaderThatIsNotPresent) {
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs, rpc_options);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -460,7 +466,8 @@ TEST_P(RingHashTest, UnsupportedHashPolicyDefaultToRandomHashing) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -496,7 +503,8 @@ TEST_P(RingHashTest, RandomHashingDistributionAccordingToEndpointWeight) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int weight_33_request_count =
@ -535,7 +543,8 @@ TEST_P(RingHashTest,
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int weight_20_request_count =
@ -912,7 +921,8 @@ TEST_P(RingHashTest, TransientFailureCheckNextOne) {
{"address_hash",
CreateMetadataValueThatHashesToBackendPort(unused_port)}};
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
}
@ -940,12 +950,15 @@ TEST_P(RingHashTest, SwitchToLowerPrioirtyAndThenBack) {
std::vector<std::pair<std::string, std::string>> metadata = {
{"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true), rpc_options);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
EXPECT_EQ(100, backends_[0]->backend_service()->request_count());
EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
@ -1029,7 +1042,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) {
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
// RPCs should go to backend 0.
gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 0 ===");
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
EXPECT_EQ(GRPC_CHANNEL_READY, channel_->GetState(false));
// Bring down backend 0 and bring up backend 1.
// Note the RPC contains a header value that will always be hashed to
@ -1059,7 +1073,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) {
EXPECT_TRUE(channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 1 ===");
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
gpr_log(GPR_INFO, "=== DONE ===");
}

8
test/cpp/end2end/xds/xds_rls_end2end_test.cc
View File

@ -153,8 +153,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPlugin) {
kRlsClusterSpecifierPluginInstanceName);
SetRouteConfiguration(balancer_.get(), new_route_config);
auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}});
WaitForAllBackends(DEBUG_LOCATION, 1, 2, WaitForBackendOptions(),
rpc_options);
WaitForAllBackends(DEBUG_LOCATION, 1, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs, rpc_options);
// Make sure RPCs all go to the correct backend.
EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count());
@ -338,8 +338,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPluginDisabled) {
// Ensure we ignore the cluster specifier plugin and send traffic according to
// the default route.
auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}});
WaitForAllBackends(DEBUG_LOCATION, 0, 1, WaitForBackendOptions(),
rpc_options);
WaitForAllBackends(DEBUG_LOCATION, 0, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
}
} // namespace

302
test/cpp/end2end/xds/xds_routing_end2end_test.cc
View File

@ -515,11 +515,15 @@ TEST_P(LdsRdsTest, ListenerRemoved) {
// Unset LDS resource.
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
// Wait for RPCs to start failing.
do {
} while (SendRpc(RpcOptions(), nullptr).ok());
// Make sure RPCs are still failing.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(1000));
SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
return false;
});
// Make sure we ACK'ed the update.
auto response_state = balancer_->ads_service()->lds_response_state();
ASSERT_TRUE(response_state.has_value());
@ -533,7 +537,13 @@ TEST_P(LdsRdsTest, NoMatchedDomain) {
route_config.mutable_virtual_hosts(0)->clear_domains();
route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
SetRouteConfiguration(balancer_.get(), route_config);
CheckRpcSendFailure(DEBUG_LOCATION);
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat(
(GetParam().enable_rds_testing() ? kDefaultRouteConfigurationName
: kServerName),
": UNAVAILABLE: could not find VirtualHost for ", kServerName,
" in RouteConfiguration"));
// Do a bit of polling, to allow the ACK to get to the ADS server.
channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100));
auto response_state = RouteConfigurationResponseState(balancer_.get());
@ -746,9 +756,8 @@ TEST_P(LdsRdsTest, MatchingRouteHasNoRouteAction) {
route->mutable_match()->set_prefix("");
route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), route_config);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_expected_error_code(
StatusCode::UNAVAILABLE));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
"Matching route has inappropriate action");
}
TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) {
@ -1324,7 +1333,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForAllBackends(DEBUG_LOCATION, 0, 1);
WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs,
@ -1495,7 +1505,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForAllBackends(DEBUG_LOCATION, 0, 1);
WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1626,7 +1637,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForBackend(DEBUG_LOCATION, 0);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1654,7 +1666,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
weighted_cluster2->mutable_weight()->set_value(kWeight50);
SetRouteConfiguration(balancer_.get(), new_route_config);
ResetBackendCounters();
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs5050,
@ -1682,7 +1695,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
weighted_cluster2->mutable_weight()->set_value(kWeight25);
SetRouteConfiguration(balancer_.get(), new_route_config);
ResetBackendCounters();
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1744,17 +1758,28 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) {
}
TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
CreateAndStartBackends(2);
// Start with only backend 1 up, but the default cluster pointing to
// backend 0, which is down.
CreateBackends(2);
StartBackend(1);
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Start an RPC with wait_for_ready=true and no deadline. This will
// stay pending until backend 0 is reachable.
LongRunningRpc rpc;
rpc.StartRpc(stub_.get(),
RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
// Send a non-wait_for_ready RPC, which should fail. This tells us
// that the client has received the update and attempted to connect.
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Now create a new cluster, pointing to backend 1.
const char* kNewClusterName = "new_cluster";
const char* kNewEdsServiceName = "new_eds_service_name";
// Populate new EDS resources.
EdsResourceArgs args({
{"locality0", CreateEndpointsForBackends(0, 1)},
});
EdsResourceArgs args1({
{"locality0", CreateEndpointsForBackends(1, 2)},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
EdsResourceArgs args1({{"locality0", CreateEndpointsForBackends(1, 2)}});
balancer_->ads_service()->SetEdsResource(
BuildEdsResource(args1, kNewEdsServiceName));
// Populate new CDS resources.
@ -1763,24 +1788,8 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
new_cluster.mutable_eds_cluster_config()->set_service_name(
kNewEdsServiceName);
balancer_->ads_service()->SetCdsResource(new_cluster);
// Bring down the current backend: 0, this will delay route picking time,
// resulting in un-committed RPCs.
ShutdownBackend(0);
// Send a RouteConfiguration with a default route that points to
// backend 0.
RouteConfiguration new_route_config = default_route_config_;
SetRouteConfiguration(balancer_.get(), new_route_config);
// Send exactly one RPC with no deadline and with wait_for_ready=true.
// This RPC will not complete until after backend 0 is started.
std::thread sending_rpc([this]() {
CheckRpcSendOk(DEBUG_LOCATION, 1,
RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
});
// Send a non-wait_for_ready RPC which should fail, this will tell us
// that the client has received the update and attempted to connect.
const Status status = SendRpc(RpcOptions().set_timeout_ms(0));
EXPECT_FALSE(status.ok());
// Send a update RouteConfiguration to use backend 1.
RouteConfiguration new_route_config = default_route_config_;
auto* default_route =
new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
default_route->mutable_route()->set_cluster(kNewClusterName);
@ -1789,13 +1798,20 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
// has processed the update.
WaitForBackend(
DEBUG_LOCATION, 1,
WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
true));
// Bring up the previous backend: 0, this will allow the delayed RPC to
// finally call on_call_committed upon completion.
[&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
},
WaitForBackendOptions().set_reset_counters(false));
// Bring up the backend 0. Yhis will allow the delayed RPC to finally
// complete.
StartBackend(0);
sending_rpc.join();
// Make sure RPCs go to the correct backend:
Status status = rpc.GetStatus();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
// Make sure RPCs went to the correct backends.
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
EXPECT_EQ(1, backends_[1]->backend_service()->request_count());
}
@ -1893,16 +1909,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(grpc_core::Duration::Seconds(
kTimeoutApplicationSecond)
.millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test max_stream_duration of 2.5 seconds applied
t0 = NowFromCycleCounter();
@ -1911,16 +1925,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(grpc_core::Duration::Seconds(
kTimeoutApplicationSecond)
.millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test http_stream_duration of 3.5 seconds applied
t0 = NowFromCycleCounter();
@ -1929,11 +1941,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
}
@ -2007,16 +2017,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
new_route_config);
// Test application timeout is applied for route 1
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2024,16 +2031,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
kTimeoutApplicationSecond * 1000000000);
// Test application timeout is applied for route 2
t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
system_clock::now() - t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
@ -2063,11 +2067,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
// Test application timeout is applied for route 1
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2084,11 +2086,9 @@ TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2115,52 +2115,34 @@ TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// Ensure we retried the correct number of times on all supported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_server_expected_error(StatusCode::CANCELLED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::INTERNAL))
.set_expected_error_code(StatusCode::INTERNAL));
DEBUG_LOCATION, StatusCode::INTERNAL, "",
RpcOptions().set_server_expected_error(StatusCode::INTERNAL));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::RESOURCE_EXHAUSTED))
.set_expected_error_code(StatusCode::RESOURCE_EXHAUSTED));
DEBUG_LOCATION, StatusCode::RESOURCE_EXHAUSTED, "",
RpcOptions().set_server_expected_error(StatusCode::RESOURCE_EXHAUSTED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE))
.set_expected_error_code(StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE, "",
RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
// Ensure we don't retry on an unsupported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::UNAUTHENTICATED))
.set_expected_error_code(StatusCode::UNAUTHENTICATED));
DEBUG_LOCATION, StatusCode::UNAUTHENTICATED, "",
RpcOptions().set_server_expected_error(StatusCode::UNAUTHENTICATED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2182,11 +2164,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyAtVirtualHostLevel) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// Ensure we retried the correct number of times on a supported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
}
@ -2217,12 +2196,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
// No need to set max interval and just let it be the default of 10x of base.
// We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED));
EXPECT_EQ(1 + 1, backends_[0]->backend_service()->request_count());
}
@ -2260,12 +2236,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect 2 retry before the RPC times out with DEADLINE_EXCEEDED.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED));
EXPECT_EQ(2 + 1, backends_[0]->backend_service()->request_count());
}
@ -2286,11 +2259,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyUnsupportedStatusCode) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect no retry.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2319,11 +2289,8 @@ TEST_P(LdsRdsTest,
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect no retry.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2447,8 +2414,8 @@ TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) {
.set_metadata(std::move(metadata));
// Make sure all backends are up.
WaitForBackend(DEBUG_LOCATION, 0);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
header_match_rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), header_match_rpc_options);
// Send RPCs.
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs, header_match_rpc_options);
@ -2744,12 +2711,12 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
SetRouteConfiguration(balancer_.get(), route_config);
// Make sure all backends are up and that requests for each RPC
// service go to the right backends.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Requests for services Echo and Echo2 should have gone to backend 0.
@ -2764,16 +2731,17 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
// different RPC service, and wait for the client to make the change.
route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
SetRouteConfiguration(balancer_.get(), route_config);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Now repeat the earlier test, making sure all traffic goes to the
// right place.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Requests for services Echo and Echo1 should have gone to backend 0.