diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index dacf89e754..be38308f3a 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -718,6 +718,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( PEER_LATENCY_DEGRADATION_PERCENTILE, 0.90 ); init( PEER_LATENCY_DEGRADATION_THRESHOLD, 0.05 ); init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD, 0.1 ); + init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT, 1 ); // Test harness init( WORKER_POLL_DELAY, 1.0 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index e15a3100b5..2d87507198 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -660,6 +660,8 @@ public: double PEER_LATENCY_DEGRADATION_PERCENTILE; // The percentile latency used to check peer health. double PEER_LATENCY_DEGRADATION_THRESHOLD; // The latency threshold to consider a peer degraded. double PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD; // The percentage of timeout to consider a peer degraded. + int PEER_DEGRADATION_CONNECTION_FAILURE_COUNT; // The number of connection failures experienced during measurement + // period to consider a peer degraded. // Test harness double WORKER_POLL_DELAY; diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index bd721b437b..b3dcd3e10f 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -892,7 +892,8 @@ ACTOR Future healthMonitor(ReferencepingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) { + if (peer->connectFailedCount == 0 && + peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) { // Ignore peers that don't have enough samples. // TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a // regular @@ -909,7 +910,8 @@ ACTOR Future healthMonitor(ReferencepingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) > + if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT || + peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) > SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD || peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) > SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) { @@ -926,8 +928,9 @@ ACTOR Future healthMonitor(ReferencepingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)) - .detail("Count", peer->pingLatencies.getPopulationSize()) - .detail("TimeoutCount", peer->timeoutCount); + .detail("PingCount", peer->pingLatencies.getPopulationSize()) + .detail("PingTimeoutCount", peer->timeoutCount) + .detail("ConnectionFailureCount", peer->connectFailedCount); req.degradedPeers.push_back(address); }