wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

latency tracking for dataDistributionRelocator (#13072) (#13103) 

Cherrypick PR#13072 into release-7.4.

This function does a lot of stuff and has been observed to be expensive from time to time. This PR tracks its latency explicitly. This is done in 5 minute windows which seems about the right granularity for the sort of incidents which we might anticipate. Note too short, not too long.

Testing (7.4 specific):
20260428-142636-gglass-3b52791173c99235 compressed=True data_size=41487471 duration=4813496 ended=100000 fail_fast=1000 max_runs=100000 pass=100000 priority=100 remaining=0 runtime=1:00:16 sanity=False started=100000 stopped=20260428-152652 submitted=20260428-142636 timeout=5400 username=gglass

* latency tracking for dataDistributionRelocator (#13072)

* add latency tracking for dataDistributionRelocator

* simplify approach, and try to avoid negative latency samples

* formatting
This commit is contained in:
gxglass 2026-04-28 16:10:52 -07:00 committed by GitHub
parent cec76934b9
commit 9aaa5222f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fdbclient/ServerKnobs.cpp
View File

@ -138,6 +138,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( BG_REBALANCE_MAX_POLLING_INTERVAL, 10.0 );
init( BG_REBALANCE_SWITCH_CHECK_INTERVAL, 5.0 ); if (randomize && BUGGIFY) BG_REBALANCE_SWITCH_CHECK_INTERVAL = 1.0;
init( DD_QUEUE_LOGGING_INTERVAL, 5.0 );
init( DD_RELOCATOR_LATENCY_LOGGING_INTERVAL, 300.0 );
init( DD_QUEUE_COUNTER_REFRESH_INTERVAL, 60.0 );
// 100 / 60 < 2 trace/sec ~ 2 * 200 = 400b/sec
init( DD_QUEUE_COUNTER_MAX_LOG, 100 ); if( randomize && BUGGIFY ) DD_QUEUE_COUNTER_MAX_LOG = 1;

1
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -124,6 +124,7 @@ public:
double BG_REBALANCE_MAX_POLLING_INTERVAL;
double BG_REBALANCE_SWITCH_CHECK_INTERVAL;
double DD_QUEUE_LOGGING_INTERVAL;
double DD_RELOCATOR_LATENCY_LOGGING_INTERVAL;
double DD_QUEUE_COUNTER_REFRESH_INTERVAL;
double DD_QUEUE_COUNTER_MAX_LOG; // max number of servers for which trace events will be generated in each round of
// DD_QUEUE_COUNTER_REFRESH_INTERVAL duration

31
fdbserver/DDRelocationQueue.actor.cpp
View File

@ -2360,6 +2360,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
.detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId());
}
}
self->relocatorLatency.addSample(now() - startTime);
return Void();
} else {
if (doBulkLoading) {
@ -2408,6 +2410,9 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
}
} catch (Error& e) {
state Error err = e;
if (err.code() != error_code_actor_cancelled && err.code() != error_code_data_move_cancelled) {
self->relocatorErrorLatency.addSample(now() - startTime);
}
TraceEvent(relocateShardInterval.end(), distributorId)
.errorUnsuppressed(err)
.detail("Duration", now() - startTime);
@ -2816,6 +2821,7 @@ struct DDQueueImpl {
state KeyRange keysToLaunchFrom;
state RelocateData launchData;
state Future<Void> recordMetrics = delay(SERVER_KNOBS->DD_QUEUE_LOGGING_INTERVAL);
state Future<Void> recordRelocatorLatency = delay(SERVER_KNOBS->DD_RELOCATOR_LATENCY_LOGGING_INTERVAL);
state std::vector<Future<Void>> ddQueueFutures;
@ -2993,6 +2999,31 @@ struct DDQueueImpl {
}
}
}
when(wait(recordRelocatorLatency)) {
recordRelocatorLatency =
delay(SERVER_KNOBS->DD_RELOCATOR_LATENCY_LOGGING_INTERVAL, TaskPriority::FlushTrace);
auto& s = self->relocatorLatency;
auto& e = self->relocatorErrorLatency;
TraceEvent("RelocatorLatency", self->distributorId)
.detail("Count", s.getPopulationSize())
.detail("Mean", s.mean())
.detail("Min", s.getPopulationSize() > 0 ? s.min() : 0)
.detail("P50", s.median())
.detail("P90", s.percentile(0.9))
.detail("P95", s.percentile(0.95))
.detail("P99", s.percentile(0.99))
.detail("Max", s.getPopulationSize() > 0 ? s.max() : 0)
.detail("ErrorCount", e.getPopulationSize())
.detail("ErrorMean", e.mean())
.detail("ErrorMin", e.getPopulationSize() > 0 ? e.min() : 0)
.detail("ErrorP50", e.median())
.detail("ErrorP90", e.percentile(0.9))
.detail("ErrorP95", e.percentile(0.95))
.detail("ErrorP99", e.percentile(0.99))
.detail("ErrorMax", e.getPopulationSize() > 0 ? e.max() : 0);
s.clear();
e.clear();
}
when(wait(self->error.getFuture())) {} // Propagate errors from dataDistributionRelocator
when(wait(waitForAll(ddQueueFutures))) {}
when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) {

5
fdbserver/include/fdbserver/DDRelocationQueue.h
View File

@ -22,6 +22,7 @@
#include <numeric>
#include "fdbrpc/DDSketch.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbserver/MovingWindow.h"
@ -273,6 +274,10 @@ public:
std::map<UID, double> lastAsSource;
ServerCounter serverCounter;
// Declared before inFlightActors so it outlives the relocator actors (reverse destruction order).
DDSketch<double> relocatorLatency;
DDSketch<double> relocatorErrorLatency;
KeyRangeMap<RelocateData> inFlight;
// Track all actors that relocates specified keys to a good place; Key: keyRange; Value: actor
KeyRangeActorMap inFlightActors;