diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index bc463f68c7..e5945f1a27 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -138,6 +138,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( BG_REBALANCE_MAX_POLLING_INTERVAL, 10.0 ); init( BG_REBALANCE_SWITCH_CHECK_INTERVAL, 5.0 ); if (randomize && BUGGIFY) BG_REBALANCE_SWITCH_CHECK_INTERVAL = 1.0; init( DD_QUEUE_LOGGING_INTERVAL, 5.0 ); + init( DD_RELOCATOR_LATENCY_LOGGING_INTERVAL, 300.0 ); init( DD_QUEUE_COUNTER_REFRESH_INTERVAL, 60.0 ); // 100 / 60 < 2 trace/sec ~ 2 * 200 = 400b/sec init( DD_QUEUE_COUNTER_MAX_LOG, 100 ); if( randomize && BUGGIFY ) DD_QUEUE_COUNTER_MAX_LOG = 1; diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index b1d125d926..7c4c0ba538 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -124,6 +124,7 @@ public: double BG_REBALANCE_MAX_POLLING_INTERVAL; double BG_REBALANCE_SWITCH_CHECK_INTERVAL; double DD_QUEUE_LOGGING_INTERVAL; + double DD_RELOCATOR_LATENCY_LOGGING_INTERVAL; double DD_QUEUE_COUNTER_REFRESH_INTERVAL; double DD_QUEUE_COUNTER_MAX_LOG; // max number of servers for which trace events will be generated in each round of // DD_QUEUE_COUNTER_REFRESH_INTERVAL duration diff --git a/fdbserver/DDRelocationQueue.actor.cpp b/fdbserver/DDRelocationQueue.actor.cpp index cda28312b0..41556dad0c 100644 --- a/fdbserver/DDRelocationQueue.actor.cpp +++ b/fdbserver/DDRelocationQueue.actor.cpp @@ -2360,6 +2360,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId()); } } + + self->relocatorLatency.addSample(now() - startTime); return Void(); } else { if (doBulkLoading) { @@ -2408,6 +2410,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } } catch (Error& e) { state Error err = e; + if (err.code() != error_code_actor_cancelled && err.code() != error_code_data_move_cancelled) { + self->relocatorErrorLatency.addSample(now() - startTime); + } TraceEvent(relocateShardInterval.end(), distributorId) .errorUnsuppressed(err) .detail("Duration", now() - startTime); @@ -2816,6 +2821,7 @@ struct DDQueueImpl { state KeyRange keysToLaunchFrom; state RelocateData launchData; state Future recordMetrics = delay(SERVER_KNOBS->DD_QUEUE_LOGGING_INTERVAL); + state Future recordRelocatorLatency = delay(SERVER_KNOBS->DD_RELOCATOR_LATENCY_LOGGING_INTERVAL); state std::vector> ddQueueFutures; @@ -2993,6 +2999,31 @@ struct DDQueueImpl { } } } + when(wait(recordRelocatorLatency)) { + recordRelocatorLatency = + delay(SERVER_KNOBS->DD_RELOCATOR_LATENCY_LOGGING_INTERVAL, TaskPriority::FlushTrace); + auto& s = self->relocatorLatency; + auto& e = self->relocatorErrorLatency; + TraceEvent("RelocatorLatency", self->distributorId) + .detail("Count", s.getPopulationSize()) + .detail("Mean", s.mean()) + .detail("Min", s.getPopulationSize() > 0 ? s.min() : 0) + .detail("P50", s.median()) + .detail("P90", s.percentile(0.9)) + .detail("P95", s.percentile(0.95)) + .detail("P99", s.percentile(0.99)) + .detail("Max", s.getPopulationSize() > 0 ? s.max() : 0) + .detail("ErrorCount", e.getPopulationSize()) + .detail("ErrorMean", e.mean()) + .detail("ErrorMin", e.getPopulationSize() > 0 ? e.min() : 0) + .detail("ErrorP50", e.median()) + .detail("ErrorP90", e.percentile(0.9)) + .detail("ErrorP95", e.percentile(0.95)) + .detail("ErrorP99", e.percentile(0.99)) + .detail("ErrorMax", e.getPopulationSize() > 0 ? e.max() : 0); + s.clear(); + e.clear(); + } when(wait(self->error.getFuture())) {} // Propagate errors from dataDistributionRelocator when(wait(waitForAll(ddQueueFutures))) {} when(Promise r = waitNext(getUnhealthyRelocationCount)) { diff --git a/fdbserver/include/fdbserver/DDRelocationQueue.h b/fdbserver/include/fdbserver/DDRelocationQueue.h index 4c18dfabf4..aed4f295e6 100644 --- a/fdbserver/include/fdbserver/DDRelocationQueue.h +++ b/fdbserver/include/fdbserver/DDRelocationQueue.h @@ -22,6 +22,7 @@ #include +#include "fdbrpc/DDSketch.h" #include "fdbserver/DataDistribution.actor.h" #include "fdbserver/MovingWindow.h" @@ -273,6 +274,10 @@ public: std::map lastAsSource; ServerCounter serverCounter; + // Declared before inFlightActors so it outlives the relocator actors (reverse destruction order). + DDSketch relocatorLatency; + DDSketch relocatorErrorLatency; + KeyRangeMap inFlight; // Track all actors that relocates specified keys to a good place; Key: keyRange; Value: actor KeyRangeActorMap inFlightActors;