wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add knob

This commit is contained in:
Xiaoxi Wang 2021-10-21 22:48:21 -07:00
parent c320391c4c
commit 1a2a838df3
3 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fdbclient/ServerKnobs.cpp
View File

@ -212,6 +212,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( INITIAL_FAILURE_REACTION_DELAY, 30.0 ); if( randomize && BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0;
init( CHECK_TEAM_DELAY, 30.0 );
init( PERPETUAL_WIGGLE_DELAY, 50.0 );
init( PERPETUAL_WIGGLE_DISABLE_REMOVER, true );
init( LOG_ON_COMPLETION_DELAY, DD_QUEUE_LOGGING_INTERVAL );
init( BEST_TEAM_MAX_TEAM_TRIES, 10 );
init( BEST_TEAM_OPTION_COUNT, 4 );

1
fdbclient/ServerKnobs.h
View File

@ -165,6 +165,7 @@ public:
double INITIAL_FAILURE_REACTION_DELAY;
double CHECK_TEAM_DELAY;
double PERPETUAL_WIGGLE_DELAY;
bool PERPETUAL_WIGGLE_DISABLE_REMOVER;
double LOG_ON_COMPLETION_DELAY;
int BEST_TEAM_MAX_TEAM_TRIES;
int BEST_TEAM_OPTION_COUNT;

23
fdbserver/DataDistribution.actor.cpp
View File

@ -3047,10 +3047,11 @@ ACTOR Future<Void> updateServerMetrics(Reference<TCServerInfo> server) {
// NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period)
// processingWiggle and processingUnhealthy indicate that some servers are going to be removed.
ACTOR Future<Void> waitUntilHealthy(DDTeamCollection* self, double extraDelay = 0) {
ACTOR Future<Void> waitUntilHealthy(DDTeamCollection* self, double extraDelay = 0, bool waitWiggle = false) {
state int waitCount = 0;
loop {
while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() || self->processingWiggle->get()) {
while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() ||
(waitWiggle && self->processingWiggle->get())) {
// processingUnhealthy: true when there exists data movement
// processingWiggle: true when there exists data movement because we want to wiggle a SS
TraceEvent("WaitUntilHealthyStalled", self->distributorId)
@ -3066,7 +3067,8 @@ ACTOR Future<Void> waitUntilHealthy(DDTeamCollection* self, double extraDelay =
TaskPriority::Low)); // After the team trackers wait on the initial failure reaction delay, they
// yield. We want to make sure every tracker has had the opportunity to send
// their relocations to the queue.
if (!self->zeroHealthyTeams->get() && !self->processingUnhealthy->get() && !self->processingWiggle->get()) {
if (!self->zeroHealthyTeams->get() && !self->processingUnhealthy->get() &&
(!waitWiggle || !self->processingWiggle->get())) {
if (extraDelay <= 0.01 || waitCount >= 1) {
// Return healthy if we do not need extraDelay or when DD are healthy in at least two consecutive check
return Void();
@ -3368,10 +3370,12 @@ ACTOR Future<Void> machineTeamRemover(DDTeamCollection* self) {
wait(delay(SERVER_KNOBS->TR_REMOVE_MACHINE_TEAM_DELAY, TaskPriority::DataDistribution));
if (self->pauseWiggle && self->pauseWiggle->get()) {
wait(waitUntilHealthy(
self, SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
wait(
waitUntilHealthy(self,
SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY,
SERVER_KNOBS->PERPETUAL_WIGGLE_DISABLE_REMOVER));
} else {
wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY, false));
}
// Wait for the badTeamRemover() to avoid the potential race between adding the bad team (add the team tracker)
// and remove bad team (cancel the team tracker).
@ -3497,8 +3501,10 @@ ACTOR Future<Void> serverTeamRemover(DDTeamCollection* self) {
wait(delay(removeServerTeamDelay, TaskPriority::DataDistribution));
if (self->pauseWiggle && self->pauseWiggle->get()) {
wait(waitUntilHealthy(
self, SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
wait(
waitUntilHealthy(self,
SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY,
SERVER_KNOBS->PERPETUAL_WIGGLE_DISABLE_REMOVER));
} else {
wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
}
@ -4170,7 +4176,6 @@ ACTOR Future<Void> perpetualStorageWiggleIterator(AsyncVar<bool>* stopSignal,
// there must not have other teams to place wiggled data
takeRest = teamCollection->server_info.size() <= teamCollection->configuration.storageTeamSize ||
teamCollection->machine_info.size() < teamCollection->configuration.storageTeamSize;
teamCollection->restartRecruiting.trigger();
if (takeRest &&
teamCollection->configuration.storageMigrationType == StorageMigrationType::GRADUAL) {
TraceEvent(SevWarn, "PerpetualWiggleSleep", teamCollection->distributorId)