diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index f1cae7207e..44f0ec6e2c 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -212,6 +212,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( INITIAL_FAILURE_REACTION_DELAY, 30.0 ); if( randomize && BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0; init( CHECK_TEAM_DELAY, 30.0 ); init( PERPETUAL_WIGGLE_DELAY, 50.0 ); + init( PERPETUAL_WIGGLE_DISABLE_REMOVER, true ); init( LOG_ON_COMPLETION_DELAY, DD_QUEUE_LOGGING_INTERVAL ); init( BEST_TEAM_MAX_TEAM_TRIES, 10 ); init( BEST_TEAM_OPTION_COUNT, 4 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index c2e441dbbd..6a35065204 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -165,6 +165,7 @@ public: double INITIAL_FAILURE_REACTION_DELAY; double CHECK_TEAM_DELAY; double PERPETUAL_WIGGLE_DELAY; + bool PERPETUAL_WIGGLE_DISABLE_REMOVER; double LOG_ON_COMPLETION_DELAY; int BEST_TEAM_MAX_TEAM_TRIES; int BEST_TEAM_OPTION_COUNT; diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index b4ef7ac173..35240f94d8 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -3047,10 +3047,11 @@ ACTOR Future updateServerMetrics(Reference server) { // NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period) // processingWiggle and processingUnhealthy indicate that some servers are going to be removed. -ACTOR Future waitUntilHealthy(DDTeamCollection* self, double extraDelay = 0) { +ACTOR Future waitUntilHealthy(DDTeamCollection* self, double extraDelay = 0, bool waitWiggle = false) { state int waitCount = 0; loop { - while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() || self->processingWiggle->get()) { + while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() || + (waitWiggle && self->processingWiggle->get())) { // processingUnhealthy: true when there exists data movement // processingWiggle: true when there exists data movement because we want to wiggle a SS TraceEvent("WaitUntilHealthyStalled", self->distributorId) @@ -3066,7 +3067,8 @@ ACTOR Future waitUntilHealthy(DDTeamCollection* self, double extraDelay = TaskPriority::Low)); // After the team trackers wait on the initial failure reaction delay, they // yield. We want to make sure every tracker has had the opportunity to send // their relocations to the queue. - if (!self->zeroHealthyTeams->get() && !self->processingUnhealthy->get() && !self->processingWiggle->get()) { + if (!self->zeroHealthyTeams->get() && !self->processingUnhealthy->get() && + (!waitWiggle || !self->processingWiggle->get())) { if (extraDelay <= 0.01 || waitCount >= 1) { // Return healthy if we do not need extraDelay or when DD are healthy in at least two consecutive check return Void(); @@ -3368,10 +3370,12 @@ ACTOR Future machineTeamRemover(DDTeamCollection* self) { wait(delay(SERVER_KNOBS->TR_REMOVE_MACHINE_TEAM_DELAY, TaskPriority::DataDistribution)); if (self->pauseWiggle && self->pauseWiggle->get()) { - wait(waitUntilHealthy( - self, SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY)); + wait( + waitUntilHealthy(self, + SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY, + SERVER_KNOBS->PERPETUAL_WIGGLE_DISABLE_REMOVER)); } else { - wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY)); + wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY, false)); } // Wait for the badTeamRemover() to avoid the potential race between adding the bad team (add the team tracker) // and remove bad team (cancel the team tracker). @@ -3497,8 +3501,10 @@ ACTOR Future serverTeamRemover(DDTeamCollection* self) { wait(delay(removeServerTeamDelay, TaskPriority::DataDistribution)); if (self->pauseWiggle && self->pauseWiggle->get()) { - wait(waitUntilHealthy( - self, SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY)); + wait( + waitUntilHealthy(self, + SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY + SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY, + SERVER_KNOBS->PERPETUAL_WIGGLE_DISABLE_REMOVER)); } else { wait(waitUntilHealthy(self, SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY)); } @@ -4170,7 +4176,6 @@ ACTOR Future perpetualStorageWiggleIterator(AsyncVar* stopSignal, // there must not have other teams to place wiggled data takeRest = teamCollection->server_info.size() <= teamCollection->configuration.storageTeamSize || teamCollection->machine_info.size() < teamCollection->configuration.storageTeamSize; - teamCollection->restartRecruiting.trigger(); if (takeRest && teamCollection->configuration.storageMigrationType == StorageMigrationType::GRADUAL) { TraceEvent(SevWarn, "PerpetualWiggleSleep", teamCollection->distributorId)