From 951bc4acd78cbeaf23c3c8415f6d8ff481087013 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <evan.tschannen@snowflake.com>
Date: Mon, 6 Dec 2021 13:12:27 -0800
Subject: [PATCH] fix: do not call better master exists until the long lived
 stateless processes have settled into their desired locations

---
 fdbclient/ServerKnobs.cpp             |  1 +
 fdbclient/ServerKnobs.h               |  1 +
 fdbserver/ClusterController.actor.cpp | 19 ++++++++++++++++---
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 38b1bbb18c..4011c3c80a 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -509,6 +509,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( POLICY_GENERATIONS,                                    100 ); if( randomize && BUGGIFY ) POLICY_GENERATIONS = 10;
 	init( DBINFO_SEND_AMOUNT,                                      5 );
 	init( DBINFO_BATCH_DELAY,                                    0.1 );
+	init( SINGLETON_RECRUIT_BME_DELAY,                          10.0 );
 
 	//Move Keys
 	init( SHARD_READY_DELAY,                                    0.25 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 68f4456e1f..33e3a023cd 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -446,6 +446,7 @@ public:
 	double RECRUITMENT_TIMEOUT;
 	int DBINFO_SEND_AMOUNT;
 	double DBINFO_BATCH_DELAY;
+	double SINGLETON_RECRUIT_BME_DELAY;
 
 	// Move Keys
 	double SHARD_READY_DELAY;
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 1ab68c344a..a9412769e2 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -3195,6 +3195,7 @@ public:
 	// recruitX is used to signal when role X needs to be (re)recruited.
 	// recruitingXID is used to track the ID of X's interface which is being recruited.
 	// We use AsyncVars to kill (i.e. halt) singletons that have been replaced.
+	double lastRecruitTime = 0;
 	AsyncVar<bool> recruitDistributor;
 	Optional<UID> recruitingDistributorID;
 	AsyncVar<bool> recruitRatekeeper;
@@ -3299,7 +3300,10 @@ struct RatekeeperSingleton : Singleton<RatekeeperInterface> {
 			    brokenPromiseToNever(interface.get().haltRatekeeper.getReply(HaltRatekeeperRequest(cc->id)));
 		}
 	}
-	void recruit(ClusterControllerData* cc) const { cc->recruitRatekeeper.set(true); }
+	void recruit(ClusterControllerData* cc) const {
+		cc->lastRecruitTime = now();
+		cc->recruitRatekeeper.set(true);
+	}
 };
 
 struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
@@ -3320,7 +3324,10 @@ struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
 			    brokenPromiseToNever(interface.get().haltDataDistributor.getReply(HaltDataDistributorRequest(cc->id)));
 		}
 	}
-	void recruit(ClusterControllerData* cc) const { cc->recruitDistributor.set(true); }
+	void recruit(ClusterControllerData* cc) const {
+		cc->lastRecruitTime = now();
+		cc->recruitDistributor.set(true);
+	}
 };
 
 struct BlobManagerSingleton : Singleton<BlobManagerInterface> {
@@ -3341,7 +3348,10 @@ struct BlobManagerSingleton : Singleton<BlobManagerInterface> {
 			    brokenPromiseToNever(interface.get().haltBlobManager.getReply(HaltBlobManagerRequest(cc->id)));
 		}
 	}
-	void recruit(ClusterControllerData* cc) const { cc->recruitBlobManager.set(true); }
+	void recruit(ClusterControllerData* cc) const {
+		cc->lastRecruitTime = now();
+		cc->recruitBlobManager.set(true);
+	}
 };
 
 ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster, ClusterControllerData::DBInfo* db) {
@@ -3798,6 +3808,9 @@ ACTOR Future<Void> doCheckOutstandingRequests(ClusterControllerData* self) {
 		while (!self->goodRecruitmentTime.isReady()) {
 			wait(self->goodRecruitmentTime);
 		}
+		while (now() - self->lastRecruitTime < SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY) {
+			wait(delay(SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY + 0.001 - (now() - self->lastRecruitTime)));
+		}
 
 		checkOutstandingRecruitmentRequests(self);
 		checkOutstandingStorageRequests(self);