Merge ac41ec01f6 into 0bfa96f1de

With single storage replication, a rapid recovery-generation handoff during
initial database bring-up can orphan the sole seed storage server before it
makes the seed data durable: the superseding recovery pops the previous
generation's TLogs, the still-catching-up storage server hits PeekPoppedTLogData
and is removed (its KVS deleted), and because there is only one replica no
storage server remains and none is re-seeded. The database is then permanently
unavailable and the test's starting-configuration change times out
(TestFailure 'Unable to set starting configuration', e.g. seed 327963237).

This is a pre-existing single-replica recovery race, unrelated to the range-lock
feature the test exercises. Forcing at least double redundancy keeps the
database available across the handoff, matching how other startup-sensitive
tests (e.g. LowLatency) pin minimumReplication.

fdbclient/ServerKnobs.cpp

@@ -876,6 +876,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( REMOVE_RETRY_DELAY,                                    1.0 );
 	init( MOVE_KEYS_KRM_LIMIT,                                  2000 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2;
 	init( FINISH_MOVE_KEYS_MAX_RETRIES,                           50 ); // ~4 min total with exponential backoff (0.2s to 5s cap)
+	init( START_MOVE_KEYS_MAX_RETRIES,                            50 ); if( randomize && BUGGIFY ) START_MOVE_KEYS_MAX_RETRIES = 10;
 	init( MOVE_KEYS_KRM_LIMIT_BYTES,                             1e5 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
 	init( MOVE_SHARD_KRM_ROW_LIMIT,                            20000 );
  	init( MOVE_SHARD_KRM_BYTE_LIMIT,                             1e6 );

fdbclient/include/fdbclient/ServerKnobs.h

@@ -880,6 +880,7 @@ public:
 	int MOVE_KEYS_KRM_LIMIT;
 	int FINISH_MOVE_KEYS_MAX_RETRIES; // Max retries in finishMoveKeys before returning move to queue; set very high to
 	                                  // disable
+	int START_MOVE_KEYS_MAX_RETRIES; // Max retries in startMoveKeys before throwing start_move_keys_too_many_retries
 	int MOVE_KEYS_KRM_LIMIT_BYTES; // This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT
 	                               // (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an
 	                               // attempt to read a key range map

fdbserver/DDRelocationQueue.actor.cpp

@@ -2284,7 +2284,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			//TraceEvent("RelocateShardFinished", distributorId).detail("RelocateId", relocateShardInterval.pairID);
 
 			if (error.code() != error_code_move_to_removed_server &&
-			    error.code() != error_code_finish_move_keys_too_many_retries) {
+			    error.code() != error_code_finish_move_keys_too_many_retries &&
+			    error.code() != error_code_start_move_keys_too_many_retries) {
 				if (!error.code()) {
 					try {
 						wait(healthyDestinations

fdbserver/DataDistribution.actor.cpp

@@ -102,7 +102,8 @@ std::set<int> const& normalDDQueueErrors() {
 		                    error_code_broken_promise,
 		                    error_code_data_move_cancelled,
 		                    error_code_data_move_dest_team_not_found,
-		                    error_code_finish_move_keys_too_many_retries };
+		                    error_code_finish_move_keys_too_many_retries,
+		                    error_code_start_move_keys_too_many_retries };
 	return s;
 }
 

fdbserver/MoveKeys.actor.cpp

@@ -1178,8 +1178,20 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 				} catch (Error& e) {
 					txnAborted->increment(1);
 					state Error err = e;
+					if (err.code() == error_code_actor_cancelled)
+						throw err;
 					if (err.code() == error_code_move_to_removed_server)
-						throw;
+						throw err;
+					if (retries > SERVER_KNOBS->START_MOVE_KEYS_MAX_RETRIES) {
+						CODE_PROBE(true, "startMoveKeys giving up after max retries");
+						TraceEvent(SevWarnAlways, "RelocateShard_StartMoveKeysGivingUp", relocationIntervalId)
+						    .error(err)
+						    .detail("KeyBegin", keys.begin)
+						    .detail("KeyEnd", keys.end)
+						    .detail("BeginKey", begin)
+						    .detail("Retries", retries);
+						throw start_move_keys_too_many_retries();
+					}
 					wait(tr->onError(e));
 
 					if (retries % 10 == 0) {

flow/include/flow/error_definitions.h

@@ -166,6 +166,7 @@ ERROR( bulkload_manifest_decode_error, 1246, "Bulkload manifest string is failed
 ERROR( range_lock_reject, 1247, "Range lock is rejected" )
 ERROR( range_unlock_reject, 1248, "Range unlock is rejected" )
 ERROR( finish_move_keys_too_many_retries, 1249, "finishMoveKeys exceeded retry limit" )
+ERROR( start_move_keys_too_many_retries, 1250, "startMoveKeys exceeded retry limit" )
 
 // 15xx Platform errors
 ERROR( platform_error, 1500, "Platform error" )

tests/fast/RangeLockCycle.toml

@@ -1,6 +1,12 @@
 [configuration]
 tenantModes = ['disabled'] # do not support tenant
 encryptModes = ['disabled'] # do not support encryption
+# Require at least double redundancy. With single storage replication, a rapid
+# recovery-generation handoff during initial database bring-up can orphan the
+# sole (not-yet-durable) seed storage server via PeekPoppedTLogData, leaving the
+# database permanently without storage servers and timing out the starting
+# configuration. Extra replicas keep the database available across that handoff.
+minimumReplication = 2
 
 [[knobs]]
 enable_read_lock_on_range = true