From ac41ec01f6c2b74a6c1432bb75f874a9c3ab2dd2 Mon Sep 17 00:00:00 2001 From: michael stack Date: Thu, 23 Jul 2026 16:46:02 -0700 Subject: [PATCH] Backport: add retry limit to startMoveKeys to prevent infinite loops (PR 13176) Add START_MOVE_KEYS_MAX_RETRIES knob (default 50, BUGGIFY=10) and throw start_move_keys_too_many_retries when exceeded. Wire the new error into DDRelocationQueue (same light-weight re-queue handler as finish_move_keys_too_many_retries and move_to_removed_server) and normalDDQueueErrors so DD does not restart. Also propagate actor_cancelled before the retry limit check in startMoveKeys. --- fdbclient/ServerKnobs.cpp | 1 + fdbclient/include/fdbclient/ServerKnobs.h | 1 + fdbserver/DDRelocationQueue.actor.cpp | 3 ++- fdbserver/DataDistribution.actor.cpp | 3 ++- fdbserver/MoveKeys.actor.cpp | 14 +++++++++++++- flow/include/flow/error_definitions.h | 1 + 6 files changed, 20 insertions(+), 3 deletions(-) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 1374b3b8b8..6630a7d0ed 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -876,6 +876,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( REMOVE_RETRY_DELAY, 1.0 ); init( MOVE_KEYS_KRM_LIMIT, 2000 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2; init( FINISH_MOVE_KEYS_MAX_RETRIES, 50 ); // ~4 min total with exponential backoff (0.2s to 5s cap) + init( START_MOVE_KEYS_MAX_RETRIES, 50 ); if( randomize && BUGGIFY ) START_MOVE_KEYS_MAX_RETRIES = 10; init( MOVE_KEYS_KRM_LIMIT_BYTES, 1e5 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map init( MOVE_SHARD_KRM_ROW_LIMIT, 20000 ); init( MOVE_SHARD_KRM_BYTE_LIMIT, 1e6 ); diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 4de46fb5f4..f53addcb42 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -880,6 +880,7 @@ public: int MOVE_KEYS_KRM_LIMIT; int FINISH_MOVE_KEYS_MAX_RETRIES; // Max retries in finishMoveKeys before returning move to queue; set very high to // disable + int START_MOVE_KEYS_MAX_RETRIES; // Max retries in startMoveKeys before throwing start_move_keys_too_many_retries int MOVE_KEYS_KRM_LIMIT_BYTES; // This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT // (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an // attempt to read a key range map diff --git a/fdbserver/DDRelocationQueue.actor.cpp b/fdbserver/DDRelocationQueue.actor.cpp index f588f9b00a..a57b4637e3 100644 --- a/fdbserver/DDRelocationQueue.actor.cpp +++ b/fdbserver/DDRelocationQueue.actor.cpp @@ -2284,7 +2284,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, //TraceEvent("RelocateShardFinished", distributorId).detail("RelocateId", relocateShardInterval.pairID); if (error.code() != error_code_move_to_removed_server && - error.code() != error_code_finish_move_keys_too_many_retries) { + error.code() != error_code_finish_move_keys_too_many_retries && + error.code() != error_code_start_move_keys_too_many_retries) { if (!error.code()) { try { wait(healthyDestinations diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index c213da0730..dcbdd32aa4 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -102,7 +102,8 @@ std::set const& normalDDQueueErrors() { error_code_broken_promise, error_code_data_move_cancelled, error_code_data_move_dest_team_not_found, - error_code_finish_move_keys_too_many_retries }; + error_code_finish_move_keys_too_many_retries, + error_code_start_move_keys_too_many_retries }; return s; } diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp index 60cdfb4482..577c3f92b8 100644 --- a/fdbserver/MoveKeys.actor.cpp +++ b/fdbserver/MoveKeys.actor.cpp @@ -1178,8 +1178,20 @@ ACTOR static Future startMoveKeys(Database occ, } catch (Error& e) { txnAborted->increment(1); state Error err = e; + if (err.code() == error_code_actor_cancelled) + throw err; if (err.code() == error_code_move_to_removed_server) - throw; + throw err; + if (retries > SERVER_KNOBS->START_MOVE_KEYS_MAX_RETRIES) { + CODE_PROBE(true, "startMoveKeys giving up after max retries"); + TraceEvent(SevWarnAlways, "RelocateShard_StartMoveKeysGivingUp", relocationIntervalId) + .error(err) + .detail("KeyBegin", keys.begin) + .detail("KeyEnd", keys.end) + .detail("BeginKey", begin) + .detail("Retries", retries); + throw start_move_keys_too_many_retries(); + } wait(tr->onError(e)); if (retries % 10 == 0) { diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h index dcac308699..bc921a7d1b 100755 --- a/flow/include/flow/error_definitions.h +++ b/flow/include/flow/error_definitions.h @@ -166,6 +166,7 @@ ERROR( bulkload_manifest_decode_error, 1246, "Bulkload manifest string is failed ERROR( range_lock_reject, 1247, "Range lock is rejected" ) ERROR( range_unlock_reject, 1248, "Range unlock is rejected" ) ERROR( finish_move_keys_too_many_retries, 1249, "finishMoveKeys exceeded retry limit" ) +ERROR( start_move_keys_too_many_retries, 1250, "startMoveKeys exceeded retry limit" ) // 15xx Platform errors ERROR( platform_error, 1500, "Platform error" )