From d5223da940fda208e783b49c67912e6c83f56445 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Mon, 15 Jun 2026 18:57:31 -0700 Subject: [PATCH] Revert WaitStorageMetricsHandleError SevWarn upgrade (#13336) (#13338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #12913 upgraded WaitStorageMetricsHandleError from SevDebug to SevWarn after 60s of retrying, intending to give operators visibility into stuck shard metric loops. In production this fires ~10/sec/cluster, putting it in the top 35 most frequent TraceEvents. The 60s threshold doesn't filter for stuck shards. The SS-side waitMetrics handler is a long-poll with STORAGE_METRIC_TIMEOUT = 600s (fdbserver/storageserver/storageserver.actor.cpp:11476). On timeout, the SS deliberately returns wrong_shard_server with probability WAIT_METRICS_WRONG_SHARD_CHANCE = 0.1 to force clients to refresh their location cache (fdbserver/core/StorageMetrics.cpp:742). So most calls that ever reach this catch block already have Elapsed >= 600s by design, and the SevWarn fires on normal quiet-cluster operation, not stuck shards. DD-init stall visibility (the actual goal of PR #12913) is covered by the DDInitServerListAndDataMoveReadComplete / DDInitKeyServerScanComplete / DDInitSlowDataMoveRead events that PR also added — those are at the right layer. Revert this event to plain SevDebug. --- fdbclient/NativeAPI.actor.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index f8a320156f..48d54d8cca 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -8165,7 +8165,6 @@ ACTOR Future, int>> waitStorageMetrics( Optional> trState) { state Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingSample)); state double startTime = now(); - state double lastLogTime = 0; state int retryCount = 0; loop { if (trState.present()) { @@ -8215,16 +8214,7 @@ ACTOR Future, int>> waitStorageMetrics( } } catch (Error& e) { retryCount++; - // Upgrade from SevDebug to SevWarn after 60 seconds of retrying, - // but rate-limit warns to avoid flooding the trace log on fast retries. - Severity sev = SevDebug; - if (now() - startTime > 60.0) { - if (now() - lastLogTime >= 10.0) { - sev = SevWarn; - lastLogTime = now(); - } - } - TraceEvent(sev, "WaitStorageMetricsHandleError") + TraceEvent(SevDebug, "WaitStorageMetricsHandleError") .error(e) .detail("Keys", keys) .detail("Elapsed", now() - startTime)