From 99923077f6fb671cbfe6f5ab0e551ec530b017ab Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Tue, 17 Jun 2025 11:49:12 -0700 Subject: [PATCH] [Release-7.4] Cherrypick Mitigate rocksdb external timeout in AtomicBackupToDBCorrectness test (#12193) --- fdbclient/ServerKnobs.cpp | 4 +-- fdbserver/storageserver.actor.cpp | 36 +++++++++++++++++---- tests/fast/AtomicBackupToDBCorrectness.toml | 4 +-- tests/fast/PhysicalShardMove.toml | 1 + 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 043a5205ee..9c4ec5cf07 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -559,7 +559,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_SINGLEKEY_DELETES_MAX, 200 ); // Max rocksdb::delete calls in a transaction init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS, false ); init( ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE, false ); - init( ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT, 0 ); if( isSimulated ) ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT = deterministicRandom()->randomInt(1000, 10000); // Default: 0 (disabled). + init( ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT, 0 ); if( isSimulated ) ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT = deterministicRandom()->randomInt(200, 500); // Default: 0 (disabled). // ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers // Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594 init( ROCKSDB_STATS_LEVEL, 1 ); @@ -627,7 +627,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO, 0.01 ); if (isSimulated) SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO = deterministicRandom()->random01(); init( SHARD_METADATA_SCAN_BYTES_LIMIT, 10485760 ); // 10MB init( ROCKSDB_MAX_MANIFEST_FILE_SIZE, 100 << 20 ); if (isSimulated) ROCKSDB_MAX_MANIFEST_FILE_SIZE = 500 << 20; // 500MB in simulation - init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 500 ); if (isSimulated) SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS = 50; + init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 500 ); if (isSimulated) SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS = 5; init( SHARDED_ROCKSDB_AVERAGE_FILE_SIZE, 8 << 20 ); // 8MB init( SHARDED_ROCKSDB_COMPACTION_PERIOD, isSimulated? 3600 : 2592000 ); // 30d init( SHARDED_ROCKSDB_COMPACTION_ACTOR_DELAY, 3600 ); // 1h diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index a36c73524f..b000fcb61f 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -599,7 +599,9 @@ struct StorageServerDisk { Version newStorageVersion, int64_t& bytesLeft, UnlimitedCommitBytes unlimitedCommitBytes, - int64_t& clearRangesLeft); + int64_t& clearRangesLeft, + const UID& ssId, + bool verbose = false); void makeVersionDurable(Version version); void makeAccumulativeChecksumDurable(const AccumulativeChecksumState& acsState); void clearAccumulativeChecksumState(const AccumulativeChecksumState& acsState); @@ -12902,7 +12904,8 @@ ACTOR Future updateStorage(StorageServer* data) { state Version newOldestVersion = data->storageVersion(); state Version desiredVersion = data->desiredOldestVersion.get(); state int64_t bytesLeft = SERVER_KNOBS->STORAGE_COMMIT_BYTES; - state int64_t clearRangesLeft = data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1 + state int64_t clearRangesLeft = (data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1 || + data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_SHARDED_ROCKSDB) ? (SERVER_KNOBS->ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT > 0 ? SERVER_KNOBS->ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT : INT_MAX) @@ -12981,12 +12984,22 @@ ACTOR Future updateStorage(StorageServer* data) { } } + // When unlimitedCommitBytes is set to true, clearRangesLeft will be ignored. + // Make sure unlimitedCommitBytes is set to True only when storage engine is sharded rocksdb. + ASSERT(data->shardAware || unlimitedCommitBytes == UnlimitedCommitBytes::False); + // Write mutations to storage until we reach the desiredVersion or have written too much (bytesleft) // or until we reach clearRanges limit, in case of rocksdb. state double beforeStorageUpdates = now(); loop { - state bool done = data->storage.makeVersionMutationsDurable( - newOldestVersion, desiredVersion, bytesLeft, unlimitedCommitBytes, clearRangesLeft); + state bool done = data->storage.makeVersionMutationsDurable(newOldestVersion, + desiredVersion, + bytesLeft, + unlimitedCommitBytes, + clearRangesLeft, + data->thisServerID, + data->storage.getKeyValueStoreType() == + KeyValueStoreType::SSD_ROCKSDB_V1); if (data->tenantMap.getLatestVersion() < newOldestVersion) { data->tenantMap.createNewVersion(newOldestVersion); } @@ -13574,10 +13587,21 @@ bool StorageServerDisk::makeVersionMutationsDurable(Version& prevStorageVersion, Version newStorageVersion, int64_t& bytesLeft, UnlimitedCommitBytes unlimitedCommitBytes, - int64_t& clearRangesLeft) { - if ((!unlimitedCommitBytes && bytesLeft <= 0) || clearRangesLeft <= 0) + int64_t& clearRangesLeft, + const UID& ssId, + bool verbose) { + if (!unlimitedCommitBytes && (bytesLeft <= 0 || clearRangesLeft <= 0)) return true; + if (clearRangesLeft <= 0 && verbose) { + TraceEvent(SevInfo, "MakeVersionMutationsDurableClearRangesLeftZero", ssId) + .suppressFor(5.0) + .detail("PrevStorageVersion", prevStorageVersion) + .detail("NewStorageVersion", newStorageVersion) + .detail("BytesLeft", bytesLeft) + .detail("ClearRangesLeft", clearRangesLeft) + .detail("UnlimitedCommitBytes", unlimitedCommitBytes); + } // Apply mutations from the mutationLog auto u = data->getMutationLog().upper_bound(prevStorageVersion); if (u != data->getMutationLog().end() && u->first <= newStorageVersion) { diff --git a/tests/fast/AtomicBackupToDBCorrectness.toml b/tests/fast/AtomicBackupToDBCorrectness.toml index 0ac72362b3..2df9c3d361 100644 --- a/tests/fast/AtomicBackupToDBCorrectness.toml +++ b/tests/fast/AtomicBackupToDBCorrectness.toml @@ -10,8 +10,8 @@ simBackupAgents = 'BackupToDB' [[test.workload]] testName = 'AtomicOps' - nodeCount = 30000 - transactionsPerSecond = 2500.0 + nodeCount = 10000 + transactionsPerSecond = 750.0 testDuration = 30.0 [[test.workload]] diff --git a/tests/fast/PhysicalShardMove.toml b/tests/fast/PhysicalShardMove.toml index 389d4f7dfa..87787b43dd 100644 --- a/tests/fast/PhysicalShardMove.toml +++ b/tests/fast/PhysicalShardMove.toml @@ -20,6 +20,7 @@ shard_encode_location_metadata = true # min_byte_sampling_probability = 0.99 # rocksdb_read_range_reuse_iterators = false dd_physical_shard_move_probability = 1.0 +sharded_rocksdb_memtable_max_range_deletions = 50 [[test]] testTitle = 'PhysicalShardMove'