wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add 66 counters for 22 DD or related transactions to count begin, commit, and abort for each (#13062) (#13097) 

Problem: some of these transactions have been observed to involve cascades of many shards trying to execute them simultaneously owing to somewhat unpredictable DD dynamics. This then results in DD being pegged on CPU with many failing transactions.

Solution: the counters here will make it clear if this is happening and if so, where, so that further mitigations/solutions can be devised. Some of these code paths have TraceEvents, but some don't and of the existing TraceEvents, many are sampled. This PR avoids all of that and counts every time through, making us less reliant on guesswork.

An alternative approach (not taken here for many reasons) would be to instrument transaction code to let callers pass a tag. Then have the transaction client and/or server side emit a few metrics (started, committed, aborted) parameterized by tag.

One may reasonably object that the boilerplate here is kind of fugly. I guess my view is that at this late date the time for more subtle approaches has come and gone. We need this code to tell us what it is doing and if this looks a little intrusive, so be it.

Testing:
20260422-234326-gglass-2f514436068d99e6 compressed=True data_size=35519571 duration=5298275 ended=100000 fail=1 fail_fast=10 max_runs=100000 pass=99999 priority=100 remaining=0 runtime=0:55:44 sanity=False started=100000 stopped=20260423-003910 submitted=20260422-234326 timeout=5400 username=gglass
This commit is contained in:
gxglass 2026-04-28 17:37:09 -07:00 committed by GitHub
parent 9aaa5222f8
commit 0bc25f0438
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 413 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
fdbserver/DDTeamCollection.actor.cpp
View File

@ -30,6 +30,7 @@
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/network.h"
#include "flow/SimpleCounter.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -74,6 +75,55 @@ int EligibilityCounter::getCount(int combinedType) const {
} // namespace data_distribution
static SimpleCounter<int64_t>* counterUpdateNextWigglingStorageIDStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateNextWigglingStorageID/started");
return c;
}
static SimpleCounter<int64_t>* counterUpdateNextWigglingStorageIDCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateNextWigglingStorageID/committed");
return c;
}
static SimpleCounter<int64_t>* counterUpdateNextWigglingStorageIDAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateNextWigglingStorageID/aborted");
return c;
}
static SimpleCounter<int64_t>* counterPerpetualStorageWigglerStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/perpetualStorageWiggler/started");
return c;
}
static SimpleCounter<int64_t>* counterPerpetualStorageWigglerCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/perpetualStorageWiggler/committed");
return c;
}
static SimpleCounter<int64_t>* counterPerpetualStorageWigglerAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/perpetualStorageWiggler/aborted");
return c;
}
static SimpleCounter<int64_t>* counterWaitHealthyZoneChangeStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitHealthyZoneChange/started");
return c;
}
static SimpleCounter<int64_t>* counterWaitHealthyZoneChangeCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitHealthyZoneChange/committed");
return c;
}
static SimpleCounter<int64_t>* counterWaitHealthyZoneChangeAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitHealthyZoneChange/aborted");
return c;
}
static SimpleCounter<int64_t>* counterUpdateStorageMetadataStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateStorageMetadata/started");
return c;
}
static SimpleCounter<int64_t>* counterUpdateStorageMetadataCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateStorageMetadata/committed");
return c;
}
static SimpleCounter<int64_t>* counterUpdateStorageMetadataAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateStorageMetadata/aborted");
return c;
}
class DDTeamCollectionImpl {
ACTOR static Future<Void> checkAndRemoveInvalidLocalityAddr(DDTeamCollection* self) {
state double start = now();
@ -2276,6 +2326,9 @@ public:
}
ACTOR static Future<Void> updateNextWigglingStorageID(DDTeamCollection* self) {
state SimpleCounter<int64_t>* txnStarted = counterUpdateNextWigglingStorageIDStarted();
state SimpleCounter<int64_t>* txnCommitted = counterUpdateNextWigglingStorageIDCommitted();
state SimpleCounter<int64_t>* txnAborted = counterUpdateNextWigglingStorageIDAborted();
state StorageWiggleData wiggleState;
state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
wiggleState.wigglingStorageServer(PrimaryRegion(self->primary));
@ -2284,13 +2337,16 @@ public:
state StorageWiggleValue value(nextId);
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->dbContext()));
loop {
txnStarted->increment(1);
// write the next server id
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
metadataMap.set(tr, nextId, value);
wait(tr->commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr->onError(e));
}
}
@ -2560,6 +2616,9 @@ public:
// command `configure perpetual_storage_wiggle=$value` if the value is 1, this actor start 2 actors,
// `perpetualStorageWiggleIterator` and `perpetualStorageWiggler`. Otherwise, it sends stop signal to them.
ACTOR static Future<Void> monitorPerpetualStorageWiggle(DDTeamCollection* self) {
state SimpleCounter<int64_t>* txnPSWStarted = counterPerpetualStorageWigglerStarted();
state SimpleCounter<int64_t>* txnPSWCommitted = counterPerpetualStorageWigglerCommitted();
state SimpleCounter<int64_t>* txnPSWAborted = counterPerpetualStorageWigglerAborted();
state int speed = 0;
state PromiseStream<Void> finishStorageWiggleSignal;
state SignalableActorCollection collection;
@ -2569,6 +2628,7 @@ public:
loop {
state ReadYourWritesTransaction tr(self->dbContext());
loop {
txnPSWStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<Standalone<StringRef>> value = wait(tr.get(perpetualStorageWiggleKey));
@ -2578,6 +2638,7 @@ public:
}
state Future<Void> watchFuture = tr.watch(perpetualStorageWiggleKey);
wait(tr.commit());
txnPSWCommitted->increment(1);
ASSERT(speed == 1 || speed == 0);
if (speed == 1 && self->storageWiggler->isStopped()) { // avoid duplicated start
@ -2600,6 +2661,7 @@ public:
wait(watchFuture);
break;
} catch (Error& e) {
txnPSWAborted->increment(1);
wait(tr.onError(e));
}
}
@ -2607,8 +2669,12 @@ public:
}
ACTOR static Future<Void> waitHealthyZoneChange(DDTeamCollection* self) {
state SimpleCounter<int64_t>* txnStarted = counterWaitHealthyZoneChangeStarted();
state SimpleCounter<int64_t>* txnCommitted = counterWaitHealthyZoneChangeCommitted();
state SimpleCounter<int64_t>* txnAborted = counterWaitHealthyZoneChangeAborted();
state ReadYourWritesTransaction tr(self->dbContext());
loop {
txnStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@ -2649,9 +2715,11 @@ public:
state Future<Void> watchFuture = tr.watch(healthyZoneKey);
wait(tr.commit());
txnCommitted->increment(1);
wait(watchFuture || healthyZoneTimeout);
tr.reset();
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}
@ -3374,6 +3442,9 @@ public:
}
ACTOR static Future<Void> updateStorageMetadata(DDTeamCollection* self, TCServerInfo* server) {
state SimpleCounter<int64_t>* txnStarted = counterUpdateStorageMetadataStarted();
state SimpleCounter<int64_t>* txnCommitted = counterUpdateStorageMetadataCommitted();
state SimpleCounter<int64_t>* txnAborted = counterUpdateStorageMetadataAborted();
state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(
serverMetadataKeys.begin, IncludeVersion());
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dbContext());
@ -3397,6 +3468,7 @@ public:
// read storage metadata
loop {
txnStarted->increment(1);
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<Value> serverInterfaceValue = wait(tr->get(serverListKeyFor(server->getId())));
@ -3417,8 +3489,10 @@ public:
metadataMap.set(tr, server->getId(), data);
tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
wait(tr->commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr->onError(e));
}
}
@ -7212,4 +7286,4 @@ TEST_CASE("/DataDistribution/GetTeam/PreferWithinShardRange") {
}
wait(DDTeamCollectionUnitTest::GetTeam_PreferShardsWithinLimit());
return Void();
}
}

58
fdbserver/DDTxnProcessor.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbclient/DatabaseContext.h"
#include "flow/SimpleCounter.h"
#include "flow/genericactors.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -42,6 +43,43 @@ static void updateServersAndCompleteSources(std::set<UID>& servers,
}
}
static SimpleCounter<int64_t>* counterUpdateReplicaKeysStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateReplicaKeys/started");
return c;
}
static SimpleCounter<int64_t>* counterUpdateReplicaKeysCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateReplicaKeys/committed");
return c;
}
static SimpleCounter<int64_t>* counterUpdateReplicaKeysAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/updateReplicaKeys/aborted");
return c;
}
static SimpleCounter<int64_t>* counterTryUpdateReplicasKeyForDcStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/tryUpdateReplicasKeyForDc/started");
return c;
}
static SimpleCounter<int64_t>* counterTryUpdateReplicasKeyForDcCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/tryUpdateReplicasKeyForDc/committed");
return c;
}
static SimpleCounter<int64_t>* counterTryUpdateReplicasKeyForDcAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/tryUpdateReplicasKeyForDc/aborted");
return c;
}
static SimpleCounter<int64_t>* counterWaitDDTeamInfoPrintSignalStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitDDTeamInfoPrintSignal/started");
return c;
}
static SimpleCounter<int64_t>* counterWaitDDTeamInfoPrintSignalCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitDDTeamInfoPrintSignal/committed");
return c;
}
static SimpleCounter<int64_t>* counterWaitDDTeamInfoPrintSignalAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitDDTeamInfoPrintSignal/aborted");
return c;
}
class DDTxnProcessorImpl {
friend class DDTxnProcessor;
@ -178,8 +216,12 @@ class DDTxnProcessorImpl {
std::vector<Optional<Key>> primaryDcId,
std::vector<Optional<Key>> remoteDcIds,
DatabaseConfiguration configuration) {
state SimpleCounter<int64_t>* txnStarted = counterUpdateReplicaKeysStarted();
state SimpleCounter<int64_t>* txnCommitted = counterUpdateReplicaKeysCommitted();
state SimpleCounter<int64_t>* txnAborted = counterUpdateReplicaKeysAborted();
state Transaction tr(cx);
loop {
txnStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -200,8 +242,10 @@ class DDTxnProcessorImpl {
}
wait(tr.commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}
@ -209,8 +253,12 @@ class DDTxnProcessorImpl {
}
ACTOR static Future<int> tryUpdateReplicasKeyForDc(Database cx, Optional<Key> dcId, int storageTeamSize) {
state SimpleCounter<int64_t>* txnStarted = counterTryUpdateReplicasKeyForDcStarted();
state SimpleCounter<int64_t>* txnCommitted = counterTryUpdateReplicasKeyForDcCommitted();
state SimpleCounter<int64_t>* txnAborted = counterTryUpdateReplicasKeyForDcAborted();
state Transaction tr(cx);
loop {
txnStarted->increment(1);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -225,9 +273,11 @@ class DDTxnProcessorImpl {
}
tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(storageTeamSize));
wait(tr.commit());
txnCommitted->increment(1);
return oldReplicas;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}
@ -654,15 +704,21 @@ class DDTxnProcessorImpl {
}
ACTOR static Future<Void> waitDDTeamInfoPrintSignal(Database cx) {
state SimpleCounter<int64_t>* txnStarted = counterWaitDDTeamInfoPrintSignalStarted();
state SimpleCounter<int64_t>* txnCommitted = counterWaitDDTeamInfoPrintSignalCommitted();
state SimpleCounter<int64_t>* txnAborted = counterWaitDDTeamInfoPrintSignalAborted();
state ReadYourWritesTransaction tr(cx);
loop {
txnStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Future<Void> watchFuture = tr.watch(triggerDDTeamInfoPrintKey);
wait(tr.commit());
txnCommitted->increment(1);
wait(watchFuture);
return Void();
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}
@ -1194,4 +1250,4 @@ Future<Void> DDMockTxnProcessor::waitForAllDataRemoved(
shardsAffectedByTeamFailure->getNumberOfShards(serverID) == 0;
},
TaskPriority::DataDistribution);
}
}

76
fdbserver/DataDistribution.actor.cpp
View File

@ -48,8 +48,10 @@
#include "flow/Arena.h"
#include "flow/Error.h"
#include "flow/Platform.h"
#include "flow/SimpleCounter.h"
#include "flow/Trace.h"
#include "flow/UnitTest.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "flow/serialize.h"
@ -401,6 +403,19 @@ struct DDBulkDumpJobManager {
bool isValid() const { return jobState.isValid(); }
};
static SimpleCounter<int64_t>* counterRemoveDataMoveTombstoneStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/removeDataMoveTombstone/started");
return c;
}
static SimpleCounter<int64_t>* counterRemoveDataMoveTombstoneCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/removeDataMoveTombstone/committed");
return c;
}
static SimpleCounter<int64_t>* counterRemoveDataMoveTombstoneAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/removeDataMoveTombstone/aborted");
return c;
}
struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
public:
Reference<AsyncVar<ServerDBInfo> const> dbInfo;
@ -738,11 +753,15 @@ public:
}
ACTOR static Future<Void> removeDataMoveTombstoneBackground(Reference<DataDistributor> self) {
state SimpleCounter<int64_t>* txnStarted = counterRemoveDataMoveTombstoneStarted();
state SimpleCounter<int64_t>* txnCommitted = counterRemoveDataMoveTombstoneCommitted();
state SimpleCounter<int64_t>* txnAborted = counterRemoveDataMoveTombstoneAborted();
state UID currentID;
try {
state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
state Transaction tr(cx);
loop {
txnStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -752,8 +771,10 @@ public:
TraceEvent(SevDebug, "RemoveDataMoveTombstone", self->ddId).detail("DataMoveID", currentID);
}
wait(tr.commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}
@ -3124,11 +3145,43 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
}
}
static SimpleCounter<int64_t>* counterDdSnapSetRecoveryStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapSetRecovery/started");
return c;
}
static SimpleCounter<int64_t>* counterDdSnapSetRecoveryCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapSetRecovery/committed");
return c;
}
static SimpleCounter<int64_t>* counterDdSnapSetRecoveryAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapSetRecovery/aborted");
return c;
}
static SimpleCounter<int64_t>* counterDdSnapClearRecoveryStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapClearRecovery/started");
return c;
}
static SimpleCounter<int64_t>* counterDdSnapClearRecoveryCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapClearRecovery/committed");
return c;
}
static SimpleCounter<int64_t>* counterDdSnapClearRecoveryAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/ddSnapClearRecovery/aborted");
return c;
}
ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<ServerDBInfo> const> db) {
state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
state SimpleCounter<int64_t>* setRecoveryStarted = counterDdSnapSetRecoveryStarted();
state SimpleCounter<int64_t>* setRecoveryCommitted = counterDdSnapSetRecoveryCommitted();
state SimpleCounter<int64_t>* setRecoveryAborted = counterDdSnapSetRecoveryAborted();
state SimpleCounter<int64_t>* clearRecoveryStarted = counterDdSnapClearRecoveryStarted();
state SimpleCounter<int64_t>* clearRecoveryCommitted = counterDdSnapClearRecoveryCommitted();
state SimpleCounter<int64_t>* clearRecoveryAborted = counterDdSnapClearRecoveryAborted();
state ReadYourWritesTransaction tr(cx);
loop {
setRecoveryStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@ -3137,8 +3190,10 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
.detail("SnapUID", snapReq.snapUID);
tr.set(writeRecoveryKey, writeRecoveryKeyTrue);
wait(tr.commit());
setRecoveryCommitted->increment(1);
break;
} catch (Error& e) {
setRecoveryAborted->increment(1);
TraceEvent("SnapDataDistributor_WriteFlagError").error(e);
wait(tr.onError(e));
}
@ -3230,6 +3285,7 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
.detail("SnapUID", snapReq.snapUID);
tr.reset();
loop {
clearRecoveryStarted->increment(1);
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@ -3238,8 +3294,10 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
.detail("SnapUID", snapReq.snapUID);
tr.clear(writeRecoveryKey);
wait(tr.commit());
clearRecoveryCommitted->increment(1);
break;
} catch (Error& e) {
clearRecoveryAborted->increment(1);
TraceEvent("SnapDataDistributor_ClearFlagError").error(e);
wait(tr.onError(e));
}
@ -3373,18 +3431,36 @@ ACTOR Future<Void> ddExclusionSafetyCheck(DistributorExclusionSafetyCheckRequest
return Void();
}
static SimpleCounter<int64_t>* counterWaitFailCacheServerStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitFailCacheServer/started");
return c;
}
static SimpleCounter<int64_t>* counterWaitFailCacheServerCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitFailCacheServer/committed");
return c;
}
static SimpleCounter<int64_t>* counterWaitFailCacheServerAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/dd/waitFailCacheServer/aborted");
return c;
}
ACTOR Future<Void> waitFailCacheServer(Database* db, StorageServerInterface ssi) {
state SimpleCounter<int64_t>* txnStarted = counterWaitFailCacheServerStarted();
state SimpleCounter<int64_t>* txnCommitted = counterWaitFailCacheServerCommitted();
state SimpleCounter<int64_t>* txnAborted = counterWaitFailCacheServerAborted();
state Transaction tr(*db);
state Key key = storageCacheServerKey(ssi.id());
wait(waitFailureClient(ssi.waitFailure));
loop {
txnStarted->increment(1);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
tr.addReadConflictRange(storageCacheServerKeys);
tr.clear(key);
wait(tr.commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
}
}

206
fdbserver/MoveKeys.actor.cpp
View File

@ -36,6 +36,7 @@
#include "fdbclient/ReadYourWrites.h"
#include "fdbserver/BlobMigratorInterface.h"
#include "fdbserver/TSSMappingUtil.actor.h"
#include "flow/SimpleCounter.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -283,9 +284,26 @@ ACTOR Future<MoveKeysLock> readMoveKeysLock(Database cx) {
}
}
static SimpleCounter<int64_t>* counterTakeMoveKeysLockStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/takeMoveKeysLock/started");
return c;
}
static SimpleCounter<int64_t>* counterTakeMoveKeysLockCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/takeMoveKeysLock/committed");
return c;
}
static SimpleCounter<int64_t>* counterTakeMoveKeysLockAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/takeMoveKeysLock/aborted");
return c;
}
ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
state SimpleCounter<int64_t>* txnStarted = counterTakeMoveKeysLockStarted();
state SimpleCounter<int64_t>* txnCommitted = counterTakeMoveKeysLockCommitted();
state SimpleCounter<int64_t>* txnAborted = counterTakeMoveKeysLockAborted();
state Transaction tr(cx);
loop {
txnStarted->increment(1);
try {
state MoveKeysLock lock;
state UID txnId;
@ -300,6 +318,7 @@ ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
lock.myOwner = deterministicRandom()->randomUniqueID();
tr.set(moveKeysLockOwnerKey, BinaryWriter::toValue(lock.myOwner, Unversioned()));
wait(tr.commit());
txnCommitted->increment(1);
TraceEvent("TakeMoveKeysLockTransaction", ddId)
.detail("TransactionUID", txnId)
.detail("PrevOwner", lock.prevOwner.toString())
@ -307,6 +326,7 @@ ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
.detail("MyOwner", lock.myOwner.toString());
return lock;
} catch (Error& e) {
txnAborted->increment(1);
wait(tr.onError(e));
CODE_PROBE(true, "takeMoveKeysLock retry");
}
@ -667,6 +687,18 @@ ACTOR Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range,
}
// Cleans up dest servers of a single shard, and unassigns the keyrange from the dest servers if necessary.
static SimpleCounter<int64_t>* counterCleanUpSingleShardDataMoveStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpSingleShardDataMove/started");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpSingleShardDataMoveCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpSingleShardDataMove/committed");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpSingleShardDataMoveAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpSingleShardDataMove/aborted");
return c;
}
ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
KeyRange keys,
MoveKeysLock lock,
@ -675,10 +707,14 @@ ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
const DDEnabledState* ddEnabledState) {
ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
TraceEvent(SevInfo, "CleanUpSingleShardDataMoveBegin", dataMoveId).detail("Range", keys);
state SimpleCounter<int64_t>* txnStarted = counterCleanUpSingleShardDataMoveStarted();
state SimpleCounter<int64_t>* txnCommitted = counterCleanUpSingleShardDataMoveCommitted();
state SimpleCounter<int64_t>* txnAborted = counterCleanUpSingleShardDataMoveAborted();
state bool runPreCheck = true;
loop {
txnStarted->increment(1);
state Transaction tr(occ);
try {
@ -740,6 +776,7 @@ ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
wait(waitForAll(actors));
wait(tr.commit());
txnCommitted->increment(1);
// Post validate consistency of update of keyServers and serverKeys
if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) {
@ -747,6 +784,7 @@ ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
}
break;
} catch (Error& e) {
txnAborted->increment(1);
state Error err = e;
if (err.code() == error_code_location_metadata_corruption) {
throw location_metadata_corruption();
@ -961,6 +999,18 @@ ACTOR Future<Void> logWarningAfter(const char* context, double duration, std::ve
// subrange of keys that the server did not already have, = complete for each subrange that it already has. Set
// serverKeys[dest][keys] = "" for the dest servers of each existing shard in keys (unless that destination is a member
// of servers OR if the source list is sufficiently degraded)
static SimpleCounter<int64_t>* counterStartMoveKeysStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveKeys/started");
return c;
}
static SimpleCounter<int64_t>* counterStartMoveKeysCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveKeys/committed");
return c;
}
static SimpleCounter<int64_t>* counterStartMoveKeysAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveKeys/aborted");
return c;
}
ACTOR static Future<Void> startMoveKeys(Database occ,
KeyRange keys,
std::vector<UID> servers,
@ -972,6 +1022,9 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
state TraceInterval interval("RelocateShard_StartMoveKeys");
state Future<Void> warningLogger = logWarningAfter("StartMoveKeysTooLong", 600, servers);
// state TraceInterval waitInterval("");
state SimpleCounter<int64_t>* txnStarted = counterStartMoveKeysStarted();
state SimpleCounter<int64_t>* txnCommitted = counterStartMoveKeysCommitted();
state SimpleCounter<int64_t>* txnAborted = counterStartMoveKeysAborted();
wait(startMoveKeysLock->take(TaskPriority::DataDistributionLaunch));
state FlowLock::Releaser releaser(*startMoveKeysLock);
@ -997,6 +1050,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
state int retries = 0;
loop {
txnStarted->increment(1);
try {
retries++;
@ -1126,6 +1180,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
wait(waitForAll(actors));
wait(tr->commit());
txnCommitted->increment(1);
/*TraceEvent("StartMoveKeysCommitDone", relocationIntervalId)
.detail("CommitVersion", tr.getCommittedVersion())
@ -1134,6 +1189,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
shards += old.size() - 1;
break;
} catch (Error& e) {
txnAborted->increment(1);
state Error err = e;
if (err.code() == error_code_move_to_removed_server)
throw;
@ -1266,6 +1322,18 @@ ACTOR Future<Void> checkFetchingState(Database cx,
// keyServers[k].dest must be the same for all k in keys
// Set serverKeys[dest][keys] = true; serverKeys[src][keys] = false for all src not in dest
// Should be cancelled and restarted if keyServers[keys].dest changes (?so this is no longer true?)
static SimpleCounter<int64_t>* counterFinishMoveKeysStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveKeys/started");
return c;
}
static SimpleCounter<int64_t>* counterFinishMoveKeysCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveKeys/committed");
return c;
}
static SimpleCounter<int64_t>* counterFinishMoveKeysAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveKeys/aborted");
return c;
}
ACTOR static Future<Void> finishMoveKeys(Database occ,
KeyRange keys,
std::vector<UID> destinationTeam,
@ -1278,6 +1346,9 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
state TraceInterval interval("RelocateShard_FinishMoveKeys");
state TraceInterval waitInterval("");
state Future<Void> warningLogger = logWarningAfter("FinishMoveKeysTooLong", 600, destinationTeam);
state SimpleCounter<int64_t>* txnStarted = counterFinishMoveKeysStarted();
state SimpleCounter<int64_t>* txnCommitted = counterFinishMoveKeysCommitted();
state SimpleCounter<int64_t>* txnAborted = counterFinishMoveKeysAborted();
state Key begin = keys.begin;
state Key endKey;
state int retries = 0;
@ -1303,6 +1374,7 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
// printf("finishMoveKeys( '%s'-'%s' )\n", begin.toString().c_str(), keys.end.toString().c_str());
loop {
txnStarted->increment(1);
try {
tr.trState->taskID = TaskPriority::MoveKeys;
@ -1577,12 +1649,16 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
wait(waitForAll(actors));
wait(tr.commit());
txnCommitted->increment(1);
begin = endKey;
break;
}
// This leads to a count of transactions starting that exceeds the sum of
// committed or aborted, but this is intentional here.
tr.reset();
} catch (Error& error) {
txnAborted->increment(1);
if (error.code() == error_code_actor_cancelled)
throw;
state Error err = error;
@ -1616,6 +1692,18 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
// Set keyServers[keys].dest = servers Set serverKeys[servers][keys] = dataMoveId for each
// subrange of keys.
// Set dataMoves[dataMoveId] = DataMoveMetaData.
static SimpleCounter<int64_t>* counterStartMoveShardsStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveShards/started");
return c;
}
static SimpleCounter<int64_t>* counterStartMoveShardsCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveShards/committed");
return c;
}
static SimpleCounter<int64_t>* counterStartMoveShardsAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/startMoveShards/aborted");
return c;
}
ACTOR static Future<Void> startMoveShards(Database occ,
UID dataMoveId,
std::vector<KeyRange> ranges,
@ -1628,6 +1716,9 @@ ACTOR static Future<Void> startMoveShards(Database occ,
CancelConflictingDataMoves cancelConflictingDataMoves,
Optional<BulkLoadTaskState> bulkLoadTaskState) {
state Future<Void> warningLogger = logWarningAfter("StartMoveShardsTooLong", 600, servers);
state SimpleCounter<int64_t>* txnStarted = counterStartMoveShardsStarted();
state SimpleCounter<int64_t>* txnCommitted = counterStartMoveShardsCommitted();
state SimpleCounter<int64_t>* txnAborted = counterStartMoveShardsAborted();
wait(startMoveKeysLock->take(TaskPriority::DataDistributionLaunch));
state FlowLock::Releaser releaser(*startMoveKeysLock);
@ -1647,6 +1738,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
state bool runPreCheck = true;
try {
loop {
txnStarted->increment(1);
state Key begin = keys.begin;
state KeyRange currentKeys = keys;
@ -1680,6 +1772,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
dataMove.setPhase(DataMoveMetaData::Deleting);
tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
wait(tr.commit());
// Don't increment committed as we prefer to increment aborted below.
throw movekeys_conflict();
}
if (dataMove.getPhase() == DataMoveMetaData::Running) {
@ -1922,6 +2015,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
wait(waitForAll(actors));
wait(tr.commit());
txnCommitted->increment(1);
if (currentKeys.end == keys.end && bulkLoadTaskState.present()) {
Version commitVersion = tr.getCommittedVersion();
@ -1952,6 +2046,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
break;
}
} catch (Error& e) {
txnAborted->increment(1);
if (e.code() == error_code_location_metadata_corruption) {
throw location_metadata_corruption();
} else if (e.code() == error_code_retry) {
@ -2048,6 +2143,18 @@ ACTOR static Future<Void> checkDataMoveComplete(Database occ, UID dataMoveId, Ke
// keyServers[k].dest must be the same for all k in keys.
// Set serverKeys[dest][keys] = dataMoveId; serverKeys[src][keys] = false for all src not in dest.
// Clear dataMoves[dataMoveId].
static SimpleCounter<int64_t>* counterFinishMoveShardsStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveShards/started");
return c;
}
static SimpleCounter<int64_t>* counterFinishMoveShardsCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveShards/committed");
return c;
}
static SimpleCounter<int64_t>* counterFinishMoveShardsAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/finishMoveShards/aborted");
return c;
}
ACTOR static Future<Void> finishMoveShards(Database occ,
UID dataMoveId,
std::vector<KeyRange> targetRanges,
@ -2063,6 +2170,9 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
ASSERT(targetRanges.size() == 1);
state KeyRange keys = targetRanges[0];
state Future<Void> warningLogger = logWarningAfter("FinishMoveShardsTooLong", 600, destinationTeam);
state SimpleCounter<int64_t>* txnStarted = counterFinishMoveShardsStarted();
state SimpleCounter<int64_t>* txnCommitted = counterFinishMoveShardsCommitted();
state SimpleCounter<int64_t>* txnAborted = counterFinishMoveShardsAborted();
state int retries = 0;
state DataMoveMetaData dataMove;
state bool cancelDataMove = false;
@ -2084,6 +2194,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
// This process can be split up into multiple transactions if getRange() doesn't return the entire
// target range.
loop {
txnStarted->increment(1);
state std::vector<UID> completeSrc;
state std::vector<UID> destServers;
state std::unordered_set<UID> allServers;
@ -2110,6 +2221,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
dataMove.setPhase(DataMoveMetaData::Deleting);
tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
wait(tr.commit());
// Don't increment committed as we prefer to increment aborted below.
throw movekeys_conflict();
}
destServers.insert(destServers.end(), dataMove.dest.begin(), dataMove.dest.end());
@ -2370,6 +2482,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
}
wait(tr.commit());
txnCommitted->increment(1);
if (range.end == dataMove.ranges.front().end && bulkLoadTaskState.present()) {
Version commitVersion = tr.getCommittedVersion();
@ -2394,6 +2507,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
tr.reset();
}
} catch (Error& error) {
txnAborted->increment(1);
TraceEvent(SevWarn, "TryFinishMoveShardsError", relocationIntervalId)
.errorUnsuppressed(error)
.detail("DataMoveID", dataMoveId);
@ -2434,7 +2548,22 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
}; // anonymous namespace
static SimpleCounter<int64_t>* counterAddStorageServerStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/addStorageServer/started");
return c;
}
static SimpleCounter<int64_t>* counterAddStorageServerCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/addStorageServer/committed");
return c;
}
static SimpleCounter<int64_t>* counterAddStorageServerAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/addStorageServer/aborted");
return c;
}
ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server) {
state SimpleCounter<int64_t>* txnStarted = counterAddStorageServerStarted();
state SimpleCounter<int64_t>* txnCommitted = counterAddStorageServerCommitted();
state SimpleCounter<int64_t>* txnAborted = counterAddStorageServerAborted();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
@ -2443,6 +2572,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
state int maxSkipTags = 1;
loop {
txnStarted->increment(1);
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -2601,12 +2731,14 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
tr->set(serverListKeyFor(server.id()), serverListValue(server));
wait(tr->commit());
txnCommitted->increment(1);
TraceEvent("AddedStorageServerSystemKey")
.detail("ServerID", server.id())
.detail("CommitVersion", tr->getCommittedVersion());
return std::make_pair(tr->getCommittedVersion(), tag);
} catch (Error& e) {
txnAborted->increment(1);
if (e.code() == error_code_commit_unknown_result)
throw recruitment_failed(); // There is a remote possibility that we successfully added ourselves and
// then someone removed us, so we have to fail
@ -2649,11 +2781,26 @@ ACTOR Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> t
return !assigned && keys[1].key == allKeys.end;
}
static SimpleCounter<int64_t>* counterRemoveStorageServerStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeStorageServer/started");
return c;
}
static SimpleCounter<int64_t>* counterRemoveStorageServerCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeStorageServer/committed");
return c;
}
static SimpleCounter<int64_t>* counterRemoveStorageServerAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeStorageServer/aborted");
return c;
}
ACTOR Future<Void> removeStorageServer(Database cx,
UID serverID,
Optional<UID> tssPairID,
MoveKeysLock lock,
const DDEnabledState* ddEnabledState) {
state SimpleCounter<int64_t>* txnStarted = counterRemoveStorageServerStarted();
state SimpleCounter<int64_t>* txnCommitted = counterRemoveStorageServerCommitted();
state SimpleCounter<int64_t>* txnAborted = counterRemoveStorageServerAborted();
state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
state KeyBackedObjectMap<UID, StorageMigrationType, decltype(IncludeVersion())> metadataMap(
serverMetadataKeys.begin, IncludeVersion());
@ -2662,6 +2809,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
state int noCanRemoveCount = 0;
loop {
txnStarted->increment(1);
try {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -2759,6 +2907,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
retry = true;
wait(tr->commit());
txnCommitted->increment(1);
TraceEvent("RemoveStorageServer")
.detail("State", "Success")
.detail("ServerID", serverID)
@ -2766,6 +2915,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
return Void();
}
} catch (Error& e) {
txnAborted->increment(1);
state Error err = e;
wait(tr->onError(e));
TraceEvent("RemoveStorageServer").error(err).detail("State", "Retry").detail("ServerID", serverID);
@ -2776,11 +2926,26 @@ ACTOR Future<Void> removeStorageServer(Database cx,
// Changes to keyServer and serverKey must happen symmetrically in a transaction.
// If serverID is the last source server for a shard, the shard will be erased, and then be assigned
// to teamForDroppedRange.
static SimpleCounter<int64_t>* counterRemoveKeysFromFailedServerStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeKeysFromFailedServer/started");
return c;
}
static SimpleCounter<int64_t>* counterRemoveKeysFromFailedServerCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeKeysFromFailedServer/committed");
return c;
}
static SimpleCounter<int64_t>* counterRemoveKeysFromFailedServerAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/removeKeysFromFailedServer/aborted");
return c;
}
ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
UID serverID,
std::vector<UID> teamForDroppedRange,
MoveKeysLock lock,
const DDEnabledState* ddEnabledState) {
state SimpleCounter<int64_t>* txnStarted = counterRemoveKeysFromFailedServerStarted();
state SimpleCounter<int64_t>* txnCommitted = counterRemoveKeysFromFailedServerCommitted();
state SimpleCounter<int64_t>* txnAborted = counterRemoveKeysFromFailedServerAborted();
state Key begin = allKeys.begin;
state std::vector<UID> src;
@ -2791,6 +2956,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
while (begin < allKeys.end) {
state Transaction tr(cx);
loop {
txnStarted->increment(1);
try {
tr.trState->taskID = TaskPriority::MoveKeys;
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -2938,6 +3104,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
.detail("End", currentKeys.end);
wait(krmSetRangeCoalescing(&tr, serverKeysPrefixFor(serverID), currentKeys, allKeys, serverKeysFalse));
wait(tr.commit());
txnCommitted->increment(1);
TraceEvent(SevDebug, "FailedServerCommitSuccess", serverID)
.detail("Begin", currentKeys.begin)
.detail("End", currentKeys.end)
@ -2946,6 +3113,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
begin = currentKeys.end;
break;
} catch (Error& e) {
txnAborted->increment(1);
TraceEvent("FailedServerError", serverID).error(e);
wait(tr.onError(e));
}
@ -2973,6 +3141,18 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
// the place holder on the metadata put by cleanUpDataMoveCore. Then, startMoveShard gives up and exits.
// No update to the metadata by the startMoveShard
// For all three cases, the background cleanup only needs to cleanup the place holder
static SimpleCounter<int64_t>* counterCleanUpDataMoveBackgroundStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveBackground/started");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpDataMoveBackgroundCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveBackground/committed");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpDataMoveBackgroundAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveBackground/aborted");
return c;
}
ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
UID dataMoveId,
MoveKeysLock lock,
@ -2980,6 +3160,9 @@ ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
KeyRange keys,
const DDEnabledState* ddEnabledState,
double delaySeconds) {
state SimpleCounter<int64_t>* txnStarted = counterCleanUpDataMoveBackgroundStarted();
state SimpleCounter<int64_t>* txnCommitted = counterCleanUpDataMoveBackgroundCommitted();
state SimpleCounter<int64_t>* txnAborted = counterCleanUpDataMoveBackgroundAborted();
wait(delay(std::max(10.0, delaySeconds)));
TraceEvent(SevDebug, "CleanUpDataMoveBackgroundBegin", dataMoveId)
.detail("DataMoveID", dataMoveId)
@ -2989,6 +3172,7 @@ ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
state DataMoveMetaData dataMove;
state Transaction tr(occ);
loop {
txnStarted->increment(1);
try {
tr.trState->taskID = TaskPriority::MoveKeys;
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -3004,8 +3188,10 @@ ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
ASSERT(dataMove.getPhase() == DataMoveMetaData::Deleting);
tr.clear(dataMoveKeyFor(dataMoveId));
wait(tr.commit());
txnCommitted->increment(1);
break;
} catch (Error& e) {
txnAborted->increment(1);
TraceEvent(SevWarn, "CleanUpDataMoveBackgroundFail", dataMoveId).errorUnsuppressed(e);
wait(tr.onError(e));
}
@ -3018,12 +3204,27 @@ ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
return Void();
}
static SimpleCounter<int64_t>* counterCleanUpDataMoveCoreStarted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveCore/started");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpDataMoveCoreCommitted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveCore/committed");
return c;
}
static SimpleCounter<int64_t>* counterCleanUpDataMoveCoreAborted() {
static auto* c = SimpleCounter<int64_t>::makeCounter("/movekeys/cleanUpDataMoveCore/aborted");
return c;
}
ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
UID dataMoveId,
MoveKeysLock lock,
FlowLock* cleanUpDataMoveParallelismLock,
KeyRange keys,
const DDEnabledState* ddEnabledState) {
state SimpleCounter<int64_t>* txnStarted = counterCleanUpDataMoveCoreStarted();
state SimpleCounter<int64_t>* txnCommitted = counterCleanUpDataMoveCoreCommitted();
state SimpleCounter<int64_t>* txnAborted = counterCleanUpDataMoveCoreAborted();
state KeyRange range;
state Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
TraceEvent(SevInfo, "CleanUpDataMoveBegin", dataMoveId).detail("DataMoveID", dataMoveId).detail("Range", keys);
@ -3035,6 +3236,7 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
try {
loop {
txnStarted->increment(1);
state Transaction tr(occ);
state std::unordered_map<UID, std::vector<Shard>> physicalShardMap;
state std::set<UID> oldDests;
@ -3172,6 +3374,7 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
wait(waitForAll(actors));
wait(tr.commit());
txnCommitted->increment(1);
TraceEvent(sevDm, "CleanUpDataMoveCommitted", dataMoveId)
.detail("DataMoveID", dataMoveId)
@ -3186,6 +3389,7 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
break;
}
} catch (Error& e) {
txnAborted->increment(1);
if (e.code() == error_code_location_metadata_corruption) {
throw location_metadata_corruption();
} else {
@ -3504,4 +3708,4 @@ ACTOR Future<Void> prepareBlobRestore(Database occ,
}
}
}
}
}