wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove parallel restore feature (#12903) 

This parallel restore feature has been slated for removal for at least a year. This PR is closely based on earlier PR #12107.

This blog post explains some of the problems with the parallel restore feature: https://medium.com/@jingyuzhou/why-foundationdb-restore-is-slow-and-what-can-be-done-about-it-e73a821fdd33

As far as large feature removal changes go, this one is very straightforward, with most relevant files and test cases simply being deleted. There is one knob rename where storageserver.actor.cpp was using a knob with FASTRESTORE in the name. Other than that, changes to shared files mainly involve removing fastrestore-specific CLI and role support.

In progress:
20260330-222511-gglass-5ee0142213471b70 compressed=True data_size=35343375 duration=4611964 ended=100000 fail=1 fail_fast=1000 max_runs=100000 pass=99999 priority=100 remaining=0 runtime=0:58:23 sanity=False started=100000 stopped=20260330-232334 submitted=20260330-222511 timeout=5400 username=gglass

The one failure was in SwizzledCycleTest.toml with too many lines of output and a timeout. I kind of suspect it's unrelated but haven't looked further.
* Remove parallel restore feature.  This is based on earlier PR 12107.  Compiles but untested.

* AI generated commit:

⏺ The fix restores a single if block that was accidentally deleted when removing the FASTRESTORE_TOOL code:

  if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty()) {
      addDefaultBackupRanges(backupKeys);
  }

  When no explicit key ranges are specified on the command line and neither --user-data nor --system-metadata flags are set, this populates backupKeys with the default backup ranges
  (essentially all user data). Without it, backupKeys stays empty and hits the ASSERT(!backupRanges.empty()) in submitBackup().

* Remove a believed-to-be-dead code path, and update .gitignore

* Remove duplicate definition of restoreRequestDoneKey
This commit is contained in:
gxglass 2026-04-01 15:57:12 -07:00 committed by GitHub
parent daf3b81cfc
commit bca167fe96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
54 changed files with 125 additions and 9726 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -114,4 +114,5 @@ temp/
# AI assistants
*.aider*
*.fdq
*.fdq
.claude/

4
fdbbackup/CMakeLists.txt
View File

@ -37,11 +37,11 @@ if(NOT OPEN_FOR_IDE)
symlink_files(
LOCATION packages/bin
SOURCE fdbbackup
TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool)
TARGETS fdbdr dr_agent backup_agent fdbrestore)
symlink_files(
LOCATION bin
SOURCE fdbbackup
TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool)
TARGETS fdbdr dr_agent backup_agent fdbrestore)
# Test version of backup.cpp without main() function
add_flow_target(EXECUTABLE NAME backup_tests SRCS backup.cpp)

249
fdbbackup/backup.cpp
View File

@ -85,7 +85,7 @@
#include "SimpleOpt/SimpleOpt.h"
// Type of program being executed
enum class ProgramExe { AGENT, BACKUP, RESTORE, FASTRESTORE_TOOL, DR_AGENT, DB_BACKUP, UNDEFINED };
enum class ProgramExe { AGENT, BACKUP, RESTORE, DR_AGENT, DB_BACKUP, UNDEFINED };
enum class BackupType {
UNDEFINED = 0,
@ -696,7 +696,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
SO_END_OF_OPTIONS
};
// g_rgRestoreOptions is used by fdbrestore and fastrestore_tool
// g_rgRestoreOptions is used by fdbrestore
CSimpleOpt::SOption g_rgRestoreOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
@ -932,7 +932,6 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
const KeyRef exeAgent = "backup_agent"_sr;
const KeyRef exeBackup = "fdbbackup"_sr;
const KeyRef exeRestore = "fdbrestore"_sr;
const KeyRef exeFastRestoreTool = "fastrestore_tool"_sr; // must be lower case
const KeyRef exeDatabaseAgent = "dr_agent"_sr;
const KeyRef exeDatabaseBackup = "fdbdr"_sr;
@ -1257,14 +1256,6 @@ static void printRestoreUsage(bool devhelp) {
return;
}
static void printFastRestoreUsage(bool devhelp) {
printf(" NOTE: Fast restore aims to support the same fdbrestore option list.\n");
printf(" But fast restore is still under development. The options may not be fully supported.\n");
printf(" Supported options are: --dest-cluster-file, -r, --waitfordone, --logdir\n");
printRestoreUsage(devhelp);
return;
}
static void printDBAgentUsage(bool devhelp) {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str());
@ -1367,9 +1358,6 @@ static void printUsage(ProgramExe programExe, bool devhelp) {
case ProgramExe::RESTORE:
printRestoreUsage(devhelp);
break;
case ProgramExe::FASTRESTORE_TOOL:
printFastRestoreUsage(devhelp);
break;
case ProgramExe::DR_AGENT:
printDBAgentUsage(devhelp);
break;
@ -1429,14 +1417,6 @@ ProgramExe getProgramType(std::string programExe) {
enProgramExe = ProgramExe::RESTORE;
}
// Check if restore
else if ((programExe.length() >= exeFastRestoreTool.size()) &&
(programExe.compare(programExe.length() - exeFastRestoreTool.size(),
exeFastRestoreTool.size(),
(const char*)exeFastRestoreTool.begin()) == 0)) {
enProgramExe = ProgramExe::FASTRESTORE_TOOL;
}
// Check if db agent
else if ((programExe.length() >= exeDatabaseAgent.size()) &&
(programExe.compare(programExe.length() - exeDatabaseAgent.size(),
@ -2513,123 +2493,6 @@ Future<Void> runRestore(Database db,
}
}
// Fast restore agent that kicks off the restore: send restore requests to restore workers.
Future<Void> runFastRestoreTool(Database db,
std::string tagName,
std::string container,
Optional<std::string> proxy,
Standalone<VectorRef<KeyRangeRef>> ranges,
Version dbVersion,
bool performRestore,
Verbose verbose,
WaitForComplete waitForDone) {
try {
FileBackupAgent backupAgent;
Version restoreVersion = invalidVersion;
if (ranges.size() > 1) {
fprintf(stdout, "[WARNING] Currently only a single restore range is tested!\n");
}
if (ranges.size() == 0) {
ranges.push_back(ranges.arena(), normalKeys);
}
printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n",
ranges.size(),
ranges.front().toString().c_str());
TraceEvent ev("FastRestoreTool");
ev.detail("RestoreRanges", ranges.size());
for (int i = 0; i < ranges.size(); ++i) {
ev.detail(format("Range%d", i), ranges[i]);
}
if (performRestore) {
if (dbVersion == invalidVersion) {
TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", "Largest restorable version");
// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
BackupDescription desc = co_await IBackupContainer::openContainer(container, proxy, {})
->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0);
if (!desc.maxRestorableVersion.present()) {
fprintf(stderr, "The specified backup is not restorable to any version.\n");
throw restore_error();
}
dbVersion = desc.maxRestorableVersion.get();
TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", dbVersion);
}
UID randomUID = deterministicRandom()->randomUniqueID();
TraceEvent("FastRestoreTool")
.detail("SubmitRestoreRequests", ranges.size())
.detail("RestoreUID", randomUID);
co_await backupAgent.submitParallelRestore(db,
KeyRef(tagName),
ranges,
KeyRef(container),
proxy,
dbVersion,
LockDB::True,
randomUID,
""_sr,
""_sr);
// TODO: Support addPrefix and removePrefix
if (waitForDone) {
// Wait for parallel restore to finish and unlock DB after that
TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "WaitForRestoreToFinish");
co_await backupAgent.parallelRestoreFinish(db, randomUID);
TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "RestoreFinished");
} else {
TraceEvent("FastRestoreTool")
.detail("RestoreUID", randomUID)
.detail("OperationGuide", "Manually unlock DB when restore finishes");
printf("WARNING: DB will be in locked state after restore. Need UID:%s to unlock DB\n",
randomUID.toString().c_str());
}
restoreVersion = dbVersion;
} else {
Reference<IBackupContainer> bc = IBackupContainer::openContainer(container, proxy, {});
// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
BackupDescription description =
co_await bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0);
if (dbVersion <= 0) {
co_await description.resolveVersionTimes(db);
if (description.maxRestorableVersion.present())
restoreVersion = description.maxRestorableVersion.get();
else {
fprintf(stderr, "Backup is not restorable\n");
throw restore_invalid_version();
}
} else {
restoreVersion = dbVersion;
}
Optional<RestorableFileSet> rset = co_await bc->getRestoreSet(restoreVersion);
if (!rset.present()) {
fmt::print(stderr, "Insufficient data to restore to version {}\n", restoreVersion);
throw restore_invalid_version();
}
// Display the restore information, if requested
if (verbose) {
fmt::print("[DRY RUN] Restoring backup to version: {}\n", restoreVersion);
fmt::print("{}\n", description.toString());
}
}
if (waitForDone && verbose) {
// If restore completed then report version restored
fmt::print("Restored to version {0}{1}\n", restoreVersion, (performRestore) ? "" : " (DRY RUN)");
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
throw;
fprintf(stderr, "ERROR: %s\n", e.what());
throw;
}
}
Future<Void> dumpBackupData(const char* name,
std::string destinationContainer,
Optional<std::string> proxy,
@ -3661,21 +3524,6 @@ int main(int argc, char* argv[]) {
newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
}
break;
case ProgramExe::FASTRESTORE_TOOL:
if (newArgC < 2) {
printFastRestoreUsage(false);
return FDB_EXIT_ERROR;
}
// Get the restore operation type
restoreType = getRestoreType(newArgV[1]);
if (restoreType == RestoreType::UNKNOWN) {
args =
std::make_unique<CSimpleOpt>(newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
} else {
args = std::make_unique<CSimpleOpt>(
newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
}
break;
case ProgramExe::UNDEFINED:
default:
fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
@ -4178,7 +4026,7 @@ int main(int argc, char* argv[]) {
return FDB_EXIT_ERROR;
}
snapshotMode = parsedMode.get();
} else if (programExe == ProgramExe::RESTORE || programExe == ProgramExe::FASTRESTORE_TOOL) {
} else if (programExe == ProgramExe::RESTORE) {
// Validate and store mode parameter for restore mechanism
auto parsedMode = getRestoreMode(args->OptionArg());
if (!parsedMode.present()) {
@ -4227,13 +4075,6 @@ int main(int argc, char* argv[]) {
return FDB_EXIT_ERROR;
break;
case ProgramExe::FASTRESTORE_TOOL:
fprintf(
stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(newArgV[0]);
return FDB_EXIT_ERROR;
break;
case ProgramExe::DR_AGENT:
fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(newArgV[0]);
@ -4388,18 +4229,10 @@ int main(int argc, char* argv[]) {
return result.present();
};
// The fastrestore tool does not yet support multiple ranges and is incompatible with
// features that back up data in the system keys.
if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty() &&
programExe != ProgramExe::FASTRESTORE_TOOL) {
if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty()) {
addDefaultBackupRanges(backupKeys);
}
if ((restoreSystemKeys || restoreUserKeys) && programExe == ProgramExe::FASTRESTORE_TOOL) {
fprintf(stderr, "ERROR: Options: --user-data and --system-metadata are not supported with fastrestore\n");
return FDB_EXIT_ERROR;
}
if ((restoreUserKeys || restoreSystemKeys) && !backupKeys.empty()) {
fprintf(stderr,
"ERROR: Cannot specify additional ranges when using --user-data or --system-metadata "
@ -4655,80 +4488,6 @@ int main(int argc, char* argv[]) {
throw restore_error();
}
break;
case ProgramExe::FASTRESTORE_TOOL:
// Support --dest-cluster-file option as fdbrestore does
if (dryRun) {
if (restoreType != RestoreType::START) {
fprintf(stderr, "Restore dry run only works for 'start' command\n");
return FDB_EXIT_ERROR;
}
// Must explicitly call trace file options handling if not calling Database::createDatabase()
initTraceFile();
} else {
if (restoreClusterFileDest.empty()) {
fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
return FDB_EXIT_ERROR;
}
if (!fileExists(restoreClusterFileDest)) {
fprintf(stderr,
"Restore destination cluster file '%s' does not exist.\n",
restoreClusterFileDest.c_str());
return FDB_EXIT_ERROR;
}
try {
db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION);
} catch (Error& e) {
fprintf(stderr,
"Restore destination cluster file '%s' invalid: %s\n",
restoreClusterFileDest.c_str(),
e.what());
return FDB_EXIT_ERROR;
}
}
// TODO: We have not implemented the code commented out in this case
switch (restoreType) {
case RestoreType::START:
f = stopAfter(runFastRestoreTool(db,
tagName,
restoreContainer,
proxy,
backupKeys,
restoreVersion,
!dryRun,
Verbose{ !quietDisplay },
waitForDone));
break;
case RestoreType::WAIT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
throw restore_error();
// f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
break;
case RestoreType::ABORT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
throw restore_error();
// f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)),
//[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s
// State: %s\n", tagName.c_str(),
// FileBackupAgent::restoreStateText(s).toString().c_str()); return Void();
// }) );
break;
case RestoreType::STATUS:
printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n");
throw restore_error();
// If no tag is specifically provided then print all tag status, don't just use "default"
if (tagProvided)
tag = tagName;
// f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void
//{ printf("%s\n", s.c_str()); return Void();
// }) );
break;
default:
throw restore_error();
}
break;
case ProgramExe::DR_AGENT:
if (!initCluster() || !initSourceCluster(true)) {
return FDB_EXIT_ERROR;

1
fdbclient/ClientKnobs.cpp
View File

@ -203,7 +203,6 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
init( BACKUP_STATUS_DELAY, 40.0 );
init( BACKUP_STATUS_JITTER, 0.05 );
init( MIN_CLEANUP_SECONDS, 3600.0 );
init( FASTRESTORE_ATOMICOP_WEIGHT, 1 ); if( randomize && BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom()->random01() * 200 + 1; }
init( RESTORE_RANGES_READ_BATCH, 10000 );
init( BACKUP_RANGE_PARTITIONED_VDIR_INTERVAL, 100000 * 1000000LL );

526
fdbclient/FileBackupAgent.cpp
View File

@ -6835,157 +6835,6 @@ struct LogInfo : public ReferenceCounted<LogInfo> {
class FileBackupAgentImpl {
public:
// Parallel restore
static Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB = UnlockDB::True) {
ReadYourWritesTransaction tr(cx);
Optional<Value> restoreRequestDoneKeyValue;
TraceEvent("FastRestoreToolWaitForRestoreToFinish").detail("DBLock", randomUID);
// TODO: register watch first and then check if the key exist
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> _restoreRequestDoneKeyValue = co_await tr.get(restoreRequestDoneKey);
restoreRequestDoneKeyValue = _restoreRequestDoneKeyValue;
// Restore may finish before restoreTool waits on the restore finish event.
if (restoreRequestDoneKeyValue.present()) {
break;
} else {
Future<Void> watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
co_await tr.commit();
co_await watchForRestoreRequestDone;
break;
}
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
TraceEvent("FastRestoreToolRestoreFinished")
.detail("ClearRestoreRequestDoneKey", restoreRequestDoneKeyValue.present());
// Only this agent can clear the restoreRequestDoneKey
co_await runRYWTransaction(cx, [](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->clear(restoreRequestDoneKey);
return Void();
});
if (unlockDB) {
TraceEvent("FastRestoreToolRestoreFinished").detail("UnlockDBStart", randomUID);
co_await unlockDatabase(cx, randomUID);
TraceEvent("FastRestoreToolRestoreFinished").detail("UnlockDBFinish", randomUID);
} else {
TraceEvent("FastRestoreToolRestoreFinished").detail("DBLeftLockedAfterRestore", randomUID);
}
co_return;
}
static Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Optional<std::string> proxy,
Version targetVersion,
LockDB lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix) {
// Sanity check backup is valid
std::string bcUrlStr = bcUrl.toString();
Reference<IBackupContainer> bc = IBackupContainer::openContainer(bcUrlStr, proxy, {});
// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
// This is needed for S3 where metadata may not be immediately consistent
BackupDescription desc = co_await bc->describeBackup(false, isBlobstoreUrl(bcUrlStr) ? invalidVersion : 0);
co_await desc.resolveVersionTimes(cx);
if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
targetVersion = desc.maxRestorableVersion.get();
TraceEvent(SevWarn, "FastRestoreSubmitRestoreRequestWithInvalidTargetVersion")
.detail("OverrideTargetVersion", targetVersion);
}
Optional<RestorableFileSet> restoreSet = co_await bc->getRestoreSet(targetVersion);
if (!restoreSet.present()) {
TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
.detail("BackupContainer", bc->getURL())
.detail("TargetVersion", targetVersion);
throw restore_invalid_version();
}
TraceEvent("FastRestoreSubmitRestoreRequest")
.detail("BackupDesc", desc.toString())
.detail("TargetVersion", targetVersion);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
int restoreIndex = 0;
int numTries = 0;
// lock DB for restore
while (true) {
Error err;
try {
if (lockDB) {
co_await lockDatabase(cx, randomUID);
}
co_await checkDatabaseLock(tr, randomUID);
TraceEvent("FastRestoreToolSubmitRestoreRequests").detail("DBIsLocked", randomUID);
break;
} catch (Error& e) {
err = e;
}
TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsMayFail")
.error(err)
.detail("Reason", "DB is not properly locked")
.detail("ExpectedLockID", randomUID);
numTries++;
co_await tr->onError(err);
}
// set up restore request
tr->reset();
numTries = 0;
while (true) {
Error err;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
try {
// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
auto range = backupRanges[restoreIndex];
Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
// Register the request request in DB, which will be picked up by restore worker leader
struct RestoreRequest restoreRequest(restoreIndex,
restoreTag,
bcUrl,
proxy,
targetVersion,
range,
deterministicRandom()->randomUniqueID(),
addPrefix,
removePrefix);
tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
}
tr->set(restoreRequestTriggerKey,
restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
co_await tr->commit(); // Trigger restore
break;
} catch (Error& e) {
err = e;
}
TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsRetry")
.error(err)
.detail("RestoreIndex", restoreIndex);
numTries++;
co_await tr->onError(err);
}
co_return;
}
// This method will return the final status of the backup at tag, and return the URL that was used on the tag
// when that status value was read.
static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent,
@ -8146,8 +7995,7 @@ public:
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix,
UsePartitionedLog fastRestore) {
Key removePrefix) {
auto ryw_tr = makeReference<ReadYourWritesTransaction>(cx);
BackupConfig backupConfig;
DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
@ -8255,140 +8103,76 @@ public:
bc = fileBackup::getBackupContainerWithProxy(bc);
if (fastRestore) {
TraceEvent("AtomicParallelRestoreStartRestore").log();
Version targetVersion = ::invalidVersion;
co_await submitParallelRestore(cx,
tagName,
ranges,
KeyRef(bc->getURL()),
bc->getProxy(),
targetVersion,
LockDB::True,
randomUid,
addPrefix,
removePrefix);
bool hasPrefix = (addPrefix.size() > 0 || removePrefix.size() > 0);
TraceEvent("AtomicParallelRestoreWaitForRestoreFinish").detail("HasPrefix", hasPrefix);
co_await parallelRestoreFinish(cx, randomUid, UnlockDB{ !hasPrefix });
// If addPrefix or removePrefix set, we want to transform the effect by copying data
if (hasPrefix) {
co_await transformRestoredDatabase(cx, ranges, addPrefix, removePrefix);
co_await unlockDatabase(cx, randomUid);
}
co_return -1;
} else {
TraceEvent("AS_StartRestore").log();
Standalone<VectorRef<KeyRangeRef>> restoreRange;
Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
for (auto r : ranges) {
restoreRange.push_back_deep(restoreRange.arena(), r);
}
if (!systemRestoreRange.empty()) {
// restore system keys
co_await restore(backupAgent,
cx,
cx,
"system_restore"_sr,
KeyRef(bc->getURL()),
bc->getProxy(),
systemRestoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::False,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
auto rywTransaction = makeReference<ReadYourWritesTransaction>(cx);
// clear old restore config associated with system keys
while (true) {
Error err;
try {
rywTransaction->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
rywTransaction->setOption(FDBTransactionOptions::LOCK_AWARE);
RestoreConfig oldRestore(randomUid);
oldRestore.clear(rywTransaction);
co_await rywTransaction->commit();
break;
} catch (Error& e) {
err = e;
}
co_await rywTransaction->onError(err);
}
}
// restore user data
Version ver = co_await restore(backupAgent,
cx,
cx,
tagName,
KeyRef(bc->getURL()),
bc->getProxy(),
restoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::True,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
co_return ver;
TraceEvent("AS_StartRestore").log();
Standalone<VectorRef<KeyRangeRef>> restoreRange;
Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
for (auto r : ranges) {
restoreRange.push_back_deep(restoreRange.arena(), r);
}
}
// Similar to atomicRestore, only used in simulation test.
// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
// the tagname of the backup must be the same as the restore.
static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent,
Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix) {
return success(
atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::True));
if (!systemRestoreRange.empty()) {
// restore system keys
co_await restore(backupAgent,
cx,
cx,
"system_restore"_sr,
KeyRef(bc->getURL()),
bc->getProxy(),
systemRestoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::False,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
auto rywTransaction = makeReference<ReadYourWritesTransaction>(cx);
// clear old restore config associated with system keys
while (true) {
Error err;
try {
rywTransaction->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
rywTransaction->setOption(FDBTransactionOptions::LOCK_AWARE);
RestoreConfig oldRestore(randomUid);
oldRestore.clear(rywTransaction);
co_await rywTransaction->commit();
break;
} catch (Error& e) {
err = e;
}
co_await rywTransaction->onError(err);
}
}
// restore user data
Version ver = co_await restore(backupAgent,
cx,
cx,
tagName,
KeyRef(bc->getURL()),
bc->getProxy(),
restoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::True,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
co_return ver;
}
};
const int FileBackupAgent::dataFooterSize = 20;
// Return if parallel restore has finished
Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB) {
return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID, unlockDB);
}
Future<Void> FileBackupAgent::submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Optional<std::string> proxy,
Version targetVersion,
LockDB lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix) {
return FileBackupAgentImpl::submitParallelRestore(
cx, backupTag, backupRanges, bcUrl, proxy, targetVersion, lockDB, randomUID, addPrefix, removePrefix);
}
Future<Void> FileBackupAgent::atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix) {
return FileBackupAgentImpl::atomicParallelRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
}
Future<Version> FileBackupAgent::restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
@ -8540,8 +8324,7 @@ Future<Version> FileBackupAgent::atomicRestore(Database cx,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix) {
return FileBackupAgentImpl::atomicRestore(
this, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::False);
return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
}
Future<ERestoreState> FileBackupAgent::abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@ -8704,183 +8487,6 @@ static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef>> kvs
TraceEvent(SevFRTestInfo, "TransformDatabaseContentsWriteKVDone").detail("Begin", begin).detail("End", end);
}
// restoreRanges is the actual range that has applied removePrefix and addPrefix processed by restore system
// Assume: restoreRanges do not overlap which is achieved by ensuring backup ranges do not overlap
static Future<Void> transformDatabaseContents(Database cx,
Key addPrefix,
Key removePrefix,
Standalone<VectorRef<KeyRangeRef>> restoreRanges) {
ReadYourWritesTransaction tr(cx);
Standalone<VectorRef<KeyValueRef>> oldData;
TraceEvent("FastRestoreWorkloadTransformDatabaseContents")
.detail("AddPrefix", addPrefix)
.detail("RemovePrefix", removePrefix);
int i = 0;
while (true) { // Read all data from DB
Error err;
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
for (i = 0; i < restoreRanges.size(); ++i) {
RangeResult kvs = co_await tr.getRange(restoreRanges[i], CLIENT_KNOBS->TOO_MANY);
ASSERT(!kvs.more);
for (auto kv : kvs) {
oldData.push_back_deep(oldData.arena(), KeyValueRef(kv.key, kv.value));
}
}
break;
} catch (Error& e) {
err = e;
}
TraceEvent("FastRestoreWorkloadTransformDatabaseContentsGetAllKeys")
.error(err)
.detail("Index", i)
.detail("RestoreRange", restoreRanges[i]);
oldData = Standalone<VectorRef<KeyValueRef>>(); // clear the vector
co_await tr.onError(err);
}
// Convert data by removePrefix and addPrefix in memory
Standalone<VectorRef<KeyValueRef>> newKVs;
for (int i = 0; i < oldData.size(); ++i) {
Key newKey(oldData[i].key);
TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
.detail("Keys", oldData.size())
.detail("Index", i)
.detail("GetKey", oldData[i].key)
.detail("GetValue", oldData[i].value);
if (newKey.size() < removePrefix.size()) { // If true, must check why.
TraceEvent(SevError, "TransformDatabaseContents")
.detail("Key", newKey)
.detail("RemovePrefix", removePrefix);
continue;
}
newKey = newKey.removePrefix(removePrefix).withPrefix(addPrefix);
newKVs.push_back_deep(newKVs.arena(), KeyValueRef(newKey.contents(), oldData[i].value));
TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
.detail("Keys", newKVs.size())
.detail("Index", i)
.detail("NewKey", newKVs.back().key)
.detail("NewValue", newKVs.back().value);
}
Standalone<VectorRef<KeyRangeRef>> backupRanges; // dest. ranges
for (auto& range : restoreRanges) {
KeyRange tmpRange = range;
backupRanges.push_back_deep(backupRanges.arena(), tmpRange.removePrefix(removePrefix).withPrefix(addPrefix));
}
// Clear the transformed data (original data with removePrefix and addPrefix) in restoreRanges
co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
for (int i = 0; i < restoreRanges.size(); i++) {
TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
.detail("ClearRestoreRange", restoreRanges[i])
.detail("ClearBackupRange", backupRanges[i]);
tr->clear(restoreRanges[i]); // Clear the range.removePrefix().withPrefix()
tr->clear(backupRanges[i]);
}
return Void();
});
// Sanity check to ensure no data in the ranges
tr.reset();
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
RangeResult emptyData = co_await tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY);
for (int i = 0; i < emptyData.size(); ++i) {
TraceEvent(SevError, "ExpectEmptyData")
.detail("Index", i)
.detail("Key", emptyData[i].key)
.detail("Value", emptyData[i].value);
}
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
// Write transformed KVs (i.e., kv backup took) back to DB
while (true) {
Error err;
try {
std::vector<Future<Void>> fwrites;
int begin = 0;
while (begin < newKVs.size()) {
int len = std::min(100, newKVs.size() - begin);
fwrites.push_back(writeKVs(cx, newKVs, begin, begin + len));
begin += len;
}
co_await waitForAll(fwrites);
break;
} catch (Error& e) {
err = e;
}
TraceEvent(SevError, "FastRestoreWorkloadTransformDatabaseContentsUnexpectedErrorOnWriteKVs").error(err);
co_await tr.onError(err);
}
// Sanity check
tr.reset();
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
RangeResult allData = co_await tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY);
TraceEvent(SevFRTestInfo, "SanityCheckData").detail("Size", allData.size());
for (int i = 0; i < allData.size(); ++i) {
std::pair<bool, bool> backupRestoreValid = insideValidRange(allData[i], restoreRanges, backupRanges);
TraceEvent(backupRestoreValid.first ? SevFRTestInfo : SevError, "SanityCheckData")
.detail("Index", i)
.detail("Key", allData[i].key)
.detail("Value", allData[i].value)
.detail("InsideBackupRange", backupRestoreValid.first)
.detail("InsideRestoreRange", backupRestoreValid.second);
}
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
TraceEvent("FastRestoreWorkloadTransformDatabaseContentsFinish")
.detail("AddPrefix", addPrefix)
.detail("RemovePrefix", removePrefix);
}
// addPrefix and removePrefix are the options used in the restore request:
// every backup key applied removePrefix and addPrefix in restore;
// transformRestoredDatabase actor will revert it by remove addPrefix and add removePrefix.
Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix) {
try {
Standalone<VectorRef<KeyRangeRef>> restoreRanges;
for (int i = 0; i < backupRanges.size(); ++i) {
KeyRange range(backupRanges[i]);
Key begin = range.begin.removePrefix(removePrefix).withPrefix(addPrefix);
Key end = range.end.removePrefix(removePrefix).withPrefix(addPrefix);
TraceEvent("FastRestoreTransformRestoredDatabase")
.detail("From", KeyRangeRef(begin.contents(), end.contents()))
.detail("To", range);
restoreRanges.push_back_deep(restoreRanges.arena(), KeyRangeRef(begin.contents(), end.contents()));
}
co_await transformDatabaseContents(cx, removePrefix, addPrefix, restoreRanges);
} catch (Error& e) {
TraceEvent(SevError, "FastRestoreTransformRestoredDatabaseUnexpectedError").error(e);
throw;
}
}
void simulateBlobFailure() {
if (BUGGIFY && deterministicRandom()->random01() < 0.01) { // Simulate blob failures
double i = deterministicRandom()->random01();

47
fdbclient/ServerKnobs.cpp
View File

@ -1119,6 +1119,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( FETCH_SHARD_BUFFER_BYTE_LIMIT, 20e6 ); if( randomize && BUGGIFY ) FETCH_SHARD_BUFFER_BYTE_LIMIT = 1;
init( FETCH_SHARD_UPDATES_BYTE_LIMIT, 2500000 ); if( randomize && BUGGIFY ) FETCH_SHARD_UPDATES_BYTE_LIMIT = 100;
init( TRACK_READ_LATENCIES_PER_TYPE, false ); if( randomize && BUGGIFY ) TRACK_READ_LATENCIES_PER_TYPE = true;
init( STORAGE_UPDATE_PROCESS_STATS_INTERVAL, 5 ); if( randomize && BUGGIFY ) STORAGE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom()->random01() * 60 + 1;
//Wait Failure
init( MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS, 250 ); if( randomize && BUGGIFY ) MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS = 2;
@ -1190,52 +1191,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( TIME_KEEPER_DELAY, 10 );
init( TIME_KEEPER_MAX_ENTRIES, 3600 * 24 * 30 * 6 ); if( randomize && BUGGIFY ) { TIME_KEEPER_MAX_ENTRIES = 2; }
// Fast Restore
init( FASTRESTORE_FAILURE_TIMEOUT, 3600 );
init( FASTRESTORE_HEARTBEAT_INTERVAL, 60 );
init( FASTRESTORE_SAMPLING_PERCENT, 100 ); if( randomize && BUGGIFY ) { FASTRESTORE_SAMPLING_PERCENT = deterministicRandom()->random01() * 100; }
init( FASTRESTORE_NUM_LOADERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_NUM_APPLIERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_TXN_BATCH_MAX_BYTES, 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; }
init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 50 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
init( FASTRESTORE_VB_PARALLELISM, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() < 0.2 ? 2 : deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_VB_MONITOR_DELAY, 30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; }
init( FASTRESTORE_VB_LAUNCH_DELAY, 1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() < 0.2 ? 0.1 : deterministicRandom()->random01() * 10.0 + 1; }
init( FASTRESTORE_ROLE_LOGGING_DELAY, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_ROLE_LOGGING_DELAY = deterministicRandom()->random01() * 60 + 1; }
init( FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom()->random01() * 60 + 1; }
init( FASTRESTORE_MONITOR_LEADER_DELAY, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS, 60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
init( FASTRESTORE_TRACK_REQUEST_LATENCY, false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT, 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
init( FASTRESTORE_WAIT_FOR_MEMORY_LATENCY, 10 ); if( randomize && BUGGIFY ) { FASTRESTORE_WAIT_FOR_MEMORY_LATENCY = 60; }
init( FASTRESTORE_HEARTBEAT_DELAY, 10 ); if( randomize && BUGGIFY ) { FASTRESTORE_HEARTBEAT_DELAY = deterministicRandom()->random01() * 120 + 2; }
init( FASTRESTORE_HEARTBEAT_MAX_DELAY, 10 ); if( randomize && BUGGIFY ) { FASTRESTORE_HEARTBEAT_MAX_DELAY = FASTRESTORE_HEARTBEAT_DELAY * 10; }
init( FASTRESTORE_APPLIER_FETCH_KEYS_SIZE, 100 ); if( randomize && BUGGIFY ) { FASTRESTORE_APPLIER_FETCH_KEYS_SIZE = deterministicRandom()->random01() * 10240 + 1; }
init( FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES, 1.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES = deterministicRandom()->random01() < 0.2 ? 1024 : deterministicRandom()->random01() * 5.0 * 1024.0 * 1024.0 + 1; }
init( FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE, false ); if( randomize && BUGGIFY ) { FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE = deterministicRandom()->random01() < 0.5 ? true : false; }
init( FASTRESTORE_REQBATCH_PARALLEL, 50 ); if( randomize && BUGGIFY ) { FASTRESTORE_REQBATCH_PARALLEL = deterministicRandom()->random01() * 100 + 1; }
init( FASTRESTORE_REQBATCH_LOG, false ); if( randomize && BUGGIFY ) { FASTRESTORE_REQBATCH_LOG = deterministicRandom()->random01() < 0.2 ? true : false; }
init( FASTRESTORE_TXN_CLEAR_MAX, 100 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_CLEAR_MAX = deterministicRandom()->random01() * 100 + 1; }
init( FASTRESTORE_TXN_RETRY_MAX, 10 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_RETRY_MAX = deterministicRandom()->random01() * 100 + 1; }
init( FASTRESTORE_TXN_EXTRA_DELAY, 0.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_EXTRA_DELAY = deterministicRandom()->random01() * 1 + 0.001;}
init( FASTRESTORE_NOT_WRITE_DB, false ); // Perf test only: set it to true will cause simulation failure
init( FASTRESTORE_USE_RANGE_FILE, true ); // Perf test only: set it to false will cause simulation failure
init( FASTRESTORE_USE_LOG_FILE, true ); // Perf test only: set it to false will cause simulation failure
init( FASTRESTORE_SAMPLE_MSG_BYTES, 1048576 ); if( randomize && BUGGIFY ) { FASTRESTORE_SAMPLE_MSG_BYTES = deterministicRandom()->random01() * 2048;}
init( FASTRESTORE_SCHED_UPDATE_DELAY, 0.1 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_UPDATE_DELAY = deterministicRandom()->random01() * 2;}
init( FASTRESTORE_SCHED_TARGET_CPU_PERCENT, 70 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_TARGET_CPU_PERCENT = deterministicRandom()->random01() * 100 + 50;} // simulate cpu usage can be larger than 100
init( FASTRESTORE_SCHED_MAX_CPU_PERCENT, 90 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_MAX_CPU_PERCENT = FASTRESTORE_SCHED_TARGET_CPU_PERCENT + deterministicRandom()->random01() * 100;}
init( FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS, 50 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 30 + 1;}
init( FASTRESTORE_SCHED_INFLIGHT_SEND_REQS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SEND_REQS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 10 + 1;}
init( FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 10 + 1;}
init( FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD, 10 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 15 + 1;}
init( FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH, 2 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 15 + 1;}
init( FASTRESTORE_NUM_TRACE_EVENTS, 100 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_TRACE_EVENTS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 500 + 1;}
init( FASTRESTORE_EXPENSIVE_VALIDATION, false ); if( randomize && BUGGIFY ) { FASTRESTORE_EXPENSIVE_VALIDATION = deterministicRandom()->random01() < 0.5 ? true : false;}
init( FASTRESTORE_WRITE_BW_MB, 70 ); if( randomize && BUGGIFY ) { FASTRESTORE_WRITE_BW_MB = deterministicRandom()->random01() < 0.5 ? 2 : 100;}
init( FASTRESTORE_RATE_UPDATE_SECONDS, 1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_RATE_UPDATE_SECONDS = deterministicRandom()->random01() < 0.5 ? 0.1 : 2;}
init( FASTRESTORE_DUMP_INSERT_RANGE_VERSION, false );
init( REDWOOD_DEFAULT_PAGE_SIZE, 8192 );
init( REDWOOD_DEFAULT_EXTENT_SIZE, 32 * 1024 * 1024 );
init( REDWOOD_DEFAULT_EXTENT_READ_SIZE, 1024 * 1024 );

2
fdbclient/SystemData.cpp
View File

@ -1512,8 +1512,6 @@ const KeyRef mustContainSystemMutationsKey = "\xff/mustContainSystemMutations"_s
const KeyRangeRef monitorConfKeys("\xff\x02/monitorConf/"_sr, "\xff\x02/monitorConf0"_sr);
const KeyRef restoreRequestDoneKey = "\xff\x02/restoreRequestDone"_sr;
const KeyRef healthyZoneKey = "\xff\x02/healthyZone"_sr;
const StringRef ignoreSSFailuresZoneString = "IgnoreSSFailures"_sr;
const KeyRef rebalanceDDIgnoreKey = "\xff\x02/rebalanceDDIgnored"_sr;

27
fdbclient/include/fdbclient/BackupAgent.h
View File

@ -171,24 +171,6 @@ public:
static StringRef restoreStateText(ERestoreState id);
static Key getPauseKey();
// parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB = UnlockDB::True);
Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Optional<std::string> proxy,
Version targetVersion,
LockDB lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix);
// restore() will
// - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid
@ -1063,15 +1045,6 @@ Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<
Value makePadding(int size);
} // namespace fileBackup
// For fast restore simulation test
// For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked
Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure();
// Add the set of ranges that are backed up in a default backup to the given vector. This consists of all normal keys

1
fdbclient/include/fdbclient/ClientKnobs.h
View File

@ -202,7 +202,6 @@ public:
double BACKUP_STATUS_DELAY;
double BACKUP_STATUS_JITTER;
double MIN_CLEANUP_SECONDS;
int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
int RESTORE_RANGES_READ_BATCH;
// interval for version directory bucketing in range-partitioned backup.

10
fdbclient/include/fdbclient/CommitTransaction.h
View File

@ -113,16 +113,6 @@ struct MutationRef {
(accumulativeChecksumIndex.present() ? sizeof(uint16_t) + 1 : 1);
}
int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const {
// AtomicOp can cause more workload to FDB cluster than the same-size set mutation;
// Amplify atomicOp size to consider such extra workload.
// A good value for FASTRESTORE_ATOMICOP_WEIGHT needs experimental evaluations.
if (isAtomicOp()) {
return totalSize() * CLIENT_KNOBS->FASTRESTORE_ATOMICOP_WEIGHT;
} else {
return totalSize();
}
}
std::string toString() const {
std::string checksumStr;

54
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -1176,6 +1176,7 @@ public:
int FETCH_SHARD_BUFFER_BYTE_LIMIT;
int FETCH_SHARD_UPDATES_BYTE_LIMIT;
bool TRACK_READ_LATENCIES_PER_TYPE;
int64_t STORAGE_UPDATE_PROCESS_STATS_INTERVAL;
// Wait Failure
int MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS;
@ -1266,59 +1267,6 @@ public:
int64_t TIME_KEEPER_DELAY;
int64_t TIME_KEEPER_MAX_ENTRIES;
// Fast Restore
// TODO: After 6.3, review FR knobs, remove unneeded ones and change default value
// TODO(gglass): revisit the above FR
int64_t FASTRESTORE_FAILURE_TIMEOUT;
int64_t FASTRESTORE_HEARTBEAT_INTERVAL;
double FASTRESTORE_SAMPLING_PERCENT;
int64_t FASTRESTORE_NUM_LOADERS;
int64_t FASTRESTORE_NUM_APPLIERS;
// FASTRESTORE_TXN_BATCH_MAX_BYTES is target txn size used by appliers to apply mutations
double FASTRESTORE_TXN_BATCH_MAX_BYTES;
// FASTRESTORE_VERSIONBATCH_MAX_BYTES is the maximum data size in each version batch
double FASTRESTORE_VERSIONBATCH_MAX_BYTES;
// FASTRESTORE_VB_PARALLELISM is the number of concurrently running version batches
int64_t FASTRESTORE_VB_PARALLELISM;
int64_t FASTRESTORE_VB_MONITOR_DELAY; // How quickly monitor finished version batch
double FASTRESTORE_VB_LAUNCH_DELAY;
int64_t FASTRESTORE_ROLE_LOGGING_DELAY;
int64_t FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL; // How quickly to update process metrics for restore
int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
int64_t FASTRESTORE_WAIT_FOR_MEMORY_LATENCY;
int64_t FASTRESTORE_HEARTBEAT_DELAY; // interval for master to ping loaders and appliers
int64_t
FASTRESTORE_HEARTBEAT_MAX_DELAY; // master claim a node is down if no heart beat from the node for this delay
int64_t FASTRESTORE_APPLIER_FETCH_KEYS_SIZE; // number of keys to fetch in a txn on applier
int64_t FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES; // desired size of mutation message sent from loader to appliers
bool FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE; // parse each range file to get (range, version) it has?
int64_t FASTRESTORE_REQBATCH_PARALLEL; // number of requests to wait on for getBatchReplies()
bool FASTRESTORE_REQBATCH_LOG; // verbose log information for getReplyBatches
int FASTRESTORE_TXN_CLEAR_MAX; // threshold to start tracking each clear op in a txn
int FASTRESTORE_TXN_RETRY_MAX; // threshold to start output error on too many retries
double FASTRESTORE_TXN_EXTRA_DELAY; // extra delay to avoid overwhelming fdb
bool FASTRESTORE_NOT_WRITE_DB; // do not write result to DB. Only for dev testing
bool FASTRESTORE_USE_RANGE_FILE; // use range file in backup
bool FASTRESTORE_USE_LOG_FILE; // use log file in backup
int64_t FASTRESTORE_SAMPLE_MSG_BYTES; // sample message desired size
double FASTRESTORE_SCHED_UPDATE_DELAY; // delay in seconds in updating process metrics
int FASTRESTORE_SCHED_TARGET_CPU_PERCENT; // release as many requests as possible when cpu usage is below the knob
int FASTRESTORE_SCHED_MAX_CPU_PERCENT; // max cpu percent when scheduler shall not release non-urgent requests
int FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS; // number of inflight requests to load backup files
int FASTRESTORE_SCHED_INFLIGHT_SEND_REQS; // number of inflight requests for loaders to send mutations to appliers
int FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE; // number of load request to release at once
int FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD; // we can send future VB requests if it is less than this knob
int FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH; // number of future VB sendLoadingParam requests to process at once
int FASTRESTORE_NUM_TRACE_EVENTS;
bool FASTRESTORE_EXPENSIVE_VALIDATION; // when set true, performance will be heavily affected
double FASTRESTORE_WRITE_BW_MB; // target aggregated write bandwidth from all appliers
double FASTRESTORE_RATE_UPDATE_SECONDS; // how long to update appliers target write rate
bool FASTRESTORE_DUMP_INSERT_RANGE_VERSION; // Dump all the range version after insertion. This is for debugging
// purpose.
int REDWOOD_DEFAULT_PAGE_SIZE; // Page size for new Redwood files
int REDWOOD_DEFAULT_EXTENT_SIZE; // Extent size for new Redwood files
int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files

5
fdbrpc/Locality.cpp
View File

@ -60,7 +60,7 @@ ProcessClass::ProcessClass(std::string s, ClassSource source) : _source(source)
else if (s == "cluster_controller")
_class = ClusterControllerClass;
else if (s == "fast_restore")
_class = FastRestoreClass;
ASSERT(false); // deprecated
else if (s == "data_distributor")
_class = DataDistributorClass;
else if (s == "coordinator")
@ -113,7 +113,7 @@ ProcessClass::ProcessClass(std::string classStr, std::string sourceStr) {
else if (classStr == "cluster_controller")
_class = ClusterControllerClass;
else if (classStr == "fast_restore")
_class = FastRestoreClass;
ASSERT(false); // deprecated
else if (classStr == "data_distributor")
_class = DataDistributorClass;
else if (classStr == "coordinator")
@ -172,6 +172,7 @@ std::string ProcessClass::toString() const {
case ClusterControllerClass:
return "cluster_controller";
case FastRestoreClass:
ASSERT(false);
return "fast_restore";
case DataDistributorClass:
return "data_distributor";

2
fdbrpc/include/fdbrpc/Locality.h
View File

@ -39,7 +39,7 @@ struct ProcessClass {
LogClass,
ClusterControllerClass,
LogRouterClass,
FastRestoreClass,
FastRestoreClass, // deprecated
DataDistributorClass,
CoordinatorClass,
RatekeeperClass,

2
fdbserver/CMakeLists.txt
View File

@ -28,7 +28,6 @@ add_subdirectory(core)
add_subdirectory(logsystem)
add_subdirectory(kvstore)
add_subdirectory(mocks3)
add_subdirectory(restoreworker)
add_subdirectory(clustercontroller)
add_subdirectory(backupworker)
add_subdirectory(commitproxy)
@ -172,7 +171,6 @@ target_link_libraries(fdbserver PRIVATE
"$<LINK_LIBRARY:WHOLE_ARCHIVE,fdbserver_workloads>"
fdbserver_worker
fdbserver_backupworker
fdbserver_restoreworker
fdbserver_clustercontroller
fdbserver_commitproxy
fdbserver_consistencyscan

3
fdbserver/commitproxy/CommitProxyServer.actor.cpp
View File

@ -53,7 +53,6 @@
#include "fdbserver/core/ProxyCommitData.h"
#include "fdbserver/core/RatekeeperInterface.h"
#include "fdbserver/core/RecoveryState.h"
#include "fdbserver/core/RestoreCoreUtil.h"
#include "fdbserver/core/ServerDBInfo.h"
#include "fdbserver/core/WaitFailure.h"
#include "fdbserver/commitproxy/CommitProxyServer.actor.h"
@ -2631,7 +2630,7 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
MutationsVec mutations;
Standalone<VectorRef<MutationRef>> mutations;
std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
std::vector<UID> src, dest;
ServerCacheInfo info;

64
fdbserver/core/RestoreCoreUtil.cpp
View File

@ -1,64 +0,0 @@
/*
* RestoreCoreUtil.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/core/RestoreCoreUtil.h"
// Similar to debugMutation(), we use debugFRMutation to track mutations for fast restore systems only.
#if CENABLED(0, NOT_IN_CLEAN)
StringRef debugFRKey = "\xff\xff\xff\xff"_sr;
// Track any mutation in fast restore that has overlap with debugFRKey
bool debugFRMutation(const char* context, Version version, MutationRef const& mutation) {
if (mutation.type != mutation.ClearRange && mutation.param1 == debugFRKey) { // Single key mutation
TraceEvent("FastRestoreMutationTracking")
.detail("At", context)
.detail("Version", version)
.detail("MutationType", getTypeString((MutationRef::Type)mutation.type))
.detail("Key", mutation.param1)
.detail("Value", mutation.param2);
} else if (mutation.type == mutation.ClearRange && debugFRKey >= mutation.param1 &&
debugFRKey < mutation.param2) { // debugFRKey is in the range mutation
TraceEvent("FastRestoreMutationTracking")
.detail("At", context)
.detail("Version", version)
.detail("MutationType", getTypeString((MutationRef::Type)mutation.type))
.detail("Begin", mutation.param1)
.detail("End", mutation.param2);
} else {
return false;
}
return true;
}
#else
bool debugFRMutation(const char* context, Version version, MutationRef const& mutation) {
return false;
}
#endif
bool isRangeMutation(MutationRef m) {
if (m.type == MutationRef::Type::ClearRange) {
ASSERT(m.type != MutationRef::Type::DebugKeyRange);
return true;
} else {
ASSERT(m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type)m.type));
return false;
}
}

69
fdbserver/core/include/fdbserver/core/RestoreCoreUtil.h
View File

@ -1,69 +0,0 @@
/*
* RestoreCoreUtil.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_RESTORECOREUTIL_H
#define FDBSERVER_RESTORECOREUTIL_H
#pragma once
#include "fdbclient/CommitTransaction.h"
#include "flow/flow.h"
struct VersionedMutationSerialized {
MutationRef mutation;
LogMessageVersion version;
VersionedMutationSerialized() = default;
explicit VersionedMutationSerialized(MutationRef mutation, LogMessageVersion version)
: mutation(mutation), version(version) {}
explicit VersionedMutationSerialized(Arena& arena, const VersionedMutationSerialized& vm)
: mutation(arena, vm.mutation), version(vm.version) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, mutation, version);
}
};
struct SampledMutation {
KeyRef key;
long size;
explicit SampledMutation(KeyRef key, long size) : key(key), size(size) {}
explicit SampledMutation(Arena& arena, const SampledMutation& sm) : key(arena, sm.key), size(sm.size) {}
SampledMutation() = default;
int totalSize() { return key.size() + sizeof(size); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, size);
}
};
using MutationsVec = Standalone<VectorRef<MutationRef>>;
using LogMessageVersionVec = Standalone<VectorRef<LogMessageVersion>>;
using VersionedMutationsVec = Standalone<VectorRef<VersionedMutationSerialized>>;
using SampledMutationsVec = Standalone<VectorRef<SampledMutation>>;
bool debugFRMutation(const char* context, Version version, MutationRef const& mutation);
bool isRangeMutation(MutationRef m);
#endif

78
fdbserver/fdbserver.actor.cpp
View File

@ -61,7 +61,6 @@
#include "fdbserver/core/MoveKeys.h"
#include "fdbserver/NetworkTest.h"
#include "fdbserver/kvstore/KVFileUtils.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
#include "fdbserver/core/ServerDBInfo.h"
#include "fdbserver/datadistributor/SimulatedCluster.h"
#include "fdbserver/tester/TestEncryptionUtils.h"
@ -1334,8 +1333,6 @@ private:
role = ServerRole::NetworkTestClient;
else if (!strcmp(sRole, "networktestserver"))
role = ServerRole::NetworkTestServer;
else if (!strcmp(sRole, "restore"))
role = ServerRole::Restore;
else if (!strcmp(sRole, "kvfileintegritycheck"))
role = ServerRole::KVFileIntegrityCheck;
else if (!strcmp(sRole, "kvfilegeneratesums"))
@ -2084,8 +2081,8 @@ int main(int argc, char* argv[]) {
FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT, &opts.allowList);
opts.buildNetwork(argv[0]);
const bool expectsPublicAddress = (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer ||
role == ServerRole::Restore || role == ServerRole::MockS3Server);
const bool expectsPublicAddress =
(role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::MockS3Server);
if (opts.publicAddressStrs.empty()) {
if (expectsPublicAddress) {
fprintf(stderr, "ERROR: The -p or --public-address option is required\n");
@ -2368,54 +2365,40 @@ int main(int argc, char* argv[]) {
auto* pProxy = static_cast<Optional<std::string>*>(g_network->global(INetwork::enProxy));
*pProxy = opts.proxy;
// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
if (opts.processClass == ProcessClass::FastRestoreClass) {
printf("Run as fast restore worker\n");
ASSERT(opts.connectionFile);
auto dataFolder = opts.dataFolder;
if (!dataFolder.size())
dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
ASSERT(opts.connectionFile);
std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back(restoreWorker(opts.connectionFile, opts.localities, dataFolder));
f = stopAfter(waitForAll(actors));
printf("Fast restore worker started\n");
g_network->run();
printf("g_network->run() done\n");
} else { // Call fdbd roles in conventional way
ASSERT(opts.connectionFile);
setupRunLoopProfiler();
setupRunLoopProfiler();
auto dataFolder = opts.dataFolder;
if (!dataFolder.size())
dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
auto dataFolder = opts.dataFolder;
if (!dataFolder.size())
dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back(fdbd(opts.connectionFile,
opts.localities,
opts.processClass,
dataFolder,
dataFolder,
opts.storageMemLimit,
opts.metricsConnFile,
opts.metricsPrefix,
opts.rsssize,
opts.whitelistBinPaths,
opts.consistencyCheckUrgentMode));
actors.push_back(histogramReport());
actors.push_back(metricsReport());
#ifdef FLOW_GRPC_ENABLED
if (opts.grpcAddressStrs.size() > 0) {
FlowGrpc::init(&opts.tlsConfig, NetworkAddress::parse(opts.grpcAddressStrs[0]));
actors.push_back(GrpcServer::instance()->run());
}
#endif
actors.push_back(fdbd(opts.connectionFile,
opts.localities,
opts.processClass,
dataFolder,
dataFolder,
opts.storageMemLimit,
opts.metricsConnFile,
opts.metricsPrefix,
opts.rsssize,
opts.whitelistBinPaths,
opts.consistencyCheckUrgentMode));
actors.push_back(histogramReport());
actors.push_back(metricsReport());
f = stopAfter(waitForAll(actors));
g_network->run();
if (opts.grpcAddressStrs.size() > 0) {
FlowGrpc::init(&opts.tlsConfig, NetworkAddress::parse(opts.grpcAddressStrs[0]));
actors.push_back(GrpcServer::instance()->run());
}
#endif
f = stopAfter(waitForAll(actors));
g_network->run();
} else if (role == ServerRole::MultiTester) {
setupRunLoopProfiler();
f = stopAfter(runTests(opts.connectionFile,
@ -2481,9 +2464,6 @@ int main(int argc, char* argv[]) {
} else if (role == ServerRole::NetworkTestServer) {
f = stopAfter(networkTestServer());
g_network->run();
} else if (role == ServerRole::Restore) {
f = stopAfter(restoreWorker(opts.connectionFile, opts.localities, opts.dataFolder));
g_network->run();
} else if (role == ServerRole::KVFileIntegrityCheck) {
f = stopAfter(KVFileCheck(opts.kvFile, true));
g_network->run();

3
fdbserver/resolver/Resolver.actor.cpp
View File

@ -35,7 +35,6 @@
#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
#include "fdbserver/core/MasterInterface.h"
#include "fdbserver/core/ResolverInterface.h"
#include "fdbserver/core/RestoreCoreUtil.h"
#include "fdbserver/core/ServerDBInfo.h"
#include "fdbserver/core/StorageMetrics.actor.h"
#include "fdbserver/core/WaitFailure.h"
@ -619,7 +618,7 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(Reference<Resolver> se
((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
MutationsVec mutations;
Standalone<VectorRef<MutationRef>> mutations;
std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
std::vector<UID> src, dest;
ServerCacheInfo info;

16
fdbserver/restoreworker/CMakeLists.txt
View File

@ -1,16 +0,0 @@
fdb_find_sources(FDBSERVER_RESTOREWORKER_SRCS)
add_flow_target(STATIC_LIBRARY NAME fdbserver_restoreworker SRCS ${FDBSERVER_RESTOREWORKER_SRCS})
add_fdbserver_link_test(fdbserver_restoreworkerlinktest
fdbserver_restoreworker
fdbserver_core)
configure_fdbserver_common_includes(fdbserver_restoreworker)
target_include_directories(fdbserver_restoreworker
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_BINARY_DIR}/include
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR})
target_link_libraries(fdbserver_restoreworker PRIVATE fdbserver_core)

816
fdbserver/restoreworker/RestoreApplier.cpp
View File

@ -1,816 +0,0 @@
/*
* RestoreApplier.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file defines the functions used by the RestoreApplier role.
// RestoreApplier role starts at restoreApplierCore actor
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/BackupAgent.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/core/Knobs.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "RestoreRoleCommon.h"
#include "RestoreApplier.h"
#include "flow/network.h"
#include "flow/CoroUtils.h"
static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
Reference<RestoreApplierData> self);
static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req,
Reference<RestoreApplierData> self,
Database cx);
void handleUpdateRateRequest(RestoreUpdateRateRequest req, Reference<RestoreApplierData> self);
Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx) {
auto self = makeReference<RestoreApplierData>(applierInterf.id(), nodeIndex);
ActorCollection actors(false);
Future<Void> exitRole = Never();
actors.add(updateProcessMetrics(self));
actors.add(traceProcessMetrics(self, "RestoreApplier"));
actors.add(traceRoleVersionBatchProgress(self, "RestoreApplier"));
while (true) {
std::string requestTypeStr = "[Init]";
try {
auto res = co_await race(applierInterf.heartbeat.getFuture(),
applierInterf.sendMutationVector.getFuture(),
applierInterf.applyToDB.getFuture(),
applierInterf.updateRate.getFuture(),
applierInterf.initVersionBatch.getFuture(),
applierInterf.finishRestore.getFuture(),
actors.getResult(),
exitRole);
if (res.index() == 0) {
RestoreSimpleRequest req = std::get<0>(std::move(res));
requestTypeStr = "heartbeat";
actors.add(handleHeartbeat(req, applierInterf.id()));
} else if (res.index() == 1) {
RestoreSendVersionedMutationsRequest req = std::get<1>(std::move(res));
requestTypeStr = "sendMutationVector";
actors.add(handleSendMutationVectorRequest(req, self));
} else if (res.index() == 2) {
RestoreVersionBatchRequest req = std::get<2>(std::move(res));
requestTypeStr = "applyToDB";
actors.add(handleApplyToDBRequest(
req, self, cx)); // TODO: Check how FDB uses TaskPriority for ACTORS. We may need to add
// priority here to avoid requests at later VB block requests at earlier VBs
} else if (res.index() == 3) {
RestoreUpdateRateRequest req = std::get<3>(std::move(res));
requestTypeStr = "updateRate";
handleUpdateRateRequest(req, self);
} else if (res.index() == 4) {
RestoreVersionBatchRequest req = std::get<4>(std::move(res));
requestTypeStr = "initVersionBatch";
actors.add(handleInitVersionBatchRequest(req, self));
} else if (res.index() == 5) {
RestoreFinishRequest req = std::get<5>(std::move(res));
requestTypeStr = "finishRestore";
actors.clear(false); // cancel all pending actors
handleFinishRestoreRequest(req, self);
if (req.terminate) {
exitRole = Void();
}
} else if (res.index() == 6) {
} else if (res.index() == 7) {
TraceEvent("RestoreApplierCoreExitRole", self->id());
break;
} else {
UNREACHABLE();
}
//TraceEvent("RestoreApplierCore", self->id()).detail("Request", requestTypeStr); // For debug only
} catch (Error& e) {
bool isError = e.code() != error_code_operation_cancelled;
TraceEvent(isError ? SevError : SevWarnAlways, "FastRestoreApplierError", self->id())
.errorUnsuppressed(e)
.detail("RequestType", requestTypeStr);
actors.clear(false);
break;
}
}
}
// The actor may be invoked multiple times and executed async.
// No race condition as long as we do not wait or yield when operate the shared
// data. Multiple such actors can run on different fileIDs.
// Different files may contain mutations of the same commit versions, but with
// different subsequence number.
// Only one actor can process mutations from the same file.
static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
Reference<RestoreApplierData> self) {
if (req.batchIndex <= self->finishedBatch.get()) { // Handle duplicate request from batchIndex that has finished
TraceEvent(SevWarn, "FastRestoreApplierRestoreSendVersionedMutationsRequestTooLate")
.detail("RequestBatchIndex", req.batchIndex)
.detail("FinishedBatchIndex", self->finishedBatch.get());
req.reply.send(RestoreCommonReply(self->id(), true));
ASSERT_WE_THINK(false); // Test to see if simulation can reproduce this
co_return;
}
Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
ASSERT(batchData.isValid());
ASSERT(self->finishedBatch.get() < req.batchIndex);
// wait(delay(0.0, TaskPriority::RestoreApplierReceiveMutations)); // This hurts performance from 100MB/s to 60MB/s
// on circus
batchData->receiveMutationReqs += 1;
// Trace when the receive phase starts at a VB and when it finishes.
// This can help check if receiveMutations block applyMutation phase.
// If so, we need more sophisticated scheduler to ensure priority execution
bool printTrace = (batchData->receiveMutationReqs % SERVER_KNOBS->FASTRESTORE_NUM_TRACE_EVENTS == 0);
TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("RestoreAsset", req.asset.toString())
.detail("RestoreAssetMesssageIndex", batchData->processedFileState[req.asset].get())
.detail("Request", req.toString())
.detail("CurrentMemory", getSystemStatistics().processMemory)
.detail("PreviousVersionBatchState", batchData->vbState.get())
.detail("ReceiveMutationRequests", batchData->receiveMutationReqs);
co_await isSchedulable(self, req.batchIndex, __FUNCTION__);
ASSERT(batchData.isValid());
ASSERT(req.batchIndex > self->finishedBatch.get());
// Assume: processedFileState[req.asset] will not be erased while the actor is active.
// Note: Insert new items into processedFileState will not invalidate the reference.
NotifiedVersion* curMsgIndex = &batchData->processedFileState[req.asset];
co_await curMsgIndex->whenAtLeast(req.msgIndex - 1);
batchData->vbState = ApplierVersionBatchState::RECEIVE_MUTATIONS;
bool isDuplicated = true;
if (curMsgIndex->get() == req.msgIndex - 1) {
isDuplicated = false;
for (int mIndex = 0; mIndex < req.versionedMutations.size(); mIndex++) {
const VersionedMutationSerialized& versionedMutation = req.versionedMutations[mIndex];
TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
.detail("RestoreAsset", req.asset.toString())
.detail("Version", versionedMutation.version.toString())
.detail("Index", mIndex)
.detail("MutationReceived", versionedMutation.mutation.toString());
batchData->receivedBytes += versionedMutation.mutation.totalSize();
batchData->counters.receivedBytes += versionedMutation.mutation.totalSize();
batchData->counters.receivedWeightedBytes +=
versionedMutation.mutation.weightedTotalSize(); // atomicOp will be amplified
batchData->counters.receivedMutations += 1;
batchData->counters.receivedAtomicOps +=
isAtomicOp((MutationRef::Type)versionedMutation.mutation.type) ? 1 : 0;
// Sanity check
ASSERT_WE_THINK(req.asset.isInVersionRange(versionedMutation.version.version));
ASSERT_WE_THINK(req.asset.isInKeyRange(
versionedMutation.mutation)); // mutation is already applied removePrefix and addPrefix
// Note: Log and range mutations may be delivered out of order. Can we handle it?
batchData->addMutation(versionedMutation.mutation, versionedMutation.version);
ASSERT(versionedMutation.mutation.type != MutationRef::SetVersionstampedKey &&
versionedMutation.mutation.type != MutationRef::SetVersionstampedValue);
}
curMsgIndex->set(req.msgIndex);
}
req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutationsDone", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("RestoreAsset", req.asset.toString())
.detail("ProcessedMessageIndex", curMsgIndex->get())
.detail("Request", req.toString());
}
// Clear all ranges in input ranges
static Future<Void> applyClearRangeMutations(Standalone<VectorRef<KeyRangeRef>> ranges,
double delayTime,
Database cx,
UID applierID,
int batchIndex,
ApplierBatchData::Counters* cc) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
int retries = 0;
double numOps = 0;
co_await delay(delayTime + deterministicRandom()->random01() * delayTime);
TraceEvent(delayTime > 5 ? SevWarnAlways : SevDebug, "FastRestoreApplierClearRangeMutationsStart", applierID)
.detail("BatchIndex", batchIndex)
.detail("Ranges", ranges.size())
.detail("DelayTime", delayTime);
if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) {
TraceEvent("FastRestoreApplierClearRangeMutationsNotWriteDB", applierID)
.detail("BatchIndex", batchIndex)
.detail("Ranges", ranges.size());
ASSERT(!g_network->isSimulated());
co_return;
}
while (true) {
Error err;
try {
// TODO: Consider clearrange traffic in write traffic control
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
for (auto& range : ranges) {
debugFRMutation("FastRestoreApplierApplyClearRangeMutation",
0,
MutationRef(MutationRef::ClearRange, range.begin, range.end));
tr->clear(range);
cc->clearOps += 1;
++numOps;
if (numOps >= SERVER_KNOBS->FASTRESTORE_TXN_CLEAR_MAX) {
TraceEvent(SevWarn, "FastRestoreApplierClearRangeMutationsTooManyClearsInTxn")
.suppressFor(5.0)
.detail("Clears", numOps)
.detail("Ranges", ranges.size())
.detail("Range", range.toString());
}
}
co_await tr->commit();
cc->clearTxns += 1;
break;
} catch (Error& e) {
err = e;
}
retries++;
if (retries > SERVER_KNOBS->FASTRESTORE_TXN_RETRY_MAX) {
TraceEvent(SevWarnAlways, "RestoreApplierApplyClearRangeMutationsStuck", applierID)
.error(err)
.detail("BatchIndex", batchIndex)
.detail("ClearRanges", ranges.size());
}
co_await tr->onError(err);
}
}
// Get keys in incompleteStagingKeys and precompute the stagingKey which is stored in batchData->stagingKeys
static Future<Void> getAndComputeStagingKeys(std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys,
double delayTime,
Database cx,
UID applierID,
int batchIndex,
ApplierBatchData::Counters* cc) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
std::vector<Future<Optional<Value>>> fValues(incompleteStagingKeys.size(), Never());
int retries = 0;
UID randomID = deterministicRandom()->randomUniqueID();
co_await delay(delayTime + deterministicRandom()->random01() * delayTime);
if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) { // Get dummy value to short-circut DB
TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStartNotUseDB", applierID)
.detail("RandomUID", randomID)
.detail("BatchIndex", batchIndex)
.detail("GetKeys", incompleteStagingKeys.size())
.detail("DelayTime", delayTime);
ASSERT(!g_network->isSimulated());
for (auto& [stagingKey, stagingKeyIter] : incompleteStagingKeys) {
MutationRef m(MutationRef::SetValue, stagingKey, "0"_sr);
stagingKeyIter->second.add(m, LogMessageVersion(1));
stagingKeyIter->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
}
co_return;
}
TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID)
.detail("RandomUID", randomID)
.detail("BatchIndex", batchIndex)
.detail("GetKeys", incompleteStagingKeys.size())
.detail("DelayTime", delayTime);
while (true) {
Error err;
try {
int i = 0;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
for (auto& [stagingKey, _stagingKeyIter] : incompleteStagingKeys) {
fValues[i++] = tr->get(stagingKey);
cc->fetchKeys += 1;
}
co_await waitForAll(fValues);
cc->fetchTxns += 1;
break;
} catch (Error& e) {
err = e;
}
cc->fetchTxnRetries += 1;
if (retries++ > incompleteStagingKeys.size()) {
if (err.code() != error_code_actor_cancelled) {
TraceEvent(SevWarnAlways, "GetAndComputeStagingKeys", applierID)
.errorUnsuppressed(err)
.suppressFor(1.0)
.detail("RandomUID", randomID)
.detail("BatchIndex", batchIndex);
}
}
co_await tr->onError(err);
}
ASSERT(fValues.size() == incompleteStagingKeys.size());
int i = 0;
for (auto& [stagingKey, stagingKeyIter] : incompleteStagingKeys) {
if (!fValues[i].get().present()) { // Key not exist in DB
// if condition: fValues[i].Valid() && fValues[i].isReady() && !fValues[i].isError() &&
TraceEvent(SevDebug, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID)
.suppressFor(5.0)
.detail("BatchIndex", batchIndex)
.detail("Key", stagingKey)
.detail("IsReady", fValues[i].isReady())
.detail("PendingMutations", stagingKeyIter->second.pendingMutations.size())
.detail("StagingKeyType", getTypeString(stagingKeyIter->second.type));
for (auto& [pendingVersion, pendingMutation] : stagingKeyIter->second.pendingMutations) {
TraceEvent(SevDebug, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB")
.detail("PendingMutationVersion", pendingVersion.toString())
.detail("PendingMutation", pendingMutation.toString());
}
stagingKeyIter->second.precomputeResult("GetAndComputeStagingKeysNoBaseValueInDB", applierID, batchIndex);
} else {
// The key's version ideally should be the most recently committed version.
// But as long as it is > 1 and less than the start version of the version batch, it is the same result.
MutationRef m(MutationRef::SetValue, stagingKey, fValues[i].get().get());
stagingKeyIter->second.add(m, LogMessageVersion(1));
stagingKeyIter->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
}
i++;
}
TraceEvent("FastRestoreApplierGetAndComputeStagingKeysDone", applierID)
.detail("RandomUID", randomID)
.detail("BatchIndex", batchIndex)
.detail("GetKeys", incompleteStagingKeys.size())
.detail("DelayTime", delayTime);
}
static Future<Void> precomputeMutationsResult(Reference<ApplierBatchData> batchData,
UID applierID,
int64_t batchIndex,
Database cx) {
// Apply range mutations (i.e., clearRange) to database cx
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResultStart", applierID)
.detail("BatchIndex", batchIndex)
.detail("Step", "Applying clear range mutations to DB")
.detail("ClearRanges", batchData->stagingKeyRanges.size());
std::vector<Future<Void>> fClearRanges;
Standalone<VectorRef<KeyRangeRef>> clearRanges;
double curTxnSize = 0;
{
double delayTime = 0;
for (auto& rangeMutation : batchData->stagingKeyRanges) {
KeyRangeRef range(rangeMutation.mutation.param1, rangeMutation.mutation.param2);
debugFRMutation("FastRestoreApplierPrecomputeMutationsResultClearRange",
rangeMutation.version.version,
MutationRef(MutationRef::ClearRange, range.begin, range.end));
clearRanges.push_back_deep(clearRanges.arena(), range);
curTxnSize += range.expectedSize();
if (curTxnSize >= SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
fClearRanges.push_back(
applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
clearRanges = Standalone<VectorRef<KeyRangeRef>>();
curTxnSize = 0;
}
}
if (curTxnSize > 0) {
fClearRanges.push_back(
applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
}
}
// Apply range mutations (i.e., clearRange) to stagingKeyRanges
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
.detail("BatchIndex", batchIndex)
.detail("Step", "Applying clear range mutations to staging keys")
.detail("ClearRanges", batchData->stagingKeyRanges.size())
.detail("FutureClearRanges", fClearRanges.size());
for (auto& rangeMutation : batchData->stagingKeyRanges) {
ASSERT(rangeMutation.mutation.param1 <= rangeMutation.mutation.param2);
auto lb = batchData->stagingKeys.lower_bound(rangeMutation.mutation.param1);
auto ub = batchData->stagingKeys.lower_bound(rangeMutation.mutation.param2);
while (lb != ub) {
if (lb->first >= rangeMutation.mutation.param2) {
TraceEvent(SevError, "FastRestoreApplerPhasePrecomputeMutationsResultIncorrectUpperBound")
.detail("Key", lb->first)
.detail("ClearRangeUpperBound", rangeMutation.mutation.param2)
.detail("UsedUpperBound", ub->first);
}
// We make the beginKey = endKey for the ClearRange on purpose so that
// we can sanity check ClearRange mutation when we apply it to DB.
MutationRef clearKey(MutationRef::ClearRange, lb->first, lb->first);
lb->second.add(clearKey, rangeMutation.version);
lb++;
}
}
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
.detail("BatchIndex", batchIndex)
.detail("Step", "Wait on applying clear range mutations to DB")
.detail("FutureClearRanges", fClearRanges.size());
co_await waitForAll(fClearRanges);
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
.detail("BatchIndex", batchIndex)
.detail("Step", "Getting and computing staging keys")
.detail("StagingKeys", batchData->stagingKeys.size());
// Get keys in stagingKeys which does not have a baseline key by reading database cx, and precompute the key's value
std::vector<Future<Void>> fGetAndComputeKeys;
std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys;
auto stagingKeyIter = batchData->stagingKeys.begin();
int numKeysInBatch = 0;
int numGetTxns = 0;
{
double delayTime = 0; // Start transactions at different time to avoid overwhelming FDB.
for (; stagingKeyIter != batchData->stagingKeys.end(); stagingKeyIter++) {
if (!stagingKeyIter->second.hasBaseValue()) {
incompleteStagingKeys.emplace(stagingKeyIter->first, stagingKeyIter);
numKeysInBatch++;
}
if (numKeysInBatch == SERVER_KNOBS->FASTRESTORE_APPLIER_FETCH_KEYS_SIZE) {
fGetAndComputeKeys.push_back(getAndComputeStagingKeys(
incompleteStagingKeys, delayTime, cx, applierID, batchIndex, &batchData->counters));
numGetTxns++;
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
numKeysInBatch = 0;
incompleteStagingKeys.clear();
}
}
if (numKeysInBatch > 0) {
numGetTxns++;
fGetAndComputeKeys.push_back(getAndComputeStagingKeys(
incompleteStagingKeys, delayTime, cx, applierID, batchIndex, &batchData->counters));
}
}
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
.detail("BatchIndex", batchIndex)
.detail("Step", "Compute the other staging keys")
.detail("StagingKeys", batchData->stagingKeys.size())
.detail("GetStagingKeyBatchTxns", numGetTxns);
// Pre-compute pendingMutations to other keys in stagingKeys that has base value
for (stagingKeyIter = batchData->stagingKeys.begin(); stagingKeyIter != batchData->stagingKeys.end();
stagingKeyIter++) {
if (stagingKeyIter->second.hasBaseValue()) {
stagingKeyIter->second.precomputeResult("HasBaseValue", applierID, batchIndex);
}
}
TraceEvent("FastRestoreApplierGetAndComputeStagingKeysWaitOn", applierID).log();
co_await waitForAll(fGetAndComputeKeys);
// Sanity check all stagingKeys have been precomputed
ASSERT_WE_THINK(batchData->allKeysPrecomputed());
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResultDone", applierID).detail("BatchIndex", batchIndex);
}
bool okToReleaseTxns(double targetMB, double applyingDataBytes) {
return applyingDataBytes < targetMB * 1024 * 1024;
}
static Future<Void> shouldReleaseTransaction(double* targetMB, double* applyingDataBytes, AsyncTrigger* releaseTxns) {
while (true) {
if (okToReleaseTxns(*targetMB, *applyingDataBytes)) {
break;
} else {
co_await releaseTxns->onTrigger();
co_await delay(0.0); // Avoid all waiting txns are triggered at the same time and all decide to proceed
// before applyingDataBytes has a chance to update
}
}
}
// Apply mutations in batchData->stagingKeys [begin, end).
static Future<Void> applyStagingKeysBatch(std::map<Key, StagingKey>::iterator begin,
std::map<Key, StagingKey>::iterator end,
Database cx,
UID applierID,
ApplierBatchData::Counters* cc,
double* appliedBytes,
double* applyingDataBytes,
double* targetMB,
AsyncTrigger* releaseTxnTrigger) {
if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) {
TraceEvent("FastRestoreApplierPhaseApplyStagingKeysBatchSkipped", applierID).detail("Begin", begin->first);
ASSERT(!g_network->isSimulated());
co_return;
}
co_await shouldReleaseTransaction(targetMB, applyingDataBytes, releaseTxnTrigger);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
int sets = 0;
int clears = 0;
Key endKey = begin->first;
double txnSize = 0;
double txnSizeUsed = 0; // txn size accounted in applyingDataBytes
TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID).detail("Begin", begin->first);
while (true) {
Error err;
try {
txnSize = 0;
txnSizeUsed = 0;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
auto iter = begin;
while (iter != end) {
if (iter->second.type == MutationRef::SetValue) {
tr->set(iter->second.key, iter->second.val);
txnSize += iter->second.totalSize();
cc->appliedMutations += 1;
TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID)
.detail("SetKey", iter->second.key);
sets++;
} else if (iter->second.type == MutationRef::ClearRange) {
if (iter->second.key != iter->second.val) {
TraceEvent(SevError, "FastRestoreApplierPhaseApplyStagingKeysBatchClearTooMuchData", applierID)
.detail("KeyBegin", iter->second.key)
.detail("KeyEnd", iter->second.val)
.detail("Version", iter->second.version.version)
.detail("SubVersion", iter->second.version.sub);
}
tr->clear(singleKeyRange(iter->second.key));
txnSize += iter->second.totalSize();
cc->appliedMutations += 1;
TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID)
.detail("ClearKey", iter->second.key);
clears++;
} else {
ASSERT(false);
}
endKey = iter != end ? iter->first : endKey;
iter++;
if (sets > 10000000 || clears > 10000000) {
TraceEvent(SevError, "FastRestoreApplierPhaseApplyStagingKeysBatchInfiniteLoop", applierID)
.detail("Begin", begin->first)
.detail("Sets", sets)
.detail("Clears", clears);
}
}
TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseApplyStagingKeysBatchPrecommit", applierID)
.detail("Begin", begin->first)
.detail("End", endKey)
.detail("Sets", sets)
.detail("Clears", clears);
tr->addWriteConflictRange(KeyRangeRef(begin->first, keyAfter(endKey))); // Reduce resolver load
txnSizeUsed = txnSize;
*applyingDataBytes += txnSizeUsed; // Must account for applying bytes before wait for write traffic control
co_await tr->commit();
cc->appliedTxns += 1;
cc->appliedBytes += txnSize;
*appliedBytes += txnSize;
*applyingDataBytes -= txnSizeUsed;
if (okToReleaseTxns(*targetMB, *applyingDataBytes)) {
releaseTxnTrigger->trigger();
}
break;
} catch (Error& e) {
err = e;
}
cc->appliedTxnRetries += 1;
co_await tr->onError(err);
*applyingDataBytes -= txnSizeUsed;
}
}
// Apply mutations in stagingKeys in batches in parallel
static Future<Void> applyStagingKeys(Reference<ApplierBatchData> batchData,
UID applierID,
int64_t batchIndex,
Database cx) {
auto begin = batchData->stagingKeys.begin();
auto cur = begin;
int txnBatches = 0;
double txnSize = 0;
std::vector<Future<Void>> fBatches;
TraceEvent("FastRestoreApplerPhaseApplyStagingKeysStart", applierID)
.detail("BatchIndex", batchIndex)
.detail("StagingKeys", batchData->stagingKeys.size());
batchData->totalBytesToWrite = 0;
while (cur != batchData->stagingKeys.end()) {
txnSize += cur->second.totalSize(); // should be consistent with receivedBytes accounting method
if (txnSize > SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
fBatches.push_back(applyStagingKeysBatch(begin,
cur,
cx,
applierID,
&batchData->counters,
&batchData->appliedBytes,
&batchData->applyingDataBytes,
&batchData->targetWriteRateMB,
&batchData->releaseTxnTrigger));
batchData->totalBytesToWrite += txnSize;
begin = cur;
txnSize = 0;
txnBatches++;
}
cur++;
}
if (begin != batchData->stagingKeys.end()) {
fBatches.push_back(applyStagingKeysBatch(begin,
cur,
cx,
applierID,
&batchData->counters,
&batchData->appliedBytes,
&batchData->applyingDataBytes,
&batchData->targetWriteRateMB,
&batchData->releaseTxnTrigger));
batchData->totalBytesToWrite += txnSize;
txnBatches++;
}
co_await waitForAll(fBatches);
TraceEvent("FastRestoreApplerPhaseApplyStagingKeysDone", applierID)
.detail("BatchIndex", batchIndex)
.detail("StagingKeys", batchData->stagingKeys.size())
.detail("TransactionBatches", txnBatches)
.detail("TotalBytesToWrite", batchData->totalBytesToWrite);
}
// Write mutations to the destination DB
Future<Void> writeMutationsToDB(UID applierID, int64_t batchIndex, Reference<ApplierBatchData> batchData, Database cx) {
TraceEvent("FastRestoreApplierPhaseApplyTxnStart", applierID).detail("BatchIndex", batchIndex);
co_await precomputeMutationsResult(batchData, applierID, batchIndex, cx);
co_await applyStagingKeys(batchData, applierID, batchIndex, cx);
TraceEvent("FastRestoreApplierPhaseApplyTxnDone", applierID)
.detail("BatchIndex", batchIndex)
.detail("AppliedBytes", batchData->appliedBytes)
.detail("ReceivedBytes", batchData->receivedBytes);
}
void handleUpdateRateRequest(RestoreUpdateRateRequest req, Reference<RestoreApplierData> self) {
TraceEvent ev("FastRestoreApplierUpdateRateRequest", self->id());
ev.suppressFor(10)
.detail("BatchIndex", req.batchIndex)
.detail("FinishedBatch", self->finishedBatch.get())
.detail("WriteMB", req.writeMB);
double remainingDataMB = 0;
if (self->finishedBatch.get() == req.batchIndex - 1) { // current applying batch
Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
ASSERT(batchData.isValid());
batchData->targetWriteRateMB = req.writeMB;
remainingDataMB = batchData->totalBytesToWrite > 0
? std::max(0.0, batchData->totalBytesToWrite - batchData->appliedBytes) / 1024 / 1024
: batchData->receivedBytes / 1024 / 1024;
ev.detail("TotalBytesToWrite", batchData->totalBytesToWrite)
.detail("AppliedBytes", batchData->appliedBytes)
.detail("ReceivedBytes", batchData->receivedBytes)
.detail("TargetWriteRateMB", batchData->targetWriteRateMB)
.detail("RemainingDataMB", remainingDataMB);
}
req.reply.send(RestoreUpdateRateReply(self->id(), remainingDataMB));
return;
}
static Future<Void> traceRate(const char* context,
Reference<ApplierBatchData> batchData,
int batchIndex,
UID nodeID,
NotifiedVersion* finishedVB,
bool once = false) {
while (true) {
if ((finishedVB->get() != batchIndex - 1) || !batchData.isValid()) {
break;
}
TraceEvent(context, nodeID)
.suppressFor(10)
.detail("BatchIndex", batchIndex)
.detail("FinishedBatchIndex", finishedVB->get())
.detail("TotalDataToWriteMB", batchData->totalBytesToWrite / 1024 / 1024)
.detail("AppliedBytesMB", batchData->appliedBytes / 1024 / 1024)
.detail("TargetBytesMB", batchData->targetWriteRateMB)
.detail("InflightBytesMB", batchData->applyingDataBytes)
.detail("ReceivedBytes", batchData->receivedBytes);
if (once) {
break;
}
co_await delay(5.0);
}
}
static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req,
Reference<RestoreApplierData> self,
Database cx) {
TraceEvent("FastRestoreApplierPhaseHandleApplyToDBStart", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("FinishedBatch", self->finishedBatch.get());
// Ensure batch (i-1) is applied before batch i
// TODO: Add a counter to warn when too many requests are waiting on the actor
co_await self->finishedBatch.whenAtLeast(req.batchIndex - 1);
bool isDuplicated = true;
if (self->finishedBatch.get() == req.batchIndex - 1) {
// duplicate request from earlier version batch will be ignored
Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
ASSERT(batchData.isValid());
TraceEvent("FastRestoreApplierPhaseHandleApplyToDBRunning", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("FinishedBatch", self->finishedBatch.get())
.detail("HasStarted", batchData->dbApplier.present())
.detail("WroteToDBDone", batchData->dbApplier.present() ? batchData->dbApplier.get().isReady() : 0)
.detail("PreviousVersionBatchState", batchData->vbState.get());
ASSERT(batchData.isValid());
if (!batchData->dbApplier.present()) {
isDuplicated = false;
batchData->dbApplier = Never();
batchData->dbApplier = writeMutationsToDB(self->id(), req.batchIndex, batchData, cx);
batchData->vbState = ApplierVersionBatchState::WRITE_TO_DB;
batchData->rateTracer = traceRate("FastRestoreApplierTransactionRateControl",
batchData,
req.batchIndex,
self->id(),
&self->finishedBatch);
}
ASSERT(batchData->dbApplier.present());
ASSERT(!batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error.
// We cannot blindly retry because it is not idempotent
co_await batchData->dbApplier.get();
// Multiple actors can wait on req.batchIndex-1;
// Avoid setting finishedBatch when finishedBatch > req.batchIndex
if (self->finishedBatch.get() == req.batchIndex - 1) {
batchData->rateTracer = traceRate("FastRestoreApplierTransactionRateControlDone",
batchData,
req.batchIndex,
self->id(),
&self->finishedBatch,
true /*print once*/); // Track the last rate info
self->finishedBatch.set(req.batchIndex);
// self->batch[req.batchIndex]->vbState = ApplierVersionBatchState::DONE;
// Free memory for the version batch
self->batch.erase(req.batchIndex);
if (self->delayedActors > 0) {
self->checkMemory.trigger();
}
}
}
req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
TraceEvent("FastRestoreApplierPhaseHandleApplyToDBDone", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("FinishedBatch", self->finishedBatch.get())
.detail("IsDuplicated", isDuplicated);
}
// Copy from WriteDuringRead.cpp with small modifications
// Not all AtomicOps are handled in this function: SetVersionstampedKey, SetVersionstampedValue, and CompareAndClear
Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type) {
Arena arena;
if (type == MutationRef::AddValue)
return doLittleEndianAdd(existingValue, value, arena);
else if (type == MutationRef::AppendIfFits)
return doAppendIfFits(existingValue, value, arena);
else if (type == MutationRef::And || type == MutationRef::AndV2)
return doAndV2(existingValue, value, arena);
else if (type == MutationRef::Or)
return doOr(existingValue, value, arena);
else if (type == MutationRef::Xor)
return doXor(existingValue, value, arena);
else if (type == MutationRef::Max)
return doMax(existingValue, value, arena);
else if (type == MutationRef::Min || type == MutationRef::MinV2)
return doMinV2(existingValue, value, arena);
else if (type == MutationRef::ByteMin)
return doByteMin(existingValue, value, arena);
else if (type == MutationRef::ByteMax)
return doByteMax(existingValue, value, arena);
else {
TraceEvent(SevError, "ApplyAtomicOpUnhandledType")
.detail("TypeCode", (int)type)
.detail("TypeName", getTypeString(type));
ASSERT(false);
}
return Value();
}

407
fdbserver/restoreworker/RestoreApplier.h
View File

@ -1,407 +0,0 @@
/*
* RestoreApplier.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file declears RestoreApplier interface and actors
#pragma once
#include <sstream>
#include "fdbclient/Atomic.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
#include "fdbrpc/Stats.h"
#include "fdbserver/core/CoordinationInterface.h"
#include "fdbserver/core/MutationTracking.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "RestoreRoleCommon.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type);
// Key whose mutations are buffered on applier.
// key, value, type and version defines the parsed mutation at version.
// pendingMutations has all versioned mutations to be applied.
// Mutations in pendingMutations whose version is below the version in StagingKey can be ignored in applying phase.
struct StagingKey {
Key key; // TODO: Maybe not needed?
Value val;
MutationRef::Type type; // set or clear
LogMessageVersion version; // largest version of set or clear for the key
std::map<LogMessageVersion, Standalone<MutationRef>> pendingMutations; // mutations not set or clear type
explicit StagingKey(Key key) : key(key), type(MutationRef::MAX_ATOMIC_OP), version(0) {}
// Add mutation m at newVersion to stagingKey
// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
void add(const MutationRef& m, LogMessageVersion newVersion) {
ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m)
.detail("SubVersion", version.toString())
.detail("NewSubVersion", newVersion.toString());
if (version == newVersion) {
// This could happen because the same mutation can be present in
// overlapping mutation logs, because new TLogs can copy mutations
// from old generation TLogs (or backup worker is recruited without
// knowning previously saved progress).
ASSERT(type == m.type && key == m.param1 && val == m.param2);
TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m);
return;
}
// newVersion can be smaller than version as different loaders can send
// mutations out of order.
if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
if (m.type == MutationRef::ClearRange) {
// We should only clear this key! Otherwise, it causes side effect to other keys
ASSERT(m.param1 == m.param2);
}
if (version < newVersion) {
DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m)
.detail("SubVersion", version.toString())
.detail("NewSubVersion", newVersion.toString())
.detail("MType", getTypeString(type))
.detail("Key", key)
.detail("Val", val)
.detail("NewMutation", m.toString());
key = m.param1;
val = m.param2;
type = (MutationRef::Type)m.type;
version = newVersion;
}
} else {
auto it = pendingMutations.find(newVersion);
if (it == pendingMutations.end()) {
pendingMutations.emplace(newVersion, m);
} else {
// Duplicated mutation ignored.
// TODO: Add SevError here
TraceEvent("SameVersion")
.detail("Version", version.toString())
.detail("NewVersion", newVersion.toString())
.detail("OldMutation", it->second)
.detail("NewMutation", m);
ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2);
}
}
}
// Precompute the final value of the key.
// TODO: Look at the last LogMessageVersion, if it set or clear, we can ignore the rest of versions.
void precomputeResult(const char* context, UID applierID, int batchIndex) {
TraceEvent(SevFRMutationInfo, "FastRestoreApplierPrecomputeResult", applierID)
.detail("BatchIndex", batchIndex)
.detail("Context", context)
.detail("Version", version.toString())
.detail("Key", key)
.detail("Value", val)
.detail("MType", type < MutationRef::MAX_ATOMIC_OP ? getTypeString(type) : "[Unset]")
.detail("LargestPendingVersion",
(pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString()))
.detail("PendingMutations", pendingMutations.size());
std::map<LogMessageVersion, Standalone<MutationRef>>::iterator lb = pendingMutations.lower_bound(version);
if (lb == pendingMutations.end()) {
return;
}
ASSERT(!pendingMutations.empty());
if (lb->first == version) {
// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
MutationRef m = lb->second;
if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation", applierID)
.detail("BatchIndex", batchIndex)
.detail("Context", context)
.detail("BufferedType", getTypeString(type))
.detail("PendingType", getTypeString(m.type))
.detail("BufferedVal", val.toString())
.detail("PendingVal", m.param2.toString());
}
}
lb++;
}
for (; lb != pendingMutations.end(); lb++) {
MutationRef mutation = lb->second;
if (mutation.type == MutationRef::CompareAndClear) { // Special atomicOp
Arena arena;
Optional<StringRef> inputVal;
if (hasBaseValue()) {
inputVal = val;
}
Optional<ValueRef> retVal = doCompareAndClear(inputVal, mutation.param2, arena);
if (!retVal.present()) {
val = key;
type = MutationRef::ClearRange;
} // else no-op
} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
Optional<StringRef> inputVal;
if (hasBaseValue()) {
inputVal = val;
}
val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
type = MutationRef::SetValue; // Precomputed result should be set to DB.
} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
type = MutationRef::SetValue;
TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet", applierID)
.detail("BatchIndex", batchIndex)
.detail("Context", context)
.detail("MutationType", getTypeString(mutation.type))
.detail("Version", lb->first.toString());
} else {
TraceEvent(SevError, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation", applierID)
.detail("BatchIndex", batchIndex)
.detail("Context", context)
.detail("MutationType", getTypeString(mutation.type))
.detail("Version", lb->first.toString());
}
ASSERT(lb->first > version);
version = lb->first;
}
}
// Does the key has at least 1 set or clear mutation to get the base value
bool hasBaseValue() const {
if (version.version > 0) {
ASSERT(type == MutationRef::SetValue || type == MutationRef::ClearRange);
}
return version.version > 0;
}
// Has all pendingMutations been pre-applied to the val?
bool hasPrecomputed() const {
ASSERT(pendingMutations.empty() || pendingMutations.rbegin()->first >= pendingMutations.begin()->first);
return pendingMutations.empty() || version >= pendingMutations.rbegin()->first;
}
int totalSize() const { return MutationRef::OVERHEAD_BYTES + key.size() + val.size(); }
};
// The range mutation received on applier.
// Range mutations should be applied both to the destination DB and to the StagingKeys
struct StagingKeyRange {
Standalone<MutationRef> mutation;
LogMessageVersion version;
explicit StagingKeyRange(MutationRef m, LogMessageVersion newVersion) : mutation(m), version(newVersion) {}
bool operator<(const StagingKeyRange& rhs) const {
return std::tie(version, mutation.type, mutation.param1, mutation.param2) <
std::tie(rhs.version, rhs.mutation.type, rhs.mutation.param1, rhs.mutation.param2);
}
};
// Applier state in each version batch
class ApplierVersionBatchState : RoleVersionBatchState {
public:
static const int NOT_INIT = 0;
static const int INIT = 1;
static const int RECEIVE_MUTATIONS = 2;
static const int WRITE_TO_DB = 3;
static const int DONE = 4;
static const int INVALID = 5;
explicit ApplierVersionBatchState(int newState) { vbState = newState; }
~ApplierVersionBatchState() override = default;
void operator=(int newState) override { vbState = newState; }
int get() const override { return vbState; }
};
struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
// processedFileState: key: RestoreAsset; value: largest version of mutation received on the applier
std::map<RestoreAsset, NotifiedVersion> processedFileState;
Optional<Future<Void>> dbApplier;
VersionedMutationsMap kvOps; // Mutations at each version
std::map<Key, StagingKey> stagingKeys;
std::set<StagingKeyRange> stagingKeyRanges;
Future<Void> pollMetrics;
RoleVersionBatchState vbState;
long receiveMutationReqs;
// Stats
double receivedBytes; // received mutation size
double appliedBytes; // after coalesce, how many bytes to write to DB
double targetWriteRateMB; // target amount of data outstanding for DB;
double totalBytesToWrite; // total amount of data in bytes to write
double applyingDataBytes; // amount of data in flight of committing
AsyncTrigger releaseTxnTrigger; // trigger to release more txns
Future<Void> rateTracer; // trace transaction rate control info
// Status counters
struct Counters {
CounterCollection cc;
Counter receivedBytes, receivedWeightedBytes, receivedMutations, receivedAtomicOps;
Counter appliedBytes, appliedWeightedBytes, appliedMutations, appliedAtomicOps;
Counter appliedTxns, appliedTxnRetries;
Counter fetchKeys, fetchTxns, fetchTxnRetries; // number of keys to fetch from dest. FDB cluster.
Counter clearOps, clearTxns;
Counters(ApplierBatchData* self, UID applierInterfID, int batchIndex)
: cc("ApplierBatch", applierInterfID.toString() + ":" + std::to_string(batchIndex)),
receivedBytes("ReceivedBytes", cc), receivedWeightedBytes("ReceivedWeightedMutations", cc),
receivedMutations("ReceivedMutations", cc), receivedAtomicOps("ReceivedAtomicOps", cc),
appliedBytes("AppliedBytes", cc), appliedWeightedBytes("AppliedWeightedBytes", cc),
appliedMutations("AppliedMutations", cc), appliedAtomicOps("AppliedAtomicOps", cc),
appliedTxns("AppliedTxns", cc), appliedTxnRetries("AppliedTxnRetries", cc), fetchKeys("FetchKeys", cc),
fetchTxns("FetchTxns", cc), fetchTxnRetries("FetchTxnRetries", cc), clearOps("ClearOps", cc),
clearTxns("ClearTxns", cc) {}
} counters;
void addref() { return ReferenceCounted<ApplierBatchData>::addref(); }
void delref() { return ReferenceCounted<ApplierBatchData>::delref(); }
explicit ApplierBatchData(UID nodeID, int batchIndex)
: vbState(ApplierVersionBatchState::NOT_INIT), receiveMutationReqs(0), receivedBytes(0), appliedBytes(0),
targetWriteRateMB(SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB / SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS),
totalBytesToWrite(-1), applyingDataBytes(0), counters(this, nodeID, batchIndex) {
pollMetrics =
counters.cc.traceCounters(format("FastRestoreApplierMetrics%d", batchIndex),
nodeID,
SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY,
nodeID.toString() + "/RestoreApplierMetrics/" + std::to_string(batchIndex));
TraceEvent("FastRestoreApplierMetricsCreated").detail("Node", nodeID);
}
~ApplierBatchData() {
rateTracer = Void(); // cancel actor
}
void addMutation(MutationRef m, LogMessageVersion ver) {
if (!isRangeMutation(m)) {
auto item = stagingKeys.emplace(m.param1, StagingKey(m.param1));
item.first->second.add(m, ver);
} else {
stagingKeyRanges.insert(StagingKeyRange(m, ver));
}
}
// Return true if all staging keys have been precomputed
bool allKeysPrecomputed() {
for (auto& stagingKey : stagingKeys) {
if (!stagingKey.second.hasPrecomputed()) {
TraceEvent("FastRestoreApplierAllKeysPrecomputedFalse")
.detail("Key", stagingKey.first)
.detail("BufferedVersion", stagingKey.second.version.toString())
.detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first.toString());
return false;
}
}
TraceEvent("FastRestoreApplierAllKeysPrecomputed").log();
return true;
}
void reset() {
kvOps.clear();
dbApplier = Optional<Future<Void>>();
}
void sanityCheckMutationOps() const {
if (kvOps.empty())
return;
ASSERT_WE_THINK(isKVOpsSorted());
ASSERT_WE_THINK(allOpsAreKnown());
}
bool isKVOpsSorted() const {
auto prev = kvOps.begin();
for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
if (prev->first > it->first) {
return false;
}
prev = it;
}
return true;
}
bool allOpsAreKnown() const {
for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
for (auto m = it->second.begin(); m != it->second.end(); ++m) {
if (m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange ||
isAtomicOp((MutationRef::Type)m->type))
continue;
else {
TraceEvent(SevError, "FastRestoreApplier").detail("UnknownMutationType", m->type);
return false;
}
}
}
return true;
}
};
struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreApplierData> {
// Buffer for uncommitted data at ongoing version batches
std::map<int, Reference<ApplierBatchData>> batch;
void addref() { return ReferenceCounted<RestoreApplierData>::addref(); }
void delref() { return ReferenceCounted<RestoreApplierData>::delref(); }
explicit RestoreApplierData(UID applierInterfID, int assignedIndex) {
nodeID = applierInterfID;
nodeIndex = assignedIndex;
// Q: Why do we need to initMetric?
// version.initMetric("RestoreApplier.Version"_sr, cc.id);
role = RestoreRole::Applier;
}
~RestoreApplierData() override = default;
// getVersionBatchState may be called periodically to dump version batch state,
// even when no version batch has been started.
int getVersionBatchState(int batchIndex) const final {
auto item = batch.find(batchIndex);
if (item == batch.end()) { // Batch has not been initialized when we blindly profile the state
return ApplierVersionBatchState::INVALID;
} else {
return item->second->vbState.get();
}
}
void setVersionBatchState(int batchIndex, int vbState) final {
std::map<int, Reference<ApplierBatchData>>::iterator item = batch.find(batchIndex);
ASSERT(item != batch.end());
item->second->vbState = vbState;
}
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreApplierInitVersionBatch", id()).detail("BatchIndex", batchIndex);
batch[batchIndex] = makeReference<ApplierBatchData>(nodeID, batchIndex);
}
void resetPerRestoreRequest() override {
batch.clear();
finishedBatch = NotifiedVersion(0);
}
std::string describeNode() const override {
std::stringstream ss;
ss << "NodeID:" << nodeID.toString() << " nodeIndex:" << nodeIndex;
return ss.str();
}
};
Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx);

364
fdbserver/restoreworker/RestoreCommon.cpp
View File

@ -1,364 +0,0 @@
/*
* RestoreCommon.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file implements the functions defined in RestoreCommon.h
// The functions in this file are copied from BackupAgent
#include "fdbserver/restoreworker/RestoreCommon.h"
// Backup agent header
#include "fdbclient/BackupAgent.h"
#include "fdbclient/BackupContainer.h"
#include "fdbclient/KeyBackedTypes.actor.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "flow/CoroUtils.h"
// Split RestoreConfigFR defined in FileBackupAgent.cpp to declaration in Restore.actor.h and implementation in
// RestoreCommon.cpp
KeyBackedProperty<ERestoreState> RestoreConfigFR::stateEnum() {
return configSpace.pack(__FUNCTION__sr);
}
Future<StringRef> RestoreConfigFR::stateText(Reference<ReadYourWritesTransaction> tr) {
return map(stateEnum().getD(tr), [](ERestoreState s) -> StringRef { return FileBackupAgent::restoreStateText(s); });
}
KeyBackedProperty<Key> RestoreConfigFR::addPrefix() {
return configSpace.pack(__FUNCTION__sr);
}
KeyBackedProperty<Key> RestoreConfigFR::removePrefix() {
return configSpace.pack(__FUNCTION__sr);
}
// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
KeyBackedProperty<KeyRange> RestoreConfigFR::restoreRange() {
return configSpace.pack(__FUNCTION__sr);
}
KeyBackedProperty<std::vector<KeyRange>> RestoreConfigFR::restoreRanges() {
return configSpace.pack(__FUNCTION__sr);
}
KeyBackedProperty<Key> RestoreConfigFR::batchFuture() {
return configSpace.pack(__FUNCTION__sr);
}
KeyBackedProperty<Version> RestoreConfigFR::restoreVersion() {
return configSpace.pack(__FUNCTION__sr);
}
KeyBackedProperty<Reference<IBackupContainer>> RestoreConfigFR::sourceContainer() {
return configSpace.pack(__FUNCTION__sr);
}
// Get the source container as a bare URL, without creating a container instance
KeyBackedProperty<Value> RestoreConfigFR::sourceContainerURL() {
return configSpace.pack("sourceContainer"_sr);
}
// Total bytes written by all log and range restore tasks.
KeyBackedBinaryValue<int64_t> RestoreConfigFR::bytesWritten() {
return configSpace.pack(__FUNCTION__sr);
}
// File blocks that have had tasks created for them by the Dispatch task
KeyBackedBinaryValue<int64_t> RestoreConfigFR::filesBlocksDispatched() {
return configSpace.pack(__FUNCTION__sr);
}
// File blocks whose tasks have finished
KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlocksFinished() {
return configSpace.pack(__FUNCTION__sr);
}
// Total number of files in the fileMap
KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileCount() {
return configSpace.pack(__FUNCTION__sr);
}
// Total number of file blocks in the fileMap
KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlockCount() {
return configSpace.pack(__FUNCTION__sr);
}
Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
return getRestoreRangesOrDefault_impl(this, tr);
}
Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault_impl(RestoreConfigFR* self,
Reference<ReadYourWritesTransaction> tr) {
std::vector<KeyRange> ranges = co_await self->restoreRanges().getD(tr);
if (ranges.empty()) {
KeyRange range = co_await self->restoreRange().getD(tr);
ranges.push_back(range);
}
co_return ranges;
}
KeyBackedSet<RestoreConfigFR::RestoreFile> RestoreConfigFR::fileSet() {
return configSpace.pack(__FUNCTION__sr);
}
Future<bool> RestoreConfigFR::isRunnable(Reference<ReadYourWritesTransaction> tr) {
return map(stateEnum().getD(tr), [](ERestoreState s) -> bool {
return s != ERestoreState::ABORTED && s != ERestoreState::COMPLETED && s != ERestoreState::UNINITIALIZED;
});
}
Future<Void> RestoreConfigFR::logError(Database cx, Error e, std::string const& details, void* taskInstance) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileRestoreErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileRestoreError");
t.error(e).detail("RestoreUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if (e.code() == error_code_key_not_found)
t.backtrace();
return updateErrorInfo(cx, e, details);
}
Key RestoreConfigFR::mutationLogPrefix() {
return uidPrefixKey(applyLogKeys.begin, uid);
}
Key RestoreConfigFR::applyMutationsMapPrefix() {
return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
}
Future<int64_t> RestoreConfigFR::getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
// Both of these are snapshot reads
Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True);
Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True);
co_await (success(beginVal) && success(endVal));
if (!beginVal.get().present() || !endVal.get().present())
co_return 0;
Version beginVersion = BinaryReader::fromStringRef<Version>(beginVal.get().get(), Unversioned());
Version endVersion = BinaryReader::fromStringRef<Version>(endVal.get().get(), Unversioned());
co_return endVersion - beginVersion;
}
Future<int64_t> RestoreConfigFR::getApplyVersionLag(Reference<ReadYourWritesTransaction> tr) {
return getApplyVersionLag_impl(tr, uid);
}
void RestoreConfigFR::initApplyMutations(Reference<ReadYourWritesTransaction> tr, Key addPrefix, Key removePrefix) {
// Set these because they have to match the applyMutations values.
this->addPrefix().set(tr, addPrefix);
this->removePrefix().set(tr, removePrefix);
clearApplyMutationsKeys(tr);
// Initialize add/remove prefix, range version map count and set the map's start key to InvalidVersion
tr->set(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid), addPrefix);
tr->set(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid), removePrefix);
int64_t startCount = 0;
tr->set(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid), StringRef((uint8_t*)&startCount, 8));
Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
tr->set(mapStart, BinaryWriter::toValue<Version>(invalidVersion, Unversioned()));
}
void RestoreConfigFR::clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
// Clear add/remove prefix keys
tr->clear(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid));
tr->clear(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid));
// Clear range version map and count key
tr->clear(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid));
Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
tr->clear(KeyRangeRef(mapStart, strinc(mapStart)));
// Clear any loaded mutations that have not yet been applied
Key mutationPrefix = mutationLogPrefix();
tr->clear(KeyRangeRef(mutationPrefix, strinc(mutationPrefix)));
// Clear end and begin versions (intentionally in this order)
tr->clear(uidPrefixKey(applyMutationsEndRange.begin, uid));
tr->clear(uidPrefixKey(applyMutationsBeginRange.begin, uid));
}
void RestoreConfigFR::setApplyBeginVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
tr->set(uidPrefixKey(applyMutationsBeginRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
}
void RestoreConfigFR::setApplyEndVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
tr->set(uidPrefixKey(applyMutationsEndRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
}
Future<Version> RestoreConfigFR::getApplyEndVersion(Reference<ReadYourWritesTransaction> tr) {
return map(tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid)), [=](Optional<Value> const& value) -> Version {
return value.present() ? BinaryReader::fromStringRef<Version>(value.get(), Unversioned()) : 0;
});
}
// Meng: Change RestoreConfigFR to Reference<RestoreConfigFR> because FastRestore pass the Reference<RestoreConfigFR>
// around
Future<std::string> RestoreConfigFR::getProgress_impl(Reference<RestoreConfigFR> restore,
Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<int64_t> fileCount = restore->fileCount().getD(tr);
Future<int64_t> fileBlockCount = restore->fileBlockCount().getD(tr);
Future<int64_t> fileBlocksDispatched = restore->filesBlocksDispatched().getD(tr);
Future<int64_t> fileBlocksFinished = restore->fileBlocksFinished().getD(tr);
Future<int64_t> bytesWritten = restore->bytesWritten().getD(tr);
Future<StringRef> status = restore->stateText(tr);
Future<Version> lag = restore->getApplyVersionLag(tr);
Future<std::string> tag = restore->tag().getD(tr);
Future<std::pair<std::string, Version>> lastError = restore->lastError().getD(tr);
// restore might no longer be valid after the first wait so make sure it is not needed anymore.
UID uid = restore->getUid();
co_await (success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(lag) && success(tag) &&
success(lastError));
std::string errstr = "None";
if (lastError.get().second != 0)
errstr = format("'%s' %llds ago.\n",
lastError.get().first.c_str(),
(tr->getReadVersion().get() - lastError.get().second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
TraceEvent("FileRestoreProgress")
.detail("RestoreUID", uid)
.detail("Tag", tag.get())
.detail("State", status.get().toString())
.detail("FileCount", fileCount.get())
.detail("FileBlocksFinished", fileBlocksFinished.get())
.detail("FileBlocksTotal", fileBlockCount.get())
.detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
.detail("BytesWritten", bytesWritten.get())
.detail("ApplyLag", lag.get())
.detail("TaskInstance", uintptr_t(restore.getPtr()))
.backtrace();
co_return format(
"Tag: %s UID: %s State: %s Blocks: %lld/%lld BlocksInProgress: %lld Files: %lld BytesWritten: "
"%lld ApplyVersionLag: %lld LastError: %s",
tag.get().c_str(),
uid.toString().c_str(),
status.get().toString().c_str(),
fileBlocksFinished.get(),
fileBlockCount.get(),
fileBlocksDispatched.get() - fileBlocksFinished.get(),
fileCount.get(),
bytesWritten.get(),
lag.get(),
errstr.c_str());
}
Future<std::string> RestoreConfigFR::getProgress(Reference<ReadYourWritesTransaction> tr) {
Reference<RestoreConfigFR> restore = Reference<RestoreConfigFR>(this);
return getProgress_impl(restore, tr);
}
// Meng: Change RestoreConfigFR to Reference<RestoreConfigFR>
Future<std::string> RestoreConfigFR::getFullStatus_impl(Reference<RestoreConfigFR> restore,
Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<std::vector<KeyRange>> ranges = restore->getRestoreRangesOrDefault(tr);
Future<Key> addPrefix = restore->addPrefix().getD(tr);
Future<Key> removePrefix = restore->removePrefix().getD(tr);
Future<Key> url = restore->sourceContainerURL().getD(tr);
Future<Version> restoreVersion = restore->restoreVersion().getD(tr);
Future<std::string> progress = restore->getProgress(tr);
// restore might no longer be valid after the first wait so make sure it is not needed anymore.
co_await (success(ranges) && success(addPrefix) && success(removePrefix) && success(url) &&
success(restoreVersion) && success(progress));
std::string returnStr;
returnStr = format("%s URL: %s", progress.get().c_str(), url.get().toString().c_str());
for (auto& range : ranges.get()) {
returnStr += format(" Range: '%s'-'%s'", printable(range.begin).c_str(), printable(range.end).c_str());
}
returnStr += format(" AddPrefix: '%s' RemovePrefix: '%s' Version: %lld",
printable(addPrefix.get()).c_str(),
printable(removePrefix.get()).c_str(),
restoreVersion.get());
co_return returnStr;
}
Future<std::string> RestoreConfigFR::getFullStatus(Reference<ReadYourWritesTransaction> tr) {
Reference<RestoreConfigFR> restore = Reference<RestoreConfigFR>(this);
return getFullStatus_impl(restore, tr);
}
std::string RestoreConfigFR::toString() {
std::stringstream ss;
ss << "uid:" << uid.toString() << " prefix:" << subspace.key().contents().toString();
return ss.str();
}
// parallelFileRestore is copied from FileBackupAgent.cpp for the same reason as RestoreConfigFR is copied
// The implementation of parallelFileRestore is copied from FileBackupAgent.cpp
// parallelFileRestore is copied from FileBackupAgent.cpp for the same reason as RestoreConfigFR is copied
namespace parallelFileRestore {
Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len) {
Standalone<StringRef> buf = makeString(len);
int rLen = co_await file->read(mutateString(buf), len, offset);
if (rLen != len)
throw restore_bad_read();
simulateBlobFailure();
Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
StringRefReader reader(buf, restore_corrupted_data());
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
const uint8_t* k = reader.consume(kLen);
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t* v = reader.consume(vLen);
results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
}
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder())
if (b != 0xFF)
throw restore_corrupted_data_padding();
co_return results;
} catch (Error& e) {
TraceEvent(SevError, "FileRestoreCorruptLogFileBlock")
.error(e)
.detail("Filename", file->getFilename())
.detail("BlockOffset", offset)
.detail("BlockLen", len)
.detail("ErrorRelativeOffset", reader.rptr - buf.begin())
.detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
throw;
}
}
} // namespace parallelFileRestore

1199
fdbserver/restoreworker/RestoreController.cpp
View File

File diff suppressed because it is too large Load Diff

456
fdbserver/restoreworker/RestoreController.h
View File

@ -1,456 +0,0 @@
/*
* RestoreController.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file declear RestoreController interface and actors
#pragma once
#include <sstream>
#include "flow/Platform.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
#include "fdbrpc/Stats.h"
#include "fdbserver/core/CoordinationInterface.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "RestoreRoleCommon.h"
struct RestoreWorkerData;
struct VersionBatch {
Version beginVersion; // Inclusive
Version endVersion; // exclusive
std::set<RestoreFileFR> logFiles;
std::set<RestoreFileFR> rangeFiles;
double size; // size of data in range and log files
int batchIndex; // Never reset
VersionBatch() : beginVersion(0), endVersion(0), size(0) {};
bool operator<(const VersionBatch& rhs) const {
return std::tie(batchIndex, beginVersion, endVersion, logFiles, rangeFiles, size) <
std::tie(rhs.batchIndex, rhs.beginVersion, rhs.endVersion, rhs.logFiles, rhs.rangeFiles, rhs.size);
}
bool isEmpty() const { return logFiles.empty() && rangeFiles.empty(); }
void reset() {
beginVersion = 0;
endVersion = 0;
logFiles.clear();
rangeFiles.clear();
size = 0;
}
// RestoreAsset and VersionBatch both use endVersion as exclusive in version range
bool isInVersionRange(Version version) const { return version >= beginVersion && version < endVersion; }
};
struct ControllerBatchData : public ReferenceCounted<ControllerBatchData> {
// rangeToApplier is in controller and loader node. Loader uses this to determine which applier a mutation should be
// sent.
// KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Key, UID> rangeToApplier;
Optional<Future<Void>> applyToDB;
IndexedSet<Key, int64_t> samples; // sample of range and log files
double samplesSize; // sum of the metric of all samples
std::set<UID> sampleMsgs; // deduplicate sample messages
ControllerBatchData() = default;
~ControllerBatchData() = default;
// Return true if pass the sanity check
bool sanityCheckApplierKeyRange() {
bool ret = true;
// An applier should only appear once in rangeToApplier
std::map<UID, Key> applierToRange;
for (auto& applier : rangeToApplier) {
if (applierToRange.find(applier.second) == applierToRange.end()) {
applierToRange[applier.second] = applier.first;
} else {
TraceEvent(SevError, "FastRestoreController")
.detail("SanityCheckApplierKeyRange", applierToRange.size())
.detail("ApplierID", applier.second)
.detail("Key1", applierToRange[applier.second])
.detail("Key2", applier.first);
ret = false;
}
}
return ret;
}
void logApplierKeyRange(int batchIndex) {
TraceEvent("FastRestoreLogApplierKeyRange")
.detail("BatchIndex", batchIndex)
.detail("ApplierKeyRangeNum", rangeToApplier.size());
for (auto& applier : rangeToApplier) {
TraceEvent("FastRestoreLogApplierKeyRange")
.detail("BatchIndex", batchIndex)
.detail("KeyRangeLowerBound", applier.first)
.detail("Applier", applier.second);
}
}
};
enum class RestoreAssetStatus { Loading, Loaded };
enum class RestoreSendStatus { SendingLogs, SendedLogs, SendingRanges, SendedRanges };
enum class RestoreApplyStatus { Applying, Applied };
// Track restore progress of each RestoreAsset (RA) and
// Use status to sanity check restore property, e.g., each RA should be processed exactly once.
struct ControllerBatchStatus : public ReferenceCounted<ControllerBatchStatus> {
std::map<RestoreAsset, RestoreAssetStatus> raStatus;
std::map<UID, RestoreSendStatus> loadStatus;
std::map<UID, RestoreApplyStatus> applyStatus;
void addref() { return ReferenceCounted<ControllerBatchStatus>::addref(); }
void delref() { return ReferenceCounted<ControllerBatchStatus>::delref(); }
ControllerBatchStatus() = default;
~ControllerBatchStatus() = default;
};
struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreControllerData> {
std::map<Version, VersionBatch> versionBatches; // key is the beginVersion of the version batch
Reference<IBackupContainer> bc; // Backup container is used to read backup files
Key bcUrl; // The url used to get the bc
std::map<int, Reference<ControllerBatchData>> batch;
std::map<int, Reference<ControllerBatchStatus>> batchStatus;
AsyncVar<int> runningVersionBatches; // Currently running version batches
std::map<UID, double> rolesHeartBeatTime; // Key: role id; Value: most recent time controller receives heart beat
// addActor: add to actorCollection so that when an actor has error, the ActorCollection can catch the error.
// addActor is used to create the actorCollection when the RestoreController is created
PromiseStream<Future<Void>> addActor;
void addref() { return ReferenceCounted<RestoreControllerData>::addref(); }
void delref() { return ReferenceCounted<RestoreControllerData>::delref(); }
explicit(false) RestoreControllerData(UID interfId) {
role = RestoreRole::Controller;
nodeID = interfId;
runningVersionBatches.set(0);
}
~RestoreControllerData() override = default;
int getVersionBatchState(int batchIndex) const final { return RoleVersionBatchState::INVALID; }
void setVersionBatchState(int batchIndex, int vbState) final {}
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreControllerInitVersionBatch", id()).detail("VersionBatchIndex", batchIndex);
}
// Reset controller data at the beginning of each restore request
void resetPerRestoreRequest() override {
TraceEvent("FastRestoreControllerReset").detail("OldVersionBatches", versionBatches.size());
versionBatches.clear();
batch.clear();
batchStatus.clear();
finishedBatch = NotifiedVersion(0);
versionBatchId = NotifiedVersion(0);
ASSERT(runningVersionBatches.get() == 0);
}
std::string describeNode() const override {
std::stringstream ss;
ss << "Controller";
return ss.str();
}
void dumpVersionBatches(const std::map<Version, VersionBatch>& versionBatches) const {
int i = 1;
double rangeFiles = 0;
double rangeSize = 0;
double logFiles = 0;
double logSize = 0;
for (auto& vb : versionBatches) {
TraceEvent("FastRestoreVersionBatches")
.detail("BatchIndex", vb.second.batchIndex)
.detail("ExpectedBatchIndex", i)
.detail("BeginVersion", vb.second.beginVersion)
.detail("EndVersion", vb.second.endVersion)
.detail("Size", vb.second.size);
for (auto& f : vb.second.rangeFiles) {
bool invalidVersion = (f.beginVersion != f.endVersion) || (f.beginVersion >= vb.second.endVersion ||
f.beginVersion < vb.second.beginVersion);
TraceEvent(invalidVersion ? SevError : SevInfo, "FastRestoreVersionBatches")
.detail("BatchIndex", i)
.detail("RangeFile", f.toString());
rangeSize += f.fileSize;
rangeFiles++;
}
for (auto& f : vb.second.logFiles) {
bool outOfRange = (f.beginVersion >= vb.second.endVersion || f.endVersion <= vb.second.beginVersion);
TraceEvent(outOfRange ? SevError : SevInfo, "FastRestoreVersionBatches")
.detail("BatchIndex", i)
.detail("LogFile", f.toString());
logSize += f.fileSize;
logFiles++;
}
++i;
}
TraceEvent("FastRestoreVersionBatchesSummary")
.detail("VersionBatches", versionBatches.size())
.detail("LogFiles", logFiles)
.detail("RangeFiles", rangeFiles)
.detail("LogBytes", logSize)
.detail("RangeBytes", rangeSize);
}
// Input: Get the size of data in backup files in version range [prevVersion, nextVersion)
// Return: param1: the size of data at nextVersion, param2: the minimum range file index whose version >
// nextVersion, param3: log files with data in [prevVersion, nextVersion)
std::tuple<double, int, std::vector<RestoreFileFR>> getVersionSize(Version prevVersion,
Version nextVersion,
const std::vector<RestoreFileFR>& rangeFiles,
int rangeIdx,
const std::vector<RestoreFileFR>& logFiles) {
double size = 0;
TraceEvent(SevVerbose, "FastRestoreGetVersionSize")
.detail("PreviousVersion", prevVersion)
.detail("NextVersion", nextVersion)
.detail("RangeFiles", rangeFiles.size())
.detail("RangeIndex", rangeIdx)
.detail("LogFiles", logFiles.size());
ASSERT(prevVersion <= nextVersion);
while (rangeIdx < rangeFiles.size()) {
TraceEvent(SevVerbose, "FastRestoreGetVersionSize").detail("RangeFile", rangeFiles[rangeIdx].toString());
if (rangeFiles[rangeIdx].version < nextVersion) {
ASSERT(rangeFiles[rangeIdx].version >= prevVersion);
size += rangeFiles[rangeIdx].fileSize;
} else {
break;
}
++rangeIdx;
}
std::vector<RestoreFileFR> retLogs;
// Scan all logFiles every time to avoid assumption on log files' version ranges.
// For example, we do not assume each version range only exists in one log file
for (const auto& file : logFiles) {
Version begin = std::max(prevVersion, file.beginVersion);
Version end = std::min(nextVersion, file.endVersion);
if (begin < end) { // logIdx file overlap in [prevVersion, nextVersion)
double ratio = (end - begin) * 1.0 / (file.endVersion - file.beginVersion);
size += file.fileSize * ratio;
retLogs.push_back(file);
}
}
return std::make_tuple(size, rangeIdx, retLogs);
}
// Split backup files into version batches, each of which has similar data size
// Input: sorted range files, sorted log files;
// Output: a set of version batches whose size is less than SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES
// and each mutation in backup files is included in the version batches exactly once.
// Assumption 1: input files has no empty files;
// Assumption 2: range files at one version <= FASTRESTORE_VERSIONBATCH_MAX_BYTES.
// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range
// file size at a version depends on the number of backupAgents and its upper bound is hard to get.
void buildVersionBatches(const std::vector<RestoreFileFR>& rangeFiles,
const std::vector<RestoreFileFR>& logFiles,
std::map<Version, VersionBatch>* versionBatches,
Version targetVersion) {
bool rewriteNextVersion = false;
int rangeIdx = 0;
int logIdx = 0; // Ensure each log file is included in version batch
Version prevEndVersion = 0;
Version nextVersion = 0; // Used to calculate the batch's endVersion
VersionBatch vb;
Version maxVBVersion = 0;
bool lastLogFile = false;
vb.beginVersion = 0; // Version batch range [beginVersion, endVersion)
vb.batchIndex = 1;
while (rangeIdx < rangeFiles.size() || logIdx < logFiles.size()) {
if (!rewriteNextVersion) {
if (rangeIdx < rangeFiles.size() && logIdx < logFiles.size()) {
// nextVersion as endVersion is exclusive in the version range
nextVersion = std::max(rangeFiles[rangeIdx].version + 1, nextVersion);
} else if (rangeIdx < rangeFiles.size()) { // i.e., logIdx >= logFiles.size()
nextVersion = rangeFiles[rangeIdx].version + 1;
} else if (logIdx < logFiles.size()) {
while (logIdx < logFiles.size() && logFiles[logIdx].endVersion <= nextVersion) {
logIdx++;
}
if (logIdx < logFiles.size()) {
nextVersion = logFiles[logIdx].endVersion;
} else {
TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
.detail("FinishAllLogFiles", logIdx)
.detail("CurBatchIndex", vb.batchIndex)
.detail("CurBatchSize", vb.size);
if (prevEndVersion < nextVersion) {
// Ensure the last log file is included in version batch
lastLogFile = true;
} else {
break; // Finished all log files
}
}
} else {
// TODO: Check why this may happen?!
TraceEvent(SevError, "FastRestoreBuildVersionBatch")
.detail("RangeIndex", rangeIdx)
.detail("RangeFiles", rangeFiles.size())
.detail("LogIndex", logIdx)
.detail("LogFiles", logFiles.size());
}
} else {
rewriteNextVersion = false;
}
double nextVersionSize;
int nextRangeIdx;
std::vector<RestoreFileFR> curLogFiles;
std::tie(nextVersionSize, nextRangeIdx, curLogFiles) =
getVersionSize(prevEndVersion, nextVersion, rangeFiles, rangeIdx, logFiles);
TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
.detail("BatchIndex", vb.batchIndex)
.detail("VersionBatchBeginVersion", vb.beginVersion)
.detail("PreviousEndVersion", prevEndVersion)
.detail("NextVersion", nextVersion)
.detail("TargetVersion", targetVersion)
.detail("RangeIndex", rangeIdx)
.detail("RangeFiles", rangeFiles.size())
.detail("LogIndex", logIdx)
.detail("LogFiles", logFiles.size())
.detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
.detail("CurrentBatchSize", vb.size)
.detail("NextVersionIntervalSize", nextVersionSize)
.detail("NextRangeIndex", nextRangeIdx)
.detail("UsedLogFiles", curLogFiles.size())
.detail("VersionBatchCurRangeFiles", vb.rangeFiles.size())
.detail("VersionBatchCurLogFiles", vb.logFiles.size())
.detail("LastLogFile", lastLogFile);
ASSERT(prevEndVersion < nextVersion); // Ensure progress
if (vb.size + nextVersionSize <= SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES ||
(vb.size < 1 && prevEndVersion + 1 == nextVersion) || lastLogFile) {
// In case the batch size at a single version > FASTRESTORE_VERSIONBATCH_MAX_BYTES,
// the version batch should include the single version to avoid false positive in simulation.
if (vb.size + nextVersionSize > SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES) {
TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "FastRestoreBuildVersionBatch")
.detail("NextVersion", nextVersion)
.detail("PreviousEndVersion", prevEndVersion)
.detail("NextVersionIntervalSize", nextVersionSize)
.detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
.detail("SuggestedMinimumVersionBatchSizeThreshold", nextVersionSize * 2);
}
// nextVersion should be included in this batch
vb.size += nextVersionSize;
while (rangeIdx < nextRangeIdx && rangeIdx < rangeFiles.size()) {
ASSERT(rangeFiles[rangeIdx].fileSize > 0);
vb.rangeFiles.insert(rangeFiles[rangeIdx]);
++rangeIdx;
}
for (auto& log : curLogFiles) {
ASSERT(log.beginVersion < nextVersion);
ASSERT(log.endVersion > prevEndVersion);
ASSERT(log.fileSize > 0);
vb.logFiles.insert(log);
}
vb.endVersion = std::min(nextVersion, targetVersion + 1);
maxVBVersion = std::max(maxVBVersion, vb.endVersion);
prevEndVersion = vb.endVersion;
} else {
if (vb.size < 1) {
// [vb.endVersion, nextVersion) > SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES. We should split
// the version range
if (prevEndVersion >= nextVersion) {
// If range files at one version > FASTRESTORE_VERSIONBATCH_MAX_BYTES, DBA should increase
// FASTRESTORE_VERSIONBATCH_MAX_BYTES to some value larger than nextVersion
TraceEvent(SevError, "FastRestoreBuildVersionBatch")
.detail("NextVersion", nextVersion)
.detail("PreviousEndVersion", prevEndVersion)
.detail("NextVersionIntervalSize", nextVersionSize)
.detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
.detail("SuggestedMinimumVersionBatchSizeThreshold", nextVersionSize * 2);
// Exit restore early if it won't succeed
flushAndExit(FDB_EXIT_ERROR);
}
ASSERT(prevEndVersion < nextVersion); // Ensure progress
nextVersion = (prevEndVersion + nextVersion) / 2;
rewriteNextVersion = true;
TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
.detail("NextVersionIntervalSize", nextVersionSize); // Duplicate Trace
continue;
}
// Finalize the current version batch
versionBatches->emplace(vb.beginVersion, vb); // copy vb to versionBatch
TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
.detail("FinishBatchIndex", vb.batchIndex)
.detail("VersionBatchBeginVersion", vb.beginVersion)
.detail("VersionBatchEndVersion", vb.endVersion)
.detail("VersionBatchLogFiles", vb.logFiles.size())
.detail("VersionBatchRangeFiles", vb.rangeFiles.size())
.detail("VersionBatchSize", vb.size)
.detail("RangeIndex", rangeIdx)
.detail("LogIndex", logIdx)
.detail("NewVersionBatchBeginVersion", prevEndVersion)
.detail("RewriteNextVersion", rewriteNextVersion);
// start finding the next version batch
vb.reset();
vb.size = 0;
vb.beginVersion = prevEndVersion;
vb.batchIndex++;
}
}
// The last wip version batch has some files
if (vb.size > 0) {
vb.endVersion = std::min(nextVersion, targetVersion + 1);
maxVBVersion = std::max(maxVBVersion, vb.endVersion);
versionBatches->emplace(vb.beginVersion, vb);
}
// Invariant: The last vb endverion should be no smaller than targetVersion
if (maxVBVersion < targetVersion) {
// Q: Is the restorable version always less than the maximum version from all backup filenames?
// A: This is true for the raw backup files returned by backup container before we remove the empty files.
TraceEvent(SevWarnAlways, "FastRestoreBuildVersionBatch")
.detail("TargetVersion", targetVersion)
.detail("MaxVersionBatchVersion", maxVBVersion);
}
}
void initBackupContainer(Key url, Optional<std::string> proxy) {
if (bcUrl == url && bc.isValid()) {
return;
}
TraceEvent("FastRestoreControllerInitBackupContainer")
.detail("URL", url)
.detail("Proxy", proxy.present() ? proxy.get() : "");
bcUrl = url;
bc = IBackupContainer::openContainer(url.toString(), proxy, {});
}
};
Future<Void> startRestoreController(Reference<RestoreWorkerData> controllerWorker, Database cx);

1583
fdbserver/restoreworker/RestoreLoader.cpp
View File

File diff suppressed because it is too large Load Diff

233
fdbserver/restoreworker/RestoreLoader.h
View File

@ -1,233 +0,0 @@
/*
* RestoreLoader.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file declares the functions used by the RestoreLoader role
#pragma once
#include <sstream>
#include "fdbclient/FDBTypes.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Stats.h"
#include "fdbserver/core/CoordinationInterface.h"
#include "fdbrpc/Locality.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "RestoreRoleCommon.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
#include "fdbclient/BackupContainer.h"
class LoaderVersionBatchState : RoleVersionBatchState {
public:
static const int NOT_INIT = 0;
static const int INIT = 1;
static const int LOAD_FILE = 2;
static const int SEND_MUTATIONS = 3;
static const int INVALID = 4;
explicit LoaderVersionBatchState(int newState) { vbState = newState; }
~LoaderVersionBatchState() override = default;
void operator=(int newState) override { vbState = newState; }
int get() const override { return vbState; }
};
struct LoaderBatchData : public ReferenceCounted<LoaderBatchData> {
std::map<LoadingParam, Future<Void>> processedFileParams;
std::map<LoadingParam, VersionedMutationsMap> kvOpsPerLP; // Buffered kvOps for each loading param
// rangeToApplier is in controller and loader. Loader uses this to determine which applier a mutation should be sent
// Key is the inclusive lower bound of the key range the applier (UID) is responsible for
std::map<Key, UID> rangeToApplier;
// Sampled mutations to be sent back to restore controller
std::map<LoadingParam, SampledMutationsVec> sampleMutations;
int numSampledMutations; // The total number of mutations received from sampled data.
Future<Void> pollMetrics;
LoaderVersionBatchState vbState;
long loadFileReqs;
// Status counters
struct Counters {
CounterCollection cc;
Counter loadedRangeBytes, loadedLogBytes, sentBytes;
Counter sampledRangeBytes, sampledLogBytes;
Counter oldLogMutations;
Counters(LoaderBatchData* self, UID loaderInterfID, int batchIndex)
: cc("LoaderBatch", loaderInterfID.toString() + ":" + std::to_string(batchIndex)),
loadedRangeBytes("LoadedRangeBytes", cc), loadedLogBytes("LoadedLogBytes", cc), sentBytes("SentBytes", cc),
sampledRangeBytes("SampledRangeBytes", cc), sampledLogBytes("SampledLogBytes", cc),
oldLogMutations("OldLogMutations", cc) {}
} counters;
explicit LoaderBatchData(UID nodeID, int batchIndex)
: vbState(LoaderVersionBatchState::NOT_INIT), loadFileReqs(0), counters(this, nodeID, batchIndex) {
pollMetrics =
counters.cc.traceCounters(format("FastRestoreLoaderMetrics%d", batchIndex),
nodeID,
SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY,
nodeID.toString() + "/RestoreLoaderMetrics/" + std::to_string(batchIndex));
TraceEvent("FastRestoreLoaderMetricsCreated").detail("Node", nodeID);
}
void reset() {
processedFileParams.clear();
kvOpsPerLP.clear();
sampleMutations.clear();
numSampledMutations = 0;
rangeToApplier.clear();
}
};
using LoaderCounters = LoaderBatchData::Counters;
struct LoaderBatchStatus : public ReferenceCounted<LoaderBatchStatus> {
Optional<Future<Void>> sendAllRanges;
Optional<Future<Void>> sendAllLogs;
void addref() { return ReferenceCounted<LoaderBatchStatus>::addref(); }
void delref() { return ReferenceCounted<LoaderBatchStatus>::delref(); }
std::string toString() const {
std::stringstream ss;
ss << "sendAllRanges: "
<< (!sendAllRanges.present() ? "invalid" : (sendAllRanges.get().isReady() ? "ready" : "notReady"))
<< " sendAllLogs: "
<< (!sendAllLogs.present() ? "invalid" : (sendAllLogs.get().isReady() ? "ready" : "notReady"));
return ss.str();
}
};
// Each request for each loadingParam, so that scheduler can control which requests in which version batch to send first
struct RestoreLoaderSchedSendLoadParamRequest {
int batchIndex;
Promise<Void> toSched;
double start;
explicit RestoreLoaderSchedSendLoadParamRequest(int batchIndex, Promise<Void> toSched, double start)
: batchIndex(batchIndex), toSched(toSched), start(start) {};
RestoreLoaderSchedSendLoadParamRequest() = default;
bool operator<(RestoreLoaderSchedSendLoadParamRequest const& rhs) const {
return batchIndex > rhs.batchIndex || (batchIndex == rhs.batchIndex && start > rhs.start);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreLoaderSchedSendLoadParamRequest: " << " batchIndex:" << batchIndex
<< " toSchedFutureIsReady:" << toSched.getFuture().isReady() << " start:" << start;
return ss.str();
}
};
struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoaderData> {
// buffered data per version batch
std::map<int, Reference<LoaderBatchData>> batch;
std::map<int, Reference<LoaderBatchStatus>> status;
RestoreControllerInterface ci;
KeyRangeMap<Version> rangeVersions;
Reference<IBackupContainer> bc; // Backup container is used to read backup files
Key bcUrl; // The url used to get the bc
// Request scheduler
std::priority_queue<RestoreLoadFileRequest> loadingQueue; // request queue of loading files
std::priority_queue<RestoreSendMutationsToAppliersRequest>
sendingQueue; // request queue of sending mutations to appliers
std::priority_queue<RestoreLoaderSchedSendLoadParamRequest> sendLoadParamQueue;
int finishedLoadingVB; // the max version batch index that finished loading file phase
int finishedSendingVB; // the max version batch index that finished sending mutations phase
int inflightSendingReqs; // number of sendingMutations requests released
int inflightLoadingReqs; // number of load backup file requests released
std::map<int, int> inflightSendLoadParamReqs; // key: batchIndex, value: inflightSendLoadParamReqs
Reference<AsyncVar<bool>> hasPendingRequests; // are there pending requests for loader
// addActor: add to actorCollection so that when an actor has error, the ActorCollection can catch the error.
// addActor is used to create the actorCollection when the RestoreController is created
PromiseStream<Future<Void>> addActor;
void addref() { return ReferenceCounted<RestoreLoaderData>::addref(); }
void delref() { return ReferenceCounted<RestoreLoaderData>::delref(); }
explicit RestoreLoaderData(UID loaderInterfID, int assignedIndex, RestoreControllerInterface ci)
: ci(ci), finishedLoadingVB(0), finishedSendingVB(0), inflightSendingReqs(0), inflightLoadingReqs(0) {
nodeID = loaderInterfID;
nodeIndex = assignedIndex;
role = RestoreRole::Loader;
hasPendingRequests = makeReference<AsyncVar<bool>>(false);
}
~RestoreLoaderData() override = default;
std::string describeNode() const override {
std::stringstream ss;
ss << "[Role: Loader] [NodeID:" << nodeID.toString().c_str() << "] [NodeIndex:" << std::to_string(nodeIndex)
<< "]";
return ss.str();
}
int getVersionBatchState(int batchIndex) const final {
auto item = batch.find(batchIndex);
if (item == batch.end()) { // Batch has not been initialized when we blindly profile the state
return LoaderVersionBatchState::INVALID;
} else {
return item->second->vbState.get();
}
}
void setVersionBatchState(int batchIndex, int vbState) final {
std::map<int, Reference<LoaderBatchData>>::iterator item = batch.find(batchIndex);
ASSERT(item != batch.end());
item->second->vbState = vbState;
}
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreLoaderInitVersionBatch", nodeID).detail("BatchIndex", batchIndex);
batch[batchIndex] = makeReference<LoaderBatchData>(nodeID, batchIndex);
status[batchIndex] = makeReference<LoaderBatchStatus>();
}
void resetPerRestoreRequest() override {
batch.clear();
status.clear();
finishedBatch = NotifiedVersion(0);
}
void initBackupContainer(Key url, Optional<std::string> proxy) {
if (bcUrl == url && bc.isValid()) {
return;
}
bcUrl = url;
bc = IBackupContainer::openContainer(url.toString(), proxy, {});
}
};
Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
int nodeIndex,
Database cx,
RestoreControllerInterface ci);

196
fdbserver/restoreworker/RestoreRoleCommon.cpp
View File

@ -1,196 +0,0 @@
/*
* RestoreRoleCommon.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "RestoreRoleCommon.h"
#include "RestoreLoader.h"
#include "RestoreApplier.h"
#include "RestoreController.h"
class Database;
struct RestoreWorkerData;
// id is the id of the worker to be monitored
// This actor is used for both restore loader and restore applier
Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id) {
co_await delayJittered(5.0); // Random jitter reduces heat beat monitor's pressure
req.reply.send(RestoreCommonReply(id));
}
void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self) {
self->resetPerRestoreRequest();
TraceEvent("FastRestoreRolePhaseFinishRestoreRequest", self->id())
.detail("FinishRestoreRequest", req.terminate)
.detail("Role", getRoleStr(self->role));
req.reply.send(RestoreCommonReply(self->id()));
}
// Multiple version batches may execute in parallel and init their version batches
Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self) {
TraceEvent("FastRestoreRolePhaseInitVersionBatch", self->id())
.detail("BatchIndex", req.batchIndex)
.detail("Role", getRoleStr(self->role))
.detail("VersionBatchNotifiedVersion", self->versionBatchId.get());
// Loader destroy batchData once the batch finishes and self->finishedBatch.set(req.batchIndex);
ASSERT(self->finishedBatch.get() < req.batchIndex);
// batchId is continuous. (req.batchIndex-1) is the id of the just finished batch.
co_await self->versionBatchId.whenAtLeast(req.batchIndex - 1);
if (self->versionBatchId.get() == req.batchIndex - 1) {
self->initVersionBatch(req.batchIndex);
self->setVersionBatchState(req.batchIndex, ApplierVersionBatchState::INIT);
TraceEvent("FastRestoreInitVersionBatch")
.detail("BatchIndex", req.batchIndex)
.detail("Role", getRoleStr(self->role))
.detail("Node", self->id());
self->versionBatchId.set(req.batchIndex);
}
req.reply.send(RestoreCommonReply(self->id()));
}
void updateProcessStats(Reference<RestoreRoleData> self) {
if (g_network->isSimulated()) {
// memUsage and cpuUsage are not relevant in the simulator,
// and relying on the actual values could break seed determinism
if (deterministicRandom()->random01() < 0.2) { // not fully utilized cpu
self->cpuUsage = deterministicRandom()->random01() * SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT;
} else if (deterministicRandom()->random01() < 0.6) { // achieved target cpu but cpu is not busy
self->cpuUsage = SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT +
deterministicRandom()->random01() * (SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT -
SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT);
} else { // reach desired max cpu usage; use max cpu as 200 to simulate incorrect cpu profiling
self->cpuUsage =
SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT +
deterministicRandom()->random01() * (200 - SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT);
}
self->memory = 100.0;
self->residentMemory = 100.0;
return;
}
SystemStatistics sysStats = getSystemStatistics();
if (sysStats.initialized) {
self->cpuUsage = 100 * sysStats.processCPUSeconds / sysStats.elapsed;
self->memory = sysStats.processMemory;
self->residentMemory = sysStats.processResidentMemory;
}
}
// An actor is schedulable to run if the current worker has enough resources, i.e.,
// the worker's memory usage is below the threshold;
// Exception: If the actor is working on the current version batch, we have to schedule
// the actor to run to avoid dead-lock.
// Future: When we release the actors that are blocked by memory usage, we should release them
// in increasing order of their version batch.
Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name) {
self->delayedActors++;
double memoryThresholdBytes = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT * 1024 * 1024;
while (true) {
double memory = getSystemStatistics().processMemory;
if (g_network->isSimulated() && BUGGIFY) {
// Intentionally randomly block actors for low memory reason.
// memory will be larger than threshold when deterministicRandom()->random01() > 1/2
if (deterministicRandom()->random01() < 0.4) { // enough memory
memory = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT * deterministicRandom()->random01();
} else { // used too much memory, needs throttling
memory = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT +
deterministicRandom()->random01() * SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT;
}
}
if (memory < memoryThresholdBytes || self->finishedBatch.get() + 1 == actorBatchIndex) {
if (memory >= memoryThresholdBytes) {
TraceEvent(SevWarn, "FastRestoreMemoryUsageAboveThreshold", self->id())
.suppressFor(5.0)
.detail("Role", getRoleStr(self->role))
.detail("BatchIndex", actorBatchIndex)
.detail("FinishedBatch", self->finishedBatch.get())
.detail("Actor", name)
.detail("Memory", memory);
}
self->delayedActors--;
break;
} else {
TraceEvent(SevInfo, "FastRestoreMemoryUsageAboveThresholdWait", self->id())
.suppressFor(5.0)
.detail("Role", getRoleStr(self->role))
.detail("BatchIndex", actorBatchIndex)
.detail("Actor", name)
.detail("CurrentMemory", memory);
// TODO: Set FASTRESTORE_WAIT_FOR_MEMORY_LATENCY to a large value. It should be able to avoided
co_await (delay(SERVER_KNOBS->FASTRESTORE_WAIT_FOR_MEMORY_LATENCY) || self->checkMemory.onTrigger());
}
}
}
// Updated process metrics will be used by scheduler for throttling as well
Future<Void> updateProcessMetrics(Reference<RestoreRoleData> self) {
while (true) {
updateProcessStats(self);
co_await delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL);
}
}
Future<Void> traceProcessMetrics(Reference<RestoreRoleData> self, std::string role) {
while (true) {
TraceEvent("FastRestoreTraceProcessMetrics", self->nodeID)
.detail("Role", role)
.detail("PipelinedMaxVersionBatchIndex", self->versionBatchId.get())
.detail("FinishedVersionBatchIndex", self->finishedBatch.get())
.detail("CurrentVersionBatchPhase", self->getVersionBatchState(self->finishedBatch.get() + 1))
.detail("CpuUsage", self->cpuUsage)
.detail("UsedMemory", self->memory)
.detail("ResidentMemory", self->residentMemory);
co_await delay(SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY);
}
}
Future<Void> traceRoleVersionBatchProgress(Reference<RestoreRoleData> self, std::string role) {
while (true) {
int batchIndex = self->finishedBatch.get();
int maxBatchIndex = self->versionBatchId.get();
int maxPrintBatchIndex = batchIndex + SERVER_KNOBS->FASTRESTORE_VB_PARALLELISM;
TraceEvent ev("FastRestoreVersionBatchProgressState", self->nodeID);
ev.detail("Role", role)
.detail("Node", self->nodeID)
.detail("FinishedBatch", batchIndex)
.detail("InitializedBatch", maxBatchIndex);
while (batchIndex <= maxBatchIndex) {
if (batchIndex > maxPrintBatchIndex) {
ev.detail("SkipVersionBatches", maxBatchIndex - batchIndex + 1);
break;
}
std::stringstream typeName;
typeName << "VersionBatch" << batchIndex;
ev.detail(typeName.str(), self->getVersionBatchState(batchIndex));
batchIndex++;
}
co_await delay(SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY);
}
}

119
fdbserver/restoreworker/RestoreRoleCommon.h
View File

@ -1,119 +0,0 @@
/*
* RestoreRoleCommon.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file declares common struct and functions shared by restore roles, i.e.,
// RestoreController, RestoreLoader, RestoreApplier
#pragma once
#include <sstream>
#include "flow/SystemMonitor.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbclient/Notified.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
#include "fdbrpc/Stats.h"
#include "fdbserver/core/CoordinationInterface.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
struct RestoreRoleInterface;
struct RestoreLoaderInterface;
struct RestoreApplierInterface;
struct RestoreRoleData;
struct RestoreControllerData;
struct RestoreSimpleRequest;
// Key is the (version, subsequence) of parsed backup mutations.
// Value MutationsVec is the vector of parsed backup mutations.
// For old mutation logs, the subsequence number is always 0.
// For partitioned mutation logs, each mutation has a unique LogMessageVersion.
// Note for partitioned logs, one LogMessageVersion can have multiple mutations,
// because a clear mutation may be split into several smaller clear mutations by
// backup workers.
using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);
Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self);
class RoleVersionBatchState {
public:
static const int INVALID = -1;
virtual int get() const { return vbState; }
virtual void operator=(int newState) { vbState = newState; }
explicit RoleVersionBatchState() : vbState(INVALID) {}
explicit RoleVersionBatchState(int newState) : vbState(newState) {}
virtual ~RoleVersionBatchState() = default;
int vbState;
};
struct RestoreRoleData : NonCopyable, public ReferenceCounted<RestoreRoleData> {
public:
RestoreRole role;
UID nodeID;
int nodeIndex;
double cpuUsage;
double memory;
double residentMemory;
AsyncTrigger checkMemory;
int delayedActors; // actors that are delayed to release because of low memory
std::map<UID, RestoreLoaderInterface> loadersInterf; // UID: loaderInterf's id
std::map<UID, RestoreApplierInterface> appliersInterf; // UID: applierInterf's id
Promise<Void> recruitedRoles; // sent when loaders and appliers are recruited
NotifiedVersion versionBatchId; // The index of the version batch that has been initialized and put into pipeline
NotifiedVersion finishedBatch; // The highest batch index all appliers have applied mutations
RestoreRoleData()
: role(RestoreRole::Invalid), cpuUsage(0.0), memory(0.0), residentMemory(0.0), delayedActors(0) {};
virtual ~RestoreRoleData() = default;
UID id() const { return nodeID; }
virtual void initVersionBatch(int batchIndex) = 0;
virtual void resetPerRestoreRequest() = 0;
virtual int getVersionBatchState(int batchIndex) const = 0;
virtual void setVersionBatchState(int batchIndex, int vbState) = 0;
void clearInterfaces() {
loadersInterf.clear();
appliersInterf.clear();
}
virtual std::string describeNode() const = 0;
};
void updateProcessStats(Reference<RestoreRoleData> self);
Future<Void> updateProcessMetrics(Reference<RestoreRoleData> self);
Future<Void> traceProcessMetrics(Reference<RestoreRoleData> self, std::string role);
Future<Void> traceRoleVersionBatchProgress(Reference<RestoreRoleData> self, std::string role);

52
fdbserver/restoreworker/RestoreUtil.cpp
View File

@ -1,52 +0,0 @@
/*
* RestoreUtil.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/restoreworker/RestoreUtil.h"
#include <cstdio>
#include <iomanip>
const std::vector<std::string> RestoreRoleStr = { "Invalid", "Controller", "Loader", "Applier" };
int numRoles = RestoreRoleStr.size();
std::string getRoleStr(RestoreRole role) {
if ((int)role >= numRoles || (int)role < 0) {
printf("[ERROR] role:%d is out of scope\n", (int)role);
return "[Unset]";
}
return RestoreRoleStr[(int)role];
}
std::string getHexString(StringRef input) {
std::stringstream ss;
for (int i = 0; i < input.size(); i++) {
if (i % 4 == 0)
ss << " ";
if (i == 12) { // The end of 12bytes, which is the version size for value
ss << "|";
}
if (i == (12 + 12)) { // The end of version + header
ss << "@";
}
ss << std::setfill('0') << std::setw(2) << std::hex
<< (int)input[i]; // [] operator moves the pointer in step of unit8
}
return ss.str();
}

430
fdbserver/restoreworker/RestoreWorker.cpp
View File

@ -1,430 +0,0 @@
/*
* RestoreWorker.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <ctime>
#include <climits>
#include <numeric>
#include <algorithm>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/BackupAgent.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/BackupContainer.h"
#include "flow/ApiVersion.h"
#include "flow/IAsyncFile.h"
#include "fdbrpc/simulator.h"
#include "flow/genericactors.actor.h"
#include "flow/Hash3.h"
#include "flow/ActorCollection.h"
#include "RestoreWorker.h"
#include "RestoreLoader.h"
#include "RestoreApplier.h"
#include "RestoreController.h"
#include "fdbrpc/SimulatorProcessInfo.h"
#include "flow/CoroUtils.h"
class RestoreConfigFR;
struct RestoreWorkerData; // Only declare the struct exist but we cannot use its field
Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req,
Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf,
Database cx);
Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self);
void handleRecruitRoleRequest(RestoreRecruitRoleRequest req,
Reference<RestoreWorkerData> self,
ActorCollection* actors,
Database cx);
Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers = 2);
Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader,
Database cx,
RestoreWorkerInterface myWorkerInterf);
Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf,
Database cx);
// Remove the worker interface from restoreWorkerKey and remove its roles interfaces from their keys.
Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req,
Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf,
Database cx) {
ReadYourWritesTransaction tr(cx);
while (true) {
Error err;
try {
tr.reset();
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.clear(restoreWorkerKeyFor(workerInterf.id()));
co_await tr.commit();
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
TraceEvent("FastRestoreWorker").detail("HandleTerminateWorkerReq", self->id());
}
// Assume only 1 role on a restore worker.
// Future: Multiple roles in a restore worker
void handleRecruitRoleRequest(RestoreRecruitRoleRequest req,
Reference<RestoreWorkerData> self,
ActorCollection* actors,
Database cx) {
// Future: Allow multiple restore roles on a restore worker. The design should easily allow this.
ASSERT(!self->loaderInterf.present() || !self->applierInterf.present()); // Only one role per worker for now
// Already recruited a role
if (self->loaderInterf.present()) {
ASSERT(req.role == RestoreRole::Loader);
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Loader, self->loaderInterf.get()));
return;
} else if (self->applierInterf.present()) {
req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Applier, self->applierInterf.get()));
return;
}
if (req.role == RestoreRole::Loader) {
ASSERT(!self->loaderInterf.present());
self->controllerInterf = req.ci;
self->loaderInterf = RestoreLoaderInterface();
self->loaderInterf.get().initEndpoints();
RestoreLoaderInterface& recruited = self->loaderInterf.get();
DUMPTOKEN(recruited.heartbeat);
DUMPTOKEN(recruited.updateRestoreSysInfo);
DUMPTOKEN(recruited.initVersionBatch);
DUMPTOKEN(recruited.loadFile);
DUMPTOKEN(recruited.sendMutations);
DUMPTOKEN(recruited.initVersionBatch);
DUMPTOKEN(recruited.finishVersionBatch);
DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
DUMPTOKEN(recruited.finishRestore);
actors->add(restoreLoaderCore(self->loaderInterf.get(), req.nodeIndex, cx, req.ci));
TraceEvent("FastRestoreWorker").detail("RecruitedLoaderNodeIndex", req.nodeIndex);
req.reply.send(
RestoreRecruitRoleReply(self->loaderInterf.get().id(), RestoreRole::Loader, self->loaderInterf.get()));
} else if (req.role == RestoreRole::Applier) {
ASSERT(!self->applierInterf.present());
self->controllerInterf = req.ci;
self->applierInterf = RestoreApplierInterface();
self->applierInterf.get().initEndpoints();
RestoreApplierInterface& recruited = self->applierInterf.get();
DUMPTOKEN(recruited.heartbeat);
DUMPTOKEN(recruited.sendMutationVector);
DUMPTOKEN(recruited.applyToDB);
DUMPTOKEN(recruited.initVersionBatch);
DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
DUMPTOKEN(recruited.finishRestore);
actors->add(restoreApplierCore(self->applierInterf.get(), req.nodeIndex, cx));
TraceEvent("FastRestoreWorker").detail("RecruitedApplierNodeIndex", req.nodeIndex);
req.reply.send(
RestoreRecruitRoleReply(self->applierInterf.get().id(), RestoreRole::Applier, self->applierInterf.get()));
} else {
TraceEvent(SevError, "FastRestoreWorkerHandleRecruitRoleRequestUnknownRole").detail("Request", req.toString());
}
return;
}
// Read restoreWorkersKeys from DB to get each restore worker's workerInterface and set it to self->workerInterfaces;
// This is done before we assign restore roles for restore workers.
Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers) {
Transaction tr(cx);
std::vector<RestoreWorkerInterface> agents; // agents is cmdsInterf
while (true) {
Error err;
try {
self->workerInterfaces.clear();
agents.clear();
tr.reset();
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
RangeResult agentValues = co_await tr.getRange(restoreWorkersKeys, CLIENT_KNOBS->TOO_MANY);
ASSERT(!agentValues.more);
// If agentValues.size() < min_num_workers, we should wait for coming workers to register their
// workerInterface before we read them once for all
if (agentValues.size() >= min_num_workers) {
for (auto& it : agentValues) {
agents.push_back(BinaryReader::fromStringRef<RestoreWorkerInterface>(it.value, IncludeVersion()));
// Save the RestoreWorkerInterface for the later operations
self->workerInterfaces.insert(std::make_pair(agents.back().id(), agents.back()));
}
break;
}
TraceEvent("FastRestoreWorker")
.suppressFor(10.0)
.detail("NotEnoughWorkers", agentValues.size())
.detail("MinWorkers", min_num_workers);
co_await delay(5.0);
continue;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
ASSERT(agents.size() >= min_num_workers); // ASSUMPTION: We must have at least 1 loader and 1 applier
TraceEvent("FastRestoreWorker").detail("CollectWorkerInterfaceNumWorkers", self->workerInterfaces.size());
}
// Periodically send worker heartbeat to
Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self) {
ASSERT(!self->workerInterfaces.empty());
while (true) {
std::vector<std::pair<UID, RestoreSimpleRequest>> requests;
for (auto& worker : self->workerInterfaces) {
requests.emplace_back(worker.first, RestoreSimpleRequest());
}
co_await sendBatchRequests(&RestoreWorkerInterface::heartbeat, self->workerInterfaces, requests);
co_await delay(60.0);
}
}
// RestoreWorkerLeader is the worker that runs RestoreController role
Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self,
RestoreWorkerInterface workerInterf,
Database cx) {
// We must wait for enough time to make sure all restore workers have registered their workerInterfaces into the DB
TraceEvent("FastRestoreWorker")
.detail("Controller", workerInterf.id())
.detail("WaitForRestoreWorkerInterfaces",
SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
co_await delay(10.0);
TraceEvent("FastRestoreWorker")
.detail("Controller", workerInterf.id())
.detail("CollectRestoreWorkerInterfaces",
SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
co_await collectRestoreWorkerInterface(
self, cx, SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
// TODO: Needs to keep this monitor's future. May use actorCollection
Future<Void> workersFailureMonitor = monitorWorkerLiveness(self);
RestoreControllerInterface recruited;
DUMPTOKEN(recruited.samples);
self->controllerInterf = recruited;
co_await (startRestoreController(self, cx) || workersFailureMonitor);
}
Future<Void> startRestoreWorker(Reference<RestoreWorkerData> self, RestoreWorkerInterface interf, Database cx) {
double lastLoopTopTime = now();
ActorCollection actors(false); // Collect the main actor for each role
Future<Void> exitRole = Never();
while (true) {
double loopTopTime = now();
double elapsedTime = loopTopTime - lastLoopTopTime;
if (elapsedTime > 0.050) {
if (deterministicRandom()->random01() < 0.01)
TraceEvent(SevWarn, "SlowRestoreWorkerLoopx100")
.detail("NodeDesc", self->describeNode())
.detail("Elapsed", elapsedTime);
}
lastLoopTopTime = loopTopTime;
std::string requestTypeStr = "[Init]";
try {
auto res = co_await race(interf.heartbeat.getFuture(),
interf.recruitRole.getFuture(),
interf.terminateWorker.getFuture(),
exitRole);
if (res.index() == 0) {
RestoreSimpleRequest req = std::get<0>(std::move(res));
requestTypeStr = "heartbeat";
actors.add(handleHeartbeat(req, interf.id()));
} else if (res.index() == 1) {
RestoreRecruitRoleRequest req = std::get<1>(std::move(res));
requestTypeStr = "recruitRole";
handleRecruitRoleRequest(req, self, &actors, cx);
} else if (res.index() == 2) {
RestoreSimpleRequest req = std::get<2>(std::move(res));
// Destroy the worker at the end of the restore
requestTypeStr = "terminateWorker";
exitRole = handlerTerminateWorkerRequest(req, self, interf, cx);
} else if (res.index() == 3) {
TraceEvent("FastRestoreWorkerCoreExitRole", self->id());
break;
} else {
UNREACHABLE();
}
} catch (Error& e) {
TraceEvent(SevWarn, "FastRestoreWorkerError").errorUnsuppressed(e).detail("RequestType", requestTypeStr);
break;
}
}
}
static Future<Void> waitOnRestoreRequests(Database cx, UID nodeID = UID()) {
ReadYourWritesTransaction tr(cx);
Optional<Value> numRequests;
// wait for the restoreRequestTriggerKey to be set by the client/test workload
TraceEvent("FastRestoreWaitOnRestoreRequest", nodeID).log();
while (true) {
Error err;
try {
tr.reset();
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
numRequests = co_await tr.get(restoreRequestTriggerKey);
if (!numRequests.present()) {
Future<Void> watchForRestoreRequest = tr.watch(restoreRequestTriggerKey);
co_await tr.commit();
TraceEvent(SevInfo, "FastRestoreWaitOnRestoreRequestTriggerKey", nodeID).log();
co_await watchForRestoreRequest;
TraceEvent(SevInfo, "FastRestoreDetectRestoreRequestTriggerKeyChanged", nodeID).log();
continue;
} else {
TraceEvent(SevInfo, "FastRestoreRestoreRequestTriggerKey", nodeID)
.detail("TriggerKey", numRequests.get().toString());
break;
}
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
}
// RestoreController is the leader
Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader,
Database cx,
RestoreWorkerInterface myWorkerInterf) {
co_await delay(SERVER_KNOBS->FASTRESTORE_MONITOR_LEADER_DELAY);
TraceEvent("FastRestoreWorker", myWorkerInterf.id()).detail("MonitorLeader", "StartLeaderElection");
int count = 0;
RestoreWorkerInterface leaderInterf;
ReadYourWritesTransaction tr(cx); // MX: Somewhere here program gets stuck
while (true) {
Error err;
try {
count++;
tr.reset();
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> leaderValue = co_await tr.get(restoreLeaderKey);
TraceEvent(SevInfo, "FastRestoreLeaderElection")
.detail("Round", count)
.detail("LeaderExisted", leaderValue.present());
if (leaderValue.present()) {
leaderInterf = BinaryReader::fromStringRef<RestoreWorkerInterface>(leaderValue.get(), IncludeVersion());
// Register my interface as an worker if I am not the leader
if (leaderInterf != myWorkerInterf) {
tr.set(restoreWorkerKeyFor(myWorkerInterf.id()), restoreWorkerInterfaceValue(myWorkerInterf));
}
} else {
// Workers compete to be the leader
tr.set(restoreLeaderKey,
BinaryWriter::toValue(myWorkerInterf,
IncludeVersion(ProtocolVersion::withRestoreWorkerInterfaceValue())));
leaderInterf = myWorkerInterf;
}
co_await tr.commit();
leader->set(leaderInterf);
break;
} catch (Error& e) {
err = e;
}
TraceEvent(SevInfo, "FastRestoreLeaderElection").detail("ErrorCode", err.code()).detail("Error", err.what());
co_await tr.onError(err);
}
TraceEvent("FastRestoreWorker", myWorkerInterf.id())
.detail("MonitorLeader", "FinishLeaderElection")
.detail("Leader", leaderInterf.id())
.detail("IamLeader", leaderInterf == myWorkerInterf);
}
Future<Void> _restoreWorker(Database cx, LocalityData locality) {
Future<Void> myWork = Never();
auto leader = makeReference<AsyncVar<RestoreWorkerInterface>>();
RestoreWorkerInterface myWorkerInterf;
auto self = makeReference<RestoreWorkerData>();
myWorkerInterf.initEndpoints();
self->workerID = myWorkerInterf.id();
// Protect restore worker from being killed in simulation;
// Future: Remove the protection once restore can tolerate failure
if (g_network->isSimulated()) {
auto addresses = g_simulator->getProcessByAddress(myWorkerInterf.address())->addresses;
g_simulator->protectedAddresses.insert(addresses.address);
if (addresses.secondaryAddress.present()) {
g_simulator->protectedAddresses.insert(addresses.secondaryAddress.get());
}
ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(myWorkerInterf.address());
TraceEvent("ProtectRestoreWorker")
.detail("Address", addresses.toString())
.detail("IsReliable", p->isReliable())
.detail("ReliableInfo", p->getReliableInfo())
.backtrace();
ASSERT(p->isReliable());
}
TraceEvent("FastRestoreWorkerKnobs", myWorkerInterf.id())
.detail("FailureTimeout", SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT)
.detail("HeartBeat", SERVER_KNOBS->FASTRESTORE_HEARTBEAT_INTERVAL)
.detail("SamplePercentage", SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT)
.detail("NumLoaders", SERVER_KNOBS->FASTRESTORE_NUM_LOADERS)
.detail("NumAppliers", SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS)
.detail("TxnBatchSize", SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES)
.detail("VersionBatchSize", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES);
co_await waitOnRestoreRequests(cx, myWorkerInterf.id());
co_await monitorleader(leader, cx, myWorkerInterf);
TraceEvent("FastRestoreWorker", myWorkerInterf.id()).detail("LeaderElection", "WaitForLeader");
if (leader->get() == myWorkerInterf) {
// Restore controller worker: doLeaderThings();
myWork = startRestoreWorkerLeader(self, myWorkerInterf, cx);
} else {
// Restore normal worker (for RestoreLoader and RestoreApplier roles): doWorkerThings();
myWork = startRestoreWorker(self, myWorkerInterf, cx);
}
co_await myWork;
}
Future<Void> restoreWorker(Reference<IClusterConnectionRecord> connRecord,
LocalityData locality,
std::string coordFolder) {
try {
Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
co_await reportErrors(_restoreWorker(cx, locality), "RestoreWorker");
} catch (Error& e) {
TraceEvent("FastRestoreWorker").detail("Error", e.what());
throw e;
}
}

64
fdbserver/restoreworker/RestoreWorker.h
View File

@ -1,64 +0,0 @@
/*
* RestoreWorker.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "fdbclient/Tuple.h"
#include <cstdint>
#include <cstdarg>
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Stats.h"
#include "flow/flow.h"
#include "flow/IAsyncFile.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "RestoreRoleCommon.h"
#include "RestoreLoader.h"
#include "RestoreApplier.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
// Each restore worker (a process) is assigned for a role.
// MAYBE Later: We will support multiple restore roles on a worker
struct RestoreWorkerData : NonCopyable, public ReferenceCounted<RestoreWorkerData> {
UID workerID;
std::map<UID, RestoreWorkerInterface>
workerInterfaces; // UID is worker's node id, RestoreWorkerInterface is worker's communication workerInterface
// Restore Roles
Optional<RestoreControllerInterface> controllerInterf;
Optional<RestoreLoaderInterface> loaderInterf;
Optional<RestoreApplierInterface> applierInterf;
UID id() const { return workerID; };
RestoreWorkerData() = default;
~RestoreWorkerData() {
TraceEvent("RestoreWorkerDataDeleted").detail("WorkerID", workerID.toString());
printf("[Exit] Worker:%s RestoreWorkerData is deleted\n", workerID.toString().c_str());
}
std::string describeNode() {
std::stringstream ss;
ss << "RestoreWorker workerID:" << workerID.toString();
return ss.str();
}
};

100
fdbserver/restoreworker/RestoreWorkerInterface.cpp
View File

@ -1,100 +0,0 @@
/*
* RestoreWorkerInterface.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
const KeyRef restoreLeaderKey = "\xff\x02/restoreLeader"_sr;
const KeyRangeRef restoreWorkersKeys("\xff\x02/restoreWorkers/"_sr, "\xff\x02/restoreWorkers0"_sr);
const KeyRef restoreStatusKey = "\xff\x02/restoreStatus/"_sr;
const KeyRangeRef restoreApplierKeys("\xff\x02/restoreApplier/"_sr, "\xff\x02/restoreApplier0"_sr);
const KeyRef restoreApplierTxnValue = "1"_sr;
// restoreApplierKeys: track atomic transaction progress to ensure applying atomicOp exactly once
// Version and batchIndex are passed in as LittleEndian,
// they must be converted to BigEndian to maintain ordering in lexical order
const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version) {
BinaryWriter wr(Unversioned());
wr.serializeBytes(restoreApplierKeys.begin);
wr << applierID << bigEndian64(batchIndex) << bigEndian64(version);
return wr.toValue();
}
std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key) {
BinaryReader rd(key, Unversioned());
UID applierID;
int64_t batchIndex;
Version version;
rd >> applierID >> batchIndex >> version;
return std::make_tuple(applierID, bigEndian64(batchIndex), bigEndian64(version));
}
// Encode restore worker key for workerID
const Key restoreWorkerKeyFor(UID const& workerID) {
BinaryWriter wr(Unversioned());
wr.serializeBytes(restoreWorkersKeys.begin);
wr << workerID;
return wr.toValue();
}
// Encode restore agent value
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& cmdInterf) {
BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreWorkerInterfaceValue()));
wr << cmdInterf;
return wr.toValue();
}
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value) {
RestoreWorkerInterface s;
BinaryReader reader(value, IncludeVersion());
reader >> s;
return s;
}
Value restoreRequestDoneVersionValue(Version readVersion) {
BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreRequestDoneVersionValue()));
wr << readVersion;
return wr.toValue();
}
Version decodeRestoreRequestDoneVersionValue(ValueRef const& value) {
Version v;
BinaryReader reader(value, IncludeVersion());
reader >> v;
return v;
}
RestoreRequest decodeRestoreRequestValue(ValueRef const& value) {
RestoreRequest s;
BinaryReader reader(value, IncludeVersion());
reader >> s;
return s;
}
// TODO: Register restore performance data to restoreStatus key
const Key restoreStatusKeyFor(StringRef statusType) {
BinaryWriter wr(Unversioned());
wr.serializeBytes(restoreStatusKey);
wr << statusType;
return wr.toValue();
}
const Value restoreStatusValue(double val) {
BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreStatusValue()));
wr << StringRef(std::to_string(val));
return wr.toValue();
}

376
fdbserver/restoreworker/include/fdbserver/restoreworker/RestoreCommon.h
View File

@ -1,376 +0,0 @@
/*
* RestoreCommon.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file includes the code copied from the old restore in FDB 5.2
// The functions and structure declared in this file can be shared by
// the old restore and the new performant restore systems
#pragma once
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "flow/CoroUtils.h"
#include "fdbclient/Tuple.h"
#include "fdbclient/NativeAPI.actor.h"
#include "flow/IAsyncFile.h"
#include "fdbclient/BackupAgent.h"
#include "fdbserver/core/Knobs.h"
// RestoreConfig copied from FileBackupAgent.cpp
// We copy RestoreConfig instead of using (and potentially changing) it in place
// to avoid conflict with the existing code.
// We also made minor changes to allow RestoreConfig to be ReferenceCounted
// TODO: Merge this RestoreConfig with the original RestoreConfig in FileBackupAgent.cpp
// For convenience
typedef FileBackupAgent::ERestoreState ERestoreState;
struct RestoreFileFR;
// We copy RestoreConfig copied from FileBackupAgent.cpp instead of using (and potentially changing) it in place
// to avoid conflict with the existing code Split RestoreConfig defined in FileBackupAgent.cpp to declaration in
// Restore.actor.h and implementation in RestoreCommon.cpp, so that we can use in both the existing restore and
// the new fast restore subsystems. We use RestoreConfig as a Reference<RestoreConfig>, which leads to some
// non-functional changes in RestoreConfig
class RestoreConfigFR : public KeyBackedTaskConfig, public ReferenceCounted<RestoreConfigFR> {
public:
explicit(false) RestoreConfigFR(UID uid = UID()) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, uid) {}
explicit(false) RestoreConfigFR(Reference<Task> task) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, task) {}
KeyBackedProperty<ERestoreState> stateEnum();
Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr);
KeyBackedProperty<Key> addPrefix();
KeyBackedProperty<Key> removePrefix();
// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
KeyBackedProperty<KeyRange> restoreRange();
KeyBackedProperty<std::vector<KeyRange>> restoreRanges();
KeyBackedProperty<Key> batchFuture();
KeyBackedProperty<Version> restoreVersion();
KeyBackedProperty<Reference<IBackupContainer>> sourceContainer();
// Get the source container as a bare URL, without creating a container instance
KeyBackedProperty<Value> sourceContainerURL();
// Total bytes written by all log and range restore tasks.
KeyBackedBinaryValue<int64_t> bytesWritten();
// File blocks that have had tasks created for them by the Dispatch task
KeyBackedBinaryValue<int64_t> filesBlocksDispatched();
// File blocks whose tasks have finished
KeyBackedBinaryValue<int64_t> fileBlocksFinished();
// Total number of files in the fileMap
KeyBackedBinaryValue<int64_t> fileCount();
// Total number of file blocks in the fileMap
KeyBackedBinaryValue<int64_t> fileBlockCount();
Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr);
static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfigFR* self,
Reference<ReadYourWritesTransaction> tr);
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
// incrementally advancing through the map, saving the version and path of the next starting point.
struct RestoreFile {
Version version;
std::string fileName;
bool isRange; // false for log file
int64_t blockSize;
int64_t fileSize;
Version endVersion; // not meaningful for range files
Tuple pack() const {
// fprintf(stderr, "Filename:%s\n", fileName.c_str());
return Tuple::makeTuple(version, fileName, (int)isRange, fileSize, blockSize, endVersion);
}
static RestoreFile unpack(Tuple const& t) {
RestoreFile r;
int i = 0;
r.version = t.getInt(i++);
r.fileName = t.getString(i++).toString();
r.isRange = t.getInt(i++) != 0;
r.fileSize = t.getInt(i++);
r.blockSize = t.getInt(i++);
r.endVersion = t.getInt(i++);
return r;
}
};
// typedef KeyBackedSet<RestoreFile> FileSetT;
KeyBackedSet<RestoreFile> fileSet();
Future<bool> isRunnable(Reference<ReadYourWritesTransaction> tr);
Future<Void> logError(Database cx, Error e, std::string const& details, void* taskInstance = nullptr);
Key mutationLogPrefix();
Key applyMutationsMapPrefix();
Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid);
Future<int64_t> getApplyVersionLag(Reference<ReadYourWritesTransaction> tr);
void initApplyMutations(Reference<ReadYourWritesTransaction> tr, Key addPrefix, Key removePrefix);
void clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr);
void setApplyBeginVersion(Reference<ReadYourWritesTransaction> tr, Version ver);
void setApplyEndVersion(Reference<ReadYourWritesTransaction> tr, Version ver);
Future<Version> getApplyEndVersion(Reference<ReadYourWritesTransaction> tr);
static Future<std::string> getProgress_impl(Reference<RestoreConfigFR> restore,
Reference<ReadYourWritesTransaction> tr);
Future<std::string> getProgress(Reference<ReadYourWritesTransaction> tr);
static Future<std::string> getFullStatus_impl(Reference<RestoreConfigFR> restore,
Reference<ReadYourWritesTransaction> tr);
Future<std::string> getFullStatus(Reference<ReadYourWritesTransaction> tr);
std::string toString(); // Added by Meng
};
// typedef RestoreConfigFR::RestoreFile RestoreFile;
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
// incrementally advancing through the map, saving the version and path of the next starting point.
// NOTE: The struct RestoreFileFR can NOT be named RestoreFile, because compiler will get confused in linking which
// RestoreFile should be used. If we use RestoreFile, compilation succeeds, but weird segmentation fault will happen.
struct RestoreFileFR {
Version version;
std::string fileName;
bool isRange; // false for log file
int64_t blockSize;
int64_t fileSize;
Version endVersion; // not meaningful for range files
Version beginVersion; // range file's beginVersion == endVersion; log file contains mutations in version
// [beginVersion, endVersion)
int64_t cursor; // The start block location to be restored. All blocks before cursor have been scheduled to load and
// restore
int fileIndex; // index of backup file. Must be identical per file.
int partitionId = -1; // Partition ID (Log Router Tag ID) for mutation files.
Tuple pack() const {
return Tuple::makeTuple(version,
fileName,
(int)isRange,
fileSize,
blockSize,
endVersion,
beginVersion,
cursor,
fileIndex,
partitionId);
}
static RestoreFileFR unpack(Tuple const& t) {
RestoreFileFR r;
int i = 0;
r.version = t.getInt(i++);
r.fileName = t.getString(i++).toString();
r.isRange = t.getInt(i++) != 0;
r.fileSize = t.getInt(i++);
r.blockSize = t.getInt(i++);
r.endVersion = t.getInt(i++);
r.beginVersion = t.getInt(i++);
r.cursor = t.getInt(i++);
r.fileIndex = t.getInt(i++);
r.partitionId = t.getInt(i++);
return r;
}
bool operator<(const RestoreFileFR& rhs) const {
return std::tie(beginVersion, endVersion, fileIndex, fileName) <
std::tie(rhs.beginVersion, rhs.endVersion, rhs.fileIndex, rhs.fileName);
}
RestoreFileFR()
: version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion),
beginVersion(invalidVersion), cursor(0), fileIndex(0) {}
explicit RestoreFileFR(const RangeFile& f)
: version(f.version), fileName(f.fileName), isRange(true), blockSize(f.blockSize), fileSize(f.fileSize),
endVersion(f.version), beginVersion(f.version), cursor(0), fileIndex(0) {}
explicit RestoreFileFR(const LogFile& f)
: version(f.beginVersion), fileName(f.fileName), isRange(false), blockSize(f.blockSize), fileSize(f.fileSize),
endVersion(f.endVersion), beginVersion(f.beginVersion), cursor(0), fileIndex(0), partitionId(f.tagId) {}
std::string toString() const {
std::stringstream ss;
ss << "version:" << version << " fileName:" << fileName << " isRange:" << isRange << " blockSize:" << blockSize
<< " fileSize:" << fileSize << " endVersion:" << endVersion << " beginVersion:" << beginVersion
<< " cursor:" << cursor << " fileIndex:" << fileIndex << " partitionId:" << partitionId;
return ss.str();
}
};
namespace parallelFileRestore {
Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file, int64_t offset, int len);
} // namespace parallelFileRestore
// Send each request in requests via channel of the request's interface.
// Save replies to replies if replies != nullptr
// The UID in a request is the UID of the interface to handle the request
template <class Interface, class Request>
Future<Void> getBatchReplies(RequestStream<Request> Interface::* channel,
std::map<UID, Interface> interfaces,
std::vector<std::pair<UID, Request>> requests,
std::vector<REPLY_TYPE(Request)>* replies,
TaskPriority taskID = TaskPriority::Low,
bool trackRequestLatency = true) {
if (requests.empty()) {
co_return;
}
double start = now();
int oustandingReplies = requests.size();
while (true) {
try {
std::vector<Future<REPLY_TYPE(Request)>> cmdReplies;
std::vector<std::tuple<UID, Request, double>> replyDurations; // double is end time of the request
for (auto& [requestId, request] : requests) {
RequestStream<Request> const* stream = &(interfaces[requestId].*channel);
cmdReplies.push_back(stream->getReply(request, taskID));
replyDurations.emplace_back(requestId, request, 0);
}
std::vector<Future<REPLY_TYPE(Request)>> ongoingReplies;
std::vector<int> ongoingRepliesIndex;
while (true) {
ongoingReplies.clear();
ongoingRepliesIndex.clear();
for (int i = 0; i < cmdReplies.size(); ++i) {
if (SERVER_KNOBS->FASTRESTORE_REQBATCH_LOG) {
TraceEvent(SevInfo, "FastRestoreGetBatchReplies")
.suppressFor(1.0)
.detail("Requests", requests.size())
.detail("OutstandingReplies", oustandingReplies)
.detail("ReplyIndex", i)
.detail("ReplyIsReady", cmdReplies[i].isReady())
.detail("ReplyIsError", cmdReplies[i].isError())
.detail("RequestNode", requests[i].first)
.detail("Request", requests[i].second.toString());
}
if (!cmdReplies[i].isReady()) { // still wait for reply
ongoingReplies.push_back(cmdReplies[i]);
ongoingRepliesIndex.push_back(i);
}
}
ASSERT(ongoingReplies.size() == oustandingReplies);
if (ongoingReplies.empty()) {
break;
} else {
co_await (
quorum(ongoingReplies,
std::min((int)SERVER_KNOBS->FASTRESTORE_REQBATCH_PARALLEL, (int)ongoingReplies.size())));
}
// At least one reply is received; Calculate the reply duration
for (int j = 0; j < ongoingReplies.size(); ++j) {
if (ongoingReplies[j].isReady()) {
std::get<2>(replyDurations[ongoingRepliesIndex[j]]) = now();
--oustandingReplies;
} else if (ongoingReplies[j].isError()) {
// When this happens,
// the above assertion ASSERT(ongoingReplies.size() == oustandingReplies) will fail
TraceEvent(SevError, "FastRestoreGetBatchRepliesReplyError")
.detail("OngoingReplyIndex", j)
.detail("FutureError", ongoingReplies[j].getError().what());
}
}
}
ASSERT(oustandingReplies == 0);
if (trackRequestLatency && SERVER_KNOBS->FASTRESTORE_TRACK_REQUEST_LATENCY) {
// Calculate the latest end time for each interface
std::map<UID, double> maxEndTime;
UID bathcID = deterministicRandom()->randomUniqueID();
for (int i = 0; i < replyDurations.size(); ++i) {
double endTime = std::get<2>(replyDurations[i]);
TraceEvent(SevInfo, "ProfileSendRequestBatchLatency", bathcID)
.detail("Node", std::get<0>(replyDurations[i]))
.detail("Request", std::get<1>(replyDurations[i]).toString())
.detail("Duration", endTime - start);
auto item = maxEndTime.emplace(std::get<0>(replyDurations[i]), endTime);
item.first->second = std::max(item.first->second, endTime);
}
// Check the time gap between the earliest and latest node
double earliest = std::numeric_limits<double>::max();
double latest = std::numeric_limits<double>::min();
UID earliestNode, latestNode;
for (const auto& [nodeId, endTime] : maxEndTime) {
if (earliest > endTime) {
earliest = endTime;
earliestNode = nodeId;
}
if (latest < endTime) {
latest = endTime;
latestNode = nodeId;
}
}
if (latest - earliest > SERVER_KNOBS->FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS) {
TraceEvent(SevWarn, "ProfileSendRequestBatchLatencyFoundStraggler", bathcID)
.detail("SlowestNode", latestNode)
.detail("FatestNode", earliestNode)
.detail("EarliestEndtime", earliest)
.detail("LagTime", latest - earliest);
}
}
// Update replies
if (replies != nullptr) {
for (int i = 0; i < cmdReplies.size(); ++i) {
replies->emplace_back(cmdReplies[i].get());
}
}
break;
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled)
break;
// fprintf(stdout, "sendBatchRequests Error code:%d, error message:%s\n", e.code(), e.what());
TraceEvent(SevWarn, "FastRestoreSendBatchRequests").error(e);
for (auto& [requestId, request] : requests) {
TraceEvent(SevWarn, "FastRestoreSendBatchRequests")
.detail("SendBatchRequests", requests.size())
.detail("RequestID", requestId)
.detail("Request", request.toString());
resetReply(request);
}
}
}
}
// Similar to getBatchReplies except that the caller does not expect to process the reply info.
template <class Interface, class Request>
Future<Void> sendBatchRequests(RequestStream<Request> Interface::* channel,
std::map<UID, Interface> interfaces,
std::vector<std::pair<UID, Request>> requests,
TaskPriority taskID = TaskPriority::Low,
bool trackRequestLatency = true) {
co_await getBatchReplies(channel, interfaces, requests, nullptr, taskID, trackRequestLatency);
}

71
fdbserver/restoreworker/include/fdbserver/restoreworker/RestoreUtil.h
View File

@ -1,71 +0,0 @@
/*
* RestoreUtil.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file defines restoreworker-specific data structures and functions
// shared by RestoreWorker and RestoreRoles (Controller, Loader, and Applier).
#ifndef FDBSERVER_RESTOREWORKER_RESTOREUTIL_H
#define FDBSERVER_RESTOREWORKER_RESTOREUTIL_H
#pragma once
#include <cstdint>
#include <sstream>
#include <string>
#include <vector>
#include "fdbclient/RestoreInterface.h"
#include "fdbserver/core/RestoreCoreUtil.h"
#include "fdbrpc/TimedRequest.h"
#include "fdbrpc/fdbrpc.h"
#define SevFRMutationInfo SevVerbose
// #define SevFRMutationInfo SevInfo
#define SevFRDebugInfo SevVerbose
// #define SevFRDebugInfo SevInfo
enum class RestoreRole { Invalid = 0, Controller = 1, Loader, Applier };
std::string getRoleStr(RestoreRole role);
extern const std::vector<std::string> RestoreRoleStr;
extern int numRoles;
std::string getHexString(StringRef input);
struct RestoreSimpleRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 16448937;
ReplyPromise<RestoreCommonReply> reply;
RestoreSimpleRequest() = default;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreSimpleRequest";
return ss.str();
}
};
#endif // FDBSERVER_RESTOREWORKER_RESTOREUTIL_H

725
fdbserver/restoreworker/include/fdbserver/restoreworker/RestoreWorkerInterface.h
View File

@ -1,725 +0,0 @@
/*
* RestoreWorkerInterface.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file declare and define the interface for RestoreWorker and restore roles
// which are RestoreController, RestoreLoader, and RestoreApplier
#pragma once
#include <sstream>
#include <string>
#include "flow/flow.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
#include "fdbrpc/Stats.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbserver/core/CoordinationInterface.h"
#include "fdbserver/core/Knobs.h"
#include "fdbserver/restoreworker/RestoreUtil.h"
class RestoreConfigFR;
struct RestoreCommonReply;
struct RestoreRecruitRoleRequest;
struct RestoreSysInfoRequest;
struct RestoreLoadFileRequest;
struct RestoreVersionBatchRequest;
struct RestoreSendMutationsToAppliersRequest;
struct RestoreSendVersionedMutationsRequest;
struct RestoreSysInfo;
struct RestoreApplierInterface;
struct RestoreFinishRequest;
struct RestoreSamplesRequest;
struct RestoreUpdateRateRequest;
// RestoreSysInfo includes information each (type of) restore roles should know.
// At this moment, it only include appliers. We keep the name for future extension.
// TODO: If it turns out this struct only has appliers in the final version, we will rename it to a more specific name,
// e.g., AppliersMap
struct RestoreSysInfo {
constexpr static FileIdentifier file_identifier = 68098739;
std::map<UID, RestoreApplierInterface> appliers;
RestoreSysInfo() = default;
explicit RestoreSysInfo(const std::map<UID, RestoreApplierInterface> appliers) : appliers(appliers) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, appliers);
}
};
struct RestoreWorkerInterface {
constexpr static FileIdentifier file_identifier = 15715718;
UID interfID;
RequestStream<RestoreSimpleRequest> heartbeat;
RequestStream<RestoreRecruitRoleRequest> recruitRole;
RequestStream<RestoreSimpleRequest> terminateWorker;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
UID id() const { return interfID; } // cmd.getEndpoint().token;
NetworkAddress address() const { return recruitRole.getEndpoint().addresses.address; }
void initEndpoints() {
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
recruitRole.getEndpoint(TaskPriority::LoadBalancedEndpoint); // Q: Why do we need this?
terminateWorker.getEndpoint(TaskPriority::LoadBalancedEndpoint);
interfID = deterministicRandom()->randomUniqueID();
}
// To change this serialization, ProtocolVersion::RestoreWorkerInterfaceValue must be updated, and downgrades need
// to be considered
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, interfID, heartbeat, recruitRole, terminateWorker);
}
};
struct RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 12199691;
UID nodeID;
RestoreRole role;
RestoreRoleInterface() { role = RestoreRole::Invalid; }
explicit RestoreRoleInterface(RestoreRoleInterface const& interf) : nodeID(interf.nodeID), role(interf.role) {};
UID id() const { return nodeID; }
std::string toString() const {
std::stringstream ss;
ss << "Role:" << getRoleStr(role) << " interfID:" << nodeID.toString();
return ss.str();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, nodeID, role);
}
};
struct RestoreLoaderInterface : RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 358571;
RequestStream<RestoreSimpleRequest> heartbeat;
RequestStream<RestoreSysInfoRequest> updateRestoreSysInfo;
RequestStream<RestoreLoadFileRequest> loadFile;
RequestStream<RestoreSendMutationsToAppliersRequest> sendMutations;
RequestStream<RestoreVersionBatchRequest> initVersionBatch;
RequestStream<RestoreVersionBatchRequest> finishVersionBatch;
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
RequestStream<RestoreFinishRequest> finishRestore;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreLoaderInterface() {
role = RestoreRole::Loader;
nodeID = deterministicRandom()->randomUniqueID();
}
NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
void initEndpoints() {
// Endpoint in a later restore phase has higher priority
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
updateRestoreSysInfo.getEndpoint(TaskPriority::LoadBalancedEndpoint);
initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
loadFile.getEndpoint(TaskPriority::RestoreLoaderLoadFiles);
sendMutations.getEndpoint(TaskPriority::RestoreLoaderSendMutations);
finishVersionBatch.getEndpoint(TaskPriority::RestoreLoaderFinishVersionBatch);
collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar,
*(RestoreRoleInterface*)this,
heartbeat,
updateRestoreSysInfo,
loadFile,
sendMutations,
initVersionBatch,
finishVersionBatch,
collectRestoreRoleInterfaces,
finishRestore);
}
};
struct RestoreApplierInterface : RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 3921400;
RequestStream<RestoreSimpleRequest> heartbeat;
RequestStream<RestoreSendVersionedMutationsRequest> sendMutationVector;
RequestStream<RestoreVersionBatchRequest> applyToDB;
RequestStream<RestoreVersionBatchRequest> initVersionBatch;
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
RequestStream<RestoreFinishRequest> finishRestore;
RequestStream<RestoreUpdateRateRequest> updateRate;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreApplierInterface() {
role = RestoreRole::Applier;
nodeID = deterministicRandom()->randomUniqueID();
}
NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
void initEndpoints() {
// Endpoint in a later restore phase has higher priority
heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
sendMutationVector.getEndpoint(TaskPriority::RestoreApplierReceiveMutations);
applyToDB.getEndpoint(TaskPriority::RestoreApplierWriteDB);
initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
updateRate.getEndpoint(TaskPriority::LoadBalancedEndpoint);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar,
*(RestoreRoleInterface*)this,
heartbeat,
sendMutationVector,
applyToDB,
initVersionBatch,
collectRestoreRoleInterfaces,
finishRestore,
updateRate);
}
std::string toString() const { return nodeID.toString(); }
};
struct RestoreControllerInterface : RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 11642024;
RequestStream<RestoreSamplesRequest> samples;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreControllerInterface() {
role = RestoreRole::Controller;
nodeID = deterministicRandom()->randomUniqueID();
}
NetworkAddress address() const { return samples.getEndpoint().addresses.address; }
void initEndpoints() { samples.getEndpoint(TaskPriority::LoadBalancedEndpoint); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, samples);
}
std::string toString() const { return nodeID.toString(); }
};
// RestoreAsset uniquely identifies the work unit done by restore roles;
// It is used to ensure exact-once processing on restore loader and applier;
// By combining all RestoreAssets across all version batches, restore should process all mutations in
// backup range and log files up to the target restore version.
struct RestoreAsset {
UID uid;
Version beginVersion, endVersion; // Only use mutation in [begin, end) versions;
KeyRange range; // Only use mutations in range
int fileIndex;
// Partition ID for mutation log files, which is also encoded in the filename of mutation logs.
int partitionId = -1;
std::string filename;
int64_t offset;
int64_t len;
Key addPrefix;
Key removePrefix;
int batchIndex; // for progress tracking and performance investigation
RestoreAsset() = default;
// Q: Can we simply use uid for == and use different comparison rule for less than operator.
// The ordering of RestoreAsset may change, will that affect correctness or performance?
bool operator==(const RestoreAsset& r) const {
return batchIndex == r.batchIndex && beginVersion == r.beginVersion && endVersion == r.endVersion &&
range == r.range && fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename &&
offset == r.offset && len == r.len && addPrefix == r.addPrefix && removePrefix == r.removePrefix;
}
bool operator!=(const RestoreAsset& r) const { return !(*this == r); }
bool operator<(const RestoreAsset& r) const {
return std::make_tuple(batchIndex,
fileIndex,
filename,
offset,
len,
beginVersion,
endVersion,
range.begin,
range.end,
addPrefix,
removePrefix) < std::make_tuple(r.batchIndex,
r.fileIndex,
r.filename,
r.offset,
r.len,
r.beginVersion,
r.endVersion,
r.range.begin,
r.range.end,
r.addPrefix,
r.removePrefix);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar,
uid,
beginVersion,
endVersion,
range,
filename,
fileIndex,
partitionId,
offset,
len,
addPrefix,
removePrefix,
batchIndex);
}
std::string toString() const {
std::stringstream ss;
ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion
<< " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex
<< " partitionId:" << partitionId << " offset:" << offset << " len:" << len
<< " addPrefix:" << addPrefix.toString() << " removePrefix:" << removePrefix.toString()
<< " BatchIndex:" << batchIndex;
return ss.str();
}
bool hasPrefix() const { return addPrefix.size() > 0 || removePrefix.size() > 0; }
// RestoreAsset and VersionBatch both use endVersion as exclusive in version range
bool isInVersionRange(Version commitVersion) const {
return commitVersion >= beginVersion && commitVersion < endVersion;
}
// Is mutation's begin and end keys are in RestoreAsset's range
bool isInKeyRange(MutationRef mutation) const {
if (hasPrefix()) {
Key begin = range.begin; // Avoid creating new keys if we do not have addPrefix or removePrefix
Key end = range.end;
begin = begin.removePrefix(removePrefix).withPrefix(addPrefix);
end = end.removePrefix(removePrefix).withPrefix(addPrefix);
if (isRangeMutation(mutation)) {
// Range mutation's right side is exclusive
return mutation.param1 >= begin && mutation.param2 <= end;
} else {
return mutation.param1 >= begin && mutation.param1 < end;
}
} else {
if (isRangeMutation(mutation)) {
// Range mutation's right side is exclusive
return mutation.param1 >= range.begin && mutation.param2 <= range.end;
} else {
return mutation.param1 >= range.begin && mutation.param1 < range.end;
}
}
}
};
struct LoadingParam {
constexpr static FileIdentifier file_identifier = 246621;
bool isRangeFile;
Key url;
Optional<std::string> proxy;
Optional<Version> rangeVersion; // range file's version
int64_t blockSize;
RestoreAsset asset;
LoadingParam() = default;
// TODO: Compare all fields for loadingParam
bool operator==(const LoadingParam& r) const { return isRangeFile == r.isRangeFile && asset == r.asset; }
bool operator!=(const LoadingParam& r) const { return isRangeFile != r.isRangeFile || asset != r.asset; }
bool operator<(const LoadingParam& r) const {
return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset);
}
bool isPartitionedLog() const { return !isRangeFile && asset.partitionId >= 0; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, isRangeFile, url, proxy, rangeVersion, blockSize, asset);
}
std::string toString() const {
std::stringstream str;
str << "isRangeFile:" << isRangeFile << " url:" << url.toString()
<< " proxy:" << (proxy.present() ? proxy.get() : "")
<< " rangeVersion:" << (rangeVersion.present() ? rangeVersion.get() : -1) << " blockSize:" << blockSize
<< " RestoreAsset:" << asset.toString();
return str.str();
}
};
struct RestoreRecruitRoleReply : TimedRequest {
constexpr static FileIdentifier file_identifier = 13532876;
UID id;
RestoreRole role;
Optional<RestoreLoaderInterface> loader;
Optional<RestoreApplierInterface> applier;
RestoreRecruitRoleReply() = default;
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreLoaderInterface const& loader)
: id(id), role(role), loader(loader) {}
explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreApplierInterface const& applier)
: id(id), role(role), applier(applier) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, role, loader, applier);
}
std::string toString() const {
std::stringstream ss;
ss << "roleInterf role:" << getRoleStr(role) << " replyID:" << id.toString();
if (loader.present()) {
ss << "loader:" << loader.get().toString();
}
if (applier.present()) {
ss << "applier:" << applier.get().toString();
}
return ss.str();
}
};
struct RestoreRecruitRoleRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 3136280;
RestoreControllerInterface ci;
RestoreRole role;
int nodeIndex; // Each role is a node
ReplyPromise<RestoreRecruitRoleReply> reply;
RestoreRecruitRoleRequest() : role(RestoreRole::Invalid) {}
explicit RestoreRecruitRoleRequest(RestoreControllerInterface ci, RestoreRole role, int nodeIndex)
: ci(ci), role(role), nodeIndex(nodeIndex) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ci, role, nodeIndex, reply);
}
std::string printable() const {
std::stringstream ss;
ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex
<< " RestoreController:" << ci.id().toString();
return ss.str();
}
std::string toString() const { return printable(); }
};
// Static info. across version batches
struct RestoreSysInfoRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 8851877;
RestoreSysInfo sysInfo;
Standalone<VectorRef<std::pair<KeyRangeRef, Version>>> rangeVersions;
ReplyPromise<RestoreCommonReply> reply;
RestoreSysInfoRequest() = default;
explicit RestoreSysInfoRequest(RestoreSysInfo sysInfo,
Standalone<VectorRef<std::pair<KeyRangeRef, Version>>> rangeVersions)
: sysInfo(sysInfo), rangeVersions(rangeVersions) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, sysInfo, rangeVersions, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreSysInfoRequest " << "rangeVersions.size:" << rangeVersions.size();
return ss.str();
}
};
struct RestoreSamplesRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 10751035;
UID id; // deduplicate data
int batchIndex;
SampledMutationsVec samples; // sampled mutations
ReplyPromise<RestoreCommonReply> reply;
RestoreSamplesRequest() = default;
explicit RestoreSamplesRequest(UID id, int batchIndex, SampledMutationsVec samples)
: id(id), batchIndex(batchIndex), samples(samples) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, batchIndex, samples, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "ID:" << id.toString() << " BatchIndex:" << batchIndex << " samples:" << samples.size();
return ss.str();
}
};
struct RestoreLoadFileReply : TimedRequest {
constexpr static FileIdentifier file_identifier = 523470;
LoadingParam param;
bool isDuplicated; // true if loader thinks the request is a duplicated one
RestoreLoadFileReply() = default;
explicit RestoreLoadFileReply(LoadingParam param, bool isDuplicated) : param(param), isDuplicated(isDuplicated) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, isDuplicated);
}
std::string toString() const {
std::stringstream ss;
ss << "LoadingParam:" << param.toString() << " isDuplicated:" << isDuplicated;
return ss.str();
}
};
// Sample_Range_File and Assign_Loader_Range_File, Assign_Loader_Log_File
struct RestoreLoadFileRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 9780148;
int batchIndex;
LoadingParam param;
ReplyPromise<RestoreLoadFileReply> reply;
RestoreLoadFileRequest() = default;
explicit RestoreLoadFileRequest(int batchIndex, LoadingParam& param) : batchIndex(batchIndex), param(param) {};
bool operator<(RestoreLoadFileRequest const& rhs) const { return batchIndex > rhs.batchIndex; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchIndex, param, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreLoadFileRequest batchIndex:" << batchIndex << " param:" << param.toString();
return ss.str();
}
};
struct RestoreSendMutationsToAppliersRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 1718441;
int batchIndex; // version batch index
std::map<Key, UID> rangeToApplier;
bool useRangeFile; // Send mutations parsed from range file?
ReplyPromise<RestoreCommonReply> reply;
RestoreSendMutationsToAppliersRequest() = default;
explicit RestoreSendMutationsToAppliersRequest(int batchIndex, std::map<Key, UID> rangeToApplier, bool useRangeFile)
: batchIndex(batchIndex), rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {}
bool operator<(RestoreSendMutationsToAppliersRequest const& rhs) const { return batchIndex > rhs.batchIndex; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchIndex, rangeToApplier, useRangeFile, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreSendMutationsToAppliersRequest batchIndex:" << batchIndex
<< " keyToAppliers.size:" << rangeToApplier.size() << " useRangeFile:" << useRangeFile;
return ss.str();
}
};
struct RestoreSendVersionedMutationsRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 2655701;
int batchIndex; // version batch index
RestoreAsset asset; // Unique identifier for the current restore asset
Version msgIndex; // Monitonically increasing index of mutation messages
bool isRangeFile;
VersionedMutationsVec versionedMutations; // Versioned mutations may be at different versions parsed by one loader
ReplyPromise<RestoreCommonReply> reply;
RestoreSendVersionedMutationsRequest() = default;
explicit RestoreSendVersionedMutationsRequest(int batchIndex,
const RestoreAsset& asset,
Version msgIndex,
bool isRangeFile,
VersionedMutationsVec versionedMutations)
: batchIndex(batchIndex), asset(asset), msgIndex(msgIndex), isRangeFile(isRangeFile),
versionedMutations(versionedMutations) {}
std::string toString() const {
std::stringstream ss;
ss << "VersionBatchIndex:" << batchIndex << " msgIndex:" << msgIndex << " isRangeFile:" << isRangeFile
<< " versionedMutations.size:" << versionedMutations.size() << " RestoreAsset:" << asset.toString();
return ss.str();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchIndex, asset, msgIndex, isRangeFile, versionedMutations, reply);
}
};
struct RestoreVersionBatchRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13337457;
int batchIndex;
ReplyPromise<RestoreCommonReply> reply;
RestoreVersionBatchRequest() = default;
explicit RestoreVersionBatchRequest(int batchIndex) : batchIndex(batchIndex) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchIndex, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreVersionBatchRequest batchIndex:" << batchIndex;
return ss.str();
}
};
struct RestoreFinishRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018413;
bool terminate; // role exits if terminate = true
ReplyPromise<RestoreCommonReply> reply;
RestoreFinishRequest() = default;
explicit RestoreFinishRequest(bool terminate) : terminate(terminate) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, terminate, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreFinishRequest terminate:" << terminate;
return ss.str();
}
};
struct RestoreUpdateRateReply : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018414;
UID id;
double remainMB; // remaining data in MB to write to DB;
RestoreUpdateRateReply() = default;
explicit RestoreUpdateRateReply(UID id, double remainMB) : id(id), remainMB(remainMB) {}
std::string toString() const {
std::stringstream ss;
ss << "RestoreUpdateRateReply NodeID:" << id.toString() << " remainMB:" << remainMB;
return ss.str();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, remainMB);
}
};
struct RestoreUpdateRateRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018415;
int batchIndex;
double writeMB;
ReplyPromise<RestoreUpdateRateReply> reply;
RestoreUpdateRateRequest() = default;
explicit RestoreUpdateRateRequest(int batchIndex, double writeMB) : batchIndex(batchIndex), writeMB(writeMB) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchIndex, writeMB, reply);
}
std::string toString() const {
std::stringstream ss;
ss << "RestoreUpdateRateRequest batchIndex:" << batchIndex << " writeMB:" << writeMB;
return ss.str();
}
};
////--- Interface functions
Future<Void> _restoreWorker(Database cx, LocalityData locality);
Future<Void> restoreWorker(Reference<IClusterConnectionRecord> ccr, LocalityData locality, std::string coordFolder);
extern const KeyRef restoreLeaderKey;
extern const KeyRangeRef restoreWorkersKeys;
extern const KeyRef restoreStatusKey; // To be used when we measure fast restore performance
extern const KeyRangeRef restoreRequestKeys;
extern const KeyRangeRef restoreApplierKeys;
extern const KeyRef restoreApplierTxnValue;
const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version);
std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key);
const Key restoreWorkerKeyFor(UID const& workerID);
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server);
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value);
Version decodeRestoreRequestDoneVersionValue(ValueRef const& value);
RestoreRequest decodeRestoreRequestValue(ValueRef const& value);
const Key restoreStatusKeyFor(StringRef statusType);
const Value restoreStatusValue(double val);
Value restoreRequestDoneVersionValue(Version readVersion);

4
fdbserver/storageserver/storageserver.actor.cpp
View File

@ -11897,7 +11897,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
state double lastLoopTopTime = now();
state Future<Void> dbInfoChange = Void();
state Future<Void> checkLastUpdate = Void();
state Future<Void> updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL);
state Future<Void> updateProcessStatsTimer = delay(SERVER_KNOBS->STORAGE_UPDATE_PROCESS_STATS_INTERVAL);
self->actors.add(updateStorage(self));
self->actors.add(waitFailureServer(ssi.waitFailure.getFuture()));
@ -12077,7 +12077,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
}
when(wait(updateProcessStatsTimer)) {
updateProcessStats(self);
updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL);
updateProcessStatsTimer = delay(SERVER_KNOBS->STORAGE_UPDATE_PROCESS_STATS_INTERVAL);
}
when(GetHotShardsRequest req = waitNext(ssi.getHotShards.getFuture())) {
struct ComparePair {

2
fdbserver/worker/CMakeLists.txt
View File

@ -4,7 +4,6 @@ add_flow_target(STATIC_LIBRARY NAME fdbserver_worker SRCS ${FDBSERVER_WORKER_SRC
add_fdbserver_link_test(fdbserver_workerlinktest
fdbserver_worker
fdbserver_backupworker
fdbserver_restoreworker
fdbserver_clustercontroller
fdbserver_commitproxy
fdbserver_consistencyscan
@ -40,7 +39,6 @@ target_link_libraries(fdbserver_worker
fdbctl
PRIVATE
fdbserver_backupworker
fdbserver_restoreworker
fdbserver_clustercontroller
fdbserver_commitproxy
fdbserver_consistencyscan

34
fdbserver/workloads/AtomicRestore.cpp
View File

@ -23,7 +23,6 @@
#include "fdbclient/BackupAgent.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbserver/core/Knobs.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "fdbserver/tester/workloads.h"
#include "BulkSetup.h"
@ -32,7 +31,6 @@
struct AtomicRestoreWorkload : TestWorkload {
static constexpr auto NAME = "AtomicRestore";
double startAfter, restoreAfter;
bool fastRestore; // true: use fast restore, false: use old style restore
Standalone<VectorRef<KeyRangeRef>> backupRanges;
UsePartitionedLog usePartitionedLogs{ false };
Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
@ -42,13 +40,7 @@ struct AtomicRestoreWorkload : TestWorkload {
startAfter = getOption(options, "startAfter"_sr, 10.0);
restoreAfter = getOption(options, "restoreAfter"_sr, 20.0);
fastRestore = getOption(options, "fastRestore"_sr, false);
if (!fastRestore) {
addDefaultBackupRanges(backupRanges);
} else {
// Fast restore doesn't support multiple ranges yet
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
}
addDefaultBackupRanges(backupRanges);
usePartitionedLogs.set(
getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->random01() < 0.5 ? true : false));
@ -119,22 +111,16 @@ struct AtomicRestoreWorkload : TestWorkload {
co_await delay(restoreAfter * deterministicRandom()->random01());
TraceEvent("AtomicRestore_RestoreStart").log();
if (fastRestore) { // New fast parallel restore
TraceEvent(SevInfo, "AtomicParallelRestore").log();
co_await backupAgent.atomicParallelRestore(
cx, BackupAgentBase::getDefaultTag(), backupRanges, addPrefix, removePrefix);
} else { // Old style restore
while (true) {
try {
co_await backupAgent.atomicRestore(
cx, BackupAgentBase::getDefaultTag(), backupRanges, StringRef(), StringRef());
break;
} catch (Error& e) {
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
while (true) {
try {
co_await backupAgent.atomicRestore(
cx, BackupAgentBase::getDefaultTag(), backupRanges, StringRef(), StringRef());
break;
} catch (Error& e) {
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
}
// SOMEDAY: Remove after backup agents can exist quiescently

803
fdbserver/workloads/BackupAndParallelRestoreCorrectness.cpp
View File

@ -1,803 +0,0 @@
/*
* BackupAndParallelRestoreCorrectness.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.h"
#include "fdbclient/BackupContainer.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
#include "fdbclient/RunRYWTransaction.h"
#include "fdbserver/restoreworker/RestoreCommon.h"
#include "fdbserver/tester/workloads.h"
#include "fdbserver/tester/TestEncryptionUtils.h"
#include "BulkSetup.h"
#define TEST_ABORT_FASTRESTORE 0
struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
static constexpr auto NAME = "BackupAndParallelRestoreCorrectness";
double backupAfter, restoreAfter, abortAndRestartAfter;
double backupStartAt, restoreStartAfterBackupFinished, stopDifferentialAfter;
Key backupTag;
int backupRangesCount, backupRangeLengthMax;
bool differentialBackup, performRestore, agentRequest;
Standalone<VectorRef<KeyRangeRef>> backupRanges;
static int backupAgentRequests;
LockDB locked{ false };
bool allowPauses;
bool shareLogRange;
UsePartitionedLog usePartitionedLogs{ false };
Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
// CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix
Optional<std::string> encryptionKeyFileName;
std::map<Standalone<KeyRef>, Standalone<ValueRef>> dbKVs;
// This workload is not compatible with RandomRangeLock workload because they will race in locked range
void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
out.insert({ "RandomRangeLock" });
}
BackupAndParallelRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
locked.set(sharedRandomNumber % 2);
backupAfter = getOption(options, "backupAfter"_sr, 10.0);
restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
performRestore = getOption(options, "performRestore"_sr, true);
backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
abortAndRestartAfter =
getOption(options,
"abortAndRestartAfter"_sr,
deterministicRandom()->random01() < 0.5
? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter
: 0.0);
differentialBackup =
getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false);
stopDifferentialAfter =
getOption(options,
"stopDifferentialAfter"_sr,
differentialBackup ? deterministicRandom()->random01() *
(restoreAfter - std::max(abortAndRestartAfter, backupAfter)) +
std::max(abortAndRestartAfter, backupAfter)
: 0.0);
agentRequest = getOption(options, "simBackupAgents"_sr, true);
allowPauses = getOption(options, "allowPauses"_sr, true);
shareLogRange = getOption(options, "shareLogRange"_sr, false);
usePartitionedLogs.set(getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->coinflip()));
addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.5)) {
encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
}
KeyRef beginRange;
KeyRef endRange;
UID randomID = nondeterministicRandom()->randomUniqueID();
// Correctness is not clean for addPrefix feature yet. Uncomment below to enable the test
// Generate addPrefix
// if (addPrefix.size() == 0 && removePrefix.size() == 0) {
// if (deterministicRandom()->random01() < 0.5) { // Generate random addPrefix
// int len = deterministicRandom()->randomInt(1, 100);
// std::string randomStr = deterministicRandom()->randomAlphaNumeric(len);
// TraceEvent("BackupAndParallelRestoreCorrectness")
// .detail("GenerateAddPrefix", randomStr)
// .detail("Length", len)
// .detail("StrLen", randomStr.size());
// addPrefix = Key(randomStr);
// }
// }
TraceEvent("BackupAndParallelRestoreCorrectness")
.detail("AddPrefix", addPrefix)
.detail("RemovePrefix", removePrefix);
ASSERT(addPrefix.size() == 0 && removePrefix.size() == 0);
// Do not support removePrefix right now because we must ensure all backup keys have the removePrefix
// otherwise, test will fail because fast restore will simply add the removePrefix to every key in the end.
ASSERT(removePrefix.size() == 0);
if (shareLogRange) {
bool beforePrefix = sharedRandomNumber & 1;
if (beforePrefix)
backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
else
backupRanges.push_back_deep(backupRanges.arena(),
KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end));
} else if (backupRangesCount <= 0) {
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
} else {
// Add backup ranges
std::set<std::string> rangeEndpoints;
while (rangeEndpoints.size() < backupRangesCount * 2) {
rangeEndpoints.insert(deterministicRandom()->randomAlphaNumeric(
deterministicRandom()->randomInt(1, backupRangeLengthMax + 1)));
}
// Create ranges from the keys, in order, to prevent overlaps
std::vector<std::string> sortedEndpoints(rangeEndpoints.begin(), rangeEndpoints.end());
sort(sortedEndpoints.begin(), sortedEndpoints.end());
for (auto i = sortedEndpoints.begin(); i != sortedEndpoints.end(); ++i) {
const std::string& start = *i++;
backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(start, *i));
// Track the added range
TraceEvent("BARW_BackupCorrectnessRange", randomID)
.detail("RangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange))
.detail("RangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange));
}
}
}
Future<Void> setup(Database const& cx) override { return Void(); }
Future<Void> start(Database const& cx) override {
if (clientId != 0)
return Void();
TraceEvent(SevInfo, "BARW_Param").detail("Locked", locked);
TraceEvent(SevInfo, "BARW_Param").detail("BackupAfter", backupAfter);
TraceEvent(SevInfo, "BARW_Param").detail("RestoreAfter", restoreAfter);
TraceEvent(SevInfo, "BARW_Param").detail("PerformRestore", performRestore);
TraceEvent(SevInfo, "BARW_Param").detail("BackupTag", printable(backupTag).c_str());
TraceEvent(SevInfo, "BARW_Param").detail("BackupRangesCount", backupRangesCount);
TraceEvent(SevInfo, "BARW_Param").detail("BackupRangeLengthMax", backupRangeLengthMax);
TraceEvent(SevInfo, "BARW_Param").detail("AbortAndRestartAfter", abortAndRestartAfter);
TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup);
TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter);
TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest);
TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present());
return _start(cx);
}
bool hasPrefix() const { return addPrefix != ""_sr || removePrefix != ""_sr; }
Future<bool> check(Database const& cx) override { return true; }
void getMetrics(std::vector<PerfMetric>& m) override {}
static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
while (true) {
co_await backupAgent->changePause(cx, true);
co_await delay(30 * deterministicRandom()->random01());
co_await backupAgent->changePause(cx, false);
co_await delay(120 * deterministicRandom()->random01());
}
}
static Future<Void> statusLoop(Database cx, std::string tag) {
FileBackupAgent agent;
while (true) {
std::string status = co_await agent.getStatus(cx, ShowErrors::True, tag);
puts(status.c_str());
co_await delay(2.0);
}
}
Future<Void> doBackup(double startDelay,
FileBackupAgent* backupAgent,
Database cx,
Key tag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
double stopDifferentialDelay,
Promise<Void> submitted) {
UID randomID = nondeterministicRandom()->randomUniqueID();
Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
co_await delay(startDelay);
if (startDelay || BUGGIFY) {
TraceEvent("BARW_DoBackupAbortBackup1", randomID)
.detail("Tag", printable(tag))
.detail("StartDelay", startDelay);
try {
co_await backupAgent->abortBackup(cx, tag.toString());
} catch (Error& e) {
TraceEvent("BARW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
if (e.code() != error_code_backup_unneeded)
throw;
}
}
TraceEvent("BARW_DoBackupSubmitBackup", randomID)
.detail("Tag", printable(tag))
.detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
std::string backupContainer = "file://simfdb/backups/";
Future<Void> status = statusLoop(cx, tag.toString());
try {
co_await backupAgent->submitBackup(cx,
StringRef(backupContainer),
{},
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
tag.toString(),
backupRanges,
StopWhenDone{ !stopDifferentialDelay },
usePartitionedLogs,
IncrementalBackupOnly::False,
encryptionKeyFileName);
} catch (Error& e) {
TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
submitted.send(Void());
// Stop the differential backup, if enabled
if (stopDifferentialDelay) {
CODE_PROBE(!stopDifferentialFuture.isReady(),
"Restore starts at specified time - stopDifferential not ready");
co_await stopDifferentialFuture;
TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
.detail("Tag", printable(tag))
.detail("DifferentialAfter", stopDifferentialDelay);
try {
if (BUGGIFY) {
KeyBackedTag backupTag = makeBackupTag(tag.toString());
TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
// Wait until the backup is in a restorable state and get the status, URL, and UID atomically
Reference<IBackupContainer> lastBackupContainer;
UID lastBackupUID;
EBackupState resultWait = co_await backupAgent->waitBackup(
cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID);
TraceEvent("BARW_DoBackupWaitForRestorable", randomID)
.detail("Tag", backupTag.tagName)
.detail("Result", BackupAgentBase::getStateText(resultWait));
bool restorable = false;
if (lastBackupContainer) {
Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
co_await ready(fdesc);
if (!fdesc.isError()) {
BackupDescription desc = fdesc.get();
co_await desc.resolveVersionTimes(cx);
printf("BackupDescription:\n%s\n", desc.toString().c_str());
restorable = desc.maxRestorableVersion.present();
}
}
TraceEvent("BARW_LastBackupContainer", randomID)
.detail("BackupTag", printable(tag))
.detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
.detail("LastBackupUID", lastBackupUID)
.detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
.detail("Restorable", restorable);
// Do not check the backup, if aborted
if (resultWait == EBackupState::STATE_ABORTED) {
}
// Ensure that a backup container was found
else if (!lastBackupContainer) {
TraceEvent(SevError, "BARW_MissingBackupContainer", randomID)
.detail("LastBackupUID", lastBackupUID)
.detail("BackupTag", printable(tag))
.detail("WaitStatus", resultWait);
printf("BackupCorrectnessMissingBackupContainer tag: %s status: %s\n",
printable(tag).c_str(),
BackupAgentBase::getStateText(resultWait));
}
// Check that backup is restorable
else if (!restorable) {
TraceEvent(SevError, "BARW_NotRestorable", randomID)
.detail("LastBackupUID", lastBackupUID)
.detail("BackupTag", printable(tag))
.detail("BackupFolder", lastBackupContainer->getURL())
.detail("WaitStatus", BackupAgentBase::getStateText(resultWait));
printf("BackupCorrectnessNotRestorable: tag: %s\n", printable(tag).c_str());
}
// Abort the backup, if not the first backup because the second backup may have aborted the backup
// by now
if (startDelay) {
TraceEvent("BARW_DoBackupAbortBackup2", randomID)
.detail("Tag", printable(tag))
.detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
.detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
.detail("Restorable", restorable);
co_await backupAgent->abortBackup(cx, tag.toString());
} else {
TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
.detail("Tag", printable(tag))
.detail("DifferentialAfter", stopDifferentialDelay);
co_await backupAgent->discontinueBackup(cx, tag);
}
}
else {
TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
.detail("Tag", printable(tag))
.detail("DifferentialAfter", stopDifferentialDelay);
co_await backupAgent->discontinueBackup(cx, tag);
}
} catch (Error& e) {
TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
}
// Wait for the backup to complete
TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
EBackupState statusValue = co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
std::string statusText;
std::string _statusText = co_await backupAgent->getStatus(cx, ShowErrors::True, tag.toString());
statusText = _statusText;
// Can we validate anything about status?
TraceEvent("BARW_DoBackupComplete", randomID)
.detail("Tag", printable(tag))
.detail("Status", statusText)
.detail("StatusValue", BackupAgentBase::getStateText(statusValue));
}
// This actor attempts to restore the database without clearing the keyspace.
// TODO: Enable this function in correctness test
Future<Void> attemptDirtyRestore(Database cx,
FileBackupAgent* backupAgent,
Standalone<StringRef> lastBackupContainer,
UID randomID) {
Transaction tr(cx);
int rowCount = 0;
while (true) {
Error err;
try {
RangeResult existingRows = co_await tr.getRange(normalKeys, 1);
rowCount = existingRows.size();
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
// Try doing a restore without clearing the keys
if (rowCount > 0) {
try {
// TODO: Change to my restore agent code
TraceEvent(SevError, "MXFastRestore").detail("RestoreFunction", "ShouldChangeToMyOwnRestoreLogic");
co_await backupAgent->restore(cx,
cx,
backupTag,
KeyRef(lastBackupContainer),
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
normalKeys,
Key(),
Key(),
locked,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
::invalidVersion,
encryptionKeyFileName);
TraceEvent(SevError, "BARW_RestoreAllowedOverwrittingDatabase", randomID).log();
ASSERT(false);
} catch (Error& e) {
if (e.code() != error_code_restore_destination_not_empty) {
throw;
}
}
}
}
Future<Void> _start(Database cx) {
FileBackupAgent backupAgent;
Future<Void> extraBackup;
UID randomID = nondeterministicRandom()->randomUniqueID();
int restoreIndex = 0;
ReadYourWritesTransaction tr2(cx);
TraceEvent("BARW_Arguments")
.detail("BackupTag", printable(backupTag))
.detail("PerformRestore", performRestore)
.detail("BackupAfter", backupAfter)
.detail("RestoreAfter", restoreAfter)
.detail("AbortAndRestartAfter", abortAndRestartAfter)
.detail("DifferentialAfter", stopDifferentialAfter);
if (allowPauses && BUGGIFY) {
Future<Void> cp = changePaused(cx, &backupAgent);
}
// Increment the backup agent requests
if (agentRequest) {
BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests++;
}
if (encryptionKeyFileName.present()) {
co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
}
try {
Future<Void> startRestore = delay(restoreAfter);
// backup
co_await delay(backupAfter);
TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(backupTag));
Promise<Void> submitted;
Future<Void> b = doBackup(0, &backupAgent, cx, backupTag, backupRanges, stopDifferentialAfter, submitted);
if (abortAndRestartAfter) {
TraceEvent("BARW_DoBackup2", randomID)
.detail("Tag", printable(backupTag))
.detail("AbortWait", abortAndRestartAfter);
co_await submitted.getFuture();
b = b && doBackup(abortAndRestartAfter,
&backupAgent,
cx,
backupTag,
backupRanges,
stopDifferentialAfter,
Promise<Void>());
}
TraceEvent("BARW_DoBackupWait", randomID)
.detail("BackupTag", printable(backupTag))
.detail("AbortAndRestartAfter", abortAndRestartAfter);
try {
co_await b;
} catch (Error& e) {
if (e.code() != error_code_database_locked)
throw;
if (performRestore)
throw;
co_return;
}
TraceEvent("BARW_DoBackupDone", randomID)
.detail("BackupTag", printable(backupTag))
.detail("AbortAndRestartAfter", abortAndRestartAfter);
KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
UID logUid = uidFlag.first;
Key destUidValue = co_await BackupConfig(logUid).destUidValue().getD(cx.getReference());
Reference<IBackupContainer> lastBackupContainer =
co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
// Occasionally start yet another backup that might still be running when we restore
if (!locked && BUGGIFY) {
TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(backupTag));
try {
// Note the "partitionedLog" must be false, because we change
// the configuration to disable backup workers before restore.
extraBackup = backupAgent.submitBackup(cx,
"file://simfdb/backups/"_sr,
{},
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
backupTag.toString(),
backupRanges,
StopWhenDone::True,
UsePartitionedLog::False,
IncrementalBackupOnly::False,
encryptionKeyFileName);
} catch (Error& e) {
TraceEvent("BARW_SubmitBackup2Exception", randomID)
.error(e)
.detail("BackupTag", printable(backupTag));
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
}
CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
co_await startRestore;
if (lastBackupContainer && performRestore) {
if (deterministicRandom()->random01() < 0.5) {
printf("TODO: Check if restore can succeed if dirty restore is performed first\n");
// TODO: To support restore even after we attempt dirty restore. Not implemented in the 1st version
// fast restore
// wait(attemptDirtyRestore(cx, &backupAgent, StringRef(lastBackupContainer->getURL()),
// randomID));
}
// We must ensure no backup workers are running, otherwise the clear DB
// below can be picked up by backup workers and applied during restore.
co_await ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=0", true);
// Clear DB before restore
co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
for (auto& kvrange : backupRanges)
tr->clear(kvrange);
return Void();
});
// restore database
TraceEvent("BAFRW_Restore", randomID)
.detail("LastBackupContainer", lastBackupContainer->getURL())
.detail("RestoreAfter", restoreAfter)
.detail("BackupTag", printable(backupTag));
// start restoring
auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
lastBackupContainer->getProxy(),
lastBackupContainer->getEncryptionKeyFileName());
BackupDescription desc = co_await container->describeBackup();
ASSERT(usePartitionedLogs == desc.partitioned);
ASSERT(desc.minRestorableVersion.present()); // We must have a valid backup now.
Version targetVersion = -1;
if (desc.maxRestorableVersion.present()) {
if (deterministicRandom()->random01() < 0.1) {
targetVersion = desc.minRestorableVersion.get();
} else if (deterministicRandom()->random01() < 0.1) {
targetVersion = desc.maxRestorableVersion.get();
} else if (deterministicRandom()->random01() < 0.5) {
targetVersion = (desc.minRestorableVersion.get() != desc.maxRestorableVersion.get())
? deterministicRandom()->randomInt64(desc.minRestorableVersion.get(),
desc.maxRestorableVersion.get())
: desc.maxRestorableVersion.get();
}
}
TraceEvent("BAFRW_Restore", randomID)
.detail("LastBackupContainer", lastBackupContainer->getURL())
.detail("MinRestorableVersion", desc.minRestorableVersion.get())
.detail("MaxRestorableVersion", desc.maxRestorableVersion.get())
.detail("ContiguousLogEnd", desc.contiguousLogEnd.get())
.detail("TargetVersion", targetVersion);
std::vector<Future<Version>> restores;
std::vector<Standalone<StringRef>> restoreTags;
// Submit parallel restore requests
TraceEvent("BackupAndParallelRestoreWorkload")
.detail("PrepareRestores", backupRanges.size())
.detail("AddPrefix", addPrefix)
.detail("RemovePrefix", removePrefix);
co_await backupAgent.submitParallelRestore(cx,
backupTag,
backupRanges,
KeyRef(lastBackupContainer->getURL()),
lastBackupContainer->getProxy(),
targetVersion,
locked,
randomID,
addPrefix,
removePrefix);
TraceEvent("BackupAndParallelRestoreWorkload")
.detail("TriggerRestore", "Setting up restoreRequestTriggerKey");
// Sometimes kill and restart the restore
// In real cluster, aborting a restore needs:
// (1) kill restore cluster; (2) clear dest. DB restore system keyspace.
// TODO: Consider gracefully abort a restore and restart.
if (BUGGIFY && TEST_ABORT_FASTRESTORE) {
TraceEvent(SevError, "FastRestore").detail("Buggify", "NotImplementedYet");
co_await delay(deterministicRandom()->randomInt(0, 10));
for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) {
FileBackupAgent::ERestoreState rs =
co_await backupAgent.abortRestore(cx, restoreTags[restoreIndex]);
// The restore may have already completed, or the abort may have been done before the restore
// was even able to start. Only run a new restore if the previous one was actually aborted.
if (rs == FileBackupAgent::ERestoreState::ABORTED) {
co_await runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
tr->clear(backupRanges[restoreIndex]);
return Void();
});
// TODO: Not Implemented yet
// restores[restoreIndex] = backupAgent.restore(cx, restoreTags[restoreIndex],
// KeyRef(lastBackupContainer->getURL()), true, -1, true, backupRanges[restoreIndex],
// Key(), Key(), locked);
}
}
}
// Wait for parallel restore to finish before we can proceed
TraceEvent("FastRestoreWorkload").detail("WaitForRestoreToFinish", randomID);
// Do not unlock DB when restore finish because we need to transformDatabaseContents
co_await backupAgent.parallelRestoreFinish(cx, randomID, UnlockDB{ !hasPrefix() });
TraceEvent("FastRestoreWorkload").detail("RestoreFinished", randomID);
for (auto& restore : restores) {
ASSERT(!restore.isError());
}
// If addPrefix or removePrefix set, we want to transform the effect by copying data
if (hasPrefix()) {
co_await transformRestoredDatabase(cx, backupRanges, addPrefix, removePrefix);
co_await unlockDatabase(cx, randomID);
}
}
// Q: What is the extra backup and why do we need to care about it?
if (extraBackup.isValid()) { // SOMEDAY: Handle this case
TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(backupTag));
try {
co_await extraBackup;
} catch (Error& e) {
TraceEvent("BARW_ExtraBackupException", randomID)
.error(e)
.detail("BackupTag", printable(backupTag));
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw;
}
TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(backupTag));
try {
co_await backupAgent.abortBackup(cx, backupTag.toString());
} catch (Error& e) {
TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
if (e.code() != error_code_backup_unneeded)
throw;
}
}
Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
int displaySystemKeys = 0;
// Ensure that there is no left over key within the backup subspace
while (true) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(backupTag));
Error err;
try {
tr->reset();
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// Check the left over tasks
// We have to wait for the list to empty since an abort and get status
// can leave extra tasks in the queue
TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(backupTag));
int64_t taskCount = co_await backupAgent.getTaskCount(tr);
int waitCycles = 0;
while (true) {
waitCycles++;
TraceEvent("BARW_NonzeroTaskWait", randomID)
.detail("BackupTag", printable(backupTag))
.detail("TaskCount", taskCount)
.detail("WaitCycles", waitCycles);
printf("%.6f %-10s Wait #%4d for %lld tasks to end\n",
now(),
randomID.toString().c_str(),
waitCycles,
(long long)taskCount);
co_await delay(5.0);
co_await tr->commit();
tr = makeReference<ReadYourWritesTransaction>(cx);
int64_t _taskCount = co_await backupAgent.getTaskCount(tr);
taskCount = _taskCount;
if (!taskCount) {
break;
}
}
if (taskCount) {
displaySystemKeys++;
TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID)
.detail("BackupTag", printable(backupTag))
.detail("TaskCount", taskCount)
.detail("WaitCycles", waitCycles);
printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long)taskCount);
}
RangeResult agentValues =
co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
// Error if the system keyspace for the backup tag is not empty
if (agentValues.size() > 0) {
displaySystemKeys++;
printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n",
agentValues.size(),
printable(backupAgentKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID)
.detail("BackupTag", printable(backupTag))
.detail("LeftOverKeys", agentValues.size())
.detail("KeySpace", printable(backupAgentKey));
for (auto& s : agentValues) {
TraceEvent("BARW_LeftOverKey", randomID)
.detail("Key", printable(StringRef(s.key.toString())))
.detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n",
printable(StringRef(s.key.toString())).c_str(),
printable(StringRef(s.value.toString())).c_str());
}
} else {
printf("No left over backup agent configuration keys\n");
}
Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
if (latestVersion.present()) {
TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID)
.detail("BackupTag", printable(backupTag))
.detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable())
.detail("DestUidValue", destUidValue.printable());
} else {
printf("No left over backup version key\n");
}
RangeResult versions = co_await tr->getRange(
KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
if (!shareLogRange || !versions.size()) {
RangeResult logValues = co_await tr->getRange(
KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
// Error if the log/mutation keyspace for the backup tag is not empty
if (logValues.size() > 0) {
displaySystemKeys++;
printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n",
logValues.size(),
printable(backupLogValuesKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID)
.detail("BackupTag", printable(backupTag))
.detail("LeftOverKeys", logValues.size())
.detail("KeySpace", printable(backupLogValuesKey));
} else {
printf("No left over backup log keys\n");
}
}
break;
} catch (Error& e) {
err = e;
}
TraceEvent("BARW_CheckException", randomID).error(err);
co_await tr->onError(err);
}
if (displaySystemKeys) {
co_await TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef());
}
TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(backupTag));
// Decrement the backup agent requests
if (agentRequest) {
BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests--;
}
// SOMEDAY: Remove after backup agents can exist quiescently
if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
(!BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests)) {
g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
}
} catch (Error& e) {
TraceEvent(SevError, "BackupAndParallelRestoreCorrectness").error(e).GetLastError();
throw;
}
}
};
int BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests = 0;
WorkloadFactory<BackupAndParallelRestoreCorrectnessWorkload> BackupAndParallelRestoreCorrectnessWorkloadFactory;

1
fdbserver/workloads/CMakeLists.txt
View File

@ -12,6 +12,5 @@ target_link_libraries(fdbserver_workloads PRIVATE
fdbserver_worker
fdbserver_tester
fdbserver_datadistributor
fdbserver_restoreworker
fdbserver_resolver
fdbserver_mocks3)

60
fdbserver/workloads/ParallelRestore.cpp
View File

@ -1,60 +0,0 @@
/*
* ParallelRestore.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/tester/workloads.h"
#include "BulkSetup.h"
#include "fdbserver/restoreworker/RestoreWorkerInterface.h"
// A workload which test the correctness of backup and restore process
struct RunRestoreWorkerWorkload : TestWorkload {
static constexpr auto NAME = "RunRestoreWorkerWorkload";
Future<Void> worker;
RunRestoreWorkerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
TraceEvent("RunRestoreWorkerWorkloadMX").log();
}
Future<Void> setup(Database const& cx) override { return Void(); }
Future<Void> start(Database const& cx) override {
int num_myWorkers = SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS + SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + 1;
TraceEvent("RunParallelRestoreWorkerWorkload")
.detail("Start", "RestoreToolDB")
.detail("Workers", num_myWorkers);
printf("RunParallelRestoreWorkerWorkload, we will start %d restore workers\n", num_myWorkers);
std::vector<Future<Void>> myWorkers;
myWorkers.reserve(num_myWorkers);
for (int i = 0; i < num_myWorkers; ++i) {
myWorkers.push_back(_restoreWorker(cx, LocalityData()));
}
printf("RunParallelRestoreWorkerWorkload, wait on reply from %ld restore workers\n", myWorkers.size());
worker = waitForAll(myWorkers);
printf("RunParallelRestoreWorkerWorkload, got all replies from restore workers\n");
return Void();
}
Future<bool> check(Database const& cx) override { return true; }
void getMetrics(std::vector<PerfMetric>& m) override {}
};
WorkloadFactory<RunRestoreWorkerWorkload> RunRestoreWorkerWorkloadFactory;

2
packaging/docker/Dockerfile
View File

@ -133,7 +133,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
$CURL $FDB_WEBSITE/$FDB_VERSION/libfdb_c.${FDB_ARCH}.so -o /usr/lib/libfdb_c.so
# Setup all symlinks for the other binaries that are a copy of fdbbackup
RUN for file in fdbdr fdbrestore backup_agent dr_agent fastrestore_tool; do \
RUN for file in fdbdr fdbrestore backup_agent dr_agent; do \
ln -s /usr/bin/fdbbackup "/usr/bin/$file"; \
done && \
cd / && \

12
tests/CMakeLists.txt
View File

@ -89,7 +89,6 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES KVStoreTestWrite.txt UNIT IGNORE)
add_fdb_test(TEST_FILES KVStoreValueSize.txt UNIT IGNORE)
add_fdb_test(TEST_FILES LayerStatusMerge.txt IGNORE)
add_fdb_test(TEST_FILES ParallelRestoreApiCorrectnessAtomicRestore.txt IGNORE)
add_fdb_test(TEST_FILES PureNetwork.txt IGNORE)
add_fdb_test(TEST_FILES RRW2500.txt IGNORE)
add_fdb_test(TEST_FILES RandomRead.txt IGNORE)
@ -435,21 +434,12 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES slow/WriteDuringReadAtomicRestore.toml)
add_fdb_test(TEST_FILES slow/WriteDuringReadSwitchover.toml)
add_fdb_test(TEST_FILES slow/ddbalance.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessCycle.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessCycle.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml)
add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml)
add_fdb_test(TEST_FILES negative/ResolverIgnoreTooOld.toml)
add_fdb_test(TEST_FILES negative/ResolverIgnoreReads.toml)
add_fdb_test(TEST_FILES negative/ResolverIgnoreWrites.toml)
add_fdb_test(TEST_FILES negative/StorageCorruption.toml)
add_fdb_test(TEST_FILES ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml IGNORE)
# Note that status tests are not deterministic.
# Note that status tests are not deterministic.
add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
add_fdb_test(TEST_FILES status/local_6_machine_no_replicas_remain.txt)
add_fdb_test(TEST_FILES status/separate_1_of_3_coordinators_remain.txt)

36
tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
View File

@ -1,36 +0,0 @@
testTitle=ApiCorrectnessTest
clearAfterTest=false
simBackupAgents=BackupToFile
;timeout is in seconds
timeout=360000
runSetup=true
testName=ApiCorrectness
numKeys=5000
onlyLowerCase=true
shortKeysRatio=0.5
minShortKeyLength=1
maxShortKeyLength=3
minLongKeyLength=1
maxLongKeyLength=128
minValueLength=1
maxValueLength=1000
numGets=1000
numGetRanges=100
numGetRangeSelectors=100
numGetKeys=100
numClears=100
numClearRanges=10
maxTransactionBytes=500000
randomTestDuration=60
testName=AtomicRestore
startAfter=10.0
restoreAfter=50.0
fastRestore=true
usePartitionedLogs=true
; Each testName=RunRestoreWorkerWorkload creates a restore worker
; We need at least 3 restore workers: master, loader, and applier
testName=RunRestoreWorkerWorkload

40
tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
View File

@ -1,40 +0,0 @@
[[test]]
testTitle = 'ApiCorrectnessTest'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
runSetup = true
[[test.workload]]
testName = 'ApiCorrectness'
numKeys = 5000
onlyLowerCase = true
shortKeysRatio = 0.5
minShortKeyLength = 1
maxShortKeyLength = 3
minLongKeyLength = 1
maxLongKeyLength = 128
minValueLength = 1
maxValueLength = 1000
numGets = 1000
numGetRanges = 100
numGetRangeSelectors = 100
numGetKeys = 100
numClears = 100
numClearRanges = 10
maxTransactionBytes = 500000
randomTestDuration = 60
resetDBTimeout = 7200
[[test.workload]]
testName = 'AtomicRestore'
startAfter = 10.0
restoreAfter = 50.0
fastRestore = true
usePartitionedLogs = false
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'

62
tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
View File

@ -1,62 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndParallelRestoreWithAtomicOp'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'AtomicOps'
nodeCount = 30000
# Make ops space only 1 key per group
# nodeCount=100
transactionsPerSecond = 2500.0
# transactionsPerSecond=500.0
# transactionsPerSecond=500.0
# nodeCount=4
# transactionsPerSecond=250.0
testDuration = 30.0
# Specify a type of atomicOp
# opType=0
# actorsPerClient=1
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
backupRangesCount = -1
# use new backup
usePartitionedLogs = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

57
tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
View File

@ -1,57 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndRestore'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'Cycle'
# nodeCount=30000
nodeCount = 1000
# transactionsPerSecond=500.0
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
# keyPrefix=!
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
# backupRangesCount<0 means backup the entire normal keyspace
backupRangesCount = -1
usePartitionedLogs = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

80
tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
View File

@ -1,80 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndRestore'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'Cycle'
# nodeCount=30000
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = '!'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'z'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'A'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'Z'
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
# backupRangesCount<0 means backup the entire normal keyspace
backupRangesCount = -1
usePartitionedLogs = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

52
tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
View File

@ -1,52 +0,0 @@
[configuration]
StderrSeverity = 30
[[test]]
testTitle = 'WriteDuringReadTest'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'WriteDuringRead'
maximumTotalData = 1000000
testDuration = 240.0
slowModeStart = 60.0
minNode = 1
useSystemKeys = false
[[test.workload]]
testName = 'AtomicRestore'
startAfter = 10.0
restoreAfter = 50.0
fastRestore = true
usePartitionedLogs = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 60.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 60.0
testDuration = 60.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 60.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 60.0
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'

60
tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
View File

@ -1,60 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndParallelRestoreWithAtomicOp'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'AtomicOps'
nodeCount = 30000
# Make ops space only 1 key per group
transactionsPerSecond = 2500.0
# nodeCount=4
# transactionsPerSecond=250.0
testDuration = 30.0
# Specify a type of atomicOp
# Unset the following two options for debug purpose
# opType=0
# actorsPerClient=1
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
backupRangesCount = -1
# use old backup
usePartitionedLogs = false
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

57
tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
View File

@ -1,57 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndRestore'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'Cycle'
nodeCount=30000
# nodeCount = 1000
# transactionsPerSecond=500.0
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
# keyPrefix=!
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
# backupRangesCount<0 means backup the entire normal keyspace
backupRangesCount = -1
usePartitionedLogs = false
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

80
tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
View File

@ -1,80 +0,0 @@
[configuration]
[[test]]
testTitle = 'BackupAndRestore'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'Cycle'
# nodeCount=30000
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = '!'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'z'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'A'
[[test.workload]]
testName = 'Cycle'
nodeCount = 1000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
keyPrefix = 'Z'
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'
# Test case for parallel restore
[[test.workload]]
testName = 'BackupAndParallelRestoreCorrectness'
backupAfter = 10.0
restoreAfter = 60.0
# backupRangesCount<0 means backup the entire normal keyspace
backupRangesCount = -1
usePartitionedLogs = false
[[test.workload]]
testName = 'RandomClogging'
testDuration = 90.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 90.0
testDuration = 90.0
# Do NOT kill restore worker process yet
# Kill other process to ensure restore works when FDB cluster has faults
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 90.0

52
tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
View File

@ -1,52 +0,0 @@
[configuration]
StderrSeverity = 30
[[test]]
testTitle = 'WriteDuringReadTest'
clearAfterTest = false
simBackupAgents = 'BackupToFile'
#timeout is in seconds
timeout = 360000
[[test.workload]]
testName = 'WriteDuringRead'
maximumTotalData = 1000000
testDuration = 240.0
slowModeStart = 60.0
minNode = 1
useSystemKeys = false
[[test.workload]]
testName = 'AtomicRestore'
startAfter = 10.0
restoreAfter = 50.0
fastRestore = true
usePartitionedLogs = false
[[test.workload]]
testName = 'RandomClogging'
testDuration = 60.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 60.0
testDuration = 60.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 60.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 60.0
# Each testName=RunRestoreWorkerWorkload creates a restore worker
# We need at least 3 restore workers: master, loader, and applier
[[test.workload]]
testName = 'RunRestoreWorkerWorkload'