wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FastRestore:Add trace for file size and bc progress

This commit is contained in:
Meng Xu 2020-07-25 19:12:25 -07:00
parent bec2e166a3
commit 99d8399f4e
5 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fdbclient/BackupContainer.actor.cpp
View File

@ -841,10 +841,18 @@ public:
state std::vector<LogFile> logs; state std::vector<LogFile> logs;
state std::vector<LogFile> plogs; state std::vector<LogFile> plogs;
TraceEvent("BackupContainerListFiles").detail("URL", bc->getURL());
wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) && wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
store(plogs, bc->listLogFiles(scanBegin, scanEnd, true)) && store(plogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
store(desc.snapshots, bc->listKeyspaceSnapshots())); store(desc.snapshots, bc->listKeyspaceSnapshots()));
TraceEvent("BackupContainerListFiles")
.detail("URL", bc->getURL())
.detail("LogFiles", logs.size())
.detail("PLogsFiles", plogs.size())
.detail("Snapshots", desc.snapshots.size());
if (plogs.size() > 0) { if (plogs.size() > 0) {
desc.partitioned = true; desc.partitioned = true;
logs.swap(plogs); logs.swap(plogs);

2
fdbclient/MasterProxyInterface.h
View File

@ -424,7 +424,7 @@ struct ProxySnapRequest
{ {
constexpr static FileIdentifier file_identifier = 22204900; constexpr static FileIdentifier file_identifier = 22204900;
Arena arena; Arena arena;
StringRef snapPayload; StringRef snapPayload; // command used to snapshot the data folder
UID snapUID; UID snapUID;
ReplyPromise<Void> reply; ReplyPromise<Void> reply;
Optional<UID> debugID; Optional<UID> debugID;

2
fdbserver/Knobs.cpp
View File

@ -610,7 +610,7 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi
init( FASTRESTORE_NUM_LOADERS, 2 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; } init( FASTRESTORE_NUM_LOADERS, 2 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_NUM_APPLIERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; } init( FASTRESTORE_NUM_APPLIERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_TXN_BATCH_MAX_BYTES, 1048576.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; } init( FASTRESTORE_TXN_BATCH_MAX_BYTES, 1048576.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; }
init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() * 10.0 * 1024.0 * 1024.0 * 1024.0; } init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 2.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() * 10.0 * 1024.0 * 1024.0 * 1024.0; }
init( FASTRESTORE_VB_PARALLELISM, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() * 20 + 1; } init( FASTRESTORE_VB_PARALLELISM, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() * 20 + 1; }
init( FASTRESTORE_VB_MONITOR_DELAY, 30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; } init( FASTRESTORE_VB_MONITOR_DELAY, 30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; }
init( FASTRESTORE_VB_LAUNCH_DELAY, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() * 60 + 1; } init( FASTRESTORE_VB_LAUNCH_DELAY, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() * 60 + 1; }

9
fdbserver/RestoreController.actor.cpp
View File

@ -34,6 +34,7 @@
#include "fdbserver/RestoreApplier.actor.h" #include "fdbserver/RestoreApplier.actor.h"
#include "fdbserver/RestoreLoader.actor.h" #include "fdbserver/RestoreLoader.actor.h"
#include "flow/Platform.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
ACTOR static Future<Void> clearDB(Database cx); ACTOR static Future<Void> clearDB(Database cx);
@ -716,6 +717,8 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
std::set<RestoreFileFR> uniqueRangeFiles; std::set<RestoreFileFR> uniqueRangeFiles;
std::set<RestoreFileFR> uniqueLogFiles; std::set<RestoreFileFR> uniqueLogFiles;
double rangeSize = 0;
double logSize = 0;
*minRangeVersion = MAX_VERSION; *minRangeVersion = MAX_VERSION;
for (const RangeFile& f : restorable.get().ranges) { for (const RangeFile& f : restorable.get().ranges) {
TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles").detail("RangeFile", f.toString()); TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles").detail("RangeFile", f.toString());
@ -726,6 +729,7 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles") TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles")
.detail("RangeFileFR", file.toString()); .detail("RangeFileFR", file.toString());
uniqueRangeFiles.insert(file); uniqueRangeFiles.insert(file);
rangeSize += file.fileSize;
*minRangeVersion = std::min(*minRangeVersion, file.version); *minRangeVersion = std::min(*minRangeVersion, file.version);
} }
for (const LogFile& f : restorable.get().logs) { for (const LogFile& f : restorable.get().logs) {
@ -737,6 +741,7 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles").detail("LogFileFR", file.toString()); TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles").detail("LogFileFR", file.toString());
logFiles->push_back(file); logFiles->push_back(file);
uniqueLogFiles.insert(file); uniqueLogFiles.insert(file);
logSize += file.fileSize;
} }
// Assign unique range files and log files to output // Assign unique range files and log files to output
rangeFiles->assign(uniqueRangeFiles.begin(), uniqueRangeFiles.end()); rangeFiles->assign(uniqueRangeFiles.begin(), uniqueRangeFiles.end());
@ -745,7 +750,9 @@ ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
TraceEvent("FastRestoreControllerPhaseCollectBackupFilesDone") TraceEvent("FastRestoreControllerPhaseCollectBackupFilesDone")
.detail("BackupDesc", desc.toString()) .detail("BackupDesc", desc.toString())
.detail("RangeFiles", rangeFiles->size()) .detail("RangeFiles", rangeFiles->size())
.detail("LogFiles", logFiles->size()); .detail("LogFiles", logFiles->size())
.detail("RangeFileBytes", rangeSize)
.detail("LogFileBytes", logSize);
return request.targetVersion; return request.targetVersion;
} }

2
fdbserver/RestoreController.actor.h
View File

@ -424,7 +424,7 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreC
if (bcUrl == url && bc.isValid()) { if (bcUrl == url && bc.isValid()) {
return; return;
} }
printf("initBackupContainer, url:%s\n", url.toString().c_str()); TraceEvent("FastRestoreControllerInitBackupContainer").detail("URL", url);
bcUrl = url; bcUrl = url;
bc = IBackupContainer::openContainer(url.toString()); bc = IBackupContainer::openContainer(url.toString());
} }