wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbclient/FileBackupAgent.cpp

8425 lines
338 KiB
C++
Raw Blame History

/*
* FileBackupAgent.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/CommitProxyInterface.h"
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbrpc/simulator.h"
#include "flow/EncryptUtils.h"
#include "flow/FastRef.h"
#include "flow/flow.h"
#include "fmt/format.h"
#include "fdbclient/BackupAgent.h"
#include "fdbclient/BackupContainer.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/BulkDumping.h"
#include "fdbclient/BulkLoading.h"
#include "fdbclient/ClientBooleanParams.h"
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/JsonBuilder.h"
#include "fdbclient/JSONDoc.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/KeyRangeMap.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/RangeLock.h"
#include "PartitionedLogIterator.h"
#include "fdbclient/Status.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/TaskBucket.h"
#include "FileBackupAgentFileFormat.h"
#include "flow/network.h"
#include "flow/Trace.h"
#include "flow/Util.h"
#include <cinttypes>
#include <cstdint>
#include <ctime>
#include <climits>
#include "flow/IAsyncFile.h"
#include "flow/genericactors.actor.h"
#include "flow/Hash3.h"
#include "flow/xxhash.h"
#include <memory>
#include <numeric>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <algorithm>
#include <unordered_map>
#include <utility>
// Counters to verify BulkDump/BulkLoad were actually used (for test assertions)
std::atomic<int> g_bulkDumpTaskCompleteCount(0);
std::atomic<int> g_bulkLoadRestoreTaskCompleteCount(0);
// Helper function to monitor BulkDump job completion
// Returns true if job completed successfully, false if timed out
Future<bool> monitorBulkDumpJobCompletion(Database cx, UID jobId, double timeoutDuration, double pollInterval) {
double timeoutStart = now();
Transaction tr(cx);
while (true) {
Error err;
try {
Optional<BulkDumpState> currentJob = co_await getSubmittedBulkDumpJob(&tr);
bool stillRunning = currentJob.present() && currentJob.get().getJobId() == jobId;
if (!stillRunning) {
co_return true; // Job completed successfully
}
if (now() - timeoutStart > timeoutDuration) {
co_return false; // Timed out
}
co_await delay(pollInterval);
tr.reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
}
// Verify that a complete BulkDump dataset exists for BulkLoad restore
// Returns true if dataset is complete, false if incomplete
// JobId corresponds to a single BulkDump job which represents one snapshot at a specific version
Future<bool> verifyBulkDumpDatasetCompleteness(Reference<IBackupContainer> bc, std::string bulkDumpJobId) {
try {
if (bulkDumpJobId.empty()) {
TraceEvent(SevWarn, "BulkLoadVerifyDatasetEmptyJobId");
co_return false;
}
// Check if job-specific directory exists: bulkdump_data/<job-uuid>/
// BulkDump stores data under data/<container>/bulkdump_data/ via getBackupDataPath(),
// which is consistent with where BackupContainer stores other files (logs, ranges, etc.)
std::string jobDirectoryPath = "bulkdump_data/" + bulkDumpJobId + "/";
Error err;
try {
// Try to list files in the job directory to verify it exists and has content
Reference<BackupContainerFileSystem> bcfs = bc.castTo<BackupContainerFileSystem>();
if (bcfs) {
// Standard listFiles works because BulkDump now writes under data/<container>/
BackupContainerFileSystem::FilesAndSizesT files = co_await bcfs->listFiles(jobDirectoryPath);
if (files.empty()) {
TraceEvent(SevWarn, "BulkLoadVerifyDatasetJobDirectoryEmpty")
.detail("JobDirectoryPath", jobDirectoryPath)
.detail("BulkDumpJobId", bulkDumpJobId);
co_return false;
}
TraceEvent("BulkLoadVerifyDatasetJobDirectoryFound")
.detail("JobDirectoryPath", jobDirectoryPath)
.detail("BulkDumpJobId", bulkDumpJobId)
.detail("FileCount", files.size());
// Basic verification: job directory exists and contains files
// More detailed verification (parsing shard manifests) is delegated to BulkLoad system
co_return true;
} else {
// Cannot verify - backup container doesn't support file listing
// Use SevWarn (not SevError) to avoid abort in simulation
TraceEvent(SevWarn, "BulkLoadVerifyDatasetCannotCast")
.detail("BulkDumpJobId", bulkDumpJobId)
.detail("Note", "BackupContainer does not support file listing required for verification");
co_return false;
}
} catch (Error& e) {
if (e.code() == error_code_file_not_found) {
TraceEvent(SevWarn, "BulkLoadVerifyDatasetJobDirectoryNotFound")
.detail("JobDirectoryPath", jobDirectoryPath)
.detail("BulkDumpJobId", bulkDumpJobId);
co_return false;
}
throw;
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw;
}
TraceEvent(SevWarn, "BulkLoadVerifyDatasetError").error(e).detail("BulkDumpJobId", bulkDumpJobId);
co_return false;
}
}
// Note: BulkLoad configuration validation (shard_encode_location_metadata, enable_read_lock_on_range)
// is performed by the BulkLoad system on the server side. Client-side validation is not possible
// because SERVER_KNOBS are not accessible from fdbclient.
Optional<std::string> fileBackupAgentProxy = Optional<std::string>();
static std::string versionToString(Optional<Version> version) {
if (version.present())
return std::to_string(version.get());
else
return "N/A";
}
static std::string timeStampToString(Optional<int64_t> epochs) {
if (!epochs.present())
return "N/A";
return BackupAgentBase::formatTime(epochs.get());
}
static Future<Optional<int64_t>> getTimestampFromVersion(Optional<Version> ver,
Reference<ReadYourWritesTransaction> tr) {
if (!ver.present())
return Optional<int64_t>();
return timeKeeperEpochsFromVersion(ver.get(), tr);
}
// Time format :
// <= 59 seconds
// <= 59.99 minutes
// <= 23.99 hours
// N.NN days
std::string secondsToTimeFormat(int64_t seconds) {
if (seconds >= 86400)
return format("%.2f day(s)", seconds / 86400.0);
else if (seconds >= 3600)
return format("%.2f hour(s)", seconds / 3600.0);
else if (seconds >= 60)
return format("%.2f minute(s)", seconds / 60.0);
else
return format("%lld second(s)", seconds);
}
const Key FileBackupAgent::keyLastRestorable = "last_restorable"_sr;
// For convenience
using ERestoreState = FileBackupAgent::ERestoreState;
StringRef FileBackupAgent::restoreStateText(ERestoreState id) {
switch (id) {
case ERestoreState::UNINITIALIZED:
return "uninitialized"_sr;
case ERestoreState::QUEUED:
return "queued"_sr;
case ERestoreState::STARTING:
return "starting"_sr;
case ERestoreState::RUNNING:
return "running"_sr;
case ERestoreState::COMPLETED:
return "completed"_sr;
case ERestoreState::ABORTED:
return "aborted"_sr;
default:
return "Unknown"_sr;
}
}
Key FileBackupAgent::getPauseKey() {
FileBackupAgent backupAgent;
return backupAgent.taskBucket->getPauseKey();
}
Future<std::vector<KeyBackedTag>> TagUidMap::getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
Snapshot snapshot) {
Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor
TagMap::RangeResultType tagPairs = co_await tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot);
std::vector<KeyBackedTag> results;
for (auto& p : tagPairs.results)
results.push_back(KeyBackedTag(p.first, prefix));
co_return results;
}
KeyBackedTag::KeyBackedTag(std::string tagName, StringRef tagMapPrefix)
: KeyBackedProperty<UidAndAbortedFlagT>(TagUidMap(tagMapPrefix).getProperty(tagName)), tagName(tagName),
tagMapPrefix(tagMapPrefix) {}
// Lists all backups and find if any partitioned backup is running.
Future<bool> anyPartitionedBackupRunning(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
std::vector<KeyBackedTag> tags = co_await getAllBackupTags(tr);
std::vector<Future<Optional<UidAndAbortedFlagT>>> futures;
for (const auto& tag : tags) {
futures.push_back(tag.get(tr));
}
co_await waitForAll(futures);
int i = 0;
for (i = 0; i < futures.size(); i++) {
if (futures[i].get().present()) {
Optional<MutationLogType> mutationLogType;
EBackupState eState;
BackupConfig config(futures[i].get().get().first);
co_await (store(eState, config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)) &&
store(mutationLogType, config.mutationLogType().get(tr)));
if (FileBackupAgent::isRunnable(eState) &&
mutationLogType.orDefault(MutationLogType::DEFAULT) == MutationLogType::PARTITIONED_LOG) {
co_return true;
}
}
}
co_return false;
}
// Lists all backups and find if any range-partitioned backup is running.
Future<bool> anyRangePartitionedBackupRunning(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
std::vector<KeyBackedTag> tags = co_await getAllBackupTags(tr);
std::vector<Future<Optional<UidAndAbortedFlagT>>> futures;
for (const auto& tag : tags) {
futures.push_back(tag.get(tr));
}
co_await waitForAll(futures);
int i = 0;
for (i = 0; i < futures.size(); i++) {
if (futures[i].get().present()) {
Optional<MutationLogType> mutationLogType;
EBackupState eState;
BackupConfig config(futures[i].get().get().first);
co_await (store(eState, config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)) &&
store(mutationLogType, config.mutationLogType().get(tr)));
if (FileBackupAgent::isRunnable(eState) &&
mutationLogType.orDefault(MutationLogType::DEFAULT) == MutationLogType::RANGE_PARTITIONED_LOG) {
co_return true;
}
}
}
co_return false;
}
class RestoreConfig : public KeyBackedTaskConfig {
public:
explicit RestoreConfig(UID uid = UID()) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, uid) {}
explicit RestoreConfig(Reference<Task> task) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, task) {}
KeyBackedProperty<ERestoreState> stateEnum() { return configSpace.pack(__FUNCTION__sr); }
Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr) {
return map(stateEnum().getD(tr),
[](ERestoreState s) -> StringRef { return FileBackupAgent::restoreStateText(s); });
}
KeyBackedProperty<Key> addPrefix() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Key> removePrefix() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<bool> onlyApplyMutationLogs() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<bool> inconsistentSnapshotOnly() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<bool> unlockDBAfterRestore() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<MutationLogType> mutationLogType() { return configSpace.pack(__FUNCTION__sr); }
// BulkLoad integration properties
KeyBackedProperty<bool> useRangeFileRestore() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<std::string> bulkDumpJobId() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Key> bulkLoadCompleteFuture() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<bool> bulkLoadComplete() { return configSpace.pack(__FUNCTION__sr); }
// Original BulkLoad mode before restore enabled it - used to restore state after completion/crash
KeyBackedProperty<int> originalBulkLoadMode() { return configSpace.pack(__FUNCTION__sr); }
// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
KeyBackedProperty<KeyRange> restoreRange() { return configSpace.pack(__FUNCTION__sr); }
// XXX: Changed to restoreRangeSet. It can be removed.
KeyBackedProperty<std::vector<KeyRange>> restoreRanges() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedSet<KeyRange> restoreRangeSet() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Key> batchFuture() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Version> beginVersion() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Version> restoreVersion() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Version> firstConsistentVersion() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedProperty<Reference<IBackupContainer>> sourceContainer() { return configSpace.pack(__FUNCTION__sr); }
// Get the source container as a bare URL, without creating a container instance
KeyBackedProperty<Value> sourceContainerURL() { return configSpace.pack("sourceContainer"_sr); }
// Total bytes written by all log and range restore tasks.
KeyBackedBinaryValue<int64_t> bytesWritten() { return configSpace.pack(__FUNCTION__sr); }
// File blocks that have had tasks created for them by the Dispatch task
KeyBackedBinaryValue<int64_t> filesBlocksDispatched() { return configSpace.pack(__FUNCTION__sr); }
// File blocks whose tasks have finished
KeyBackedBinaryValue<int64_t> fileBlocksFinished() { return configSpace.pack(__FUNCTION__sr); }
// Total number of files in the fileMap
KeyBackedBinaryValue<int64_t> fileCount() { return configSpace.pack(__FUNCTION__sr); }
// Total number of file blocks in the fileMap
KeyBackedBinaryValue<int64_t> fileBlockCount() { return configSpace.pack(__FUNCTION__sr); }
// BulkLoad sub-phase task counts for detailed progress tracking
KeyBackedBinaryValue<int64_t> bulkLoadSubmittedTasks() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedBinaryValue<int64_t> bulkLoadTriggeredTasks() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedBinaryValue<int64_t> bulkLoadRunningTasks() { return configSpace.pack(__FUNCTION__sr); }
KeyBackedBinaryValue<int64_t> bulkLoadTotalTasks() { return configSpace.pack(__FUNCTION__sr); }
Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
return getRestoreRangesOrDefault_impl(this, tr);
}
static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfig* self,
Reference<ReadYourWritesTransaction> tr) {
std::vector<KeyRange> ranges;
int batchSize = buggify() ? 1 : CLIENT_KNOBS->RESTORE_RANGES_READ_BATCH;
Optional<KeyRange> begin;
Arena arena;
while (true) {
KeyBackedSet<KeyRange>::RangeResultType rangeResult =
co_await self->restoreRangeSet().getRange(tr, begin, {}, batchSize);
ranges.insert(ranges.end(), rangeResult.results.begin(), rangeResult.results.end());
if (!rangeResult.more) {
break;
}
ASSERT(!rangeResult.results.empty());
begin = KeyRangeRef(KeyRef(arena, ranges.back().begin), keyAfter(ranges.back().end, arena));
}
// fall back to original fields if the new field is empty
if (ranges.empty()) {
std::vector<KeyRange> _ranges = co_await self->restoreRanges().getD(tr);
ranges = _ranges;
if (ranges.empty()) {
KeyRange range = co_await self->restoreRange().getD(tr);
ranges.push_back(range);
}
}
co_return ranges;
}
// Describes a file to load blocks from during restore. Ordered by version and then fileName to enable
// incrementally advancing through the map, saving the version and path of the next starting point.
struct RestoreFile {
Version version; // this is beginVersion, not endVersion
std::string fileName;
bool isRange{ false }; // false for log file
int64_t blockSize{ 0 };
int64_t fileSize{ 0 };
Version endVersion{ ::invalidVersion }; // not meaningful for range files
int64_t tagId = -1; // only meaningful to log files, Log router tag. Non-negative for new backup format.
int64_t totalTags = -1; // only meaningful to log files, Total number of log router tags.
Tuple pack() const {
return Tuple::makeTuple(
version, fileName, (int64_t)isRange, fileSize, blockSize, endVersion, tagId, totalTags);
}
static RestoreFile unpack(Tuple const& t) {
RestoreFile r;
int i = 0;
r.version = t.getInt(i++);
r.fileName = t.getString(i++).toString();
r.isRange = t.getInt(i++) != 0;
r.fileSize = t.getInt(i++);
r.blockSize = t.getInt(i++);
r.endVersion = t.getInt(i++);
r.tagId = t.getInt(i++);
r.totalTags = t.getInt(i++);
return r;
}
};
using FileSetT = KeyBackedSet<RestoreFile>;
FileSetT fileSet() { return configSpace.pack(__FUNCTION__sr); }
FileSetT logFileSet() { return configSpace.pack(__FUNCTION__sr); }
FileSetT rangeFileSet() { return configSpace.pack(__FUNCTION__sr); }
Future<bool> isRunnable(Reference<ReadYourWritesTransaction> tr) {
return map(stateEnum().getD(tr), [](ERestoreState s) -> bool {
return s != ERestoreState::ABORTED && s != ERestoreState::COMPLETED && s != ERestoreState::UNINITIALIZED;
});
}
Future<Void> logError(Database cx, Error e, std::string const& details, void* taskInstance = nullptr) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileRestoreErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileRestoreError");
t.error(e)
.detail("RestoreUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if (e.code() == error_code_key_not_found)
t.backtrace();
return updateErrorInfo(cx, e, details);
}
Key mutationLogPrefix() { return uidPrefixKey(applyLogKeys.begin, uid); }
Key applyMutationsMapPrefix() { return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid); }
static Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True);
Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True);
co_await (success(beginVal) && success(endVal));
if (!beginVal.get().present() || !endVal.get().present())
co_return 0;
Version beginVersion = BinaryReader::fromStringRef<Version>(beginVal.get().get(), Unversioned());
Version endVersion = BinaryReader::fromStringRef<Version>(endVal.get().get(), Unversioned());
co_return endVersion - beginVersion;
}
Future<int64_t> getApplyVersionLag(Reference<ReadYourWritesTransaction> tr) {
return getApplyVersionLag_impl(tr, uid);
}
void initApplyMutations(Reference<ReadYourWritesTransaction> tr,
Key addPrefix,
Key removePrefix,
OnlyApplyMutationLogs onlyApplyMutationLogs) {
// Set these because they have to match the applyMutations values.
this->addPrefix().set(tr, addPrefix);
this->removePrefix().set(tr, removePrefix);
clearApplyMutationsKeys(tr);
// Initialize add/remove prefix, range version map count and set the map's start key to InvalidVersion
tr->set(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid), addPrefix);
tr->set(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid), removePrefix);
int64_t startCount = 0;
tr->set(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid), StringRef((uint8_t*)&startCount, 8));
Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
tr->set(mapStart, BinaryWriter::toValue<Version>(invalidVersion, Unversioned()));
}
void clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
// Clear add/remove prefix keys
tr->clear(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid));
tr->clear(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid));
// Clear range version map and count key
tr->clear(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid));
Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
tr->clear(KeyRangeRef(mapStart, strinc(mapStart)));
// Clear any loaded mutations that have not yet been applied
Key mutationPrefix = mutationLogPrefix();
tr->clear(KeyRangeRef(mutationPrefix, strinc(mutationPrefix)));
// Clear end and begin versions (intentionally in this order)
tr->clear(uidPrefixKey(applyMutationsEndRange.begin, uid));
tr->clear(uidPrefixKey(applyMutationsBeginRange.begin, uid));
}
void setApplyBeginVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
tr->set(uidPrefixKey(applyMutationsBeginRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
}
Future<Version> getApplyBeginVersion(Reference<ReadYourWritesTransaction> tr) {
return map(tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid)),
[=](Optional<Value> const& value) -> Version {
return value.present() ? BinaryReader::fromStringRef<Version>(value.get(), Unversioned()) : 0;
});
}
void setApplyEndVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
tr->set(uidPrefixKey(applyMutationsEndRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
}
Future<Version> getApplyEndVersion(Reference<ReadYourWritesTransaction> tr) {
return map(tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid)),
[=](Optional<Value> const& value) -> Version {
return value.present() ? BinaryReader::fromStringRef<Version>(value.get(), Unversioned()) : 0;
});
}
static Future<Version> getCurrentVersion_impl(RestoreConfig* self, Reference<ReadYourWritesTransaction> tr) {
ERestoreState status = co_await self->stateEnum().getD(tr);
Version version = -1;
if (status == ERestoreState::RUNNING) {
version = co_await self->getApplyBeginVersion(tr);
} else if (status == ERestoreState::COMPLETED) {
version = co_await self->restoreVersion().getD(tr);
}
co_return version;
}
Future<Version> getCurrentVersion(Reference<ReadYourWritesTransaction> tr) {
return getCurrentVersion_impl(this, tr);
}
static Future<std::string> getProgress_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
Future<std::string> getProgress(Reference<ReadYourWritesTransaction> tr) { return getProgress_impl(*this, tr); }
static Future<std::string> getFullStatus_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
Future<std::string> getFullStatus(Reference<ReadYourWritesTransaction> tr) { return getFullStatus_impl(*this, tr); }
};
using RestoreFile = RestoreConfig::RestoreFile;
// Helper to count bulkload task progress for a job
// Returns: <completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes>
// Sub-phases help track progress during the long "running" period:
// - Submitted: task created, waiting to be picked up by DD
// - Triggered: assigned to storage server, waiting to start
// - Running: storage server actively downloading/ingesting SST files
Future<std::tuple<int64_t, int64_t, int64_t, int64_t, int64_t, int64_t>> getBulkLoadTaskProgress(Database cx,
UID jobId) {
Transaction tr(cx);
Key readBegin = normalKeys.begin;
Key readEnd = normalKeys.end;
int64_t completedTasks = 0;
int64_t submittedTasks = 0;
int64_t triggeredTasks = 0;
int64_t runningTasks = 0;
int64_t totalTasks = 0;
int64_t completedBytes = 0;
while (readBegin < readEnd) {
Error err;
try {
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
RangeResult rangeResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd));
if (rangeResult.empty()) {
break;
}
for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
if (rangeResult[i].value.empty()) {
continue;
}
BulkLoadTaskState task = decodeBulkLoadTaskState(rangeResult[i].value);
if (task.getJobId() != jobId) {
// Different job, stop counting
co_return std::make_tuple(
completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes);
}
int manifestCount = task.getManifests().size();
totalTasks += manifestCount;
if (task.phase == BulkLoadPhase::Complete) {
completedTasks += manifestCount;
// Sum bytes from manifest data sizes
for (const auto& manifest : task.getManifests()) {
completedBytes += manifest.getTotalBytes();
}
} else if (task.phase == BulkLoadPhase::Submitted) {
submittedTasks += manifestCount;
} else if (task.phase == BulkLoadPhase::Triggered) {
triggeredTasks += manifestCount;
} else if (task.phase == BulkLoadPhase::Running) {
runningTasks += manifestCount;
}
}
readBegin = rangeResult.back().key;
} catch (Error& e) {
err = e;
}
if (err.isValid() && err.code() != error_code_success) {
TraceEvent(SevWarn, "BulkLoadTaskProgressRetry").error(err).detail("JobId", jobId);
co_await tr.onError(err);
}
}
co_return std::make_tuple(completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes);
}
// Monitor BulkLoad job completion and update restore progress counters
// restoreUid is used to update the RestoreConfig progress
Future<bool> monitorBulkLoadJobCompletionWithProgress(Database cx,
UID jobId,
UID restoreUid,
int64_t totalBlocks,
double timeoutDuration,
double pollInterval,
bool lockAware) {
double timeoutStart = now();
RestoreConfig restore(restoreUid);
while (true) {
Optional<BulkLoadJobState> currentJob = co_await getRunningBulkLoadJob(cx, lockAware);
bool stillRunning = currentJob.present() && currentJob.get().getJobId() == jobId;
if (!stillRunning) {
co_return true;
}
// Update progress based on completed bulkload tasks
try {
auto [completed, submitted, triggered, running, total, bytes] = co_await getBulkLoadTaskProgress(cx, jobId);
if (total > 0) {
// For bulkload restores, fileBlockCount is 0, so use task count as "blocks"
// This provides meaningful progress tracking for the restore status display
// Include all in-progress tasks in dispatched count to show scheduling progress
int64_t inProgress = submitted + triggered + running;
int64_t effectiveTotalBlocks = totalBlocks > 0 ? totalBlocks : total;
int64_t blocksFinished = totalBlocks > 0 ? (totalBlocks * completed) / total : completed;
int64_t blocksDispatched =
totalBlocks > 0 ? (totalBlocks * (completed + inProgress)) / total : (completed + inProgress);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (lockAware) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
}
restore.fileBlocksFinished().set(tr, blocksFinished);
restore.filesBlocksDispatched().set(tr, blocksDispatched);
restore.fileBlockCount().set(tr, effectiveTotalBlocks);
restore.bytesWritten().set(tr, bytes);
// Store sub-phase counts for detailed progress display
restore.bulkLoadSubmittedTasks().set(tr, submitted);
restore.bulkLoadTriggeredTasks().set(tr, triggered);
restore.bulkLoadRunningTasks().set(tr, running);
restore.bulkLoadTotalTasks().set(tr, total);
co_await tr->commit();
TraceEvent("BulkLoadRestoreProgress")
.detail("RestoreUID", restoreUid)
.detail("JobId", jobId)
.detail("CompletedTasks", completed)
.detail("SubmittedTasks", submitted)
.detail("TriggeredTasks", triggered)
.detail("RunningTasks", running)
.detail("TotalTasks", total)
.detail("BlocksFinished", blocksFinished)
.detail("BlocksDispatched", blocksDispatched)
.detail("EffectiveTotalBlocks", effectiveTotalBlocks)
.detail("BytesWritten", bytes);
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw;
}
// Log but don't fail - progress updates are best-effort
TraceEvent(SevWarn, "BulkLoadRestoreProgressError").error(e).detail("JobId", jobId);
}
if (now() - timeoutStart > timeoutDuration) {
co_return false; // Timed out
}
co_await delay(pollInterval);
}
}
Future<std::string> RestoreConfig::getProgress_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<int64_t> fileCount = restore.fileCount().getD(tr);
Future<int64_t> fileBlockCount = restore.fileBlockCount().getD(tr);
Future<int64_t> fileBlocksDispatched = restore.filesBlocksDispatched().getD(tr);
Future<int64_t> fileBlocksFinished = restore.fileBlocksFinished().getD(tr);
Future<int64_t> bytesWritten = restore.bytesWritten().getD(tr);
Future<StringRef> status = restore.stateText(tr);
Future<Version> currentVersion = restore.getCurrentVersion(tr);
Future<Version> lag = restore.getApplyVersionLag(tr);
Future<Version> firstConsistentVersion = restore.firstConsistentVersion().getD(tr);
Future<std::string> tag = restore.tag().getD(tr);
Future<std::pair<std::string, Version>> lastError = restore.lastError().getD(tr);
Future<int64_t> submittedTasks = restore.bulkLoadSubmittedTasks().getD(tr);
Future<int64_t> triggeredTasks = restore.bulkLoadTriggeredTasks().getD(tr);
Future<int64_t> runningTasks = restore.bulkLoadRunningTasks().getD(tr);
Future<int64_t> totalTasks = restore.bulkLoadTotalTasks().getD(tr);
Future<Optional<bool>> useRangeFileRestore = restore.useRangeFileRestore().get(tr);
UID uid = restore.getUid();
co_await (success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(currentVersion) &&
success(lag) && success(firstConsistentVersion) && success(tag) && success(lastError) &&
success(submittedTasks) && success(triggeredTasks) && success(runningTasks) && success(totalTasks) &&
success(useRangeFileRestore));
bool useRangeFile = !useRangeFileRestore.get().present() || useRangeFileRestore.get().get();
std::string errstr = "None";
if (lastError.get().second != 0) {
errstr = format("'%s' %" PRId64 "s ago.\n",
lastError.get().first.c_str(),
(tr->getReadVersion().get() - lastError.get().second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
}
TraceEvent("FileRestoreProgress")
.detail("RestoreUID", uid)
.detail("Tag", tag.get())
.detail("State", status.get().toString())
.detail("FileCount", fileCount.get())
.detail("FileBlocksFinished", fileBlocksFinished.get())
.detail("FileBlocksTotal", fileBlockCount.get())
.detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
.detail("SubmittedTasks", submittedTasks.get())
.detail("TriggeredTasks", triggeredTasks.get())
.detail("RunningTasks", runningTasks.get())
.detail("TotalTasks", totalTasks.get())
.detail("BytesWritten", bytesWritten.get())
.detail("CurrentVersion", currentVersion.get())
.detail("FirstConsistentVersion", firstConsistentVersion.get())
.detail("ApplyLag", lag.get());
std::string progressStr;
if (useRangeFile) {
progressStr = format("Tag: %s UID: %s State: %s\n",
tag.get().c_str(),
uid.toString().c_str(),
status.get().toString().c_str());
progressStr += format(" Blocks: %lld/%lld complete\n", fileBlocksFinished.get(), fileBlockCount.get());
progressStr += format(" Files: %lld\n", fileCount.get());
progressStr += format(" Bytes written: %s\n", formatBytesHumanReadable(bytesWritten.get()).c_str());
progressStr += format(" Apply version lag: %s\n", versionToString(lag.get()).c_str());
} else {
progressStr = format("Tag: %s UID: %s State: %s\n",
tag.get().c_str(),
uid.toString().c_str(),
status.get().toString().c_str());
progressStr += format(" Tasks submitted: %lld triggered: %lld running: %lld\n",
submittedTasks.get(),
triggeredTasks.get(),
runningTasks.get());
progressStr += format(" Tasks triggered: %lld / %lld total\n", triggeredTasks.get(), totalTasks.get());
progressStr += format(" Bytes written: %s\n", formatBytesHumanReadable(bytesWritten.get()).c_str());
double avgBytesPerTask = triggeredTasks.get() > 0 ? (double)bytesWritten.get() / triggeredTasks.get() : 0;
if (avgBytesPerTask > 0) {
progressStr += format(" Avg bytes/task: %s\n", formatBytesHumanReadable((int64_t)avgBytesPerTask).c_str());
}
}
co_return progressStr;
}
Future<std::string> RestoreConfig::getFullStatus_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<std::vector<KeyRange>> ranges = restore.getRestoreRangesOrDefault(tr);
Future<Key> addPrefix = restore.addPrefix().getD(tr);
Future<Key> removePrefix = restore.removePrefix().getD(tr);
Future<Key> url = restore.sourceContainerURL().getD(tr);
Future<Version> restoreVersion = restore.restoreVersion().getD(tr);
Future<std::string> progress = restore.getProgress(tr);
Future<ERestoreState> restoreState = restore.stateEnum().getD(tr);
Future<Optional<bool>> useRangeFileRestore = restore.useRangeFileRestore().get(tr);
Future<Optional<bool>> bulkLoadComplete = restore.bulkLoadComplete().get(tr);
// restore might no longer be valid after the first wait so make sure it is not needed anymore.
co_await (success(ranges) && success(addPrefix) && success(removePrefix) && success(url) &&
success(restoreVersion) && success(progress) && success(restoreState) && success(useRangeFileRestore) &&
success(bulkLoadComplete));
std::string returnStr;
returnStr = format("%s URL: %s", progress.get().c_str(), url.get().toString().c_str());
for (auto& range : ranges.get()) {
returnStr += format(" Range: '%s'-'%s'", printable(range.begin).c_str(), printable(range.end).c_str());
}
returnStr += format(" AddPrefix: '%s' RemovePrefix: '%s' Version: %lld",
printable(addPrefix.get()).c_str(),
printable(removePrefix.get()).c_str(),
restoreVersion.get());
// Add enhanced status fields for BulkLoad integration
bool usingBulkLoad = useRangeFileRestore.get().present() && !useRangeFileRestore.get().get();
std::string snapshotMethod = usingBulkLoad ? "bulkload" : "rangefile";
returnStr += format(" Snapshot Method: %s", snapshotMethod.c_str());
// Add phase status information
ERestoreState currentState = restoreState.get();
bool bulkLoadDone = bulkLoadComplete.get().present() && bulkLoadComplete.get().get();
if (currentState == ERestoreState::RUNNING) {
if (usingBulkLoad) {
std::string snapshotPhase = bulkLoadDone ? "complete" : "in_progress";
returnStr += format(" Snapshot Phase: %s", snapshotPhase.c_str());
std::string mutationPhase = bulkLoadDone ? "in_progress" : "not_started";
returnStr += format(" Mutation Log Phase: %s", mutationPhase.c_str());
} else {
returnStr += " Snapshot Phase: in_progress Mutation Log Phase: in_progress";
}
} else if (currentState == ERestoreState::COMPLETED) {
returnStr += " Snapshot Phase: complete Mutation Log Phase: complete";
}
co_return returnStr;
}
// two buffers are alternatively serving data and reading data from file
// thus when one buffer is serving data through peek()
// the other buffer is reading data from file to provide pipelining.
class TwoBuffers : public ReferenceCounted<TwoBuffers>, NonCopyable {
public:
class IteratorBuffer : public ReferenceCounted<IteratorBuffer> {
public:
std::shared_ptr<char[]> data;
// has_value means there is data, otherwise it means there is no data being fetched or ready
// is_valid means data is being fetched, is_ready means data is ready
std::optional<Future<Void>> fetchingData;
size_t size;
int index;
int capacity;
explicit IteratorBuffer(int _capacity) {
capacity = _capacity;
data = std::shared_ptr<char[]>(new char[capacity]());
fetchingData.reset();
size = 0;
}
bool is_valid() { return fetchingData.has_value(); }
void reset() {
size = 0;
index = 0;
fetchingData.reset();
}
};
TwoBuffers(int capacity, Reference<IBackupContainer> _bc, std::vector<RestoreConfig::RestoreFile>& _files, int tag);
// ready need to be called first before calling peek
// because a shared_ptr cannot be wrapped by a Future
// this method ensures the current buffer has available data
Future<Void> ready();
static Future<Void> ready(Reference<TwoBuffers> self);
// fill buffer[index] with the next block of file
// it has side effects to change currentFileIndex and currentFilePosition
static Future<Void> readNextBlock(Reference<TwoBuffers> self, int index);
// peek can only be called after ready is called
// it returns the pointer to the active buffer
std::shared_ptr<char[]> peek();
int getFileIndex();
void setFileIndex(int);
bool hasNext();
void reset();
// discard the current buffer and swap to the next one
void discardAndSwap();
// try to fill the buffer[index]
// but no-op if the buffer have valid data or it is actively being filled
void fillBufferIfAbsent(int index);
size_t getBufferSize();
private:
Reference<IteratorBuffer> buffers[2]; // Two buffers for alternating
size_t bufferCapacity; // Size of each buffer in bytes
Reference<IBackupContainer> bc;
std::vector<RestoreConfig::RestoreFile> files;
int tag;
int cur; // Index of the current active buffer (0 or 1)
size_t currentFileIndex; // Index of the current file being read
size_t currentFilePosition; // Current read position in the current file
};
TwoBuffers::TwoBuffers(int capacity,
Reference<IBackupContainer> _bc,
std::vector<RestoreConfig::RestoreFile>& _files,
int _tag)
: currentFileIndex(0), currentFilePosition(0), cur(0), bufferCapacity(capacity), files(_files), bc(_bc), tag(_tag) {
buffers[0] = makeReference<IteratorBuffer>(capacity);
buffers[1] = makeReference<IteratorBuffer>(capacity);
}
bool TwoBuffers::hasNext() {
// if it is being load (valid but not ready, what would be the size?)
while (currentFileIndex < files.size() && currentFilePosition >= files[currentFileIndex].fileSize) {
currentFileIndex++;
currentFilePosition = 0;
}
if (buffers[0]->is_valid() || buffers[1]->is_valid()) {
return true;
}
return currentFileIndex != files.size();
}
Future<Void> TwoBuffers::ready() {
return ready(Reference<TwoBuffers>::addRef(this));
}
Future<Void> TwoBuffers::ready(Reference<TwoBuffers> self) {
// if cur is not ready, then wait
if (!self->hasNext()) {
co_return;
}
// try to fill the current buffer, and wait before it is filled
self->fillBufferIfAbsent(self->cur);
co_await self->buffers[self->cur]->fetchingData.value();
// try to fill the next buffer, do not wait for the filling
if (self->hasNext()) {
self->fillBufferIfAbsent(1 - self->cur);
}
}
std::shared_ptr<char[]> TwoBuffers::peek() {
return buffers[cur]->data;
}
int TwoBuffers::getFileIndex() {
return buffers[cur]->index;
}
void TwoBuffers::setFileIndex(int newIndex) {
if (newIndex < 0 || newIndex >= files.size()) {
TraceEvent(SevError, "TwoBuffersFileIndexOutOfBound")
.detail("FilesSize", files.size())
.detail("NewIndex", newIndex)
.log();
}
currentFileIndex = newIndex;
}
void TwoBuffers::discardAndSwap() {
// invalidate cur and change cur to next
buffers[cur]->fetchingData.reset();
cur = 1 - cur;
}
void TwoBuffers::reset() {
// invalidate cur and change cur to next
buffers[0]->reset();
buffers[1]->reset();
cur = 0;
currentFileIndex = 0;
currentFilePosition = 0;
}
size_t TwoBuffers::getBufferSize() {
return buffers[cur]->size;
}
// only one readNextBlock can be run at a single time, otherwie the same block might be loaded twice
Future<Void> TwoBuffers::readNextBlock(Reference<TwoBuffers> self, int index) {
if (self->currentFileIndex >= self->files.size()) {
TraceEvent(SevError, "ReadNextBlockOutOfBound")
.detail("FileIndex", self->currentFileIndex)
.detail("Tag", self->tag)
.detail("Position", self->currentFilePosition)
.detail("FileSize", self->files[self->currentFileIndex].fileSize)
.detail("FilesCount", self->files.size())
.log();
co_return;
}
Reference<IAsyncFile> asyncFile = co_await self->bc->readFile(self->files[self->currentFileIndex].fileName);
size_t fileSize = self->files[self->currentFileIndex].fileSize;
size_t remaining = fileSize - self->currentFilePosition;
size_t bytesToRead = std::min(self->bufferCapacity, remaining);
int bytesRead = co_await asyncFile->read(
static_cast<void*>(self->buffers[index]->data.get()), bytesToRead, self->currentFilePosition);
if (bytesRead != bytesToRead)
throw restore_bad_read();
self->buffers[index]->index = self->currentFileIndex;
self->buffers[index]->size = bytesRead; // Set to actual bytes read
self->currentFilePosition += bytesRead;
}
void TwoBuffers::fillBufferIfAbsent(int index) {
if (buffers[index]->is_valid()) {
// if this buffer is valid, then do not overwrite it
return;
}
if (currentFileIndex == files.size()) {
// quit if no more contents
return;
}
auto self = Reference<TwoBuffers>::addRef(this);
self->buffers[index]->fetchingData = readNextBlock(self, index);
return;
}
bool endOfBlock(char* start, int offset) {
const unsigned char paddingChar = '\xff';
return (unsigned char)*(start + offset) == paddingChar;
}
class PartitionedLogIteratorSimple : public PartitionedLogIterator {
public:
const int BATCH_READ_BLOCK_COUNT = 1;
const int BLOCK_SIZE = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
const int mutationHeaderBytes = sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t);
Reference<IBackupContainer> bc;
size_t bufferCapacity;
int tag;
std::vector<RestoreConfig::RestoreFile> files;
size_t bufferOffset; // Current read offset
int bufferSize;
int fileOffset;
int fileIndex;
std::shared_ptr<char[]> buffer;
std::vector<Version> endVersions;
PartitionedLogIteratorSimple(Reference<IBackupContainer> _bc,
int _tag,
std::vector<RestoreConfig::RestoreFile> _files,
std::vector<Version> _endVersions);
bool hasNext() override;
Future<Void> loadNextBlock();
static Future<Void> loadNextBlock(Reference<PartitionedLogIteratorSimple> self);
void removeBlockHeader();
Standalone<VectorRef<VersionedMutation>> consumeData(Version firstVersion);
// find the next version without advanding the iterator
Future<Version> peekNextVersion() override;
static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorSimple> iterator);
// get all the mutations of next version and advance the iterator
// this might issue multiple consumeData() if the data of a version cross buffer boundary
Future<Standalone<VectorRef<VersionedMutation>>> getNext() override;
static Future<Standalone<VectorRef<VersionedMutation>>> getNext(Reference<PartitionedLogIteratorSimple> iterator);
};
PartitionedLogIteratorSimple::PartitionedLogIteratorSimple(Reference<IBackupContainer> _bc,
int _tag,
std::vector<RestoreConfig::RestoreFile> _files,
std::vector<Version> _endVersions)
: bc(_bc), tag(_tag), endVersions(_endVersions), files(std::move(_files)), bufferOffset(0) {
bufferCapacity = static_cast<size_t>(BATCH_READ_BLOCK_COUNT) * BLOCK_SIZE;
buffer = std::shared_ptr<char[]>(new char[bufferCapacity]());
fileOffset = 0;
fileIndex = 0;
bufferSize = 0;
}
// it will set fileOffset and fileIndex
bool PartitionedLogIteratorSimple::hasNext() {
if (bufferOffset < bufferSize) {
return true;
}
while (fileIndex < files.size() && fileOffset >= files[fileIndex].fileSize) {
TraceEvent("ReachEndOfLogFiles")
.detail("BufferOffset", bufferOffset)
.detail("BufferSize", bufferSize)
.detail("FileOffset", fileOffset)
.detail("FileSize", files[fileIndex].fileSize)
.detail("FileName", files[fileIndex].fileName)
.detail("Tag", tag)
.detail("Index", fileIndex)
.log();
fileOffset = 0;
fileIndex++;
}
return fileIndex < files.size() && fileOffset < files[fileIndex].fileSize;
}
void PartitionedLogIteratorSimple::removeBlockHeader() {
if (bufferOffset % BLOCK_SIZE == 0) {
bufferOffset += sizeof(uint32_t);
}
}
Standalone<VectorRef<VersionedMutation>> PartitionedLogIteratorSimple::consumeData(Version firstVersion) {
Standalone<VectorRef<VersionedMutation>> mutations = Standalone<VectorRef<VersionedMutation>>();
char* start = buffer.get();
bool foundNewVersion = false;
while (bufferOffset < bufferSize) {
while (bufferOffset < bufferSize && !endOfBlock(start, bufferOffset)) {
// for each block
removeBlockHeader();
// encoding format:
// wr << bigEndian64(message.version.version) << bigEndian32(message.version.sub) <<
// bigEndian32(mutation.size());
Version version;
std::memcpy(&version, start + bufferOffset, sizeof(Version));
version = bigEndian64(version);
if (version != firstVersion) {
foundNewVersion = true;
break; // Different version, stop here
}
int32_t subsequence;
std::memcpy(&subsequence, start + bufferOffset + sizeof(Version), sizeof(int32_t));
subsequence = bigEndian32(subsequence);
int32_t mutationSize;
std::memcpy(&mutationSize, start + bufferOffset + sizeof(Version) + sizeof(int32_t), sizeof(int32_t));
mutationSize = bigEndian32(mutationSize);
// assumption: the entire mutation is within the buffer
size_t mutationTotalSize = mutationHeaderBytes + mutationSize;
ASSERT(bufferOffset + mutationTotalSize <= bufferSize);
// transform from stringref to mutationref here
Standalone<StringRef> mutationData = makeString(mutationSize);
std::memcpy(mutateString(mutationData), start + bufferOffset + mutationHeaderBytes, mutationSize);
ArenaReader reader(mutationData.arena(), mutationData, AssumeVersion(g_network->protocolVersion()));
MutationRef mutation;
reader >> mutation;
VersionedMutation vm;
vm.version = version;
vm.subsequence = subsequence;
vm.mutation = mutation;
mutations.push_back_deep(mutations.arena(), vm);
// Move the bufferOffset to include this mutation
bufferOffset += mutationTotalSize;
}
if (bufferOffset < bufferSize && endOfBlock(start, bufferOffset)) {
// there are paddings
int remain = BLOCK_SIZE - (bufferOffset % BLOCK_SIZE);
bufferOffset += remain;
}
if (foundNewVersion) {
break;
}
}
return mutations;
}
Future<Void> PartitionedLogIteratorSimple::loadNextBlock() {
return loadNextBlock(Reference<PartitionedLogIteratorSimple>::addRef(this));
}
Future<Void> PartitionedLogIteratorSimple::loadNextBlock(Reference<PartitionedLogIteratorSimple> self) {
if (self->bufferOffset < self->bufferSize) {
// do nothing
co_return;
}
if (!self->hasNext()) {
co_return;
}
Reference<IAsyncFile> asyncFile = co_await self->bc->readFile(self->files[self->fileIndex].fileName);
size_t fileSize = self->files[self->fileIndex].fileSize;
size_t remaining = fileSize - self->fileOffset;
size_t bytesToRead = std::min(self->bufferCapacity, remaining);
int bytesRead = co_await asyncFile->read(static_cast<void*>((self->buffer.get())), bytesToRead, self->fileOffset);
if (bytesRead != bytesToRead)
throw restore_bad_read();
self->bufferSize = bytesRead; // Set to actual bytes read
self->bufferOffset = 0; // Reset bufferOffset for the new data
self->fileOffset += bytesRead;
}
Future<Version> PartitionedLogIteratorSimple::peekNextVersion() {
return peekNextVersion(Reference<PartitionedLogIteratorSimple>::addRef(this));
}
Future<Version> PartitionedLogIteratorSimple::peekNextVersion(Reference<PartitionedLogIteratorSimple> self) {
// Read the first mutation's version
if (!self->hasNext()) {
co_return Version(0);
}
co_await self->loadNextBlock();
self->removeBlockHeader();
Version version{ 0 };
std::memcpy(&version, self->buffer.get() + self->bufferOffset, sizeof(Version));
version = bigEndian64(version);
while (self->fileIndex < self->endVersions.size() - 1 && version >= self->endVersions[self->fileIndex]) {
TraceEvent("SimpleIteratorFindOverlapAndSkip")
.detail("Version", version)
.detail("FileIndex", self->fileIndex)
.log();
self->bufferOffset = 0;
self->bufferSize = 0;
self->fileOffset = 0;
self->fileIndex += 1;
co_await self->loadNextBlock();
self->removeBlockHeader();
std::memcpy(&version, self->buffer.get() + self->bufferOffset, sizeof(Version));
version = bigEndian64(version);
}
co_return version;
}
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSimple::getNext(
Reference<PartitionedLogIteratorSimple> self) {
Standalone<VectorRef<VersionedMutation>> mutations;
if (!self->hasNext()) {
TraceEvent(SevWarn, "SimpleIteratorExhausted")
.detail("BufferOffset", self->bufferOffset)
.detail("BufferSize", self->bufferSize)
.detail("Tag", self->tag)
.log();
co_return mutations;
}
Version firstVersion = co_await self->peekNextVersion();
Standalone<VectorRef<VersionedMutation>> firstBatch = self->consumeData(firstVersion);
mutations = firstBatch;
// If the current buffer is fully consumed, then we need to check the next buffer in case
// the version is sliced across this buffer boundary
while (self->bufferOffset >= self->bufferSize) {
// data for one version cannot exceed single buffer size
// if hitting the end of a batch, check the next batch in case version is
if (self->hasNext()) {
// now this is run for each block, but it is not necessary if it is the last block of a file
// cannot check hasMoreData here because other buffer might have the last piece
co_await self->loadNextBlock();
Standalone<VectorRef<VersionedMutation>> batch = self->consumeData(firstVersion);
for (const VersionedMutation& vm : batch) {
mutations.push_back_deep(mutations.arena(), vm);
}
} else {
break;
}
}
co_return mutations;
}
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSimple::getNext() {
return getNext(Reference<PartitionedLogIteratorSimple>::addRef(this));
}
class PartitionedLogIteratorTwoBuffers : public PartitionedLogIterator {
private:
Reference<TwoBuffers> twobuffer;
// consume single version data upto the end of the current batch
// stop if seeing a different version from the parameter.
// it has side effects to update bufferOffset after reading the data
Future<Standalone<VectorRef<VersionedMutation>>> consumeData(Version firstVersion);
static Future<Standalone<VectorRef<VersionedMutation>>> consumeData(
Reference<PartitionedLogIteratorTwoBuffers> self,
Version v);
// each block has a format of {<header>[mutations]<padding>}, need to skip the header to read mutations
// this method check if bufferOffset is at the boundary and advance it if necessary
void removeBlockHeader();
public:
// read up to a fixed number of block count
// noted that each version has to be contained within 2 blocks
const int BATCH_READ_BLOCK_COUNT = 1;
const int BLOCK_SIZE = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
const int mutationHeaderBytes = sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t);
Reference<IBackupContainer> bc;
int tag;
std::vector<RestoreConfig::RestoreFile> files;
std::vector<Version> endVersions;
bool hasMoreData; // Flag indicating if more data is available
size_t bufferOffset; // Current read offset
// empty means no data, future is valid but not ready means being fetched
// future is ready means it currently holds data
PartitionedLogIteratorTwoBuffers(Reference<IBackupContainer> _bc,
int _tag,
std::vector<RestoreConfig::RestoreFile> _files,
std::vector<Version> _endVersions);
// whether there are more contents for this tag in all files specified
bool hasNext() override;
// find the next version without advanding the iterator
Future<Version> peekNextVersion() override;
static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers> iterator);
// get all the mutations of next version and advance the iterator
// this might issue multiple consumeData() if the data of a version cross buffer boundary
Future<Standalone<VectorRef<VersionedMutation>>> getNext() override;
static Future<Standalone<VectorRef<VersionedMutation>>> getNext(
Reference<PartitionedLogIteratorTwoBuffers> iterator);
};
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::consumeData(Version firstVersion) {
return consumeData(Reference<PartitionedLogIteratorTwoBuffers>::addRef(this), firstVersion);
}
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::consumeData(
Reference<PartitionedLogIteratorTwoBuffers> self,
Version firstVersion) {
Standalone<VectorRef<VersionedMutation>> mutations = Standalone<VectorRef<VersionedMutation>>();
co_await self->twobuffer->ready();
std::shared_ptr<char[]> start = self->twobuffer->peek();
int size = self->twobuffer->getBufferSize();
bool foundNewVersion = false;
while (self->bufferOffset < size) {
while (self->bufferOffset < size && !endOfBlock(start.get(), self->bufferOffset)) {
// for each block
self->removeBlockHeader();
// encoding is:
// wr << bigEndian64(message.version.version) << bigEndian32(message.version.sub) <<
// bigEndian32(mutation.size());
Version version;
std::memcpy(&version, start.get() + self->bufferOffset, sizeof(Version));
version = bigEndian64(version);
if (version != firstVersion) {
foundNewVersion = true;
break; // Different version, stop here
}
int32_t subsequence;
std::memcpy(&subsequence, start.get() + self->bufferOffset + sizeof(Version), sizeof(int32_t));
subsequence = bigEndian32(subsequence);
int32_t mutationSize;
std::memcpy(
&mutationSize, start.get() + self->bufferOffset + sizeof(Version) + sizeof(int32_t), sizeof(int32_t));
mutationSize = bigEndian32(mutationSize);
// assumption: the entire mutation is within the buffer
size_t mutationTotalSize = self->mutationHeaderBytes + mutationSize;
ASSERT(self->bufferOffset + mutationTotalSize <= size);
Standalone<StringRef> mutationData = makeString(mutationSize);
std::memcpy(
mutateString(mutationData), start.get() + self->bufferOffset + self->mutationHeaderBytes, mutationSize);
// transform from stringref to mutationref here
ArenaReader reader(mutationData.arena(), mutationData, AssumeVersion(g_network->protocolVersion()));
MutationRef mutation;
reader >> mutation;
VersionedMutation vm;
vm.version = version;
vm.subsequence = subsequence;
vm.mutation = mutation;
mutations.push_back_deep(mutations.arena(), vm);
// Move the bufferOffset to include this mutation
self->bufferOffset += mutationTotalSize;
}
if (self->bufferOffset < size && endOfBlock(start.get(), self->bufferOffset)) {
// there are paddings, skip them
int remain = self->BLOCK_SIZE - (self->bufferOffset % self->BLOCK_SIZE);
self->bufferOffset += remain;
}
if (foundNewVersion) {
break;
}
}
co_return mutations;
}
void PartitionedLogIteratorTwoBuffers::removeBlockHeader() {
if (bufferOffset % BLOCK_SIZE == 0) {
bufferOffset += sizeof(uint32_t);
}
}
PartitionedLogIteratorTwoBuffers::PartitionedLogIteratorTwoBuffers(Reference<IBackupContainer> _bc,
int _tag,
std::vector<RestoreConfig::RestoreFile> _files,
std::vector<Version> _endVersions)
: bc(_bc), tag(_tag), files(std::move(_files)), endVersions(_endVersions), bufferOffset(0) {
int bufferCapacity = BATCH_READ_BLOCK_COUNT * BLOCK_SIZE;
twobuffer = makeReference<TwoBuffers>(bufferCapacity, _bc, files, tag);
}
bool PartitionedLogIteratorTwoBuffers::hasNext() {
return twobuffer->hasNext();
}
Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion() {
return peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers>::addRef(this));
}
Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers> self) {
// Read the first mutation's version
std::shared_ptr<char[]> start;
Version version{ 0 };
int fileIndex{ 0 };
if (!self->hasNext()) {
co_return Version(0);
}
co_await self->twobuffer->ready();
start = self->twobuffer->peek();
self->removeBlockHeader();
std::memcpy(&version, start.get() + self->bufferOffset, sizeof(Version));
version = bigEndian64(version);
fileIndex = self->twobuffer->getFileIndex();
while (fileIndex < self->endVersions.size() - 1 && version >= self->endVersions[fileIndex]) {
TraceEvent("RestoreLogFilesFoundOverlapAndSkip")
.detail("Version", version)
.detail("FileIndex", fileIndex)
.log();
// need to read from next file in the case of overlap range versions between log files
self->twobuffer->reset();
self->bufferOffset = 0;
self->twobuffer->setFileIndex(fileIndex + 1);
co_await self->twobuffer->ready();
start = self->twobuffer->peek();
self->removeBlockHeader();
std::memcpy(&version, start.get() + self->bufferOffset, sizeof(Version));
version = bigEndian64(version);
fileIndex = self->twobuffer->getFileIndex();
}
co_return version;
}
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::getNext(
Reference<PartitionedLogIteratorTwoBuffers> self) {
Standalone<VectorRef<VersionedMutation>> mutations;
if (!self->hasNext()) {
TraceEvent(SevWarn, "IteratorExhausted").log();
co_return mutations;
}
Version firstVersion = co_await self->peekNextVersion();
Standalone<VectorRef<VersionedMutation>> firstBatch = co_await self->consumeData(firstVersion);
mutations = firstBatch;
// If the current buffer is fully consumed, then we need to check the next buffer in case
// the version is sliced across this buffer boundary
while (self->bufferOffset >= self->twobuffer->getBufferSize()) {
self->twobuffer->discardAndSwap();
self->bufferOffset = 0;
// data for one version cannot exceed single buffer size
// if hitting the end of a batch, check the next batch in case version is
if (self->twobuffer->hasNext()) {
// now this is run for each block, but it is not necessary if it is the last block of a file
// cannot check hasMoreData here because other buffer might have the last piece
Version nextVersion = co_await self->peekNextVersion();
if (nextVersion != firstVersion) {
break;
}
Standalone<VectorRef<VersionedMutation>> batch = co_await self->consumeData(firstVersion);
for (const VersionedMutation& vm : batch) {
mutations.push_back_deep(mutations.arena(), vm);
}
} else {
break;
}
}
co_return mutations;
}
Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::getNext() {
return getNext(Reference<PartitionedLogIteratorTwoBuffers>::addRef(this));
}
FileBackupAgent::FileBackupAgent()
: subspace(Subspace(fileBackupPrefixRange.begin))
// The other subspaces have logUID -> value
,
config(subspace.get(BackupAgentBase::keyConfig)), lastRestorable(subspace.get(FileBackupAgent::keyLastRestorable)),
taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
AccessSystemKeys::True,
PriorityBatch::False,
LockAware::True)),
futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) {
}
namespace fileBackup {
static Reference<IBackupContainer> getBackupContainerWithProxy(Reference<IBackupContainer> _bc) {
Reference<IBackupContainer> bc = IBackupContainer::openContainer(
_bc->getURL(), fileBackupAgentProxy, _bc->getEncryptionKeyFileName(), _bc->getEncryptionBlockSize());
return bc;
}
Future<Void> checkTaskVersion(Database cx, Reference<Task> task, StringRef name, uint32_t version) {
uint32_t taskVersion = task->getVersion();
if (taskVersion > version) {
Error err = task_invalid_version();
TraceEvent(SevWarn, "BA_BackupRangeTaskFuncExecute")
.detail("TaskVersion", taskVersion)
.detail("Name", name)
.detail("Version", version);
if (KeyBackedTaskConfig::TaskParams.uid().exists(task)) {
std::string msg = format("%s task version `%lu' is greater than supported version `%lu'",
task->params[Task::reservedTaskParamKeyType].toString().c_str(),
(unsigned long)taskVersion,
(unsigned long)version);
co_await BackupConfig(task).logError(cx, err, msg);
}
throw err;
}
}
static Future<Void> abortFiveZeroBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
std::string tagName) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName);
Optional<Value> uidStr = co_await tr->get(tagNames.pack(Key(tagName)));
if (!uidStr.present()) {
TraceEvent(SevWarn, "FileBackupAbortIncompatibleBackup_TagNotFound").detail("TagName", tagName.c_str());
co_return;
}
UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned());
Subspace statusSpace = backupAgent->subspace.get(BackupAgentBase::keyStates).get(uid.toString());
Subspace globalConfig = backupAgent->subspace.get(BackupAgentBase::keyConfig).get(uid.toString());
Subspace newConfigSpace(uidPrefixKey("uid->config/"_sr.withPrefix(fileBackupPrefixRange.begin), uid));
Optional<Value> statusStr = co_await tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus));
EBackupState status =
!statusStr.present() ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup")
.detail("TagName", tagName.c_str())
.detail("Status", BackupAgentBase::getStateText(status));
// Clear the folder id to prevent future tasks from executing at all
tr->clear(singleKeyRange(StringRef(globalConfig.pack(FileBackupAgent::keyFolderId))));
// Clear the mutations logging config and data
Key configPath = uidPrefixKey(logRangesRange.begin, uid);
Key logsPath = uidPrefixKey(backupLogKeys.begin, uid);
tr->clear(KeyRangeRef(configPath, strinc(configPath)));
tr->clear(KeyRangeRef(logsPath, strinc(logsPath)));
// Clear the new-style config space
tr->clear(newConfigSpace.range());
Key statusKey = StringRef(statusSpace.pack(FileBackupAgent::keyStateStatus));
// Set old style state key to Aborted if it was Runnable
if (backupAgent->isRunnable(status))
tr->set(statusKey, StringRef(FileBackupAgent::getStateText(EBackupState::STATE_ABORTED)));
}
struct AbortFiveZeroBackupTask : TaskFuncBase {
static StringRef name;
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
FileBackupAgent backupAgent;
std::string tagName = task->params[BackupAgentBase::keyConfigBackupTag].toString();
TraceEvent(SevInfo, "FileBackupCancelOldTask")
.detail("Task", task->params[Task::reservedTaskParamKeyType])
.detail("TagName", tagName);
co_await abortFiveZeroBackup(&backupAgent, tr, tagName);
co_await taskBucket->finish(tr, task);
co_return;
}
StringRef getName() const override {
TraceEvent(SevError, "FileBackupError")
.detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called");
ASSERT(false);
return StringRef();
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Future<Void>(Void());
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef AbortFiveZeroBackupTask::name = "abort_legacy_backup"_sr;
REGISTER_TASKFUNC(AbortFiveZeroBackupTask);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_diff_logs);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_log_range);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_logs);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_range);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_backup_restorable);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_finish_full_backup);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_finished_full_backup);
REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_start_full_backup);
static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
std::string tagName) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeBackupTag(tagName);
UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
BackupConfig config(current.first);
EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
if (!backupAgent->isRunnable(status)) {
throw backup_unneeded();
}
TraceEvent(SevInfo, "FBA_AbortFileOneBackup")
.detail("TagName", tagName.c_str())
.detail("Status", BackupAgentBase::getStateText(status));
// Cancel backup task through tag
co_await tag.cancel(tr);
Key configPath = uidPrefixKey(logRangesRange.begin, config.getUid());
Key logsPath = uidPrefixKey(backupLogKeys.begin, config.getUid());
tr->clear(KeyRangeRef(configPath, strinc(configPath)));
tr->clear(KeyRangeRef(logsPath, strinc(logsPath)));
config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
}
struct AbortFiveOneBackupTask : TaskFuncBase {
static StringRef name;
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
FileBackupAgent backupAgent;
BackupConfig config(task);
std::string tagName = co_await config.tag().getOrThrow(tr);
TraceEvent(SevInfo, "FileBackupCancelFiveOneTask")
.detail("Task", task->params[Task::reservedTaskParamKeyType])
.detail("TagName", tagName);
co_await abortFiveOneBackup(&backupAgent, tr, tagName);
co_await taskBucket->finish(tr, task);
}
StringRef getName() const override {
TraceEvent(SevError, "FileBackupError")
.detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called");
ASSERT(false);
return StringRef();
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Future<Void>(Void());
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef AbortFiveOneBackupTask::name = "abort_legacy_backup_5.2"_sr;
REGISTER_TASKFUNC(AbortFiveOneBackupTask);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_write_range);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_dispatch_ranges);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_write_logs);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_erase_logs);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_dispatch_logs);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_finished);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_write_snapshot_manifest);
REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_start);
std::function<void(Reference<Task>)> NOP_SETUP_TASK_FN = [](Reference<Task> task) { /* NOP */ };
static Future<Key> addBackupTask(StringRef name,
uint32_t version,
Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
TaskCompletionKey completionKey,
BackupConfig config,
Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
std::function<void(Reference<Task>)> setupTaskFn = NOP_SETUP_TASK_FN,
int priority = 0,
SetValidation setValidation = SetValidation::True) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(name, version, doneKey, priority));
// Bind backup config to new task
// allow this new task to find the config(keyspace) of the parent task
co_await config.toTask(tr, task, setValidation);
// Set task specific params
setupTaskFn(task);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
// Clears the backup ID from "backupStartedKey" to pause backup workers.
static Future<Void> clearBackupStartID(Reference<ReadYourWritesTransaction> tr, UID backupUid) {
// If backup worker is not enabled, exit early.
Optional<Value> started = co_await tr->get(backupStartedKey);
std::vector<std::pair<UID, Version>> ids;
if (started.present()) {
ids = decodeBackupStartedValue(started.get());
}
auto it =
std::find_if(ids.begin(), ids.end(), [=](const std::pair<UID, Version>& p) { return p.first == backupUid; });
if (it != ids.end()) {
ids.erase(it);
}
if (ids.empty()) {
TraceEvent("ClearBackup").detail("BackupID", backupUid);
tr->clear(backupStartedKey);
} else {
tr->set(backupStartedKey, encodeBackupStartedValue(ids));
}
}
// Backup and Restore taskFunc definitions will inherit from one of the following classes which
// servers to catch and log to the appropriate config any error that execute/finish didn't catch and log.
struct RestoreTaskFuncBase : TaskFuncBase {
Future<Void> handleError(Database cx, Reference<Task> task, Error const& error) final {
return RestoreConfig(task).logError(
cx,
error,
format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
}
virtual std::string toString(Reference<Task> task) const { return ""; }
};
struct BackupTaskFuncBase : TaskFuncBase {
Future<Void> handleError(Database cx, Reference<Task> task, Error const& error) final {
return BackupConfig(task).logError(
cx,
error,
format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
}
virtual std::string toString(Reference<Task> task) const { return ""; }
};
static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr,
Key beginKey,
Key endKey,
int limit) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Standalone<VectorRef<KeyRef>> results;
RangeResult values = co_await tr->getRange(
KeyRangeRef(keyAfter(beginKey.withPrefix(keyServersPrefix)), endKey.withPrefix(keyServersPrefix)), limit);
for (auto& s : values) {
KeyRef k = s.key.removePrefix(keyServersPrefix);
results.push_back_deep(results.arena(), k);
}
co_return results;
}
struct BackupRangeTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Key> beginKey() { return __FUNCTION__sr; }
static TaskParam<Key> endKey() { return __FUNCTION__sr; }
static TaskParam<bool> addBackupRangeTasks() { return __FUNCTION__sr; }
} Params;
std::string toString(Reference<Task> task) const override {
return format("beginKey '%s' endKey '%s' addTasks %d",
Params.beginKey().get(task).printable().c_str(),
Params.endKey().get(task).printable().c_str(),
Params.addBackupRangeTasks().get(task));
}
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress
// durable. This means:
// - increment the backup config's range bytes written
// - update the range file map
// - update the task begin key
// - save/extend the task with the new params
// Returns whether or not the caller should continue executing the task.
static Future<bool> finishRangeFile(Reference<IBackupFile> file,
Database cx,
Reference<Task> task,
Reference<TaskBucket> taskBucket,
KeyRange range,
Version version) {
co_await file->finish();
TraceEvent("BackupRangeFileFinished")
.detail("BackupUID", BackupConfig(task).getUid())
.detail("FileName", file->getFileName())
.detail("FileSize", file->size())
.detail("RangeBegin", range.begin.printable())
.detail("RangeEnd", range.end.printable())
.detail("RangeEmpty", range.empty())
.detail("Version", version);
// Ignore empty ranges.
if (range.empty())
co_return false;
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
BackupConfig backup(task);
bool usedFile = false;
// Avoid unnecessary conflict by prevent taskbucket's automatic timeout extension
// because the following transaction loop extends and updates the task.
co_await task->extendMutex.take();
FlowLock::Releaser releaser(task->extendMutex, 1);
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
// Update the start key of the task so if this transaction completes but the task then fails
// when it is restarted it will continue where this execution left off.
Params.beginKey().set(task, range.end);
// Save and extend the task with the new begin parameter
Version newTimeout = co_await taskBucket->extendTimeout(tr, task, UpdateParams::True);
// Update the range bytes written in the backup config
backup.rangeBytesWritten().atomicOp(tr, file->size(), MutationRef::AddValue);
backup.snapshotRangeFileCount().atomicOp(tr, 1, MutationRef::AddValue);
// See if there is already a file for this key which has an earlier begin, update the map if not.
Optional<BackupConfig::RangeSlice> s = co_await backup.snapshotRangeFileMap().get(tr, range.end);
if (!s.present() || s.get().begin >= range.begin) {
backup.snapshotRangeFileMap().set(
tr, range.end, { range.begin, version, file->getFileName(), file->size() });
usedFile = true;
}
co_await tr->commit();
task->timeoutVersion = newTimeout;
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
co_return usedFile;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
int priority,
Key begin,
Key end,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
Version scheduledVersion = invalidVersion) {
Key key = co_await addBackupTask(
BackupRangeTaskFunc::name,
BackupRangeTaskFunc::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
[=](Reference<Task> task) {
Params.beginKey().set(task, begin);
Params.endKey().set(task, end);
Params.addBackupRangeTasks().set(task, false);
if (scheduledVersion != invalidVersion)
ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
},
priority);
co_return key;
}
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
co_await checkTaskVersion(cx, task, BackupRangeTaskFunc::name, BackupRangeTaskFunc::version);
Key beginKey = Params.beginKey().get(task);
Key endKey = Params.endKey().get(task);
TraceEvent("FileBackupRangeStart")
.suppressFor(60)
.detail("BackupUID", BackupConfig(task).getUid())
.detail("BeginKey", Params.beginKey().get(task).printable())
.detail("EndKey", Params.endKey().get(task).printable())
.detail("TaskKey", task->key.printable());
// When a key range task saves the last chunk of progress and then the executor dies, when the task
// continues its beginKey and endKey will be equal but there is no work to be done.
if (beginKey == endKey)
co_return;
// Find out if there is a shard boundary in(beginKey, endKey)
Standalone<VectorRef<KeyRef>> keys = co_await runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return getBlockOfShards(tr, beginKey, endKey, 1); });
if (!keys.empty()) {
Params.addBackupRangeTasks().set(task, true);
co_return;
}
// Read everything from beginKey to endKey, write it to an output file, run the output file processor, and
// then set on_done. If we are still writing after X seconds, end the output file and insert a new
// backup_range task for the remainder.
Reference<IBackupFile> outFile;
Version outVersion = invalidVersion;
Key lastKey;
// retrieve kvData
PromiseStream<RangeResultWithVersion> results;
Future<Void> rc = readCommitted(cx,
results,
lock,
KeyRangeRef(beginKey, endKey),
Terminator::True,
AccessSystemKeys::True,
LockAware::True,
ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY));
std::unique_ptr<IRangeFileWriter> rangeFile;
BackupConfig backup(task);
Arena arena;
DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
// Don't need to check keepRunning(task) here because we will do that while finishing each output file, but
// if bc is false then clearly the backup is no longer in progress
Reference<IBackupContainer> _bc = co_await backup.backupContainer().getD(cx.getReference());
if (!_bc) {
co_return;
}
Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
bool done = false;
int64_t nrKeys = 0;
while (true) {
RangeResultWithVersion values;
try {
RangeResultWithVersion _values = co_await results.getFuture();
values = _values;
lock->release(values.first.expectedSize());
} catch (Error& e) {
if (e.code() == error_code_end_of_stream)
done = true;
else
throw;
}
// If we've seen a new read version OR hit the end of the stream, then if we were writing a file finish
// it.
if (values.second != outVersion || done) {
if (outFile) {
CODE_PROBE(outVersion != invalidVersion, "Backup range task wrote multiple versions");
Key nextKey = done ? endKey : keyAfter(lastKey);
co_await rangeFile->writeKey(nextKey);
if (buggify()) {
co_await rangeFile->padEnd(true);
}
co_await rangeFile->finish();
bool usedFile = co_await finishRangeFile(
outFile, cx, task, taskBucket, KeyRangeRef(beginKey, nextKey), outVersion);
TraceEvent("FileBackupWroteRangeFile")
.suppressFor(60)
.detail("BackupUID", backup.getUid())
.detail("Size", outFile->size())
.detail("Keys", nrKeys)
.detail("ReadVersion", outVersion)
.detail("BeginKey", beginKey.printable())
.detail("EndKey", nextKey.printable())
.detail("AddedFileToMap", usedFile);
nrKeys = 0;
beginKey = nextKey;
}
if (done)
co_return;
// Start writing a new file after verifying this task should keep running as of a new read version
// (which must be >= outVersion)
outVersion = values.second;
// block size must be at least large enough for 3 max size keys and 2 max size values + overhead so
// 250k conservatively.
int blockSize = buggify() ? deterministicRandom()->randomInt(250e3, 4e6)
: CLIENT_KNOBS->BACKUP_RANGEFILE_BLOCK_SIZE;
Version snapshotBeginVersion{ 0 };
int64_t snapshotRangeFileCount{ 0 };
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await (taskBucket->keepRunning(tr, task) &&
storeOrThrow(snapshotBeginVersion, backup.snapshotBeginVersion().get(tr)) &&
store(snapshotRangeFileCount, backup.snapshotRangeFileCount().getD(tr)));
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
Reference<IBackupFile> f =
co_await bc->writeRangeFile(snapshotBeginVersion, snapshotRangeFileCount, outVersion, blockSize);
outFile = f;
// Initialize range file writer and write begin key
rangeFile = std::make_unique<RangeFileWriter>(outFile, blockSize);
co_await rangeFile->writeKey(beginKey);
}
// write kvData to file, update lastKey and key count
if (!values.first.empty()) {
for (size_t i = 0; i < values.first.size(); ++i) {
co_await rangeFile->writeKV(values.first[i].key, values.first[i].value);
}
lastKey = values.first.back().key;
nrKeys += values.first.size();
}
}
}
static Future<Void> startBackupRangeInternal(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task,
Reference<TaskFuture> onDone) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Key nextKey = Params.beginKey().get(task);
Key endKey = Params.endKey().get(task);
Standalone<VectorRef<KeyRef>> keys =
co_await getBlockOfShards(tr, nextKey, endKey, CLIENT_KNOBS->BACKUP_SHARD_TASK_LIMIT);
std::vector<Future<Key>> addTaskVector;
for (const auto& splitKey : keys) {
if (nextKey != splitKey) {
addTaskVector.push_back(addTask(
tr, taskBucket, task, task->getPriority(), nextKey, splitKey, TaskCompletionKey::joinWith(onDone)));
TraceEvent("FileBackupRangeSplit")
.suppressFor(60)
.detail("BackupUID", BackupConfig(task).getUid())
.detail("BeginKey", Params.beginKey().get(task).printable())
.detail("EndKey", Params.endKey().get(task).printable())
.detail("SliceBeginKey", nextKey.printable())
.detail("SliceEndKey", splitKey.printable());
}
nextKey = splitKey;
}
co_await waitForAll(addTaskVector);
if (nextKey != endKey) {
// Add task to cover nextKey to the end, using the priority of the current task
co_await addTask(tr,
taskBucket,
task,
task->getPriority(),
nextKey,
endKey,
TaskCompletionKey::joinWith(onDone),
Reference<TaskFuture>(),
task->getPriority());
}
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
if (Params.addBackupRangeTasks().get(task)) {
co_await startBackupRangeInternal(tr, taskBucket, futureBucket, task, taskFuture);
} else {
co_await taskFuture->set(tr, taskBucket);
}
co_await taskBucket->finish(tr, task);
TraceEvent("FileBackupRangeFinish")
.suppressFor(60)
.detail("BackupUID", BackupConfig(task).getUid())
.detail("BeginKey", Params.beginKey().get(task).printable())
.detail("EndKey", Params.endKey().get(task).printable())
.detail("TaskKey", task->key.printable());
co_return;
}
};
StringRef BackupRangeTaskFunc::name = "file_backup_write_range_5.2"_sr;
REGISTER_TASKFUNC(BackupRangeTaskFunc);
struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
// Set by Execute, used by Finish
static TaskParam<int64_t> shardsBehind() { return __FUNCTION__sr; }
// Set by Execute, used by Finish
static TaskParam<bool> snapshotFinished() { return __FUNCTION__sr; }
// Set by Execute, used by Finish
static TaskParam<Version> nextDispatchVersion() { return __FUNCTION__sr; }
} Params;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
int priority,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
Version scheduledVersion = invalidVersion) {
Key key = co_await addBackupTask(
name,
version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
[=](Reference<Task> task) {
if (scheduledVersion != invalidVersion)
ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
},
priority);
co_return key;
}
enum DispatchState { SKIP = 0, DONE = 1, NOT_DONE_MIN = 2 };
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
co_await checkTaskVersion(cx, task, name, version);
double startTime = timer();
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
// The shard map will use 3 values classes. Exactly SKIP, exactly DONE, then any number >= NOT_DONE_MIN
// which will mean not done. This is to enable an efficient coalesce() call to squash adjacent ranges which
// are not yet finished to enable efficiently finding random database shards which are not done.
int notDoneSequence = NOT_DONE_MIN;
KeyRangeMap<int> shardMap(notDoneSequence++);
Key beginKey = allKeys.begin;
// Read all shard boundaries and add them to the map
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<Standalone<VectorRef<KeyRef>>> shardBoundaries =
getBlockOfShards(tr, beginKey, allKeys.end, CLIENT_KNOBS->TOO_MANY);
co_await (success(shardBoundaries) && taskBucket->keepRunning(tr, task));
if (shardBoundaries.get().empty())
break;
for (auto& boundary : shardBoundaries.get()) {
shardMap.rawInsert(boundary, notDoneSequence++);
}
beginKey = keyAfter(shardBoundaries.get().back());
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// Read required stuff from backup config
BackupConfig config(task);
Version recentReadVersion{ 0 };
Version snapshotBeginVersion{ 0 };
Version snapshotTargetEndVersion{ 0 };
int64_t snapshotIntervalSeconds{ 0 };
Optional<Version> latestSnapshotEndVersion;
std::vector<KeyRange> backupRanges;
Optional<Key> snapshotBatchFutureKey;
Reference<TaskFuture> snapshotBatchFuture;
Optional<int64_t> snapshotBatchSize;
tr->reset();
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await (store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
store(backupRanges, config.backupRanges().getOrThrow(tr)) &&
store(snapshotIntervalSeconds, config.snapshotIntervalSeconds().getOrThrow(tr))
// The next two parameters are optional
&& store(snapshotBatchFutureKey, config.snapshotBatchFuture().get(tr)) &&
store(snapshotBatchSize, config.snapshotBatchSize().get(tr)) &&
store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
store(recentReadVersion, tr->getReadVersion()) && taskBucket->keepRunning(tr, task));
// If the snapshot batch future key does not exist, this is the first execution of this dispatch
// task so
// - create and set the snapshot batch future key
// - initialize the batch size to 0
// - initialize the target snapshot end version if it is not yet set
// - commit
if (!snapshotBatchFutureKey.present()) {
snapshotBatchFuture = futureBucket->future(tr);
config.snapshotBatchFuture().set(tr, snapshotBatchFuture->pack());
snapshotBatchSize = 0;
config.snapshotBatchSize().set(tr, snapshotBatchSize.get());
// The dispatch of this batch can take multiple separate executions if the executor fails
// so store a completion key for the dispatch finish() to set when dispatching the batch is
// done.
TaskCompletionKey dispatchCompletionKey = TaskCompletionKey::joinWith(snapshotBatchFuture);
// this is a bad hack - but flow doesn't work well with lambda functions and capturing
// state variables...
auto cfg = &config;
auto tx = &tr;
co_await map(dispatchCompletionKey.get(tr, taskBucket), [cfg, tx](Key const& k) {
cfg->snapshotBatchDispatchDoneKey().set(*tx, k);
return Void();
});
co_await tr->commit();
} else {
ASSERT(snapshotBatchSize.present());
// Batch future key exists in the config so create future from it
snapshotBatchFuture = makeReference<TaskFuture>(futureBucket, snapshotBatchFutureKey.get());
}
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// Read all dispatched ranges
std::vector<std::pair<Key, bool>> dispatchBoundaries;
tr->reset();
beginKey = allKeys.begin;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<BackupConfig::RangeDispatchMapT::RangeResultType> bounds =
config.snapshotRangeDispatchMap().getRange(
tr, beginKey, keyAfter(allKeys.end), CLIENT_KNOBS->TOO_MANY);
co_await (success(bounds) && taskBucket->keepRunning(tr, task) &&
store(recentReadVersion, tr->getReadVersion()));
if (!bounds.get().results.empty()) {
dispatchBoundaries.reserve(dispatchBoundaries.size() + bounds.get().results.size());
dispatchBoundaries.insert(
dispatchBoundaries.end(), bounds.get().results.begin(), bounds.get().results.end());
}
if (!bounds.get().more) {
break;
}
beginKey = keyAfter(bounds.get().results.back().first);
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// The next few sections involve combining the results above. Yields are used after operations
// that could have operated on many thousands of things and in loops which could have many
// thousands of iterations.
// Declare some common iterators which must be state vars and will be used multiple times.
int i{ 0 };
RangeMap<Key, int, KeyRangeRef>::iterator iShard;
RangeMap<Key, int, KeyRangeRef>::iterator iShardEnd;
// Set anything inside a dispatched range to DONE.
// Also ensure that the boundary value are true, false, [true, false]...
if (!dispatchBoundaries.empty()) {
bool lastValue = false;
Key lastKey;
for (i = 0; i < dispatchBoundaries.size(); ++i) {
const std::pair<Key, bool>& boundary = dispatchBoundaries[i];
// Values must alternate
ASSERT(boundary.second == !lastValue);
// If this was the end of a dispatched range
if (!boundary.second) {
// Ensure that the dispatched boundaries exist AND set all shard ranges in the dispatched range
// to DONE.
RangeMap<Key, int, KeyRangeRef>::Ranges shardRanges =
shardMap.modify(KeyRangeRef(lastKey, boundary.first));
iShard = shardRanges.begin();
iShardEnd = shardRanges.end();
for (; iShard != iShardEnd; ++iShard) {
iShard->value() = DONE;
co_await yield();
}
}
lastValue = dispatchBoundaries[i].second;
lastKey = dispatchBoundaries[i].first;
co_await yield();
}
ASSERT(lastValue == false);
}
// Set anything outside the backup ranges to SKIP. We can use insert() here instead of modify()
// because it's OK to delete shard boundaries in the skipped ranges.
if (!backupRanges.empty()) {
shardMap.insert(KeyRangeRef(allKeys.begin, backupRanges.front().begin), SKIP);
co_await yield();
for (i = 0; i < backupRanges.size() - 1; ++i) {
shardMap.insert(KeyRangeRef(backupRanges[i].end, backupRanges[i + 1].begin), SKIP);
co_await yield();
}
shardMap.insert(KeyRangeRef(backupRanges.back().end, allKeys.end), SKIP);
co_await yield();
}
int countShardsDone = 0;
int countShardsNotDone = 0;
// Scan through the shard map, counting the DONE and NOT_DONE shards.
RangeMap<Key, int, KeyRangeRef>::Ranges shardRanges = shardMap.ranges();
iShard = shardRanges.begin();
iShardEnd = shardRanges.end();
for (; iShard != iShardEnd; ++iShard) {
if (iShard->value() == DONE) {
++countShardsDone;
} else if (iShard->value() >= NOT_DONE_MIN) {
++countShardsNotDone;
}
co_await yield();
}
// Coalesce the shard map to make random selection below more efficient.
shardMap.coalesce(allKeys);
co_await yield();
// In this context "all" refers to all of the shards relevant for this particular backup
int countAllShards = countShardsDone + countShardsNotDone;
// Log backup ranges and shard counts for debugging mode=BOTH issues
TraceEvent("FileBackupSnapshotDispatchShardCount")
.detail("BackupUID", config.getUid())
.detail("BackupRangesCount", backupRanges.size())
.detail("FirstRangeBegin", backupRanges.empty() ? ""_sr : backupRanges.front().begin.printable())
.detail("FirstRangeEnd", backupRanges.empty() ? ""_sr : backupRanges.front().end.printable())
.detail("CountAllShards", countAllShards)
.detail("CountShardsDone", countShardsDone)
.detail("CountShardsNotDone", countShardsNotDone)
.detail("LatestSnapshotEndVersion", latestSnapshotEndVersion.orDefault(-1));
// NOTE: Don't finish here even if countShardsNotDone == 0. We need to dispatch tasks first.
// The completion check after dispatch (with dispatchedInThisIteration guard) prevents
// finishing in the same iteration we dispatch the last tasks.
if (countShardsNotDone == 0) {
TraceEvent("FileBackupSnapshotDispatchAllDoneBeforeDispatch")
.detail("BackupUID", config.getUid())
.detail("Note", "Will check again after dispatch loop");
}
// Decide when the next snapshot dispatch should run.
Version nextDispatchVersion{ 0 };
// In simulation, use snapshot interval / 5 to ensure multiple dispatches run
// Otherwise, use the knob for the number of seconds between snapshot dispatch tasks.
if (g_network->isSimulated()) {
nextDispatchVersion =
recentReadVersion + CLIENT_KNOBS->CORE_VERSIONSPERSECOND * (snapshotIntervalSeconds / 5.0);
} else {
nextDispatchVersion = recentReadVersion + CLIENT_KNOBS->CORE_VERSIONSPERSECOND *
CLIENT_KNOBS->BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
}
// If nextDispatchVersion is greater than snapshotTargetEndVersion (which could be in the past) then just
// use the greater of recentReadVersion or snapshotTargetEndVersion. Any range tasks created in this
// dispatch will be scheduled at a random time between recentReadVersion and nextDispatchVersion, so
// nextDispatchVersion shouldn't be less than recentReadVersion.
if (nextDispatchVersion > snapshotTargetEndVersion)
nextDispatchVersion = std::max(recentReadVersion, snapshotTargetEndVersion);
Params.nextDispatchVersion().set(task, nextDispatchVersion);
// Calculate number of shards that should be done before the next interval end
// timeElapsed is between 0 and 1 and represents what portion of the shards we should have completed by now
double timeElapsed;
Version snapshotScheduledVersionInterval = snapshotTargetEndVersion - snapshotBeginVersion;
if (snapshotTargetEndVersion > snapshotBeginVersion) {
timeElapsed = std::min(
1.0, (double)(nextDispatchVersion - snapshotBeginVersion) / (snapshotScheduledVersionInterval));
} else {
timeElapsed = 1.0;
}
int countExpectedShardsDone = countAllShards * timeElapsed;
int countShardsToDispatch = std::max<int>(0, countExpectedShardsDone - countShardsDone);
// Calculate the number of shards that would have been dispatched by a normal (on-schedule)
// BackupSnapshotDispatchTask given the dispatch window and the start and expected-end versions of the
// current snapshot.
int64_t dispatchWindow = nextDispatchVersion - recentReadVersion;
// If the scheduled snapshot interval is 0 (such as for initial, as-fast-as-possible snapshot) then all
// shards are considered late
int countShardsExpectedPerNormalWindow;
if (snapshotScheduledVersionInterval == 0) {
countShardsExpectedPerNormalWindow = 0;
} else {
// A dispatchWindow of 0 means the target end version is <= now which also results in all shards being
// considered late
countShardsExpectedPerNormalWindow =
(double(dispatchWindow) / snapshotScheduledVersionInterval) * countAllShards;
}
// The number of shards 'behind' the snapshot is the count of how may additional shards beyond normal are
// being dispatched, if any.
int countShardsBehind =
std::max<int64_t>(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow);
Params.shardsBehind().set(task, countShardsBehind);
TraceEvent("FileBackupSnapshotDispatchStats")
.detail("BackupUID", config.getUid())
.detail("AllShards", countAllShards)
.detail("ShardsDone", countShardsDone)
.detail("ShardsNotDone", countShardsNotDone)
.detail("ExpectedShardsDone", countExpectedShardsDone)
.detail("ShardsToDispatch", countShardsToDispatch)
.detail("ShardsBehind", countShardsBehind)
.detail("SnapshotBeginVersion", snapshotBeginVersion)
.detail("SnapshotTargetEndVersion", snapshotTargetEndVersion)
.detail("NextDispatchVersion", nextDispatchVersion)
.detail("CurrentVersion", recentReadVersion)
.detail("TimeElapsed", timeElapsed)
.detail("SnapshotIntervalSeconds", snapshotIntervalSeconds);
// Track whether we dispatched any tasks in this iteration
bool dispatchedInThisIteration = false;
// Dispatch random shards to catch up to the expected progress
while (countShardsToDispatch > 0) {
// First select ranges to add
std::vector<KeyRange> rangesToAdd;
// Limit number of tasks added per transaction
int taskBatchSize = buggify() ? deterministicRandom()->randomInt(1, countShardsToDispatch + 1)
: CLIENT_KNOBS->BACKUP_DISPATCH_ADDTASK_SIZE;
int added = 0;
while (countShardsToDispatch > 0 && added < taskBatchSize && shardMap.size() > 0) {
// Get a random range.
auto it = shardMap.randomRange();
// Find a NOT_DONE range and add it to rangesToAdd
while (1) {
if (it->value() >= NOT_DONE_MIN) {
rangesToAdd.push_back(it->range());
it->value() = DONE;
shardMap.coalesce(Key(it->begin()));
++added;
++countShardsDone;
--countShardsToDispatch;
--countShardsNotDone;
break;
}
if (it->end() == shardMap.mapEnd)
break;
++it;
}
}
int64_t oldBatchSize = snapshotBatchSize.get();
int64_t newBatchSize = oldBatchSize + rangesToAdd.size();
// Now add the selected ranges in a single transaction.
tr->reset();
while (true) {
Error err;
try {
TraceEvent("FileBackupSnapshotDispatchAddingTasks")
.suppressFor(2)
.detail("TasksToAdd", rangesToAdd.size())
.detail("NewBatchSize", newBatchSize);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
// For each range, make sure it isn't set in the dispatched range map.
std::vector<Future<Optional<bool>>> beginReads;
std::vector<Future<Optional<bool>>> endReads;
for (auto& range : rangesToAdd) {
beginReads.push_back(config.snapshotRangeDispatchMap().get(tr, range.begin));
endReads.push_back(config.snapshotRangeDispatchMap().get(tr, range.end));
}
co_await (store(snapshotBatchSize.get(), config.snapshotBatchSize().getOrThrow(tr)) &&
waitForAll(beginReads) && waitForAll(endReads) && taskBucket->keepRunning(tr, task));
// Snapshot batch size should be either oldBatchSize or newBatchSize. If new, this transaction
// is already done.
if (snapshotBatchSize.get() == newBatchSize) {
break;
} else {
ASSERT(snapshotBatchSize.get() == oldBatchSize);
config.snapshotBatchSize().set(tr, newBatchSize);
snapshotBatchSize = newBatchSize;
config.snapshotDispatchLastShardsBehind().set(tr, Params.shardsBehind().get(task));
config.snapshotDispatchLastVersion().set(tr, tr->getReadVersion().get());
}
std::vector<Future<Void>> addTaskFutures;
for (i = 0; i < beginReads.size(); ++i) {
KeyRange& range = rangesToAdd[i];
// This loop might have made changes to begin or end boundaries in a prior
// iteration. If so, the updated values exist in the RYW cache so re-read both entries.
Optional<bool> beginValue = config.snapshotRangeDispatchMap().get(tr, range.begin).get();
Optional<bool> endValue = config.snapshotRangeDispatchMap().get(tr, range.end).get();
ASSERT(!beginValue.present() || !endValue.present() || beginValue != endValue);
// If begin is present, it must be a range end so value must be false
// If end is present, it must be a range begin so value must be true
if ((!beginValue.present() || !beginValue.get()) && (!endValue.present() || endValue.get())) {
if (beginValue.present()) {
config.snapshotRangeDispatchMap().erase(tr, range.begin);
} else {
config.snapshotRangeDispatchMap().set(tr, range.begin, true);
}
if (endValue.present()) {
config.snapshotRangeDispatchMap().erase(tr, range.end);
} else {
config.snapshotRangeDispatchMap().set(tr, range.end, false);
}
Version scheduledVersion = invalidVersion;
// If the next dispatch version is in the future, choose a random version at which to
// start the new task.
if (nextDispatchVersion > recentReadVersion)
scheduledVersion = recentReadVersion + deterministicRandom()->random01() *
(nextDispatchVersion - recentReadVersion);
// Range tasks during the initial snapshot should run at a higher priority
int priority = latestSnapshotEndVersion.present() ? 0 : 1;
addTaskFutures.push_back(
success(BackupRangeTaskFunc::addTask(tr,
taskBucket,
task,
priority,
range.begin,
range.end,
TaskCompletionKey::joinWith(snapshotBatchFuture),
Reference<TaskFuture>(),
scheduledVersion)));
TraceEvent("FileBackupSnapshotRangeDispatched")
.suppressFor(2)
.detail("BackupUID", config.getUid())
.detail("CurrentVersion", recentReadVersion)
.detail("ScheduledVersion", scheduledVersion)
.detail("BeginKey", range.begin.printable())
.detail("EndKey", range.end.printable());
} else {
// This shouldn't happen because if the transaction was already done or if another
// execution of this task is making progress it should have been detected above.
ASSERT(false);
}
}
co_await waitForAll(addTaskFutures);
co_await tr->commit();
dispatchedInThisIteration = true;
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
// Only finish if all shards are done AND we didn't dispatch any tasks this iteration.
// This prevents the bug where we mark snapshot finished immediately after dispatching
// the last batch of tasks, before they actually complete.
if (countShardsNotDone == 0 && !dispatchedInThisIteration) {
TraceEvent("FileBackupSnapshotDispatchFinished")
.detail("BackupUID", config.getUid())
.detail("AllShards", countAllShards)
.detail("ShardsDone", countShardsDone)
.detail("ShardsNotDone", countShardsNotDone)
.detail("SnapshotBeginVersion", snapshotBeginVersion)
.detail("SnapshotTargetEndVersion", snapshotTargetEndVersion)
.detail("CurrentVersion", recentReadVersion)
.detail("SnapshotIntervalSeconds", snapshotIntervalSeconds)
.detail("DispatchTimeSeconds", timer() - startTime);
Params.snapshotFinished().set(task, true);
}
co_return;
}
// This function is just a wrapper for BackupSnapshotManifest::addTask() which is defined below.
// The BackupSnapshotDispatchTask and BackupSnapshotManifest tasks reference each other so in order to keep
// their execute and finish phases defined together inside their class definitions this wrapper is declared here
// but defined after BackupSnapshotManifest is defined.
static Future<Key> addSnapshotManifestTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>());
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
BackupConfig config(task);
// Get the batch future and dispatch done keys, then clear them.
Key snapshotBatchFutureKey;
Key snapshotBatchDispatchDoneKey;
co_await (store(snapshotBatchFutureKey, config.snapshotBatchFuture().getOrThrow(tr)) &&
store(snapshotBatchDispatchDoneKey, config.snapshotBatchDispatchDoneKey().getOrThrow(tr)));
Reference<TaskFuture> snapshotBatchFuture = futureBucket->unpack(snapshotBatchFutureKey);
Reference<TaskFuture> snapshotBatchDispatchDoneFuture = futureBucket->unpack(snapshotBatchDispatchDoneKey);
config.snapshotBatchFuture().clear(tr);
config.snapshotBatchDispatchDoneKey().clear(tr);
config.snapshotBatchSize().clear(tr);
// Update shardsBehind here again in case the execute phase did not actually have to create any shard tasks
config.snapshotDispatchLastShardsBehind().set(tr, Params.shardsBehind().getOrDefault(task, 0));
config.snapshotDispatchLastVersion().set(tr, tr->getReadVersion().get());
Reference<TaskFuture> snapshotFinishedFuture = task->getDoneFuture(futureBucket);
bool snapshotFinished = Params.snapshotFinished().getOrDefault(task, false);
TraceEvent("FileBackupSnapshotDispatchFinish")
.detail("BackupUID", config.getUid())
.detail("SnapshotFinished", snapshotFinished)
.detail("ShardsBehind", Params.shardsBehind().getOrDefault(task, 0))
.detail("NextDispatchVersion", Params.nextDispatchVersion().getOrDefault(task, -1));
// If the snapshot is finished, the next task is to write a snapshot manifest, otherwise it's another
// snapshot dispatch task. In either case, the task should wait for snapshotBatchFuture. The snapshot done
// key, passed to the current task, is also passed on.
if (snapshotFinished) {
TraceEvent("FileBackupSnapshotDispatchAddingManifestTask").detail("BackupUID", config.getUid());
co_await addSnapshotManifestTask(
tr, taskBucket, task, TaskCompletionKey::signal(snapshotFinishedFuture), snapshotBatchFuture);
} else {
co_await addTask(tr,
taskBucket,
task,
1,
TaskCompletionKey::signal(snapshotFinishedFuture),
snapshotBatchFuture,
Params.nextDispatchVersion().get(task));
}
// This snapshot batch is finished, so set the batch done future.
co_await snapshotBatchDispatchDoneFuture->set(tr, taskBucket);
co_await taskBucket->finish(tr, task);
co_return;
}
};
StringRef BackupSnapshotDispatchTask::name = "file_backup_dispatch_ranges_5.2"_sr;
REGISTER_TASKFUNC(BackupSnapshotDispatchTask);
struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<bool> addBackupLogRangeTasks() { return __FUNCTION__sr; }
static TaskParam<int64_t> fileSize() { return __FUNCTION__sr; }
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
} Params;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
co_await checkTaskVersion(cx, task, BackupLogRangeTaskFunc::name, BackupLogRangeTaskFunc::version);
Version beginVersion = Params.beginVersion().get(task);
Version endVersion = Params.endVersion().get(task);
BackupConfig config(task);
Reference<IBackupContainer> bc;
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
while (true) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
// Wait for the read version to pass endVersion
{
Error err;
try {
co_await taskBucket->keepRunning(tr, task);
if (!bc) {
// Backup container must be present if we're still here
Reference<IBackupContainer> _bc = co_await config.backupContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
}
Version currentVersion = tr->getReadVersion().get();
if (endVersion < currentVersion)
break;
co_await delay(
std::max(CLIENT_KNOBS->BACKUP_RANGE_MINWAIT,
(double)(endVersion - currentVersion) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
// Get the set of key ranges that hold mutations for (beginVersion, endVersion). They will be queried in
// parallel below and there is a limit on how many we want to process in a single BackupLogRangeTask so if
// that limit is exceeded then set the addBackupLogRangeTasks boolean in Params and stop, signalling the
// finish() step to break up the (beginVersion, endVersion) range into smaller intervals which are then
// processed by individual BackupLogRangeTasks.
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion, endVersion, destUidValue);
if (ranges.size() > CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES) {
Params.addBackupLogRangeTasks().set(task, true);
co_return;
}
// Block size must be at least large enough for 1 max size key, 1 max size value, and overhead, so
// conservatively 125k.
int blockSize =
buggify() ? deterministicRandom()->randomInt(125e3, 4e6) : CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
Reference<IBackupFile> outFile = co_await bc->writeLogFile(beginVersion, endVersion, blockSize);
LogFileWriter logFile(outFile, blockSize);
// Query all key ranges covering (beginVersion, endVersion) in parallel, writing their results to the
// results promise stream as they are received. Note that this means the records read from the results
// stream are not likely to be in increasing Version order.
PromiseStream<RangeResultWithVersion> results;
std::vector<Future<Void>> rc;
for (auto& range : ranges) {
rc.push_back(readCommitted(cx,
results,
lock,
range,
Terminator::False,
AccessSystemKeys::True,
LockAware::True,
ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY)));
}
Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) mutable {
if (result.isError())
results.sendError(result.getError());
else
results.sendError(end_of_stream());
return Void();
});
Version lastVersion{ 0 };
{
Error caughtErr;
bool hasCaughtErr = false;
try {
while (true) {
RangeResultWithVersion r = co_await results.getFuture();
lock->release(r.first.expectedSize());
for (int i = 0; i < r.first.size(); ++i) {
// Remove the backupLogPrefix + UID bytes from the key
co_await logFile.writeKV(r.first[i].key.substr(backupLogPrefixBytes + 16), r.first[i].value);
lastVersion = r.second;
}
}
} catch (Error& e) {
caughtErr = e;
hasCaughtErr = true;
}
if (hasCaughtErr) {
if (caughtErr.code() == error_code_actor_cancelled)
throw caughtErr;
if (caughtErr.code() != error_code_end_of_stream) {
Error err = caughtErr;
co_await config.logError(
cx, err, format("Failed to write to file `%s'", outFile->getFileName().c_str()));
throw err;
}
}
}
// Make sure this task is still alive, if it's not then the data read above could be incomplete.
co_await taskBucket->keepRunning(cx, task);
co_await outFile->finish();
TraceEvent("FileBackupWroteLogFile")
.suppressFor(60)
.detail("BackupUID", config.getUid())
.detail("Size", outFile->size())
.detail("BeginVersion", beginVersion)
.detail("EndVersion", endVersion)
.detail("LastReadVersion", lastVersion);
Params.fileSize().set(task, outFile->size());
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
int priority,
Version beginVersion,
Version endVersion,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(
BackupLogRangeTaskFunc::name,
BackupLogRangeTaskFunc::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
[=](Reference<Task> task) {
Params.beginVersion().set(task, beginVersion);
Params.endVersion().set(task, endVersion);
Params.addBackupLogRangeTasks().set(task, false);
},
priority);
co_return key;
}
static Future<Void> startBackupLogRangeInternal(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task,
Reference<TaskFuture> taskFuture,
Version beginVersion,
Version endVersion) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
std::vector<Future<Key>> addTaskVector;
int tasks = 0;
for (int64_t vblock = beginVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE - 1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
vblock += CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES) {
Version bv = std::max(beginVersion, vblock * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
if (tasks >= CLIENT_KNOBS->BACKUP_SHARD_TASK_LIMIT) {
addTaskVector.push_back(addTask(tr,
taskBucket,
task,
task->getPriority(),
bv,
endVersion,
TaskCompletionKey::joinWith(taskFuture)));
break;
}
Version ev = std::min(endVersion,
(vblock + CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES) * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
addTaskVector.push_back(
addTask(tr, taskBucket, task, task->getPriority(), bv, ev, TaskCompletionKey::joinWith(taskFuture)));
tasks++;
}
co_await waitForAll(addTaskVector);
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Version beginVersion = Params.beginVersion().get(task);
Version endVersion = Params.endVersion().get(task);
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
BackupConfig config(task);
if (Params.fileSize().exists(task)) {
config.logBytesWritten().atomicOp(tr, Params.fileSize().get(task), MutationRef::AddValue);
}
if (Params.addBackupLogRangeTasks().get(task)) {
co_await startBackupLogRangeInternal(
tr, taskBucket, futureBucket, task, taskFuture, beginVersion, endVersion);
} else {
co_await taskFuture->set(tr, taskBucket);
}
co_await taskBucket->finish(tr, task);
co_return;
}
};
StringRef BackupLogRangeTaskFunc::name = "file_backup_write_logs_5.2"_sr;
REGISTER_TASKFUNC(BackupLogRangeTaskFunc);
// This task stopped being used in 6.2, however the code remains here to handle upgrades.
struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static struct {
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
static TaskParam<Key> destUidValue() { return __FUNCTION__sr; }
} Params;
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
UID logUid,
TaskCompletionKey completionKey,
Key destUidValue,
Version endVersion = 0,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(
EraseLogRangeTaskFunc::name,
EraseLogRangeTaskFunc::version,
tr,
taskBucket,
completionKey,
BackupConfig(logUid),
waitFor,
[=](Reference<Task> task) {
Params.beginVersion().set(task,
1); // FIXME: remove in 6.X, only needed for 5.2 backward compatibility
Params.endVersion().set(task, endVersion);
Params.destUidValue().set(task, destUidValue);
},
0,
SetValidation::False);
co_return key;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
co_await checkTaskVersion(tr->getDatabase(), task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version);
Version endVersion = Params.endVersion().get(task);
Key destUidValue = Params.destUidValue().get(task);
BackupConfig config(task);
Key logUidValue = config.getUidAsKey();
co_await (
taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) &&
eraseLogData(
tr, logUidValue, destUidValue, endVersion != 0 ? Optional<Version>(endVersion) : Optional<Version>()));
co_return;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef EraseLogRangeTaskFunc::name = "file_backup_erase_logs_5.2"_sr;
REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
struct BackupLogsDispatchTask : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> prevBeginVersion() { return __FUNCTION__sr; }
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
} Params;
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
co_await checkTaskVersion(
tr->getDatabase(), task, BackupLogsDispatchTask::name, BackupLogsDispatchTask::version);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Reference<TaskFuture> onDone = task->getDoneFuture(futureBucket);
Version prevBeginVersion = Params.prevBeginVersion().get(task);
Version beginVersion = Params.beginVersion().get(task);
BackupConfig config(task);
config.latestLogEndVersion().set(tr, beginVersion);
bool stopWhenDone{ false };
Optional<Version> restorableVersion;
EBackupState backupState;
Optional<std::string> tag;
Optional<Version> latestSnapshotEndVersion;
Optional<MutationLogType> mutationLogType;
co_await (store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
store(backupState, config.stateEnum().getOrThrow(tr)) && store(tag, config.tag().get(tr)) &&
store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
store(mutationLogType, config.mutationLogType().get(tr)));
// If restorable, update the last restorable version for this tag
if (restorableVersion.present() && tag.present()) {
FileBackupAgent().setLastRestorable(tr, StringRef(tag.get()), restorableVersion.get());
}
// If the backup is restorable but the state is not differential then set state to differential
if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL)
config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL);
// If stopWhenDone is set and there is a restorable version, set the done future and do not create further
// tasks.
if (stopWhenDone && restorableVersion.present()) {
co_await (onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
TraceEvent("FileBackupLogsDispatchDone")
.detail("BackupUID", config.getUid())
.detail("BeginVersion", beginVersion)
.detail("RestorableVersion", restorableVersion.orDefault(-1));
co_return;
}
Version endVersion =
std::max<Version>(tr->getReadVersion().get() + 1,
beginVersion + static_cast<Version>(CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES - 1) *
CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
TraceEvent("FileBackupLogDispatch")
.suppressFor(60)
.detail("BeginVersion", beginVersion)
.detail("EndVersion", endVersion)
.detail("RestorableVersion", restorableVersion.orDefault(-1));
Reference<TaskFuture> logDispatchBatchFuture = futureBucket->future(tr);
// If a snapshot has ended for this backup then mutations are higher priority to reduce backup lag
int priority = latestSnapshotEndVersion.present() ? 1 : 0;
if (!mutationLogType.present() || mutationLogType.get() == MutationLogType::DEFAULT) {
// Add the initial log range task to read/copy the mutations and the next logs dispatch task which will
// run after this batch is done
// read blog/ prefix and write those (param1, param2) into files
co_await BackupLogRangeTaskFunc::addTask(tr,
taskBucket,
task,
priority,
beginVersion,
endVersion,
TaskCompletionKey::joinWith(logDispatchBatchFuture));
// issue the next key range
co_await BackupLogsDispatchTask::addTask(tr,
taskBucket,
task,
priority,
beginVersion,
endVersion,
TaskCompletionKey::signal(onDone),
logDispatchBatchFuture);
// Do not erase at the first time
if (prevBeginVersion > 0) {
Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
co_await eraseLogData(tr, config.getUidAsKey(), destUidValue, Optional<Version>(beginVersion));
}
} else {
// Skip mutation copy and erase backup mutations. Just check back periodically.
Version scheduledVersion = tr->getReadVersion().get() +
CLIENT_KNOBS->BACKUP_POLL_PROGRESS_SECONDS * CLIENT_KNOBS->VERSIONS_PER_SECOND;
co_await BackupLogsDispatchTask::addTask(tr,
taskBucket,
task,
1,
beginVersion,
endVersion,
TaskCompletionKey::signal(onDone),
Reference<TaskFuture>(),
scheduledVersion);
}
co_await taskBucket->finish(tr, task);
TraceEvent("FileBackupLogsDispatchContinuing")
.suppressFor(60)
.detail("BackupUID", config.getUid())
.detail("BeginVersion", beginVersion)
.detail("EndVersion", endVersion);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
int priority,
Version prevBeginVersion,
Version beginVersion,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
Version scheduledVersion = invalidVersion) {
Key key = co_await addBackupTask(
BackupLogsDispatchTask::name,
BackupLogsDispatchTask::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
[=](Reference<Task> task) {
Params.prevBeginVersion().set(task, prevBeginVersion);
Params.beginVersion().set(task, beginVersion);
if (scheduledVersion != invalidVersion) {
ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
}
},
priority);
co_return key;
}
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef BackupLogsDispatchTask::name = "file_backup_dispatch_logs_5.2"_sr;
REGISTER_TASKFUNC(BackupLogsDispatchTask);
struct FileBackupFinishedTask : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
co_await checkTaskVersion(
tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version);
BackupConfig backup(task);
UID uid = backup.getUid();
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
Key destUidValue = co_await backup.destUidValue().getOrThrow(tr);
co_await (eraseLogData(tr, backup.getUidAsKey(), destUidValue) && clearBackupStartID(tr, uid));
backup.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
co_await taskBucket->finish(tr, task);
TraceEvent("FileBackupFinished").detail("BackupUID", uid);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(FileBackupFinishedTask::name,
FileBackupFinishedTask::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor);
co_return key;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef FileBackupFinishedTask::name = "file_backup_finished_5.2"_sr;
REGISTER_TASKFUNC(FileBackupFinishedTask);
struct BackupSnapshotManifest : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
static TaskParam<int64_t> totalBytes() { return __FUNCTION__sr; }
} Params;
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
BackupConfig config(task);
Reference<IBackupContainer> bc;
DatabaseConfiguration dbConfig;
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
// Read the entire range file map into memory, then walk it backwards from its last entry to produce a list
// of non overlapping key range files
std::map<Key, BackupConfig::RangeSlice> localmap;
Key startKey;
int batchSize = buggify() ? 1 : 1000000;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await taskBucket->keepRunning(tr, task);
dbConfig = co_await getDatabaseConfiguration(cx);
if (!bc) {
// Backup container must be present if we're still here
Reference<IBackupContainer> _bc = co_await config.backupContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
}
BackupConfig::RangeFileMapT::RangeResultType rangeresults =
co_await config.snapshotRangeFileMap().getRange(tr, startKey, {}, batchSize);
for (auto& p : rangeresults.results) {
localmap.insert(p);
}
if (!rangeresults.more) {
break;
}
startKey = keyAfter(rangeresults.results.back().first);
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
std::vector<std::string> files;
std::vector<std::pair<Key, Key>> beginEndKeys;
Version maxVer = 0;
Version minVer = std::numeric_limits<Version>::max();
int64_t totalBytes = 0;
if (!localmap.empty()) {
// Get iterator that points to greatest key, start there.
auto ri = localmap.rbegin();
auto i = (++ri).base();
while (1) {
const BackupConfig::RangeSlice& r = i->second;
// Add file to final file list
files.push_back(r.fileName);
// Add (beginKey, endKey) pairs to the list
beginEndKeys.emplace_back(i->second.begin, i->first);
// Update version range seen
if (r.version < minVer)
minVer = r.version;
if (r.version > maxVer)
maxVer = r.version;
// Update total bytes counted.
totalBytes += r.fileSize;
// Jump to file that either ends where this file begins or has the greatest end that is less than
// the begin of this file. In other words find the map key that is <= begin of this file. To do
// this find the first end strictly greater than begin and then back up one.
i = localmap.upper_bound(i->second.begin);
// If we get begin then we're done, there are no more ranges that end at or before the last file's
// begin
if (i == localmap.begin())
break;
--i;
}
}
// Log what range files were found for debugging mode=BOTH issues
TraceEvent("BackupSnapshotManifestRangeFileSummary")
.detail("BackupUID", config.getUid())
.detail("LocalMapSize", localmap.size())
.detail("FilesFound", files.size())
.detail("TotalBytes", totalBytes)
.detail("MinVersion", minVer == std::numeric_limits<Version>::max() ? -1 : minVer)
.detail("MaxVersion", maxVer);
Params.endVersion().set(task, maxVer);
Params.totalBytes().set(task, totalBytes);
// Avoid keyRange filtering optimization for 'manifest' files
co_await bc->writeKeyspaceSnapshotFile(files, beginEndKeys, totalBytes, IncludeKeyRangeMap::True);
TraceEvent(SevInfo, "FileBackupWroteSnapshotManifest")
.detail("BackupUID", config.getUid())
.detail("BeginVersion", minVer)
.detail("EndVersion", maxVer)
.detail("TotalBytes", totalBytes);
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
co_await checkTaskVersion(
tr->getDatabase(), task, BackupSnapshotManifest::name, BackupSnapshotManifest::version);
BackupConfig config(task);
// Set the latest snapshot end version, which was set during the execute phase
config.latestSnapshotEndVersion().set(tr, Params.endVersion().get(task));
bool stopWhenDone{ false };
EBackupState backupState;
Optional<Version> restorableVersion;
Optional<Version> firstSnapshotEndVersion;
Optional<std::string> tag;
co_await (store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
store(backupState, config.stateEnum().getOrThrow(tr)) &&
store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr)) &&
store(tag, config.tag().get(tr)));
// If restorable, update the last restorable version for this tag
if (restorableVersion.present() && tag.present()) {
FileBackupAgent().setLastRestorable(tr, StringRef(tag.get()), restorableVersion.get());
}
// Always set firstSnapshotEndVersion if not already set
// This is required for getLatestRestorableVersion() to work correctly
if (!firstSnapshotEndVersion.present()) {
config.firstSnapshotEndVersion().set(tr, Params.endVersion().get(task));
}
// If the backup is restorable and the state isn't differential the set state to differential
if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL)
config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL);
// Unless we are to stop, start the next snapshot using the default interval
Reference<TaskFuture> snapshotDoneFuture = task->getDoneFuture(futureBucket);
if (!stopWhenDone) {
co_await (config.initNewSnapshot(tr) &&
success(BackupSnapshotDispatchTask::addTask(
tr, taskBucket, task, 1, TaskCompletionKey::signal(snapshotDoneFuture))));
} else {
// Set the done future as the snapshot is now complete.
co_await snapshotDoneFuture->set(tr, taskBucket);
}
co_await taskBucket->finish(tr, task);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(BackupSnapshotManifest::name,
BackupSnapshotManifest::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
NOP_SETUP_TASK_FN,
1);
co_return key;
}
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef BackupSnapshotManifest::name = "file_backup_write_snapshot_manifest_5.2"_sr;
REGISTER_TASKFUNC(BackupSnapshotManifest);
Future<Key> BackupSnapshotDispatchTask::addSnapshotManifestTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor) {
return BackupSnapshotManifest::addTask(tr, taskBucket, parentTask, completionKey, waitFor);
}
// BulkDumpTaskFunc: Creates BulkDump snapshots during backup
// Must be defined before StartFullBackupTaskFunc which references it
struct BulkDumpTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> snapshotVersion() { return __FUNCTION__sr; }
static TaskParam<std::string> bulkDumpJobId() { return __FUNCTION__sr; }
static TaskParam<bool> timeoutOccurred() { return __FUNCTION__sr; }
} Params;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
BackupConfig config(task);
Version snapshotVersion = Params.snapshotVersion().get(task);
std::string jobId = Params.bulkDumpJobId().getOrDefault(task, "");
TraceEvent("BulkDumpTaskStart")
.detail("BackupUID", config.getUid())
.detail("SnapshotVersion", snapshotVersion)
.detail("BulkDumpJobId", jobId);
// Declare state variables before try block so they're accessible in catch block
std::vector<KeyRange> backupRanges;
Reference<IBackupContainer> bc;
int originalBulkDumpMode = 0;
BulkLoadTransportMethod transportMethod = BulkLoadTransportMethod::CP;
BulkDumpState bulkDumpJob;
bool jobAlreadyRunning = false;
{
Error savedError;
try {
// Submit BulkDump job via ManagementAPI
// This is a black box delegation to the existing BulkDump system
// Get backup ranges from config
std::vector<KeyRange> ranges = co_await config.backupRanges().getOrThrow(cx.getReference());
backupRanges = ranges;
Reference<IBackupContainer> container = co_await config.backupContainer().getOrThrow(cx.getReference());
bc = container;
// Read the original BulkDump mode from config (saved by StartFullBackupTaskFunc before task creation).
// This is persisted in the database so we can restore the correct mode even after a crash.
int mode = co_await config.originalBulkDumpMode().getD(cx.getReference(), Snapshot::False, 0);
originalBulkDumpMode = mode;
// Determine transport method from backup URL
transportMethod =
isBlobstoreUrl(bc->getURL()) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
// Check if there's already a running BulkDump job (e.g., from a previous task attempt or another
// agent). If so, we'll monitor that job instead of submitting a new one. This prevents the "Conflict to
// a running BulkDump job" error that occurs when multiple backup agents try to run the same task.
Transaction checkTr(cx);
while (true) {
Error err;
try {
checkTr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
checkTr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<BulkDumpState> existingJob = co_await getSubmittedBulkDumpJob(&checkTr);
if (existingJob.present()) {
bulkDumpJob = existingJob.get();
jobAlreadyRunning = true;
TraceEvent("BulkDumpTaskUsingExistingJob")
.detail("BackupUID", config.getUid())
.detail("ExistingJobId", bulkDumpJob.getJobId());
}
break;
} catch (Error& e) {
err = e;
}
co_await checkTr.onError(err);
}
if (!jobAlreadyRunning) {
// Enable BulkDump mode at the DD level before submitting the job
co_await setBulkDumpMode(cx, 1);
// Configure BulkDump job for the full keyspace
// BulkDump/BulkLoad requires the load range to be a subset of the dump range.
// Using normalKeys for both ensures compatibility regardless of user-specified ranges.
// Store data under data/<container>/bulkdump_data/ to be consistent with backup container layout
std::string bulkDumpRoot = getBackupDataPath(bc->getURL(), "bulkdump_data");
bulkDumpJob = createBulkDumpJob(normalKeys, bulkDumpRoot, BulkLoadType::SST, transportMethod);
// Submit the BulkDump job
co_await submitBulkDumpJob(cx, bulkDumpJob);
// Set ownership so bulkdump status shows it belongs to this backup
std::string backupTag = co_await config.tag().getOrThrow(cx.getReference());
BulkDumpOwnerInfo ownerInfo(config.getUid(), "backup", backupTag, now());
co_await setBulkDumpOwner(cx, bulkDumpJob.getJobId(), ownerInfo);
}
// Store job ID for monitoring
Params.bulkDumpJobId().set(task, bulkDumpJob.getJobId().toString());
// Monitor BulkDump progress - timeout is configurable for large datasets
bool completed = co_await monitorBulkDumpJobCompletion(cx,
bulkDumpJob.getJobId(),
CLIENT_KNOBS->BULKDUMP_JOB_TIMEOUT,
5.0); // Poll every 5 seconds
if (!completed) {
TraceEvent(SevWarn, "BulkDumpTaskTimeout")
.detail("BackupUID", config.getUid())
.detail("BulkDumpJobId", bulkDumpJob.getJobId())
.detail("TimeoutDuration", 300.0);
Params.timeoutOccurred().set(task, true);
}
TraceEvent("BulkDumpTaskComplete")
.detail("BackupUID", config.getUid())
.detail("BulkDumpJobId", bulkDumpJob.getJobId())
.detail("TimeoutOccurred", Params.timeoutOccurred().getOrDefault(task, false))
.detail("JobAlreadyRunning", jobAlreadyRunning);
// Restore original BulkDump mode after job completes, but only if:
// 1. We actually enabled the mode (not if we just monitored an existing job)
// 2. The original mode was different from what we set
if (!jobAlreadyRunning && originalBulkDumpMode != 1) {
co_await setBulkDumpMode(cx, originalBulkDumpMode);
}
// Write the keyspace snapshot file to mark the backup as complete
// This is essential for the restore process to find the snapshot
if (completed) {
// Verify that BulkDump data was actually written before writing snapshot
bool datasetComplete =
co_await verifyBulkDumpDatasetCompleteness(bc, bulkDumpJob.getJobId().toString());
if (!datasetComplete) {
TraceEvent(SevError, "BulkDumpTaskDatasetIncomplete")
.detail("BackupUID", config.getUid())
.detail("BulkDumpJobId", bulkDumpJob.getJobId());
throw backup_error();
}
// Build beginEndKeys from backup ranges
std::vector<std::pair<Key, Key>> beginEndKeys;
for (const auto& range : backupRanges) {
beginEndKeys.emplace_back(range.begin, range.end);
}
// Create BulkDump snapshot metadata
// Note: totalBytes and totalKeys are set to 0 here. Unlike traditional range file backups
// which accumulate fileSize from each RangeSlice, BulkDump writes its data to manifest files
// in the backup container (one per shard). To get accurate byte/key counts, we would need to:
// 1. List all manifest files under bc->getURL()/bulkDumpJobId/
// 2. Read each BulkLoadManifest and call getTotalBytes()/getKeyCount()
// 3. Aggregate the totals
// For now, zeros are acceptable since the critical field is bulkDumpJobId which BulkLoad
// uses to locate the data. TODO: Fix. The byte counts are informational for status display.
SnapshotMetadata metadata =
SnapshotMetadata::bulkDump(bulkDumpJob.getJobId().toString(), snapshotVersion, 0, 0);
// Write the snapshot file - empty file list since BulkDump uses job manifests, not range files
co_await bc->writeKeyspaceSnapshotFile({}, // No individual range files for BulkDump
beginEndKeys,
0, // See comment above about totalBytes
IncludeKeyRangeMap::False,
metadata);
TraceEvent("BulkDumpTaskWroteSnapshot")
.detail("BackupUID", config.getUid())
.detail("BulkDumpJobId", bulkDumpJob.getJobId())
.detail("SnapshotVersion", snapshotVersion)
.detail("RangeCount", backupRanges.size());
}
// Increment counter for test assertions
g_bulkDumpTaskCompleteCount.fetch_add(1);
co_return;
} catch (Error& e) {
savedError = e;
}
TraceEvent(SevWarn, "BulkDumpTaskError")
.error(savedError)
.detail("BackupUID", config.getUid())
.detail("SnapshotVersion", snapshotVersion)
.detail("JobAlreadyRunning", jobAlreadyRunning);
// Restore original BulkDump mode on error, but only if we were the ones who enabled it
try {
if (!jobAlreadyRunning && originalBulkDumpMode != 1) {
co_await setBulkDumpMode(cx, originalBulkDumpMode);
}
} catch (Error& e2) {
if (e2.code() == error_code_actor_cancelled) {
throw;
}
TraceEvent(SevWarn, "BulkDumpTaskRestoreModeError").error(e2);
}
throw savedError;
}
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
BackupConfig config(task);
Version snapshotVersion = Params.snapshotVersion().get(task);
std::string jobId = Params.bulkDumpJobId().getOrDefault(task, "");
TraceEvent("BulkDumpTaskFinishStart")
.detail("BackupUID", config.getUid())
.detail("SnapshotVersion", snapshotVersion)
.detail("BulkDumpJobId", jobId);
// Set latestSnapshotEndVersion so BackupLogsDispatchTask knows we're restorable
// This is critical for the backup to complete when using BulkDump
config.latestSnapshotEndVersion().set(tr, snapshotVersion);
// Set bulkDumpSnapshotEndVersion to track that BulkDump data is available
// This is used by getLatestRestorableVersion() for mode=BOTH to ensure both
// rangefile and bulkdump data exist before marking backup as restorable
config.bulkDumpSnapshotEndVersion().set(tr, snapshotVersion);
// CRITICAL: Set bulkDumpJobId on the backup CONFIG so that status checks
// and restore operations can find it. Without this, the backup status shows
// "BulkLoad Compatible: no" and bulkload restore fails with "Missing backup data".
if (!jobId.empty()) {
config.bulkDumpJobId().set(tr, jobId);
}
// Set firstSnapshotEndVersion if not already set, BUT only if we're NOT in
// mode=BOTH. In mode=BOTH, firstSnapshotEndVersion tracks the rangefile snapshot
// completion and must only be set by BackupSnapshotManifest::_finish().
// Setting it here would make getLatestRestorableVersion() think both snapshots
// are complete when only bulkdump is done.
Optional<int> mode = co_await config.snapshotMode().get(tr);
if (!mode.present() || mode.get() != 2) {
Optional<Version> firstSnapshotEnd = co_await config.firstSnapshotEndVersion().get(tr);
if (!firstSnapshotEnd.present()) {
config.firstSnapshotEndVersion().set(tr, snapshotVersion);
}
}
TraceEvent("BulkDumpTaskFinishSetVersions")
.detail("BackupUID", config.getUid())
.detail("SnapshotVersion", snapshotVersion)
.detail("Mode", mode.present() ? mode.get() : 0);
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
co_await taskFuture->set(tr, taskBucket);
co_await taskBucket->finish(tr, task);
TraceEvent("BulkDumpTaskFinish")
.detail("BackupUID", config.getUid())
.detail("SnapshotVersion", snapshotVersion);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
Version snapshotVersion,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(BulkDumpTaskFunc::name,
BulkDumpTaskFunc::version,
tr,
taskBucket,
completionKey,
BackupConfig(parentTask),
waitFor,
[=](Reference<Task> task) {
Params.snapshotVersion().set(task, snapshotVersion);
Params.timeoutOccurred().set(task, false);
});
co_return key;
}
};
StringRef BulkDumpTaskFunc::name = "bulk_dump_snapshot_5.2"_sr;
REGISTER_TASKFUNC(BulkDumpTaskFunc);
struct StartFullBackupTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
} Params;
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
co_await checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
BackupConfig config(task);
Future<Optional<MutationLogType>> mutationLogType;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
mutationLogType = config.mutationLogType().get(tr);
co_await mutationLogType;
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// Enable the appropriate backup worker type if not already enabled.
if (mutationLogType.get().present()) {
if (mutationLogType.get().get() == MutationLogType::PARTITIONED_LOG) {
co_await enableBackupWorker(cx);
} else if (mutationLogType.get().get() == MutationLogType::RANGE_PARTITIONED_LOG) {
co_await enableRangePartitionedBackupWorker(cx);
}
}
// Get start version after backup worker are enabled
while (true) {
Error err;
try {
tr->reset();
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<Version> startVersionFuture = tr->getReadVersion();
co_await startVersionFuture;
Params.beginVersion().set(task, startVersionFuture.get());
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// Set the "backupStartedKey" and wait for all backup worker started
tr->reset();
while (true) {
Future<Void> watchFuture;
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Future<Void> keepRunning = taskBucket->keepRunning(tr, task);
Future<Optional<Value>> started = tr->get(backupStartedKey);
Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
mutationLogType = config.mutationLogType().get(tr);
co_await (success(started) && success(taskStarted) && success(mutationLogType));
if (!mutationLogType.get().present() || mutationLogType.get().get() == MutationLogType::DEFAULT) {
co_return; // Skip if not using partitioned or range partitioned logs
}
std::vector<std::pair<UID, Version>> ids;
if (started.get().present()) {
ids = decodeBackupStartedValue(started.get().get());
}
// First range-partitioned backup on this cluster: ask DD to compute the partition list.
if (ids.empty() && mutationLogType.get().get() == MutationLogType::RANGE_PARTITIONED_LOG) {
tr->set(backupPartitionRequiredKey, backupPartitionRequiredValue(1));
}
const UID uid = config.getUid();
auto it = std::find_if(
ids.begin(), ids.end(), [uid](const std::pair<UID, Version>& p) { return p.first == uid; });
if (it == ids.end()) {
ids.emplace_back(uid, Params.beginVersion().get(task));
} else {
Params.beginVersion().set(task, it->second);
}
tr->set(backupStartedKey, encodeBackupStartedValue(ids));
// Only PartitionedLog workers set BackupConfig.allWorkerStarted, so watch it for that mutation log
// type.
const bool isPartitionedLog = mutationLogType.get().get() == MutationLogType::PARTITIONED_LOG;
// The task may be restarted. Set the watch if started key has NOT been set.
if (isPartitionedLog && !taskStarted.get().present()) {
watchFuture = tr->watch(config.allWorkerStarted().key);
}
co_await keepRunning;
co_await tr->commit();
if (isPartitionedLog && !taskStarted.get().present()) {
co_await watchFuture;
}
co_return;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
BackupConfig config(task);
Version beginVersion = Params.beginVersion().get(task);
Future<std::vector<KeyRange>> backupRangesFuture = config.backupRanges().getOrThrow(tr);
Future<Key> destUidValueFuture = config.destUidValue().getOrThrow(tr);
Future<Optional<MutationLogType>> mutationLogType = config.mutationLogType().get(tr);
Future<Optional<bool>> incrementalBackupOnly = config.incrementalBackupOnly().get(tr);
co_await (success(backupRangesFuture) && success(destUidValueFuture) && success(mutationLogType) &&
success(incrementalBackupOnly));
std::vector<KeyRange> backupRanges = backupRangesFuture.get();
Key destUidValue = destUidValueFuture.get();
// Start logging the mutations for the specified ranges of the tag if needed
if (!mutationLogType.get().present() || mutationLogType.get().get() == MutationLogType::DEFAULT) {
for (auto& backupRange : backupRanges) {
config.startMutationLogs(tr, backupRange, destUidValue);
}
}
// Properties folder is created here to ensure backup container exists because localDirectory doesn't create
// container during in constructor.
Reference<IBackupContainer> bc = co_await config.backupContainer().getOrThrow(tr);
co_await bc->writeEncryptionMetadata(bc->getEncryptionBlockSize());
config.stateEnum().set(tr, EBackupState::STATE_RUNNING);
Reference<TaskFuture> backupFinished = futureBucket->future(tr);
// Initialize the initial snapshot and create tasks to continually write logs and snapshots.
Optional<int64_t> initialSnapshotIntervalSeconds = co_await config.initialSnapshotIntervalSeconds().get(tr);
co_await config.initNewSnapshot(tr, initialSnapshotIntervalSeconds.orDefault(0));
// Using priority 1 for both of these to at least start both tasks soon
// Do not add snapshot task if we only want the incremental backup
if (!incrementalBackupOnly.get().present() || !incrementalBackupOnly.get().get()) {
// Check snapshot mode: 0=RANGEFILE, 1=BULKDUMP, 2=BOTH
int snapshotModeValue = co_await config.snapshotMode().getD(tr, Snapshot::False, 0);
TraceEvent("StartFullBackupTaskSnapshotMode")
.detail("BackupUID", config.getUid())
.detail("SnapshotModeValue", snapshotModeValue)
.detail("IncrementalBackupOnly",
incrementalBackupOnly.get().present() && incrementalBackupOnly.get().get());
// Add BulkDump task if mode is BULKDUMP(1) or BOTH(2)
if (snapshotModeValue == 1 || snapshotModeValue == 2) {
// Save original BulkDump mode BEFORE creating the task, so it's persisted in the database.
// This allows crash recovery to restore the correct mode even if the task restarts.
int currentBulkDumpMode = co_await getBulkDumpMode(tr->getDatabase());
config.originalBulkDumpMode().set(tr, currentBulkDumpMode);
co_await BulkDumpTaskFunc::addTask(
tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(backupFinished));
}
// Add traditional range file snapshot if mode is RANGEFILE(0) or BOTH(2)
if (snapshotModeValue == 0 || snapshotModeValue == 2) {
co_await BackupSnapshotDispatchTask::addTask(
tr, taskBucket, task, 1, TaskCompletionKey::joinWith(backupFinished));
}
}
co_await BackupLogsDispatchTask::addTask(
tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished));
// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then
// the following task will clean up and set the completed state.
co_await FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished);
co_await taskBucket->finish(tr, task);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
UID uid,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = co_await addBackupTask(StartFullBackupTaskFunc::name,
StartFullBackupTaskFunc::version,
tr,
taskBucket,
completionKey,
BackupConfig(uid),
waitFor);
co_return key;
}
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef StartFullBackupTaskFunc::name = "file_backup_start_5.2"_sr;
REGISTER_TASKFUNC(StartFullBackupTaskFunc);
// BulkLoadRestoreTaskFunc: Restores data using BulkLoad instead of traditional range file restore
// This task is used when useRangeFileRestore=false is specified
struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> restoreVersion() { return __FUNCTION__sr; }
static TaskParam<std::string> backupUrl() { return __FUNCTION__sr; }
static TaskParam<std::string> bulkDumpJobId() { return __FUNCTION__sr; }
} Params;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
Version restoreVersion = Params.restoreVersion().get(task);
std::string backupUrl = Params.backupUrl().get(task);
std::string bulkDumpJobId = Params.bulkDumpJobId().getOrDefault(task, "");
TraceEvent("BulkLoadRestoreTaskStart")
.detail("RestoreUID", restore.getUid())
.detail("RestoreVersion", restoreVersion)
.detail("BackupUrl", backupUrl)
.detail("BulkDumpJobId", bulkDumpJobId);
// Declare state variable before try block so it's accessible in catch block
int originalBulkLoadMode = 0;
{
Error savedError;
try {
// Open backup container for metadata access
Reference<IBackupContainer> bcRef = IBackupContainer::openContainer(backupUrl, {}, {}, 0);
// Get restore ranges using a transaction
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
std::vector<KeyRange> restoreRanges = co_await restore.getRestoreRangesOrDefault(tr);
// If bulkDumpJobId is empty, read it from the snapshot file metadata
if (bulkDumpJobId.empty()) {
// Use the backup container we already opened to find the snapshot file for our restore version
auto* bcfsPtr = dynamic_cast<BackupContainerFileSystem*>(bcRef.getPtr());
if (bcfsPtr != nullptr) {
Reference<BackupContainerFileSystem> bcfs =
Reference<BackupContainerFileSystem>::addRef(bcfsPtr);
std::vector<KeyspaceSnapshotFile> snapshots = co_await bcfs->listKeyspaceSnapshots();
int snapIdx = snapshots.size() - 1;
while (snapIdx >= 0 && bulkDumpJobId.empty()) {
// For BulkDump snapshots: beginVersion == endVersion == snapshotVersion
// Accept any snapshot that is at or before our restore version
if (snapshots[snapIdx].endVersion <= restoreVersion) {
// Read the snapshot file to get bulkDumpJobId
std::string snapFileName = snapshots[snapIdx].fileName;
Reference<IAsyncFile> snapFile = co_await bcfs->readFile(snapFileName);
int64_t snapSize = co_await snapFile->size();
Standalone<StringRef> snapBuf = makeString(snapSize);
int bytesRead = co_await snapFile->read(mutateString(snapBuf), snapSize, 0);
if (bytesRead == snapSize) {
json_spirit::mValue json;
if (json_spirit::read_string(snapBuf.toString(), json)) {
JSONDoc doc(json);
std::string jobId;
if (doc.tryGet("bulkDumpJobId", jobId) && !jobId.empty()) {
bulkDumpJobId = jobId;
TraceEvent("BulkLoadRestoreFoundJobId")
.detail("RestoreUID", restore.getUid())
.detail("BulkDumpJobId", bulkDumpJobId)
.detail("SnapshotFile", snapFileName);
}
}
}
}
snapIdx--;
}
}
}
// Create BulkLoad job from the BulkDump data
// BulkDump stores data under data/<container>/bulkdump_data/ subdirectory.
// DD appends the jobId via getBulkLoadJobRoot(jobRoot, jobId) when accessing files.
std::string jobRoot = getBackupDataPath(backupUrl, "bulkdump_data");
UID dumpJobUid;
if (!bulkDumpJobId.empty()) {
dumpJobUid = UID::fromString(bulkDumpJobId);
} else {
// No BulkDump job ID found - this is a permanent error for BulkLoad restore.
// Abort the restore immediately instead of retrying forever.
TraceEvent(SevError, "BulkLoadRestoreNoBulkDumpJobId")
.detail("RestoreUID", restore.getUid())
.detail("BackupUrl", backupUrl)
.detail("Action", "Aborting restore - backup is not BulkLoad compatible");
co_await restore.logError(
cx, restore_missing_data(), "BulkLoad restore failed: backup has no bulkdump data", nullptr);
// Abort the restore by setting state to ABORTED
Reference<ReadYourWritesTransaction> abortTr(new ReadYourWritesTransaction(cx));
abortTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
abortTr->setOption(FDBTransactionOptions::LOCK_AWARE);
restore.stateEnum().set(abortTr, ERestoreState::ABORTED);
co_await abortTr->commit();
throw restore_missing_data();
}
// Verify BulkDump dataset completeness before proceeding
bool datasetComplete = co_await verifyBulkDumpDatasetCompleteness(bcRef, bulkDumpJobId);
if (!datasetComplete) {
// Dataset is incomplete - abort the restore permanently
TraceEvent(SevError, "BulkLoadRestoreDatasetIncomplete")
.detail("RestoreUID", restore.getUid())
.detail("BulkDumpJobId", bulkDumpJobId)
.detail("BackupUrl", backupUrl)
.detail("Action", "Aborting restore - bulkdump dataset is incomplete");
co_await restore.logError(
cx, restore_missing_data(), "BulkLoad restore failed: bulkdump dataset incomplete", nullptr);
// Abort the restore by setting state to ABORTED
Reference<ReadYourWritesTransaction> abortTr(new ReadYourWritesTransaction(cx));
abortTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
abortTr->setOption(FDBTransactionOptions::LOCK_AWARE);
restore.stateEnum().set(abortTr, ERestoreState::ABORTED);
co_await abortTr->commit();
throw restore_missing_data();
}
TraceEvent("BulkLoadRestoreDatasetVerified")
.detail("RestoreUID", restore.getUid())
.detail("BulkDumpJobId", bulkDumpJobId);
// Determine transport method from backup URL
BulkLoadTransportMethod loadTransportMethod =
isBlobstoreUrl(backupUrl) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
// BulkLoad range must match the BulkDump range (normalKeys)
// The actual restore ranges will be applied via mutation log replay
BulkLoadJobState bulkLoadJob = createBulkLoadJob(dumpJobUid, normalKeys, jobRoot, loadTransportMethod);
TraceEvent("BulkLoadRestoreJobCreated")
.detail("RestoreUID", restore.getUid())
.detail("BulkLoadJobId", bulkLoadJob.getJobId())
.detail("JobRoot", jobRoot);
// Register the BulkLoad range lock owner (required for range locking)
co_await registerRangeLockOwner(cx, "BulkLoad", "BulkLoad restore operation");
TraceEvent("BulkLoadRestoreRegisteredLockOwner")
.detail("RestoreUID", restore.getUid())
.detail("Owner", "BulkLoad");
// TODO(BulkLoad): no precondition validation happens here today.
// submitBulkLoadJob() does not check that the cluster has
// shard_encode_location_metadata=1 and enable_read_lock_on_range=1, nor
// that the storage engine supports SST ingestion. If those preconditions
// are not met, setBulkLoadMode + submitBulkLoadJob both succeed but the
// Data Distributor never dispatches any tasks, leaving the restore in
// "State: running, Tasks: 0/0" indefinitely. Validation needs to live
// somewhere with real cluster-side knob visibility (DD or a commit proxy);
// it cannot be done from fdbclient because SERVER_KNOBS here are this
// process's local defaults, not the running cluster's actual config.
// Read the original BulkLoad mode from config (saved by StartFullRestoreTaskFunc before task creation).
// This is persisted in the database so we can restore the correct mode even after a crash.
int mode = co_await restore.originalBulkLoadMode().getD(cx.getReference(), Snapshot::False, 0);
originalBulkLoadMode = mode;
// Enable BulkLoad mode at DD level so the job will be processed
co_await setBulkLoadMode(cx, 1);
TraceEvent("BulkLoadRestoreEnabledMode")
.detail("RestoreUID", restore.getUid())
.detail("BulkLoadJobId", bulkLoadJob.getJobId());
// Submit BulkLoad job (must be lockAware since DB is locked during restore)
co_await submitBulkLoadJob(cx, bulkLoadJob, true /* lockAware */);
TraceEvent("BulkLoadRestoreJobSubmitted")
.detail("RestoreUID", restore.getUid())
.detail("BulkLoadJobId", bulkLoadJob.getJobId());
// Get total block count for progress tracking
int64_t totalBlocks = co_await restore.fileBlockCount().getD(cx.getReference(), Snapshot::False, 0);
// Monitor BulkLoad progress - timeout is configurable for large datasets
// Must be lockAware since DB is locked during restore
// Use the progress-tracking version to update restore counters
bool completed = co_await monitorBulkLoadJobCompletionWithProgress(cx,
bulkLoadJob.getJobId(),
restore.getUid(),
totalBlocks,
CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT,
5.0, // Poll every 5 seconds
true); // lockAware
if (!completed) {
TraceEvent(SevWarn, "BulkLoadRestoreTimeout")
.detail("RestoreUID", restore.getUid())
.detail("BulkLoadJobId", bulkLoadJob.getJobId())
.detail("TimeoutDuration", CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT);
// Restore original BulkLoad mode before throwing
if (originalBulkLoadMode != 1) {
co_await setBulkLoadMode(cx, originalBulkLoadMode);
}
throw timed_out();
}
// Restore original BulkLoad mode now that the job is complete
if (originalBulkLoadMode != 1) {
co_await setBulkLoadMode(cx, originalBulkLoadMode);
}
TraceEvent("BulkLoadRestoreTaskComplete")
.detail("RestoreUID", restore.getUid())
.detail("BulkLoadJobId", bulkLoadJob.getJobId())
.detail("RestoreVersion", restoreVersion);
// Increment counter for test assertions
g_bulkLoadRestoreTaskCompleteCount.fetch_add(1);
co_return;
} catch (Error& e) {
savedError = e;
}
TraceEvent(SevWarn, "BulkLoadRestoreTaskError")
.error(savedError)
.detail("RestoreUID", restore.getUid())
.detail("BackupUrl", backupUrl);
// Restore original BulkLoad mode on error
try {
if (originalBulkLoadMode != 1) {
co_await setBulkLoadMode(cx, originalBulkLoadMode);
}
} catch (Error& e2) {
if (e2.code() == error_code_actor_cancelled) {
throw;
}
TraceEvent(SevWarn, "BulkLoadRestoreRestoreModeError").error(e2);
}
throw savedError;
}
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
// Mark BulkLoad phase as complete
restore.bulkLoadComplete().set(tr, true);
// Update fileBlocksFinished and filesBlocksDispatched to match fileBlockCount
// so the restore progress tracker knows all range data has been loaded.
// BulkLoad doesn't use the traditional block dispatching mechanism.
int64_t totalBlocks = co_await restore.fileBlockCount().getD(tr, Snapshot::False, 0);
int64_t finishedBlocks = co_await restore.fileBlocksFinished().getD(tr, Snapshot::False, 0);
if (finishedBlocks < totalBlocks) {
// Set both dispatched and finished to match total to indicate all range data is done
restore.filesBlocksDispatched().set(tr, totalBlocks);
restore.fileBlocksFinished().set(tr, totalBlocks);
}
// Set firstConsistentVersion if not already set
// For BulkLoad restore, this is the snapshot version from the BulkDump
Version firstConsistentVer =
co_await restore.firstConsistentVersion().getD(tr, Snapshot::False, invalidVersion);
if (firstConsistentVer == invalidVersion) {
// Get the restore version as the first consistent version
Version restoreVer = co_await restore.restoreVersion().getD(tr, Snapshot::False, invalidVersion);
if (restoreVer != invalidVersion) {
restore.firstConsistentVersion().set(tr, restoreVer);
}
}
TraceEvent("BulkLoadRestoreTaskFinished")
.detail("RestoreUID", restore.getUid())
.detail("TotalBlocks", totalBlocks)
.detail("FinishedBlocks", finishedBlocks);
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
co_await taskFuture->set(tr, taskBucket);
co_await taskBucket->finish(tr, task);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
Version restoreVersion,
std::string backupUrl,
std::string bulkDumpJobId,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(BulkLoadRestoreTaskFunc::name, BulkLoadRestoreTaskFunc::version, doneKey));
// Set task parameters
Params.restoreVersion().set(task, restoreVersion);
Params.backupUrl().set(task, backupUrl);
Params.bulkDumpJobId().set(task, bulkDumpJobId);
// Get restore config from parent task and bind it to new task
co_await RestoreConfig(parentTask).toTask(tr, task);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
};
StringRef BulkLoadRestoreTaskFunc::name = "bulk_load_restore_5.2"_sr;
REGISTER_TASKFUNC(BulkLoadRestoreTaskFunc);
struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
co_await checkTaskVersion(tr->getDatabase(), task, name, version);
RestoreConfig restore(task);
restore.stateEnum().set(tr, ERestoreState::COMPLETED);
bool unlockDB = co_await restore.unlockDBAfterRestore().getD(tr, Snapshot::False, true);
tr->atomicOp(metadataVersionKey, metadataVersionRequiredValue, MutationRef::SetVersionstampedValue);
// Clear the file map now since it could be huge.
restore.fileSet().clear(tr);
// TODO: Validate that the range version map has exactly the restored ranges in it. This means that for
// any restore operation the ranges to restore must be within the backed up ranges, otherwise from the
// restore perspective it will appear that some key ranges were missing and so the backup set is incomplete
// and the restore has failed. This validation cannot be done currently because Restore only supports a
// single restore range but backups can have many ranges.
// Clear the applyMutations stuff, including any unapplied mutations from versions beyond the restored
// version.
restore.clearApplyMutationsKeys(tr);
co_await taskBucket->finish(tr, task);
if (unlockDB) {
co_await unlockDatabase(tr, restore.getUid());
}
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(RestoreCompleteTaskFunc::name, RestoreCompleteTaskFunc::version, doneKey));
// Get restore config from parent task and bind it to new task
co_await RestoreConfig(parentTask).toTask(tr, task);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreCompleteTaskFunc::name = "restore_complete"_sr;
REGISTER_TASKFUNC(RestoreCompleteTaskFunc);
struct RestoreFileTaskFuncBase : RestoreTaskFuncBase {
struct InputParams {
static TaskParam<RestoreFile> inputFile() { return __FUNCTION__sr; }
static TaskParam<int64_t> readOffset() { return __FUNCTION__sr; }
static TaskParam<int64_t> readLen() { return __FUNCTION__sr; }
} Params;
std::string toString(Reference<Task> task) const override {
return format("fileName '%s' readLen %lld readOffset %lld",
Params.inputFile().get(task).fileName.c_str(),
Params.readLen().get(task),
Params.readOffset().get(task));
}
};
struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
static struct : InputParams {
// The range of data that the (possibly empty) data represented, which is set if it intersects the target
// restore range
static TaskParam<KeyRange> originalFileRange() { return __FUNCTION__sr; }
static TaskParam<std::vector<KeyRange>> originalFileRanges() { return __FUNCTION__sr; }
static std::vector<KeyRange> getOriginalFileRanges(Reference<Task> task) {
if (originalFileRanges().exists(task)) {
return Params.originalFileRanges().get(task);
} else {
std::vector<KeyRange> range;
if (originalFileRange().exists(task))
range.push_back(Params.originalFileRange().get(task));
return range;
}
}
} Params;
std::string toString(Reference<Task> task) const override {
std::string returnStr = RestoreFileTaskFuncBase::toString(task);
for (auto& range : Params.getOriginalFileRanges(task))
returnStr += format(" originalFileRange '%s'", printable(range).c_str());
return returnStr;
}
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
RestoreFile rangeFile = Params.inputFile().get(task);
int64_t readOffset = Params.readOffset().get(task);
int64_t readLen = Params.readLen().get(task);
TraceEvent("FileRestoreRangeStart")
.suppressFor(60)
.detail("RestoreUID", restore.getUid())
.detail("FileName", rangeFile.fileName)
.detail("FileVersion", rangeFile.version)
.detail("FileSize", rangeFile.fileSize)
.detail("ReadOffset", readOffset)
.detail("ReadLen", readLen);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
Future<Reference<IBackupContainer>> bc;
Future<std::vector<KeyRange>> restoreRanges;
Future<Key> addPrefix;
Future<Key> removePrefix;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
restoreRanges = restore.getRestoreRangesOrDefault(tr);
addPrefix = restore.addPrefix().getD(tr);
removePrefix = restore.removePrefix().getD(tr);
co_await taskBucket->keepRunning(tr, task);
co_await (success(bc) && success(restoreRanges) && success(addPrefix) && success(removePrefix) &&
checkTaskVersion(tr->getDatabase(), task, name, version));
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
Reference<IAsyncFile> inFile = co_await bc.get()->readFile(rangeFile.fileName);
Standalone<VectorRef<KeyValueRef>> blockData;
try {
// data is each real KV, not encoded mutations
Standalone<VectorRef<KeyValueRef>> data = co_await decodeRangeFileBlock(inFile, readOffset, readLen, cx);
blockData = data;
} catch (Error& e) {
if (e.code() == error_code_encrypt_keys_fetch_failed || e.code() == error_code_encrypt_key_not_found) {
co_return;
}
throw;
}
Arena arena;
DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
// First and last key are the range for this file
KeyRange fileRange;
std::vector<KeyRange> originalFileRanges;
// If fileRange doesn't intersect restore range then we're done.
int index{ 0 };
for (index = 0; index < restoreRanges.get().size(); index++) {
auto& restoreRange = restoreRanges.get()[index];
fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
if (!fileRange.intersects(restoreRange))
continue;
// We know the file range intersects the restore range but there could still be keys outside the restore
// range. Find the subvector of kv pairs that intersect the restore range. Note that the first and last
// keys are just the range endpoints for this file
int rangeStart = 1;
int rangeEnd = blockData.size() - 1;
// Slide start forward, stop if something in range is found
while (rangeStart < rangeEnd && !restoreRange.contains(blockData[rangeStart].key))
++rangeStart;
// Side end backward, stop if something in range is found
while (rangeEnd > rangeStart && !restoreRange.contains(blockData[rangeEnd - 1].key))
--rangeEnd;
VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
// Shrink file range to be entirely within restoreRange and translate it to the new prefix
// First, use the untranslated file range to create the shrunk original file range which must be used in
// the kv range version map for applying mutations
KeyRange originalFileRange =
KeyRangeRef(std::max(fileRange.begin, restoreRange.begin), std::min(fileRange.end, restoreRange.end));
originalFileRanges.push_back(originalFileRange);
// Now shrink and translate fileRange
Key fileEnd = std::min(fileRange.end, restoreRange.end);
if (fileEnd == (removePrefix.get().empty() ? allKeys.end : strinc(removePrefix.get()))) {
fileEnd = addPrefix.get().empty() ? allKeys.end : strinc(addPrefix.get());
} else {
fileEnd = fileEnd.removePrefix(removePrefix.get()).withPrefix(addPrefix.get());
}
fileRange = KeyRangeRef(std::max(fileRange.begin, restoreRange.begin)
.removePrefix(removePrefix.get())
.withPrefix(addPrefix.get()),
fileEnd);
int start = 0;
int end = data.size();
int dataSizeLimit =
buggify() ? deterministicRandom()->randomInt(256 * 1024, 10e6) : CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
tr->reset();
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
int i = start;
int txBytes = 0;
int iend = start;
// find iend that results in the desired transaction size
for (; iend < end && txBytes < dataSizeLimit; ++iend) {
txBytes += data[iend].key.expectedSize();
txBytes += data[iend].value.expectedSize();
}
// Clear the range we are about to set.
// If start == 0 then use fileBegin for the start of the range, else data[start]
// If iend == end then use fileEnd for the end of the range, else data[iend]
// clear the raw key(without alog prefix)
KeyRange trRange = KeyRangeRef(
(start == 0) ? fileRange.begin
: data[start].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()),
(iend == end) ? fileRange.end
: data[iend].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()));
tr->clear(trRange);
for (; i < iend; ++i) {
tr->setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(data[i].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()),
data[i].value);
}
// Add to bytes written count
restore.bytesWritten().atomicOp(tr, txBytes, MutationRef::Type::AddValue);
Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
co_await taskBucket->keepRunning(tr, task);
co_await checkLock;
co_await tr->commit();
TraceEvent("FileRestoreCommittedRange")
.suppressFor(60)
.detail("RestoreUID", restore.getUid())
.detail("FileName", rangeFile.fileName)
.detail("FileVersion", rangeFile.version)
.detail("FileSize", rangeFile.fileSize)
.detail("ReadOffset", readOffset)
.detail("ReadLen", readLen)
.detail("CommitVersion", tr->getCommittedVersion())
.detail("BeginRange", trRange.begin)
.detail("EndRange", trRange.end)
.detail("StartIndex", start)
.detail("EndIndex", i)
.detail("DataSize", data.size())
.detail("Bytes", txBytes)
.detail("OriginalFileRange", originalFileRange);
// Commit succeeded, so advance starting point
start = i;
if (start == end)
break;
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_transaction_too_large)
dataSizeLimit /= 2;
else
co_await tr->onError(err);
}
}
if (!originalFileRanges.empty()) {
if (buggify() && restoreRanges.get().size() == 1) {
Params.originalFileRange().set(task, originalFileRanges[0]);
} else {
Params.originalFileRanges().set(task, originalFileRanges);
}
}
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
restore.fileBlocksFinished().atomicOp(tr, 1, MutationRef::Type::AddValue);
// Update the KV range map if originalFileRange is set
std::vector<Future<Void>> updateMap;
std::vector<KeyRange> ranges = Params.getOriginalFileRanges(task);
// if to restore((a, b), (e, f), (x, y)), then there are 3 ranges
for (auto& range : ranges) {
Value versionEncoded = BinaryWriter::toValue(Params.inputFile().get(task).version, Unversioned());
updateMap.push_back(krmSetRange(tr, restore.applyMutationsMapPrefix(), range, versionEncoded));
}
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) && waitForAll(updateMap));
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
RestoreFile rf,
int64_t offset,
int64_t len,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(RestoreRangeTaskFunc::name, RestoreRangeTaskFunc::version, doneKey));
// Create a restore config from the current task and bind it to the new task.
co_await RestoreConfig(parentTask).toTask(tr, task);
Params.inputFile().set(task, rf);
Params.readOffset().set(task, offset);
Params.readLen().set(task, len);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreRangeTaskFunc::name = "restore_range_data"_sr;
REGISTER_TASKFUNC(RestoreRangeTaskFunc);
// Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and
// returns (commitVersion, chunkNumber)
std::pair<Version, int32_t> decodeMutationLogKey(const StringRef& key) {
ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t));
uint8_t hash;
Version version;
int32_t part;
BinaryReader rd(key, Unversioned());
rd >> hash >> version >> part;
version = bigEndian64(version);
part = bigEndian32(part);
int32_t v = version / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
ASSERT(((uint8_t)hashlittle(&v, sizeof(v), 0)) == hash);
return std::make_pair(version, part);
}
// Decodes an encoded list of mutations in the format of:
// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k],
// where a mutation is encoded as:
// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][param1][param2]
// noted version needs to be included here(0x0FDB00A200090001)
std::vector<MutationRef> decodeMutationLogValue(const StringRef& value) {
StringRefReader reader(value, restore_corrupted_data());
Version protocolVersion = reader.consume<uint64_t>();
if (protocolVersion <= 0x0FDB00A200090001) {
throw incompatible_protocol_version();
}
uint32_t val_length = reader.consume<uint32_t>();
if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) {
TraceEvent(SevError, "FileRestoreLogValueError")
.detail("ValueLen", val_length)
.detail("ValueSize", value.size())
.detail("Value", printable(value));
}
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
type = reader.consume<uint32_t>();
p1len = reader.consume<uint32_t>();
p2len = reader.consume<uint32_t>();
const uint8_t* key = reader.consume(p1len);
const uint8_t* val = reader.consume(p2len);
mutations.emplace_back((MutationRef::Type)type, StringRef(key, p1len), StringRef(val, p2len));
}
return mutations;
}
void AccumulatedMutations::addChunk(int chunkNumber, const KeyValueRef& kv) {
// here it validates that partition(chunk) number has to be continuous
if (chunkNumber == lastChunkNumber + 1) {
lastChunkNumber = chunkNumber;
serializedMutations += kv.value.toString();
} else {
lastChunkNumber = -2;
serializedMutations.clear();
}
kvs.push_back(kv);
}
bool AccumulatedMutations::isComplete() const {
if (lastChunkNumber >= 0) {
StringRefReader reader(serializedMutations, restore_corrupted_data());
Version protocolVersion = reader.consume<uint64_t>();
if (protocolVersion <= 0x0FDB00A200090001) {
throw incompatible_protocol_version();
}
uint32_t vLen = reader.consume<uint32_t>();
return vLen == reader.remainder().size();
}
return false;
}
// Returns true if a complete chunk contains any MutationRefs which intersect with any
// range in ranges.
// It is undefined behavior to run this if isComplete() does not return true.
bool AccumulatedMutations::matchesAnyRange(const RangeMapFilters& filters) const {
// decode param2, so that each actual mutations are in mutations variable
std::vector<MutationRef> mutations = decodeMutationLogValue(serializedMutations);
for (auto& m : mutations) {
if (filters.match(m)) {
return true;
}
}
return false;
}
bool RangeMapFilters::match(const MutationRef& m) const {
if (isSingleKeyMutation((MutationRef::Type)m.type)) {
if (match(singleKeyRange(m.param1))) {
return true;
}
} else if (m.type == MutationRef::ClearRange) {
if (match(KeyRangeRef(m.param1, m.param2))) {
return true;
}
} else {
ASSERT(false);
}
return false;
}
bool RangeMapFilters::match(const KeyValueRef& kv) const {
return match(singleKeyRange(kv.key));
}
bool RangeMapFilters::match(const KeyRangeRef& range) const {
auto ranges = rangeMap.intersectingRanges(range);
for (const auto& r : ranges) {
if (r.cvalue() == 1) {
return true;
}
}
return false;
}
// Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
// and have data relevant to one of the KV ranges in ranges
std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const RangeMapFilters& filters) {
std::unordered_map<Version, AccumulatedMutations> mutationBlocksByVersion;
// group mutations by version
for (auto& kv : data) {
// each kv is a [param1, param2]
auto versionAndChunkNumber = decodeMutationLogKey(kv.key);
mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv);
}
std::vector<KeyValueRef> output;
// then add each version to the output, and now each K in output is also a KeyValueRef,
// but mutations of the same versions stay together
for (auto& vb : mutationBlocksByVersion) {
AccumulatedMutations& m = vb.second;
// If the mutations are incomplete or match one of the ranges, include in results.
if (!m.isComplete() || m.matchesAnyRange(filters)) {
output.insert(output.end(), m.kvs.begin(), m.kvs.end());
}
}
return output;
}
struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static struct : InputParams {
} Params;
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
RestoreFile logFile = Params.inputFile().get(task);
int64_t readOffset = Params.readOffset().get(task);
int64_t readLen = Params.readLen().get(task);
TraceEvent("FileRestoreLogStart")
.suppressFor(60)
.detail("RestoreUID", restore.getUid())
.detail("FileName", logFile.fileName)
.detail("FileBeginVersion", logFile.version)
.detail("FileEndVersion", logFile.endVersion)
.detail("FileSize", logFile.fileSize)
.detail("ReadOffset", readOffset)
.detail("ReadLen", readLen);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
Reference<IBackupContainer> bc;
std::vector<KeyRange> ranges;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
ranges = co_await restore.getRestoreRangesOrDefault(tr);
co_await checkTaskVersion(tr->getDatabase(), task, name, version);
co_await taskBucket->keepRunning(tr, task);
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
Key mutationLogPrefix = restore.mutationLogPrefix();
Reference<IAsyncFile> inFile = co_await bc->readFile(logFile.fileName);
Standalone<VectorRef<KeyValueRef>> dataOriginal =
co_await decodeMutationLogFileBlock(inFile, readOffset, readLen);
// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for
// this restore based on the restore range set.
RangeMapFilters filters(ranges);
std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, filters);
int start = 0;
int end = dataFiltered.size();
int dataSizeLimit =
buggify() ? deterministicRandom()->randomInt(256 * 1024, 10e6) : CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
tr->reset();
while (true) {
Error err;
try {
if (start == end)
co_return;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
int i = start;
int txBytes = 0;
for (; i < end && txBytes < dataSizeLimit; ++i) {
Key k = dataFiltered[i].key.withPrefix(mutationLogPrefix);
ValueRef v = dataFiltered[i].value; // each KV is a [param1 with added prefix -> param2]
tr->set(k, v);
txBytes += k.expectedSize();
txBytes += v.expectedSize();
}
Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
co_await taskBucket->keepRunning(tr, task);
co_await checkLock;
// Add to bytes written count
restore.bytesWritten().atomicOp(tr, txBytes, MutationRef::Type::AddValue);
co_await tr->commit();
TraceEvent("FileRestoreCommittedLog")
.suppressFor(60)
.detail("RestoreUID", restore.getUid())
.detail("FileName", logFile.fileName)
.detail("FileBeginVersion", logFile.version)
.detail("FileEndVersion", logFile.endVersion)
.detail("FileSize", logFile.fileSize)
.detail("ReadOffset", readOffset)
.detail("ReadLen", readLen)
.detail("CommitVersion", tr->getCommittedVersion())
.detail("StartIndex", start)
.detail("EndIndex", i)
.detail("RecordCountOriginal", dataOriginal.size())
.detail("RecordCountFiltered", dataFiltered.size())
.detail("Bytes", txBytes);
// Commit succeeded, so advance starting point
start = i;
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_transaction_too_large)
dataSizeLimit /= 2;
else
co_await tr->onError(err);
}
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig(task).fileBlocksFinished().atomicOp(tr, 1, MutationRef::Type::AddValue);
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
// TODO: Check to see if there is a leak in the FutureBucket since an invalid task (validation key fails)
// will never set its taskFuture.
co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
RestoreFile lf,
int64_t offset,
int64_t len,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(RestoreLogDataTaskFunc::name, RestoreLogDataTaskFunc::version, doneKey));
// Create a restore config from the current task and bind it to the new task.
// RestoreConfig(parentTask) creates prefix of : fileRestorePrefixRange.begin/uid->config/[uid]
co_await RestoreConfig(parentTask).toTask(tr, task);
Params.inputFile().set(task, lf);
Params.readOffset().set(task, offset);
Params.readLen().set(task, len);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreLogDataTaskFunc::name = "restore_log_data"_sr;
REGISTER_TASKFUNC(RestoreLogDataTaskFunc);
// type|kLen|vLen|Key|Value
// ref: decodeBackupLogValue()
Standalone<StringRef> transformMutationToOldFormat(MutationRef m) {
BinaryWriter bw(Unversioned());
uint32_t len1, len2, type;
type = m.type;
len1 = m.param1.size();
len2 = m.param2.size();
bw << type;
bw << len1;
bw << len2;
// do not use <<, it is overloaded for stringref to write its size first
bw.serializeBytes(m.param1);
bw.serializeBytes(m.param2);
return bw.toValue();
}
// this method takes a version and a list of list of mutations of this verison,
// each list is returned from a iterator sorted by sub
// it will first add all mutations in subsequence order
// then combine them in old-format (param1, parma2) and return
// this method assumes that iterator can return a list of mutations
/*
mutations are serialized in file as below format:
`<BlockHeader>`
`<Version_1><Subseq_1><Mutation1_len><Mutation1>`
`<Version_2><Subseq_2><Mutation2_len><Mutation2>`
`…`
`<Padding>
for now, assume each iterator returns a vector<pair<subsequence, mutation> >
noted that the mutation's arena has to be valid during the execution
according to BackupWorker::addMutation, version 64-bit, sub is 32-bit and mutation length is 32-bit So iterator
will combine all mutations in the same version and return a vector iterator should also return the subsequence
together with each mutation as here we will do another mergeSort for subsequence again to decide the order and here
we will decode the stringref
*/
Standalone<VectorRef<KeyValueRef>> generateOldFormatMutations(
Version commitVersion,
const std::vector<Standalone<VectorRef<VersionedMutation>>>& newFormatMutations) {
Standalone<VectorRef<KeyValueRef>> results;
std::vector<Standalone<VectorRef<KeyValueRef>>> oldFormatMutations;
// mergeSort subversion here
// just do a global sort for everyone
int32_t totalBytes = 0;
std::map<uint32_t, std::vector<Standalone<StringRef>>> mutationsBySub;
std::map<uint32_t, std::vector<Standalone<MutationRef>>> tmpMap;
for (auto& eachTagMutations : newFormatMutations) {
for (auto& vm : eachTagMutations) {
uint32_t sub = vm.subsequence;
Standalone<StringRef> mutationOldFormat = transformMutationToOldFormat(vm.mutation);
mutationsBySub[sub].push_back(mutationOldFormat);
tmpMap[sub].push_back(vm.mutation);
totalBytes += mutationOldFormat.size();
}
}
// the list of param2 needs to have the first 64 bites as 0x0FDB00A200090001
BinaryWriter param2Writer(IncludeVersion(ProtocolVersion::withBackupMutations()));
param2Writer << totalBytes;
for (auto& mutationsForSub : mutationsBySub) {
// concatenate them to param2Str
for (auto& m : mutationsForSub.second) {
// refer to transformMutationToOldFormat
param2Writer.serializeBytes(m);
}
}
Key param2Concat = param2Writer.toValue();
// deal with param1
int32_t hashBase = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
BinaryWriter wrParam1(Unversioned()); // hash/commitVersion/part
wrParam1 << (uint8_t)hashlittle(&hashBase, sizeof(hashBase), 0);
wrParam1 << bigEndian64(commitVersion);
uint32_t* partBuffer = nullptr;
// TODO: re-use the similar logic in CommitProxyServer::addBackupMutations
// generate a list of (param1, param2)
// param2 has format: length_of_the_mutation_group | encoded_mutation_1 | … | encoded_mutation_k
// each mutation has format type|kLen|vLen|Key|Value
for (int part = 0; part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE < param2Concat.size(); part++) {
KeyValueRef backupKV;
// Assign the second parameter as the part
backupKV.value = getBackupValue(param2Concat, part);
Key key = getBackupKey(wrParam1, &partBuffer, part); // holds the memory
backupKV.key = key;
results.push_back_deep(results.arena(), backupKV);
}
return results;
}
struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static struct {
static TaskParam<int64_t> maxTagID() { return __FUNCTION__sr; }
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
static TaskParam<int64_t> bytesWritten() { return __FUNCTION__sr; }
static TaskParam<std::vector<RestoreConfig::RestoreFile>> logs() { return __FUNCTION__sr; }
} Params;
// From all iterators, find the minimum version and return it together with if this version has data.
static Future<std::pair<Version, bool>> findNextVersion(std::vector<Reference<PartitionedLogIterator>> iterators) {
Version minVersion = std::numeric_limits<int64_t>::max();
bool atLeastOneIteratorHasNext = false;
for (int k = 0; k < iterators.size(); k++) {
if (!iterators[k]->hasNext()) {
continue;
}
atLeastOneIteratorHasNext = true;
Version v = co_await iterators[k]->peekNextVersion();
minVersion = std::min(minVersion, v);
}
co_return std::make_pair(minVersion, atLeastOneIteratorHasNext);
}
// Reads from all iterators and sends a list of mutations for the given version
// to the mutationStream for consumption. This actor reads as fast as possible,
// which is fine since the task is working on the version range of size
// RESTORE_PARTITIONED_BATCH_VERSION_SIZE, thus limiting the amount of memory used.
static Future<Void> readLogData(PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream,
std::vector<Reference<PartitionedLogIterator>> iterators,
Version begin,
Version end) {
// mergeSort all iterator until all are exhausted
// it stores all mutations for the next min version, in new format
try {
bool atLeastOneIteratorHasNext = true;
Version minVersion{ 0 };
Version lastMinVersion = invalidVersion;
while (atLeastOneIteratorHasNext) {
std::pair<Version, bool> minVersionAndHasNext = co_await findNextVersion(iterators);
minVersion = minVersionAndHasNext.first;
atLeastOneIteratorHasNext = minVersionAndHasNext.second;
ASSERT_LT(lastMinVersion, minVersion);
lastMinVersion = minVersion;
if (atLeastOneIteratorHasNext) {
std::vector<Standalone<VectorRef<VersionedMutation>>> mutationsSingleVersion =
co_await getMutationsForVersion(iterators, minVersion);
if (minVersion < begin) {
// skip generating mutations, because this is not within desired range
// this is already handled by the previous taskfunc
continue;
} else if (minVersion >= end) {
// all valid data has been consumed
break;
}
// transform from new format to old format(param1, param2) for this version.
// This transformation has to be done version by version.
Standalone<VectorRef<KeyValueRef>> oldFormatMutations =
generateOldFormatMutations(minVersion, mutationsSingleVersion);
mutationStream.send(oldFormatMutations);
}
}
mutationStream.sendError(end_of_stream());
} catch (Error& e) {
TraceEvent(SevWarn, "FileRestoreLogReadError").error(e);
mutationStream.sendError(e);
}
co_return;
}
// Reads from all iterators and returns a list of mutations for the given version.
static Future<std::vector<Standalone<VectorRef<VersionedMutation>>>> getMutationsForVersion(
std::vector<Reference<PartitionedLogIterator>> iterators,
Version minVersion) {
std::vector<Standalone<VectorRef<VersionedMutation>>> mutationsSingleVersion;
for (int k = 0; k < iterators.size(); k++) {
if (!iterators[k]->hasNext()) {
continue;
}
Version v = co_await iterators[k]->peekNextVersion();
if (v == minVersion) {
Standalone<VectorRef<VersionedMutation>> tmp = co_await iterators[k]->getNext();
mutationsSingleVersion.push_back(tmp);
}
}
co_return mutationsSingleVersion;
}
// Writes backup mutations to the database
static Future<Void> writeMutations(Database cx,
std::vector<Standalone<VectorRef<KeyValueRef>>> mutations,
Key mutationLogPrefix,
Reference<Task> task,
Reference<TaskBucket> taskBucket) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
Standalone<VectorRef<KeyValueRef>> oldFormatMutations;
int mutationIndex = 0;
int mutationCount = 0;
int txBytes = 0;
int txBytesLimit = CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
for (const auto& data : mutations) {
oldFormatMutations.append(oldFormatMutations.arena(), data.begin(), data.size());
}
int totalMutation = oldFormatMutations.size();
// multiple transactions are needed, so this has to be in a _execute method rather than
// a _finish method
// this transaction does blind writes, so they are idempotent operations even if multiple instances of
// the same tasks are running these transactions.
while (true) {
Error err;
try {
if (mutationIndex == totalMutation) {
break;
}
txBytes = 0;
mutationCount = 0;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
while (mutationIndex + mutationCount < totalMutation && txBytes < txBytesLimit) {
Key k = oldFormatMutations[mutationIndex + mutationCount].key.withPrefix(mutationLogPrefix);
ValueRef v = oldFormatMutations[mutationIndex + mutationCount]
.value; // each KV is a [param1 with added prefix -> param2]
tr->set(k, v);
txBytes += k.expectedSize();
txBytes += v.expectedSize();
++mutationCount;
}
co_await taskBucket->keepRunning(tr, task);
co_await tr->commit();
int64_t oldBytes = Params.bytesWritten().get(task);
Params.bytesWritten().set(task, oldBytes + txBytes);
DisabledTraceEvent("FileRestorePartitionedLogCommittData")
.detail("MutationIndex", mutationIndex)
.detail("MutationCount", mutationCount)
.detail("TotalMutation", totalMutation)
.detail("Bytes", txBytes);
mutationIndex += mutationCount; // update mutationIndex after the commit succeeds
continue;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_transaction_too_large) {
txBytesLimit /= 2;
} else {
co_await tr->onError(err);
}
}
co_return;
}
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
int64_t maxTagID = Params.maxTagID().get(task);
std::vector<RestoreConfig::RestoreFile> logs = Params.logs().get(task);
Version begin = Params.beginVersion().get(task);
Version end = Params.endVersion().get(task);
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
Reference<IBackupContainer> bc;
std::vector<KeyRange> ranges; // this is the actual KV, not version
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
ranges = co_await restore.getRestoreRangesOrDefault(tr);
co_await checkTaskVersion(tr->getDatabase(), task, name, version);
co_await taskBucket->keepRunning(tr, task);
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
std::vector<std::vector<RestoreConfig::RestoreFile>> filesByTag(maxTagID + 1);
std::vector<std::vector<Version>> fileEndVersionByTag(maxTagID + 1);
for (RestoreConfig::RestoreFile& f : logs) {
// find the tag, aggregate files by tags
if (f.tagId == -1) {
// inconsistent data
TraceEvent(SevError, "PartitionedLogFileNoTag")
.detail("FileName", f.fileName)
.detail("FileSize", f.fileSize)
.log();
} else {
filesByTag[f.tagId].push_back(f);
}
}
for (int i = 0; i < maxTagID + 1; i++) {
std::vector<RestoreConfig::RestoreFile>& files = filesByTag[i];
int cnt = files.size();
fileEndVersionByTag[i].resize(cnt);
for (int j = 0; j < cnt; j++) {
// [10, 20), [18, 30) -> endVersion is (18, 30) for 2 files so we have [10, 18) and [18, 30)
// greedy algorithm, because sorted by beginVersion and to minimize duplicate reading
if (j != cnt - 1 && files[j].endVersion < files[j + 1].version) {
TraceEvent(SevError, "NonContinuousLog").detail("Tag", i).detail("Index", j).log();
}
fileEndVersionByTag[i][j] = (j == cnt - 1 ? end : files[j + 1].version);
}
}
std::vector<Reference<PartitionedLogIterator>> iterators(maxTagID + 1);
// for each tag, create an iterator
for (int k = 0; k < filesByTag.size(); k++) {
iterators[k] =
makeReference<PartitionedLogIteratorTwoBuffers>(bc, k, filesByTag[k], fileEndVersionByTag[k]);
}
DisabledTraceEvent("FileRestorePartitionedLogDataExeStart")
.detail("BeginVersion", begin)
.detail("EndVersion", end)
.detail("Files", logs.size());
// Converted backup mutations
PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream;
Future<Void> reader = readLogData(mutationStream, iterators, begin, end);
std::vector<Standalone<VectorRef<KeyValueRef>>> mutations;
int64_t totalBytes = 0;
Standalone<VectorRef<KeyValueRef>> oneVersionData;
while (true) {
Error err;
try {
Standalone<VectorRef<KeyValueRef>> _data = co_await mutationStream.getFuture();
oneVersionData = _data;
// batching mutations from multiple versions together before writing to the database
int64_t bytes = oneVersionData.expectedSize();
if (totalBytes + bytes > CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE) {
co_await writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket);
mutations.clear();
totalBytes = 0;
}
mutations.push_back(oneVersionData);
totalBytes += bytes;
continue;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_end_of_stream) {
if (!mutations.empty()) {
co_await writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket);
}
break;
} else {
throw err;
}
}
DisabledTraceEvent("FileRestorePartitionedLogDataExeDone")
.detail("BeginVersion", begin)
.detail("EndVersion", end)
.detail("Files", logs.size());
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
int64_t logBytesWritten = Params.bytesWritten().get(task);
RestoreConfig(task).bytesWritten().atomicOp(tr, logBytesWritten, MutationRef::Type::AddValue);
int64_t blocks =
(logBytesWritten + CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE - 1) / CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
// When dispatching, we don't know how many blocks are there, so we have to do it here
RestoreConfig(task).filesBlocksDispatched().atomicOp(tr, blocks, MutationRef::Type::AddValue);
RestoreConfig(task).fileBlocksFinished().atomicOp(tr, blocks, MutationRef::Type::AddValue);
DisabledTraceEvent("FileRestorePartitionedLogCommittedData")
.detail("Blocks", blocks)
.detail("LogBytes", logBytesWritten);
Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
int64_t maxTagID,
std::vector<RestoreConfig::RestoreFile> logs,
Version begin,
Version end,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(
new Task(RestoreLogDataPartitionedTaskFunc::name, RestoreLogDataPartitionedTaskFunc::version, doneKey));
// Create a restore config from the current task and bind it to the new task.
// RestoreConfig(parentTask) createsa prefix of : fileRestorePrefixRange.begin/uid->config/[uid]
co_await RestoreConfig(parentTask).toTask(tr, task);
Params.maxTagID().set(task, maxTagID);
Params.beginVersion().set(task, begin);
Params.endVersion().set(task, end);
Params.logs().set(task, logs);
Params.bytesWritten().set(task, 0);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreLogDataPartitionedTaskFunc::name = "restore_log_data_partitioned"_sr;
REGISTER_TASKFUNC(RestoreLogDataPartitionedTaskFunc);
// each task can be partitioned to smaller ranges because commit proxy would
// only start to commit alog/ prefix mutations to original prefix when
// the final version is set, but do it in a single task for now for simplicity
struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static struct {
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
static TaskParam<Version> firstVersion() { return __FUNCTION__sr; }
static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
} Params;
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
Version beginVersion = Params.beginVersion().get(task);
Version firstVersion = Params.firstVersion().get(task);
Version endVersion = Params.endVersion().get(task);
Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
Version restoreVersion{ 0 };
int fileLimit = 1000;
co_await (store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
checkTaskVersion(tr->getDatabase(), task, name, version));
// if current is [40, 50] and restore version is 50, we need another [50, 51] task to process data at version 50
Version nextEndVersion =
std::min(restoreVersion + 1, endVersion + CLIENT_KNOBS->RESTORE_PARTITIONED_BATCH_VERSION_SIZE);
// update the apply mutations end version so the mutations from the previous batch can be applied.
// Only do this once beginVersion is > 0 (it will be 0 for the initial dispatch).
if (beginVersion > firstVersion) {
// if the last file is [80, 100] and the restoreVersion is 90, we should use 90 here
// this is an additional taskFunc after last file
restore.setApplyEndVersion(tr, std::min(beginVersion, restoreVersion + 1));
}
// The applyLag must be retrieved AFTER potentially updating the apply end version.
int64_t applyLag = co_await restore.getApplyVersionLag(tr);
// this is to guarantee commit proxy is catching up doing apply alog -> normal key
// with this backupFile -> alog process
// If starting a new batch and the apply lag is too large then re-queue and wait
if (applyLag > (buggify() ? 1 : CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 300)) {
// Wait a small amount of time and then re-add this same task.
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
co_await RestoreDispatchPartitionedTaskFunc::addTask(
tr, taskBucket, task, firstVersion, beginVersion, endVersion);
TraceEvent("FileRestorePartitionDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "too_far_behind");
co_await taskBucket->finish(tr, task);
co_return;
}
// Get a batch of files. We're targeting batchSize blocks(30k) being dispatched so query for batchSize(150)
// files (each of which is 0 or more blocks).
// lets say files have [10, 20), [20, 30) then if our range is [15, 25], we need to include both files,
// because [15, 20] is included in the first file, and [20, 25] is included in the second file
// say we have b and e
// as a result, the first file(inclusive): largest file whose begin <= b,
// the last file(exclusive): smallest file whose begin > e
// because of the encoding, version comes before the type of files(log or range),
// begin needs to start from the beginning --
// if we have a 3 files, (log,100,200), (range, 180), (log, 200, 300), then if we want to restore[190, 250]
// it would stop at (range, 180) and not looking at (log, 100, 200)
Optional<RestoreConfig::RestoreFile> beginLogInclude = Optional<RestoreConfig::RestoreFile>{};
// because of the encoding of RestoreFile, we use greaterThanOrEqual(end + 1) instead of greaterThan(end)
// because RestoreFile::pack has the version at the most significant position, and keyAfter(end) does not result
// in a end+1
Optional<RestoreConfig::RestoreFile> endLogExclude = co_await restore.logFileSet().seekGreaterOrEqual(
tr, RestoreConfig::RestoreFile({ endVersion + 1, "", false }));
RestoreConfig::FileSetT::RangeResultType logFiles =
co_await restore.logFileSet().getRange(tr, beginLogInclude, endLogExclude, fileLimit);
Optional<RestoreConfig::RestoreFile> beginRangeInclude = co_await restore.rangeFileSet().seekGreaterOrEqual(
tr, RestoreConfig::RestoreFile({ beginVersion, "", true }));
// greaterThanOrEqual(end + 1) instead of greaterThan(end)
// because RestoreFile::pack has the version at the most significant position, and keyAfter(end) does not result
// in a end+1
Optional<RestoreConfig::RestoreFile> endRangeExclude = co_await restore.rangeFileSet().seekGreaterOrEqual(
tr, RestoreConfig::RestoreFile({ endVersion + 1, "", true }));
RestoreConfig::FileSetT::RangeResultType rangeFiles =
co_await restore.rangeFileSet().getRange(tr, beginRangeInclude, endRangeExclude, fileLimit);
int64_t maxTagID = 0;
std::vector<RestoreConfig::RestoreFile> logs;
std::vector<RestoreConfig::RestoreFile> ranges;
for (const auto& f : logFiles.results) {
if (f.endVersion > beginVersion) {
// skip all files whose endVersion is smaller or equal to beginVersion
logs.push_back(f);
maxTagID = std::max(maxTagID, f.tagId);
}
}
for (const auto& f : rangeFiles.results) {
// the getRange might get out-of-bound range file because log files need them to work
if (f.version >= beginVersion && f.version < endVersion) {
ranges.push_back(f);
}
}
// allPartsDone will be set once all block tasks in the current batch are finished.
// create a new future for the new batch
Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
restore.batchFuture().set(tr, allPartsDone->pack());
// if there are no files, if i am not the last batch, then on to the next batch
// if there are no files and i am the last batch, then just wait for applying to finish
// do we need this files.results.size() == 0 at all?
if (beginVersion > restoreVersion) {
if (applyLag == 0) {
// i am the last batch
// If apply lag is 0 then we are done so create the completion task
co_await RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal());
TraceEvent("FileRestorePartitionDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "restore_complete");
} else {
// i am the last batch, and applyLag is not zero, then I will create another dummy task to wait
// for apply log to be zero, then it will go into the branch above.
// Applying of mutations is not yet finished so wait a small amount of time and then re-add this
// same task.
// this is only to create a dummy one wait for it to finish
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
co_await RestoreDispatchPartitionedTaskFunc::addTask(
tr, taskBucket, task, firstVersion, beginVersion, endVersion);
TraceEvent("FileRestorePartitionDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "apply_still_behind");
}
co_await taskBucket->finish(tr, task);
co_return;
}
// if we reach here, this batch is not empty(i.e. we have range and/or mutation files in this)
// Start moving through the file list and queuing up blocks. Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE
// blocks per Dispatch task and target batchSize total per batch but a batch must end on a complete version
// boundary so exceed the limit if necessary to reach the end of a version of files.
std::vector<Future<Key>> addTaskFutures;
int i = 0;
// need to process all range files, keep using the same RestoreRangeTaskFunc as non-partitioned restore.
// this can be done first, because they are not overlap within a restore uid
// each task will read the file, restore those key to their original keys after clear that range
// also it will update the keyVersionMap[key -> versionFromRangeFile]
// by this time, corresponding mutation files within the same version range has not been applied yet
// because they are waiting for the singal of this RestoreDispatchPartitionedTaskFunc
// when log are being applied, they will compare version of key to the keyVersionMap updated by range file
// after each RestoreDispatchPartitionedTaskFunc, keyVersionMap will be clear if mutation version is larger.
for (; i < ranges.size(); ++i) {
RestoreConfig::RestoreFile& f = ranges[i];
// For each block of the file
for (int64_t j = 0; j < f.fileSize; j += f.blockSize) {
addTaskFutures.push_back(RestoreRangeTaskFunc::addTask(tr,
taskBucket,
task,
f,
j,
std::min<int64_t>(f.blockSize, f.fileSize - j),
TaskCompletionKey::joinWith(allPartsDone)));
}
}
addTaskFutures.push_back(RestoreLogDataPartitionedTaskFunc::addTask(
tr, taskBucket, task, maxTagID, logs, beginVersion, endVersion, TaskCompletionKey::joinWith(allPartsDone)));
// even if file exsists, but they are empty, in this case just start the next batch
addTaskFutures.push_back(RestoreDispatchPartitionedTaskFunc::addTask(tr,
taskBucket,
task,
firstVersion,
endVersion,
nextEndVersion,
TaskCompletionKey::noSignal(),
allPartsDone));
co_await waitForAll(addTaskFutures);
co_await taskBucket->finish(tr, task);
TraceEvent("FileRestorePartitionDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("EndVersion", endVersion)
.detail("NextEndVersion", nextEndVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "dispatch_batch_complete");
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
Version firstVersion,
Version beginVersion,
Version endVersion,
TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
// Use high priority for dispatch tasks that have to queue more blocks for the current batch
unsigned int priority = 0;
Reference<Task> task(new Task(
RestoreDispatchPartitionedTaskFunc::name, RestoreDispatchPartitionedTaskFunc::version, doneKey, priority));
// Create a config from the parent task and bind it to the new task
co_await RestoreConfig(parentTask).toTask(tr, task);
Params.firstVersion().set(task, firstVersion);
Params.beginVersion().set(task, beginVersion);
Params.endVersion().set(task, endVersion);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreDispatchPartitionedTaskFunc::name = "restore_dispatch_partitioned"_sr;
REGISTER_TASKFUNC(RestoreDispatchPartitionedTaskFunc);
struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const override { return name; };
static struct {
static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
static TaskParam<std::string> beginFile() { return __FUNCTION__sr; }
static TaskParam<int64_t> beginBlock() { return __FUNCTION__sr; }
static TaskParam<int64_t> batchSize() { return __FUNCTION__sr; }
static TaskParam<int64_t> remainingInBatch() { return __FUNCTION__sr; }
} Params;
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
Version beginVersion = Params.beginVersion().get(task);
Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
int64_t remainingInBatch = Params.remainingInBatch().get(task);
bool addingToExistingBatch = remainingInBatch > 0;
Version restoreVersion{ 0 };
co_await (store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
checkTaskVersion(tr->getDatabase(), task, name, version));
// If not adding to an existing batch then update the apply mutations end version so the mutations from the
// previous batch can be applied. Only do this once beginVersion is > 0 (it will be 0 for the initial
// dispatch).
if (!addingToExistingBatch && beginVersion > 0) {
// unblock apply alog to normal key space
// if the last file is [80, 100] and the restoreVersion is 90, we should use 90 here
// this call an additional call after last file
restore.setApplyEndVersion(tr, std::min(beginVersion, restoreVersion + 1));
}
// The applyLag must be retrieved AFTER potentially updating the apply end version.
int64_t applyLag = co_await restore.getApplyVersionLag(tr);
int64_t batchSize = Params.batchSize().get(task);
// If starting a new batch and the apply lag is too large then re-queue and wait
if (!addingToExistingBatch && applyLag > (buggify() ? 1 : CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 300)) {
// Wait a small amount of time and then re-add this same task.
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
co_await RestoreDispatchTaskFunc::addTask(
tr, taskBucket, task, beginVersion, "", 0, batchSize, remainingInBatch);
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("ApplyLag", applyLag)
.detail("BatchSize", batchSize)
.detail("Decision", "too_far_behind");
co_await taskBucket->finish(tr, task);
co_return;
}
// need beginFile to handle stop in the middle of version case
std::string beginFile = Params.beginFile().getOrDefault(task);
// Get a batch of files. We're targeting batchSize blocks being dispatched so query for batchSize files
// (each of which is 0 or more blocks).
int taskBatchSize = buggify() ? 1 : CLIENT_KNOBS->RESTORE_DISPATCH_ADDTASK_SIZE;
RestoreConfig::FileSetT::RangeResultType files = co_await restore.fileSet().getRange(
tr, Optional<RestoreConfig::RestoreFile>({ beginVersion, beginFile }), {}, taskBatchSize);
// allPartsDone will be set once all block tasks in the current batch are finished.
Reference<TaskFuture> allPartsDone;
// If adding to existing batch then join the new block tasks to the existing batch future
if (addingToExistingBatch) {
Key fKey = co_await restore.batchFuture().getD(tr);
allPartsDone = makeReference<TaskFuture>(futureBucket, fKey);
} else {
// Otherwise create a new future for the new batch
allPartsDone = futureBucket->future(tr);
restore.batchFuture().set(tr, allPartsDone->pack());
// Set batch quota remaining to batch size
remainingInBatch = batchSize;
}
// If there were no files to load then this batch is done and restore is almost done.
if (files.results.empty()) {
// If adding to existing batch then blocks could be in progress so create a new Dispatch task that waits
// for them to finish
if (addingToExistingBatch) {
// Setting next begin to restoreVersion + 1 so that any files in the file map at the restore version
// won't be dispatched again.
co_await RestoreDispatchTaskFunc::addTask(tr,
taskBucket,
task,
restoreVersion + 1,
"",
0,
batchSize,
0,
TaskCompletionKey::noSignal(),
allPartsDone);
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("BeginFile", Params.beginFile().get(task))
.detail("BeginBlock", Params.beginBlock().get(task))
.detail("RestoreVersion", restoreVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "end_of_final_batch");
} else if (beginVersion < restoreVersion) {
// If beginVersion is less than restoreVersion then do one more dispatch task to get there
// there are no more files between beginVersion and restoreVersion
co_await RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion, "", 0, batchSize);
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("BeginFile", Params.beginFile().get(task))
.detail("BeginBlock", Params.beginBlock().get(task))
.detail("RestoreVersion", restoreVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "apply_to_restore_version");
} else if (applyLag == 0) {
// If apply lag is 0 then we are done so create the completion task
co_await RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal());
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("BeginFile", Params.beginFile().get(task))
.detail("BeginBlock", Params.beginBlock().get(task))
.detail("ApplyLag", applyLag)
.detail("Decision", "restore_complete");
} else {
// Applying of mutations is not yet finished so wait a small amount of time and then re-add this
// same task.
// this is only to create a dummy one wait for it to finish
co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
co_await RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize);
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("ApplyLag", applyLag)
.detail("Decision", "apply_still_behind");
}
// If adding to existing batch then task is joined with a batch future so set done future
// Note that this must be done after joining at least one task with the batch future in case all other
// blockers already finished.
Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
co_await (taskBucket->finish(tr, task) && setDone);
co_return;
}
// Start moving through the file list and queuing up blocks. Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE
// blocks per Dispatch task and target batchSize total per batch but a batch must end on a complete version
// boundary so exceed the limit if necessary to reach the end of a version of files.
std::vector<Future<Key>> addTaskFutures;
Version endVersion = files.results[0].version;
int blocksDispatched = 0;
int64_t beginBlock = Params.beginBlock().getOrDefault(task);
int i = 0;
// for each file
// not creating a new task at this level because restore files are read back together -- both range and log
// so i have to process range files anyway.
for (; i < files.results.size(); ++i) {
RestoreConfig::RestoreFile& f = files.results[i];
// Here we are "between versions" (prior to adding the first block of the first file of a new version)
// so this is an opportunity to end the current dispatch batch (which must end on a version boundary) if
// the batch size has been reached or exceeded
if (f.version != endVersion && remainingInBatch <= 0) {
// Next start will be at the first version after endVersion at the first file first block
++endVersion;
// beginFile set to empty to indicate we are not in the middle of a range
// by middle of a range, we mean that we have rangeFile v=80, and logFile v=[80, 100],
// then we have to include this log file too in this batch
beginFile = "";
beginBlock = 0;
break;
}
// Set the starting point for the next task in case we stop inside this file
endVersion = f.version;
beginFile = f.fileName;
int64_t j = beginBlock * f.blockSize;
// For each block of the file
for (; j < f.fileSize; j += f.blockSize) {
// Stop if we've reached the addtask limit
if (blocksDispatched == taskBatchSize)
break;
if (f.isRange) {
addTaskFutures.push_back(
RestoreRangeTaskFunc::addTask(tr,
taskBucket,
task,
f,
j,
std::min<int64_t>(f.blockSize, f.fileSize - j),
TaskCompletionKey::joinWith(allPartsDone)));
} else {
addTaskFutures.push_back(
RestoreLogDataTaskFunc::addTask(tr,
taskBucket,
task,
f,
j,
std::min<int64_t>(f.blockSize, f.fileSize - j),
TaskCompletionKey::joinWith(allPartsDone)));
}
// Increment beginBlock for the file and total blocks dispatched for this task
++beginBlock;
++blocksDispatched;
--remainingInBatch;
}
// Stop if we've reached the addtask limit
if (blocksDispatched == taskBatchSize)
break;
// We just completed an entire file so the next task should start at the file after this one within
// endVersion (or later) if this iteration ends up being the last for this task
beginFile = beginFile + '\x00';
beginBlock = 0;
TraceEvent("FileRestoreDispatchedFile")
.suppressFor(60)
.detail("RestoreUID", restore.getUid())
.detail("FileName", f.fileName);
}
// If no blocks were dispatched then the next dispatch task should run now and be joined with the
// allPartsDone future
if (blocksDispatched == 0) {
std::string decision;
// If no files were dispatched either then the batch size wasn't large enough to catch all of the files
// at the next lowest non-dispatched version, so increase the batch size.
if (i == 0) {
batchSize *= 2;
decision = "increased_batch_size";
} else {
decision = "all_files_were_empty";
}
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("BeginFile", Params.beginFile().get(task))
.detail("BeginBlock", Params.beginBlock().get(task))
.detail("EndVersion", endVersion)
.detail("ApplyLag", applyLag)
.detail("BatchSize", batchSize)
.detail("Decision", decision)
.detail("RemainingInBatch", remainingInBatch);
co_await RestoreDispatchTaskFunc::addTask(tr,
taskBucket,
task,
endVersion,
beginFile,
beginBlock,
batchSize,
remainingInBatch,
TaskCompletionKey::joinWith((allPartsDone)));
// If adding to existing batch then task is joined with a batch future so set done future.
// Note that this must be done after joining at least one task with the batch future in case all other
// blockers already finished.
Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
co_await (setDone && taskBucket->finish(tr, task));
co_return;
}
// Increment the number of blocks dispatched in the restore config
restore.filesBlocksDispatched().atomicOp(tr, blocksDispatched, MutationRef::Type::AddValue);
// If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we
// cannot end the batch here because we do not know if we got all of the files and blocks from the last
// version queued, so make sure remainingInBatch is at least 1.
if (!beginFile.empty()) {
// this is to make sure if we stop in the middle of a version, we do not end this batch
// instead next RestoreDispatchTaskFunc should have addingToExistingBatch as true
// thus they are considered the same batch and alog will be committed only when all of them succeed
remainingInBatch = std::max<int64_t>(1, remainingInBatch);
}
// If more blocks need to be dispatched in this batch then add a follow-on task that is part of the
// allPartsDone group which will won't wait to run and will add more block tasks.
if (remainingInBatch > 0) {
addTaskFutures.push_back(RestoreDispatchTaskFunc::addTask(tr,
taskBucket,
task,
endVersion,
beginFile,
beginBlock,
batchSize,
remainingInBatch,
TaskCompletionKey::joinWith(allPartsDone)));
} else { // Otherwise, add a follow-on task to continue after all previously dispatched blocks are done
addTaskFutures.push_back(RestoreDispatchTaskFunc::addTask(tr,
taskBucket,
task,
endVersion,
beginFile,
beginBlock,
batchSize,
0,
TaskCompletionKey::noSignal(),
allPartsDone));
}
co_await waitForAll(addTaskFutures);
// If adding to existing batch then task is joined with a batch future so set done future.
Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
co_await (setDone && taskBucket->finish(tr, task));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
.detail("BeginVersion", beginVersion)
.detail("BeginFile", Params.beginFile().get(task))
.detail("BeginBlock", Params.beginBlock().get(task))
.detail("EndVersion", endVersion)
.detail("ApplyLag", applyLag)
.detail("BatchSize", batchSize)
.detail("Decision", "dispatched_files")
.detail("FilesDispatched", i)
.detail("BlocksDispatched", blocksDispatched)
.detail("RemainingInBatch", remainingInBatch);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> parentTask,
Version beginVersion,
std::string beginFile,
int64_t beginBlock,
int64_t batchSize,
int64_t remainingInBatch = 0,
TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = co_await completionKey.get(tr, taskBucket);
// Use high priority for dispatch tasks that have to queue more blocks for the current batch
auto priority = (remainingInBatch > 0) ? 1u : 0u;
Reference<Task> task(
new Task(RestoreDispatchTaskFunc::name, RestoreDispatchTaskFunc::version, doneKey, priority));
// Create a config from the parent task and bind it to the new task
co_await RestoreConfig(parentTask).toTask(tr, task);
Params.beginVersion().set(task, beginVersion);
Params.batchSize().set(task, batchSize);
Params.remainingInBatch().set(task, remainingInBatch);
Params.beginBlock().set(task, beginBlock);
Params.beginFile().set(task, beginFile);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef RestoreDispatchTaskFunc::name = "restore_dispatch"_sr;
REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
std::vector<KeyBackedTag> tags;
if (tagName.empty()) {
std::vector<KeyBackedTag> t = co_await getAllRestoreTags(tr);
tags = t;
} else {
tags.push_back(makeRestoreTag(tagName.toString()));
}
// If no tags found, return helpful message
if (tags.empty()) {
co_return "No restores found.\n\n"
"To start a restore:\n"
" fdbrestore start -r <BACKUP_URL> [-t <TAG>]\n";
}
std::string result;
for (int i = 0; i < tags.size(); ++i) {
UidAndAbortedFlagT u = co_await tags[i].getD(tr);
std::string s = co_await RestoreConfig(u.first).getFullStatus(tr);
result.append(s);
result.append("\n\n");
}
co_return result;
}
Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
KeyBackedTag tag = makeRestoreTag(tagName.toString());
Optional<UidAndAbortedFlagT> current = co_await tag.get(tr);
if (!current.present())
co_return ERestoreState::UNINITIALIZED;
RestoreConfig restore(current.get().first);
ERestoreState status = co_await restore.stateEnum().getD(tr);
bool runnable = co_await restore.isRunnable(tr);
if (!runnable)
co_return status;
restore.stateEnum().set(tr, ERestoreState::ABORTED);
// Clear all of the ApplyMutations stuff
restore.clearApplyMutationsKeys(tr);
// Cancel the backup tasks on this tag
co_await tag.cancel(tr);
co_await unlockDatabase(tr, current.get().first);
co_return ERestoreState::ABORTED;
}
Future<ERestoreState> abortRestore(Database cx, Key tagName) {
auto tr = makeReference<ReadYourWritesTransaction>(cx);
while (true) {
Error err;
try {
ERestoreState estate = co_await abortRestore(tr, tagName);
if (estate != ERestoreState::ABORTED) {
co_return estate;
}
co_await tr->commit();
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
tr = makeReference<ReadYourWritesTransaction>(cx);
// Commit a dummy transaction before returning success, to ensure the mutation applier has stopped submitting
// mutations
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
tr->addReadConflictRange(singleKeyRange(KeyRef()));
tr->addWriteConflictRange(singleKeyRange(KeyRef()));
co_await tr->commit();
co_return ERestoreState::ABORTED;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
static struct {
static TaskParam<Version> firstVersion() { return __FUNCTION__sr; }
} Params;
// Find all files needed for the restore and save them in the RestoreConfig for the task.
// Update the total number of files and blocks and change state to starting.
static Future<Void> _execute(Database cx,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
RestoreConfig restore(task);
Version restoreVersion{ 0 };
Version beginVersion{ 0 };
Reference<IBackupContainer> bc;
std::vector<KeyRange> ranges;
bool logsOnly{ false };
bool inconsistentSnapshotOnly{ false };
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await checkTaskVersion(tr->getDatabase(), task, name, version);
beginVersion = co_await restore.beginVersion().getD(tr, Snapshot::False, ::invalidVersion);
restoreVersion = co_await restore.restoreVersion().getOrThrow(tr);
ranges = co_await restore.getRestoreRangesOrDefault(tr);
logsOnly = co_await restore.onlyApplyMutationLogs().getD(tr, Snapshot::False, false);
inconsistentSnapshotOnly = co_await restore.inconsistentSnapshotOnly().getD(tr, Snapshot::False, false);
co_await taskBucket->keepRunning(tr, task);
ERestoreState oldState = co_await restore.stateEnum().getD(tr);
if (oldState != ERestoreState::QUEUED && oldState != ERestoreState::STARTING) {
co_await restore.logError(cx,
restore_error(),
format("StartFullRestore: Encountered unexpected state(%d)", oldState),
nullptr);
co_return;
}
restore.stateEnum().set(tr, ERestoreState::STARTING);
restore.fileSet().clear(tr);
restore.fileBlockCount().clear(tr);
restore.fileCount().clear(tr);
Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
bc = getBackupContainerWithProxy(_bc);
co_await tr->commit();
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
tr->reset();
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Version destVersion = co_await tr->getReadVersion();
TraceEvent("FileRestoreVersionUpgrade")
.detail("RestoreVersion", restoreVersion)
.detail("Dest", destVersion);
if (destVersion <= restoreVersion) {
CODE_PROBE(true, "Forcing restored cluster to higher version");
tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(restoreVersion + 1, Unversioned()));
co_await tr->commit();
tr->reset();
continue;
} else {
break;
}
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
Version firstConsistentVersion = invalidVersion;
if (beginVersion == invalidVersion) {
beginVersion = 0;
}
Standalone<VectorRef<KeyRangeRef>> keyRangesFilter;
for (auto const& r : ranges) {
keyRangesFilter.push_back_deep(keyRangesFilter.arena(), KeyRangeRef(r));
}
Optional<RestorableFileSet> restorable =
co_await bc->getRestoreSet(restoreVersion, keyRangesFilter, logsOnly, beginVersion);
if (!restorable.present())
throw restore_missing_data();
// Convert the two lists in restorable (logs and ranges) to a single list of RestoreFiles.
// Order does not matter, they will be put in order when written to the restoreFileMap below.
std::vector<RestoreConfig::RestoreFile> files;
std::vector<RestoreConfig::RestoreFile> logFiles;
std::vector<RestoreConfig::RestoreFile> rangeFiles;
if (!logsOnly) {
beginVersion = restorable.get().snapshot.beginVersion;
if (!inconsistentSnapshotOnly) {
for (const RangeFile& f : restorable.get().ranges) {
files.push_back({ f.version, f.fileName, true, f.blockSize, f.fileSize });
rangeFiles.push_back({ f.version, f.fileName, true, f.blockSize, f.fileSize });
// In a restore with both snapshots and logs, the firstConsistentVersion is the highest version
// of any range file.
firstConsistentVersion = std::max(firstConsistentVersion, f.version);
}
} else {
for (int i = 0; i < restorable.get().ranges.size(); ++i) {
const RangeFile& f = restorable.get().ranges[i];
files.push_back({ f.version, f.fileName, true, f.blockSize, f.fileSize });
rangeFiles.push_back({ f.version, f.fileName, true, f.blockSize, f.fileSize });
// In inconsistentSnapshotOnly mode, if all range files have the same version, then it is the
// firstConsistentVersion, otherwise unknown (use -1).
if (i != 0 && f.version != firstConsistentVersion) {
firstConsistentVersion = invalidVersion;
} else {
firstConsistentVersion = f.version;
}
}
}
} else {
// In logs-only (incremental) mode, the firstConsistentVersion should just be restore.beginVersion().
firstConsistentVersion = beginVersion;
}
if (!inconsistentSnapshotOnly) {
for (const LogFile& f : restorable.get().logs) {
files.push_back(
{ f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, f.tagId, f.totalTags });
logFiles.push_back(
{ f.beginVersion, f.fileName, false, f.blockSize, f.fileSize, f.endVersion, f.tagId, f.totalTags });
}
}
// First version for which log data should be applied
Params.firstVersion().set(task, beginVersion);
tr->reset();
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
restore.firstConsistentVersion().set(tr, firstConsistentVersion);
co_await tr->commit();
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// add log files
auto logStart = logFiles.begin();
auto logEnd = logFiles.end();
int txBytes = 0;
tr->reset();
while (logStart != logEnd) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await taskBucket->keepRunning(tr, task);
auto logIt = logStart;
txBytes = 0;
int logFileCount = 0;
auto fileSet = restore.logFileSet();
// TODO: split files into multiple keys, because files can be in the order of 10k or 100k, which
// probably can't fit due to size limit for a value in FDB. as a result, fileSet has everything,
// including [beginVersion, endVersion] for each tag
for (; logIt != logEnd && txBytes < 1e6; ++logIt) {
txBytes += fileSet.insert(tr, *logIt);
++logFileCount;
}
co_await tr->commit();
TraceEvent("FileRestoreLoadedLogFiles")
.detail("RestoreUID", restore.getUid())
.detail("FileCount", logFileCount)
.detail("TransactionBytes", txBytes);
logStart = logIt;
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
auto rangeStart = rangeFiles.begin();
auto rangeEnd = rangeFiles.end();
tr->reset();
while (rangeStart != rangeEnd) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await taskBucket->keepRunning(tr, task);
auto rangeIt = rangeStart;
txBytes = 0;
int rangeFileCount = 0;
auto fileSet = restore.rangeFileSet();
// as a result, fileSet has everything, including [beginVersion, endVersion] for each tag
for (; rangeIt != rangeEnd && txBytes < 1e6; ++rangeIt) {
txBytes += fileSet.insert(tr, *rangeIt);
// handle the remaining
++rangeFileCount;
}
co_await tr->commit();
TraceEvent("FileRestoreLoadedRangeFiles")
.detail("RestoreUID", restore.getUid())
.detail("FileCount", rangeFileCount)
.detail("TransactionBytes", txBytes);
rangeStart = rangeIt;
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
// add files
auto start = files.begin();
auto end = files.end();
tr->reset();
while (start != end) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await taskBucket->keepRunning(tr, task);
auto it = start;
txBytes = 0;
int nFileBlocks = 0;
int nFiles = 0;
auto fileSet = restore.fileSet();
// as a result, fileSet has everything, including [beginVersion, endVersion] for each tag
for (; it != end && txBytes < 1e6; ++it) {
txBytes += fileSet.insert(tr, *it);
// handle the remaining
nFileBlocks += (it->fileSize + it->blockSize - 1) / it->blockSize;
++nFiles;
}
restore.fileCount().atomicOp(tr, nFiles, MutationRef::Type::AddValue);
restore.fileBlockCount().atomicOp(tr, nFileBlocks, MutationRef::Type::AddValue);
co_await tr->commit();
TraceEvent("FileRestoreLoadedFiles")
.detail("RestoreUID", restore.getUid())
.detail("FileCount", nFiles)
.detail("FileBlockCount", nFileBlocks)
.detail("TransactionBytes", txBytes);
start = it;
tr->reset();
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
co_return;
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<FutureBucket> futureBucket,
Reference<Task> task) {
RestoreConfig restore(task);
MutationLogType mutationLogType{ MutationLogType::DEFAULT };
Version restoreVersion{ 0 };
Version firstVersion = Params.firstVersion().getOrDefault(task, invalidVersion);
bool useRangeFileRestore = false;
if (firstVersion == invalidVersion) {
co_await restore.logError(
tr->getDatabase(), restore_missing_data(), "StartFullRestore: The backup had no data.", nullptr);
std::string tag = co_await restore.tag().getD(tr);
co_await abortRestore(tr, StringRef(tag));
co_return;
}
restore.stateEnum().set(tr, ERestoreState::RUNNING);
// Check if using traditional rangefile restore instead of BulkLoad
// Default to true (traditional rangefile restore) for backward compatibility
Optional<bool> rangeFileRestore = co_await restore.useRangeFileRestore().get(tr);
useRangeFileRestore = !rangeFileRestore.present() || rangeFileRestore.get();
// Set applyMutation versions
restore.setApplyBeginVersion(tr, firstVersion);
restore.setApplyEndVersion(tr, firstVersion);
// Apply range data using either BulkLoad or traditional range file restore
mutationLogType = co_await restore.mutationLogType().getD(tr, Snapshot::False, MutationLogType::DEFAULT);
restoreVersion = co_await restore.restoreVersion().getOrThrow(tr);
if (!useRangeFileRestore) {
// Use BulkLoad for range data restoration, then apply logs
Reference<IBackupContainer> bc = co_await restore.sourceContainer().getOrThrow(tr);
// Get the BulkDump job ID - this should have been stored by the backup
// when it completed the BulkDump phase
Optional<std::string> bulkDumpJobIdOpt = co_await restore.bulkDumpJobId().get(tr);
std::string bulkDumpJobId = bulkDumpJobIdOpt.present() ? bulkDumpJobIdOpt.get() : "";
// Save original BulkLoad mode BEFORE creating the task, so it's persisted in the database.
// This allows crash recovery to restore the correct mode even if the task restarts.
int currentBulkLoadMode = co_await getBulkLoadMode(tr->getDatabase());
restore.originalBulkLoadMode().set(tr, currentBulkLoadMode);
TraceEvent("StartFullRestoreUsingBulkLoad")
.detail("RestoreUID", restore.getUid())
.detail("BackupUrl", bc->getURL())
.detail("BulkDumpJobId", bulkDumpJobId)
.detail("RestoreVersion", restoreVersion);
// Create a future that BulkLoad will signal when done
Reference<TaskFuture> bulkLoadDone = futureBucket->future(tr);
// Add BulkLoad task for range data
co_await BulkLoadRestoreTaskFunc::addTask(tr,
taskBucket,
task,
restoreVersion,
bc->getURL(),
bulkDumpJobId,
TaskCompletionKey::signal(bulkLoadDone));
// After BulkLoad completes, run RestoreDispatch to apply mutation logs
// Set onlyApplyMutationLogs so it only processes logs, not range files
restore.onlyApplyMutationLogs().set(tr, true);
// Add RestoreDispatch task that waits for BulkLoad to complete
co_await RestoreDispatchTaskFunc::addTask(tr,
taskBucket,
task,
0,
"",
0,
CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE,
0,
TaskCompletionKey::noSignal(),
bulkLoadDone);
} else if (mutationLogType == MutationLogType::PARTITIONED_LOG) {
// Traditional restore with partitioned logs
Version endVersion =
std::min(firstVersion + CLIENT_KNOBS->RESTORE_PARTITIONED_BATCH_VERSION_SIZE, restoreVersion);
co_await RestoreDispatchPartitionedTaskFunc::addTask(
tr, taskBucket, task, firstVersion, firstVersion, endVersion);
} else {
// Traditional restore with non-partitioned logs
co_await RestoreDispatchTaskFunc::addTask(
tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE);
}
// Initialize apply mutations map.
Future<Optional<bool>> logsOnly = restore.onlyApplyMutationLogs().get(tr);
co_await logsOnly;
if (logsOnly.get().present() && logsOnly.get().get()) {
// If this is an incremental restore, we need to set the applyMutationsMapPrefix
// to the earliest log version so no mutations are missed
Value versionEncoded = BinaryWriter::toValue(Params.firstVersion().get(task), Unversioned());
co_await krmSetRange(tr, restore.applyMutationsMapPrefix(), normalKeys, versionEncoded);
}
co_await taskBucket->finish(tr, task);
co_return;
}
static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
UID uid,
TaskCompletionKey completionKey,
Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Key doneKey = co_await completionKey.get(tr, taskBucket);
Reference<Task> task(new Task(StartFullRestoreTaskFunc::name, StartFullRestoreTaskFunc::version, doneKey));
RestoreConfig restore(uid);
// Bind the restore config to the new task
co_await restore.toTask(tr, task);
if (!waitFor) {
co_return taskBucket->addTask(tr, task);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task);
co_return "OnSetAddTask"_sr;
}
StringRef getName() const override { return name; };
Future<Void> execute(Database cx,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _execute(cx, tb, fb, task);
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> tb,
Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
};
StringRef StartFullRestoreTaskFunc::name = "restore_start"_sr;
REGISTER_TASKFUNC(StartFullRestoreTaskFunc);
} // namespace fileBackup
struct LogInfo : public ReferenceCounted<LogInfo> {
std::string fileName;
Reference<IAsyncFile> logFile;
Version beginVersion;
Version endVersion;
int64_t offset;
LogInfo() : offset(0) {}
};
class FileBackupAgentImpl {
public:
// This method will return the final status of the backup at tag, and return the URL that was used on the tag
// when that status value was read.
static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent,
Database cx,
std::string tagName,
StopWhenDone stopWhenDone,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr) {
std::string backTrace;
KeyBackedTag tag = makeBackupTag(tagName);
while (true) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Error err;
try {
Optional<UidAndAbortedFlagT> oldUidAndAborted = co_await tag.get(tr);
if (!oldUidAndAborted.present()) {
co_return EBackupState::STATE_NEVERRAN;
}
BackupConfig config(oldUidAndAborted.get().first);
EBackupState status =
co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
// Break, if one of the following is true
// - no longer runnable
// - in differential mode (restorable) and stopWhenDone is not enabled
if (!FileBackupAgent::isRunnable(status) ||
((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status))) {
if (pContainer != nullptr) {
Reference<IBackupContainer> c =
co_await config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info());
*pContainer = fileBackup::getBackupContainerWithProxy(c);
}
if (pUID != nullptr) {
*pUID = oldUidAndAborted.get().first;
}
co_return status;
}
Future<Void> watchFuture = tr->watch(config.stateEnum().key);
co_await tr->commit();
co_await watchFuture;
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
static Future<Void> submitBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
Key outContainer,
Optional<std::string> proxy,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
StopWhenDone stopWhenDone,
MutationLogType mutationLogType,
IncrementalBackupOnly incrementalBackupOnly,
Optional<std::string> encryptionKeyFileName,
int encryptionBlockSize,
int snapshotMode) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
TraceEvent(SevInfo, "FBA_SubmitBackup")
.detail("TagName", tagName.c_str())
.detail("StopWhenDone", stopWhenDone)
.detail("MutationLogType", mutationLogType)
.detail("OutContainer", outContainer.toString())
.detail("EncryptionKeyFileName", encryptionKeyFileName.present() ? encryptionKeyFileName.get() : "None")
.detail("EncryptionBlockSize", encryptionBlockSize);
KeyBackedTag tag = makeBackupTag(tagName);
Optional<UidAndAbortedFlagT> uidAndAbortedFlag = co_await tag.get(tr);
if (uidAndAbortedFlag.present()) {
BackupConfig prevConfig(uidAndAbortedFlag.get().first);
EBackupState prevBackupStatus =
co_await prevConfig.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
if (FileBackupAgent::isRunnable(prevBackupStatus)) {
throw backup_duplicate();
}
// Now is time to clear prev backup config space. We have no more use for it.
prevConfig.clear(tr);
}
// PARTITIONED_LOG and RANGE_PARTITIONED_LOG backups are mutually exclusive: backup worker
// recruitment is determined by the active non-default type, so both cannot run concurrently.
if (mutationLogType == MutationLogType::PARTITIONED_LOG) {
if (co_await anyRangePartitionedBackupRunning(tr)) {
TraceEvent(SevError, "FBA_SubmitBackupMutationLogTypeConflict")
.detail("TagName", tagName)
.detail("RequestedType", mutationLogTypeToString(MutationLogType::PARTITIONED_LOG))
.detail("ConflictingType", mutationLogTypeToString(MutationLogType::RANGE_PARTITIONED_LOG));
fprintf(stderr,
"ERROR: Cannot start a backup with mutation-log-type `partitioned-log' while a "
"range-partitioned-log backup is running.\n");
throw backup_error();
}
} else if (mutationLogType == MutationLogType::RANGE_PARTITIONED_LOG) {
if (co_await anyPartitionedBackupRunning(tr)) {
TraceEvent(SevError, "FBA_SubmitBackupMutationLogTypeConflict")
.detail("TagName", tagName)
.detail("RequestedType", mutationLogTypeToString(MutationLogType::RANGE_PARTITIONED_LOG))
.detail("ConflictingType", mutationLogTypeToString(MutationLogType::PARTITIONED_LOG));
fprintf(stderr,
"ERROR: Cannot start a backup with mutation-log-type `range-partitioned-log' while a "
"partitioned-log backup is running.\n");
throw backup_error();
}
}
BackupConfig config(deterministicRandom()->randomUniqueID());
UID uid = config.getUid();
// This check will ensure that current backupUid is later than the last backup Uid
Standalone<StringRef> nowStr = BackupAgentBase::getCurrentTime();
std::string backupContainer = outContainer.toString();
// To be consistent with directory handling behavior since FDB backup was first released, if the container
// string describes a local directory then "/backup-<timestamp>" will be added to it.
if (backupContainer.find("file://") == 0) {
backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString());
}
Reference<IBackupContainer> bc =
IBackupContainer::openContainer(backupContainer, proxy, encryptionKeyFileName, encryptionBlockSize);
try {
// Use longer timeout for blobstore:// URLs in simulation to handle slow S3 mock operations
double createTimeout = (g_network->isSimulated() && isBlobstoreUrl(backupContainer)) ? 300.0 : 30.0;
co_await timeoutError(bc->create(), createTimeout);
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
throw;
fprintf(stderr, "ERROR: Could not create backup container: %s\n", e.what());
throw backup_error();
}
Optional<Value> lastBackupTimestamp = co_await backupAgent->lastBackupTimestamp().get(tr);
if ((lastBackupTimestamp.present()) && (lastBackupTimestamp.get() >= nowStr)) {
fprintf(stderr,
"ERROR: The last backup `%s' happened in the future.\n",
printable(lastBackupTimestamp.get()).c_str());
throw backup_error();
}
KeyRangeMap<int> backupRangeSet;
for (auto& backupRange : backupRanges) {
backupRangeSet.insert(backupRange, 1);
}
backupRangeSet.coalesce(allKeys);
std::vector<KeyRange> normalizedRanges;
for (auto& backupRange : backupRangeSet.ranges()) {
if (backupRange.value()) {
normalizedRanges.push_back(KeyRange(KeyRangeRef(backupRange.range().begin, backupRange.range().end)));
}
}
config.clear(tr);
Key destUidValue(BinaryWriter::toValue(uid, Unversioned()));
if (normalizedRanges.size() == 1 || isDefaultBackup(normalizedRanges)) {
RangeResult existingDestUidValues = co_await tr->getRange(
KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
bool found = false;
KeyRangeRef targetRange =
normalizedRanges.size() == 1 ? normalizedRanges[0] : getDefaultBackupSharedRange();
for (auto it : existingDestUidValues) {
KeyRange uidRange =
BinaryReader::fromStringRef<KeyRange>(it.key.removePrefix(destUidLookupPrefix), IncludeVersion());
if (uidRange == targetRange) {
destUidValue = it.value;
found = true;
CODE_PROBE(isDefaultBackup(normalizedRanges), "Backup mutation sharing with default backup");
break;
}
}
if (!found) {
destUidValue = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
tr->set(BinaryWriter::toValue(targetRange, IncludeVersion(ProtocolVersion::withSharedMutations()))
.withPrefix(destUidLookupPrefix),
destUidValue);
}
}
tr->set(config.getUidAsKey().withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix),
BinaryWriter::toValue<Version>(tr->getReadVersion().get(), Unversioned()));
config.destUidValue().set(tr, destUidValue);
// Point the tag to this new uid
tag.set(tr, { uid, false });
backupAgent->lastBackupTimestamp().set(tr, nowStr);
// Set the backup keys
config.tag().set(tr, tagName);
config.stateEnum().set(tr, EBackupState::STATE_SUBMITTED);
config.backupContainer().set(tr, bc);
config.stopWhenDone().set(tr, stopWhenDone);
config.backupRanges().set(tr, normalizedRanges);
config.initialSnapshotIntervalSeconds().set(tr, initialSnapshotIntervalSeconds);
config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
config.mutationLogType().set(tr, mutationLogType);
config.incrementalBackupOnly().set(tr, incrementalBackupOnly);
config.snapshotMode().set(tr, snapshotMode);
Key taskKey = co_await fileBackup::StartFullBackupTaskFunc::addTask(
tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal());
co_return;
}
static Future<Void> submitRestore(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
Key tagName,
Key backupURL,
Optional<std::string> proxy,
Standalone<VectorRef<KeyRangeRef>> ranges,
Version restoreVersion,
Key addPrefix,
Key removePrefix,
LockDB lockDB,
UnlockDB unlockDB,
OnlyApplyMutationLogs onlyApplyMutationLogs,
InconsistentSnapshotOnly inconsistentSnapshotOnly,
Version beginVersion,
UID uid,
MutationLogType mutationLogType,
Optional<std::string> encryptionKeyFileName,
int encryptionBlockSize,
bool useRangeFileRestore = true) {
KeyRangeMap<int> restoreRangeSet;
for (auto& range : ranges) {
restoreRangeSet.insert(range, 1);
}
restoreRangeSet.coalesce(allKeys);
std::vector<KeyRange> restoreRanges;
for (auto& restoreRange : restoreRangeSet.ranges()) {
if (restoreRange.value()) {
restoreRanges.push_back(KeyRange(KeyRangeRef(restoreRange.range().begin, restoreRange.range().end)));
}
}
for (auto& restoreRange : restoreRanges) {
ASSERT(restoreRange.begin.startsWith(removePrefix) && restoreRange.end.startsWith(removePrefix));
}
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
// Get old restore config for this tag
KeyBackedTag tag = makeRestoreTag(tagName.toString());
Optional<UidAndAbortedFlagT> oldUidAndAborted = co_await tag.get(tr);
if (oldUidAndAborted.present()) {
if (oldUidAndAborted.get().first == uid) {
if (oldUidAndAborted.get().second) {
throw restore_duplicate_uid();
} else {
co_return;
}
}
RestoreConfig oldRestore(oldUidAndAborted.get().first);
// Make sure old restore for this tag is not runnable
bool runnable = co_await oldRestore.isRunnable(tr);
if (runnable) {
throw restore_duplicate_tag();
}
// Clear the old restore config
oldRestore.clear(tr);
}
// Bulkload restore (useRangeFileRestore=false) overwrites each shard via the range-lock
// mechanism in DD, so a non-empty destination is expected and required — skip the precheck.
if (!onlyApplyMutationLogs && useRangeFileRestore) {
int index{ 0 };
for (index = 0; index < restoreRanges.size(); index++) {
KeyRange restoreIntoRange = KeyRangeRef(restoreRanges[index].begin, restoreRanges[index].end)
.removePrefix(removePrefix)
.withPrefix(addPrefix);
RangeResult existingRows = co_await tr->getRange(restoreIntoRange, 1);
// Allow restoring over existing data only when using the validation restore prefix.
// validateRestoreLogKeys.begin (\xff\x02/rlog/) is the designated prefix for validation restores.
// Using any other prefix with existing data could corrupt user data.
if (!existingRows.empty() && addPrefix != validateRestoreLogKeys.begin) {
throw restore_destination_not_empty();
}
}
}
// Make new restore config
RestoreConfig restore(uid);
// Point the tag to the new uid
tag.set(tr, { uid, false });
Reference<IBackupContainer> bc =
IBackupContainer::openContainer(backupURL.toString(), proxy, encryptionKeyFileName, encryptionBlockSize);
// Configure the new restore
restore.tag().set(tr, tagName.toString());
restore.sourceContainer().set(tr, bc);
restore.stateEnum().set(tr, ERestoreState::QUEUED);
restore.restoreVersion().set(tr, restoreVersion);
restore.onlyApplyMutationLogs().set(tr, onlyApplyMutationLogs);
restore.inconsistentSnapshotOnly().set(tr, inconsistentSnapshotOnly);
restore.beginVersion().set(tr, beginVersion);
restore.unlockDBAfterRestore().set(tr, unlockDB);
restore.mutationLogType().set(tr, mutationLogType);
restore.useRangeFileRestore().set(tr, useRangeFileRestore);
if (buggify() && restoreRanges.size() == 1) {
restore.restoreRange().set(tr, restoreRanges[0]);
} else {
for (auto& range : restoreRanges) {
restore.restoreRangeSet().insert(tr, range);
}
}
// this also sets restore.add/removePrefix.
restore.initApplyMutations(tr, addPrefix, removePrefix, onlyApplyMutationLogs);
Key taskKey = co_await fileBackup::StartFullRestoreTaskFunc::addTask(
tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal());
if (lockDB)
co_await lockDatabase(tr, uid);
else
co_await checkDatabaseLock(tr, uid);
co_return;
}
// This method will return the final status of the backup
static Future<ERestoreState> waitRestore(Database cx, Key tagName, Verbose verbose) {
ERestoreState status;
while (true) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
Error err;
try {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeRestoreTag(tagName.toString());
Optional<UidAndAbortedFlagT> current = co_await tag.get(tr);
if (!current.present()) {
if (verbose) {
printf("waitRestore: Tag: %s State: %s\n",
tagName.toString().c_str(),
FileBackupAgent::restoreStateText(ERestoreState::UNINITIALIZED).toString().c_str());
}
co_return ERestoreState::UNINITIALIZED;
}
RestoreConfig restore(current.get().first);
if (verbose) {
std::string details = co_await restore.getProgress(tr);
printf("%s\n", details.c_str());
}
ERestoreState status_ = co_await restore.stateEnum().getD(tr);
status = status_;
bool runnable = co_await restore.isRunnable(tr);
// State won't change from here
if (!runnable)
break;
// Wait for a change
Future<Void> watchFuture = tr->watch(restore.stateEnum().key);
co_await tr->commit();
if (verbose)
co_await (watchFuture || delay(1));
else
co_await watchFuture;
continue;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
co_return status;
}
static Future<Void> discontinueBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
Key tagName) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeBackupTag(tagName.toString());
UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
BackupConfig config(current.first);
EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
if (!FileBackupAgent::isRunnable(status)) {
throw backup_unneeded();
}
// If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag
// and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED.
Optional<Version> latestRestorableVersion = co_await config.getLatestRestorableVersion(tr);
TraceEvent(SevInfo, "FBA_DiscontinueBackup")
.detail("AlreadyRestorable", latestRestorableVersion.present() ? "Yes" : "No")
.detail("TagName", tag.tagName.c_str())
.detail("Status", BackupAgentBase::getStateText(status));
if (latestRestorableVersion.present()) {
// Cancel all backup tasks through tag
co_await tag.cancel(tr);
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
co_await tr->getReadVersion();
co_await (eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
fileBackup::clearBackupStartID(tr, config.getUid()));
config.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
co_return;
}
bool stopWhenDone = co_await config.stopWhenDone().getOrThrow(tr);
if (stopWhenDone) {
throw backup_duplicate();
}
config.stopWhenDone().set(tr, true);
co_return;
}
static Future<Void> abortBackup(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
std::string tagName) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeBackupTag(tagName);
UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
BackupConfig config(current.first);
Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
if (!backupAgent->isRunnable(status)) {
throw backup_unneeded();
}
TraceEvent(SevInfo, "FBA_AbortBackup")
.detail("TagName", tagName.c_str())
.detail("Status", BackupAgentBase::getStateText(status));
co_await tag.cancel(tr);
co_await (eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
fileBackup::clearBackupStartID(tr, config.getUid()));
config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
co_return;
}
static Future<Void> checkAndDisableBackupWorkers(Database cx) {
bool running = co_await runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return anyPartitionedBackupRunning(tr); });
if (!running) {
co_await disableBackupWorker(cx);
}
co_return;
}
static Future<Void> checkAndDisableRangePartitionedBackupWorkers(Database cx) {
bool running = co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<bool> {
bool r = co_await anyRangePartitionedBackupRunning(tr);
if (!r) {
// Last range-partitioned backup just finished. Ask DD to clear the partition list.
tr->set(backupPartitionRequiredKey, backupPartitionRequiredValue(2));
}
co_return r;
});
if (!running) {
co_await disableRangePartitionedBackupWorker(cx);
}
co_return;
}
static Future<Void> changePause(FileBackupAgent* backupAgent, Database db, bool pause) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
while (true) {
Error err;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
try {
// This is the pause key is in the task bucket, for backup agents
backupAgent->taskBucket->changePause(tr, pause);
// This is backup workers' pause key.
tr->set(backupPausedKey, pause ? "1"_sr : "0"_sr);
co_await tr->commit();
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
TraceEvent("FileBackupAgentChangePaused").detail("Action", pause ? "Paused" : "Resumed");
co_return;
}
struct TimestampedVersion {
Optional<Version> version;
Optional<int64_t> epochs;
bool present() const { return version.present(); }
JsonBuilderObject toJSON() const {
JsonBuilderObject doc;
if (version.present()) {
doc.setKey("Version", version.get());
if (epochs.present()) {
doc.setKey("EpochSeconds", epochs.get());
doc.setKey("Timestamp", timeStampToString(epochs));
}
}
return doc;
}
};
// Helper actor for generating status
// If f is present, lookup epochs using timekeeper and tr, return TimestampedVersion
static Future<TimestampedVersion> getTimestampedVersion(Reference<ReadYourWritesTransaction> tr,
Future<Optional<Version>> f) {
TimestampedVersion tv;
tv.version = co_await f;
if (tv.version.present()) {
tv.epochs = co_await timeKeeperEpochsFromVersion(tv.version.get(), tr);
}
co_return tv;
}
static Future<std::string> getStatusJSON(FileBackupAgent* backupAgent, Database cx, std::string tagName) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
while (true) {
Error err;
try {
JsonBuilderObject doc;
doc.setKey("SchemaVersion", "1.0.0");
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeBackupTag(tagName);
Optional<UidAndAbortedFlagT> uidAndAbortedFlag;
Optional<Value> paused;
Version recentReadVersion{ 0 };
co_await (store(paused, tr->get(backupAgent->taskBucket->getPauseKey())) &&
store(uidAndAbortedFlag, tag.get(tr)) && store(recentReadVersion, tr->getReadVersion()));
doc.setKey("BackupAgentsPaused", paused.present());
doc.setKey("Tag", tag.tagName);
if (uidAndAbortedFlag.present()) {
doc.setKey("UID", uidAndAbortedFlag.get().first.toString());
BackupConfig config(uidAndAbortedFlag.get().first);
EBackupState backupState =
co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
JsonBuilderObject statusDoc;
statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState));
statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState));
statusDoc.setKey("Completed", backupState == EBackupState::STATE_COMPLETED);
statusDoc.setKey("Running", BackupAgentBase::isRunnable(backupState));
doc.setKey("Status", statusDoc);
Future<Void> done = Void();
if (backupState != EBackupState::STATE_NEVERRAN) {
Reference<IBackupContainer> bc;
TimestampedVersion latestRestorable;
co_await (
store(latestRestorable, getTimestampedVersion(tr, config.getLatestRestorableVersion(tr))) &&
store(bc, config.backupContainer().getOrThrow(tr)));
bc = fileBackup::getBackupContainerWithProxy(bc);
doc.setKey("Restorable", latestRestorable.present());
if (latestRestorable.present()) {
JsonBuilderObject o = latestRestorable.toJSON();
if (backupState != EBackupState::STATE_COMPLETED) {
o.setKey("LagSeconds",
(recentReadVersion - latestRestorable.version.get()) /
CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
}
doc.setKey("LatestRestorablePoint", o);
}
doc.setKey("DestinationURL", bc->getURL());
}
if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL ||
backupState == EBackupState::STATE_RUNNING) {
int64_t snapshotInterval{ 0 };
int64_t logBytesWritten{ 0 };
int64_t rangeBytesWritten{ 0 };
bool stopWhenDone{ false };
TimestampedVersion snapshotBegin;
TimestampedVersion snapshotTargetEnd;
TimestampedVersion latestLogEnd;
TimestampedVersion latestSnapshotEnd;
TimestampedVersion snapshotLastDispatch;
Optional<int64_t> snapshotLastDispatchShardsBehind;
co_await (
store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) &&
store(logBytesWritten, config.logBytesWritten().getD(tr)) &&
store(rangeBytesWritten, config.rangeBytesWritten().getD(tr)) &&
store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
store(snapshotBegin, getTimestampedVersion(tr, config.snapshotBeginVersion().get(tr))) &&
store(snapshotTargetEnd,
getTimestampedVersion(tr, config.snapshotTargetEndVersion().get(tr))) &&
store(latestLogEnd, getTimestampedVersion(tr, config.latestLogEndVersion().get(tr))) &&
store(latestSnapshotEnd,
getTimestampedVersion(tr, config.latestSnapshotEndVersion().get(tr))) &&
store(snapshotLastDispatch,
getTimestampedVersion(tr, config.snapshotDispatchLastVersion().get(tr))) &&
store(snapshotLastDispatchShardsBehind, config.snapshotDispatchLastShardsBehind().get(tr)));
doc.setKey("StopAfterSnapshot", stopWhenDone);
doc.setKey("SnapshotIntervalSeconds", snapshotInterval);
doc.setKey("LogBytesWritten", logBytesWritten);
doc.setKey("RangeBytesWritten", rangeBytesWritten);
if (latestLogEnd.present()) {
doc.setKey("LatestLogEnd", latestLogEnd.toJSON());
}
if (latestSnapshotEnd.present()) {
doc.setKey("LatestSnapshotEnd", latestSnapshotEnd.toJSON());
}
JsonBuilderObject snapshot;
if (snapshotBegin.present()) {
snapshot.setKey("Begin", snapshotBegin.toJSON());
if (snapshotTargetEnd.present()) {
snapshot.setKey("EndTarget", snapshotTargetEnd.toJSON());
Version interval = snapshotTargetEnd.version.get() - snapshotBegin.version.get();
snapshot.setKey("IntervalSeconds", interval / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
Version elapsed = recentReadVersion - snapshotBegin.version.get();
double progress = (interval > 0) ? (100.0 * elapsed / interval) : 100;
snapshot.setKey("ExpectedProgress", progress);
}
JsonBuilderObject dispatchDoc = snapshotLastDispatch.toJSON();
if (snapshotLastDispatchShardsBehind.present()) {
dispatchDoc.setKey("ShardsBehind", snapshotLastDispatchShardsBehind.get());
}
snapshot.setKey("LastDispatch", dispatchDoc);
}
doc.setKey("CurrentSnapshot", snapshot);
}
// Add snapshot mode information
Optional<int> snapshotModeOpt = co_await config.snapshotMode().get(tr);
int snapshotModeValue = snapshotModeOpt.present() ? snapshotModeOpt.get() : 0;
std::string snapshotModeText;
switch (snapshotModeValue) {
case 0:
snapshotModeText = "rangefile";
break;
case 1:
snapshotModeText = "bulkdump";
break;
case 2:
snapshotModeText = "both";
break;
default:
TraceEvent(SevError, "BackupInvalidSnapshotMode")
.detail("BackupUID", config.getUid())
.detail("SnapshotModeValue", snapshotModeValue)
.detail("ValidValues", "0=rangefile, 1=bulkdump, 2=both");
throw backup_error();
}
doc.setKey("SnapshotMode", snapshotModeText);
// Check if backup is BulkLoad compatible
Optional<std::string> bulkDumpJobIdOpt = co_await config.bulkDumpJobId().get(tr);
doc.setKey("BulkLoadCompatible", bulkDumpJobIdOpt.present() && !bulkDumpJobIdOpt.get().empty());
// If using bulkdump mode, get bulkdump progress
if (snapshotModeValue == 1 || snapshotModeValue == 2) {
Optional<BulkDumpProgress> bulkDumpProgressOpt = co_await getBulkDumpProgress(cx);
if (bulkDumpProgressOpt.present()) {
BulkDumpProgress bdProgress = bulkDumpProgressOpt.get();
JsonBuilderObject bdDoc;
bdDoc.setKey("JobId", bdProgress.jobId.toString());
bdDoc.setKey("TotalTasks", bdProgress.totalTasks);
bdDoc.setKey("CompleteTasks", bdProgress.completeTasks);
bdDoc.setKey("RunningTasks", bdProgress.runningTasks);
bdDoc.setKey("ErrorTasks", bdProgress.errorTasks);
bdDoc.setKey("ProgressPercent", bdProgress.progressPercent());
bdDoc.setKey("TotalBytes", bdProgress.totalBytes);
bdDoc.setKey("CompletedBytes", bdProgress.completedBytes);
bdDoc.setKey("AvgBytesPerSecond", bdProgress.avgBytesPerSecond());
if (bdProgress.etaSeconds().present()) {
bdDoc.setKey("EstimatedSecondsRemaining", bdProgress.etaSeconds().get());
}
bdDoc.setKey("ElapsedSeconds", bdProgress.elapsedSeconds);
// Add stalled tasks if any
if (!bdProgress.stalledTasks.empty()) {
JsonBuilderArray stalledArray;
for (const auto& stalled : bdProgress.stalledTasks) {
JsonBuilderObject stalledDoc;
stalledDoc.setKey("TaskId", stalled.taskId.toString());
stalledDoc.setKey("Range", stalled.range.toString());
stalledDoc.setKey("StalledSeconds", stalled.stalledSeconds);
stalledDoc.setKey("RestartCount", stalled.restartCount);
if (!stalled.lastError.empty()) {
stalledDoc.setKey("LastError", stalled.lastError);
}
stalledArray.push_back(stalledDoc);
}
bdDoc.setKey("StalledTasks", stalledArray);
}
doc.setKey("BulkDumpProgress", bdDoc);
}
}
KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
co_await config.lastErrorPerType().getRange(
tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY);
JsonBuilderArray errorList;
for (auto& e : errors.results) {
std::string msg = e.second.first;
Version ver = e.second.second;
JsonBuilderObject errDoc;
errDoc.setKey("Message", msg.c_str());
errDoc.setKey("RelativeSeconds",
(ver - recentReadVersion) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
}
doc.setKey("Errors", errorList);
}
co_return doc.getJson();
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
static Future<std::string> getStatus(FileBackupAgent* backupAgent,
Database cx,
ShowErrors showErrors,
std::string tagName) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
std::string statusText;
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag;
BackupConfig config;
EBackupState backupState;
statusText = "";
tag = makeBackupTag(tagName);
Optional<UidAndAbortedFlagT> uidAndAbortedFlag = co_await tag.get(tr);
Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
if (uidAndAbortedFlag.present()) {
config = BackupConfig(uidAndAbortedFlag.get().first);
EBackupState status =
co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
backupState = status;
}
if (!uidAndAbortedFlag.present() || backupState == EBackupState::STATE_NEVERRAN) {
statusText += "No previous backups found on tag '" + tagName + "'.\n";
statusText += "Use 'fdbbackup tags' to list all backup tags.\n";
} else {
std::string backupStatus(BackupAgentBase::getStateText(backupState));
Reference<IBackupContainer> bc;
Optional<Version> latestRestorableVersion;
Version recentReadVersion{ 0 };
co_await (store(latestRestorableVersion, config.getLatestRestorableVersion(tr)) &&
store(bc, config.backupContainer().getOrThrow(tr)) &&
store(recentReadVersion, tr->getReadVersion()));
bc = fileBackup::getBackupContainerWithProxy(bc);
bool snapshotProgress = false;
switch (backupState) {
case EBackupState::STATE_SUBMITTED:
statusText += "The backup on tag `" + tagName + "' is in progress (just started) to " +
bc->getURL() + ".\n";
break;
case EBackupState::STATE_RUNNING:
statusText += "The backup on tag `" + tagName + "' is in progress to " + bc->getURL() + ".\n";
snapshotProgress = true;
break;
case EBackupState::STATE_RUNNING_DIFFERENTIAL:
statusText += "The backup on tag `" + tagName + "' is restorable but continuing to " +
bc->getURL() + ".\n";
snapshotProgress = true;
break;
case EBackupState::STATE_COMPLETED:
statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() +
" completed at version " + format("%lld", latestRestorableVersion.orDefault(-1)) +
".\n";
break;
default:
statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " " +
backupStatus + ".\n";
break;
}
statusText += format("BackupUID: %s\n", uidAndAbortedFlag.get().first.toString().c_str());
statusText += format("BackupURL: %s\n", bc->getURL().c_str());
// Add enhanced status fields for BulkLoad integration
Optional<int> snapshotModeOpt = co_await config.snapshotMode().get(tr);
int snapshotModeValue = snapshotModeOpt.present() ? snapshotModeOpt.get() : 0;
std::string snapshotModeText;
switch (snapshotModeValue) {
case 0:
snapshotModeText = "rangefile";
break;
case 1:
snapshotModeText = "bulkdump";
break;
case 2:
snapshotModeText = "both";
break;
default:
TraceEvent(SevError, "BackupInvalidSnapshotMode")
.detail("BackupUID", config.getUid())
.detail("SnapshotModeValue", snapshotModeValue)
.detail("ValidValues", "0=rangefile, 1=bulkdump, 2=both");
throw backup_error();
}
statusText += format("Snapshot Mode: %s\n", snapshotModeText.c_str());
// Check if backup is BulkLoad compatible (has bulkdump_data/).
// In rangefile mode no bulkdump task ever runs, so the answer is
// always "no" and the line carries no information — skip it.
if (snapshotModeValue != 0) {
Optional<std::string> bulkDumpJobIdOpt = co_await config.bulkDumpJobId().get(tr);
bool bulkLoadCompatible = bulkDumpJobIdOpt.present() && !bulkDumpJobIdOpt.get().empty();
statusText += format("BulkLoad Compatible: %s\n", bulkLoadCompatible ? "yes" : "no");
}
bool showBulkDump = (snapshotModeValue == 1 || snapshotModeValue == 2);
bool showRangeFile = (snapshotModeValue == 0 || snapshotModeValue == 2);
if (snapshotProgress) {
int64_t snapshotInterval{ 0 };
Version snapshotBeginVersion{ 0 };
Version snapshotTargetEndVersion{ 0 };
Optional<Version> latestSnapshotEndVersion;
Optional<Version> latestLogEndVersion;
Optional<int64_t> logBytesWritten;
Optional<int64_t> rangeBytesWritten;
Optional<int64_t> latestSnapshotEndVersionTimestamp;
Optional<int64_t> latestLogEndVersionTimestamp;
Optional<int64_t> snapshotBeginVersionTimestamp;
Optional<int64_t> snapshotTargetEndVersionTimestamp;
bool stopWhenDone{ false };
co_await (store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) &&
store(logBytesWritten, config.logBytesWritten().get(tr)) &&
store(rangeBytesWritten, config.rangeBytesWritten().get(tr)) &&
store(latestLogEndVersion, config.latestLogEndVersion().get(tr)) &&
store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)));
co_await (
store(latestSnapshotEndVersionTimestamp,
getTimestampFromVersion(latestSnapshotEndVersion, tr)) &&
store(latestLogEndVersionTimestamp, getTimestampFromVersion(latestLogEndVersion, tr)) &&
store(snapshotBeginVersionTimestamp,
timeKeeperEpochsFromVersion(snapshotBeginVersion, tr)) &&
store(snapshotTargetEndVersionTimestamp,
timeKeeperEpochsFromVersion(snapshotTargetEndVersion, tr)));
if (showBulkDump) {
Optional<BulkDumpProgress> bulkDumpProgressOpt = co_await getBulkDumpProgress(cx);
statusText += "\nBulkDump Snapshot:\n";
if (bulkDumpProgressOpt.present()) {
BulkDumpProgress bdProgress = bulkDumpProgressOpt.get();
statusText += format(" Tasks: %d/%d complete (%.1f%%)\n",
bdProgress.completeTasks,
bdProgress.totalTasks,
bdProgress.progressPercent());
statusText += format(
" Bytes: %s\n",
formatBytesProgress(bdProgress.completedBytes, bdProgress.totalBytes).c_str());
double throughput = bdProgress.avgBytesPerSecond();
if (throughput > 0) {
statusText += format(" Throughput: %.1f MB/s\n", throughput / 1048576.0);
}
if (bdProgress.elapsedSeconds > 0) {
statusText +=
format(" Elapsed: %s\n",
formatDurationHumanReadable((int)bdProgress.elapsedSeconds).c_str());
}
if (bdProgress.etaSeconds().present()) {
statusText +=
format(" ETA: %s\n",
formatDurationHumanReadable((int)bdProgress.etaSeconds().get()).c_str());
}
if (!bdProgress.stalledTasks.empty()) {
statusText += format("\nWARNING: %zu stalled tasks (no progress > 60s):\n",
bdProgress.stalledTasks.size());
for (const auto& stalled : bdProgress.stalledTasks) {
statusText += format(" Task %s: %s, stalled %.0fs, %d restarts\n",
stalled.taskId.shortString().c_str(),
stalled.range.toString().c_str(),
stalled.stalledSeconds,
stalled.restartCount);
if (!stalled.lastError.empty()) {
statusText +=
format(" Last error: %s\n", stalled.lastError.c_str());
}
}
}
} else {
statusText += " Status: pending\n";
}
}
if (showRangeFile) {
statusText += "\nRangefile Snapshot:\n";
statusText += format(" Bytes written: %s\n",
formatBytesHumanReadable(rangeBytesWritten.orDefault(0)).c_str());
if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL) {
double pct = 100.0 * (recentReadVersion - snapshotBeginVersion) /
(snapshotTargetEndVersion - snapshotBeginVersion);
statusText += format(" Progress: %.2f%%\n", pct);
} else {
statusText += " Status: Initial snapshot still running\n";
}
statusText +=
format(" Started: %s\n", timeStampToString(snapshotBeginVersionTimestamp).c_str());
}
statusText += format("\nMutation Logs:\n");
statusText += format(" Bytes written: %s\n",
formatBytesHumanReadable(logBytesWritten.orDefault(0)).c_str());
statusText += format(" Last complete version: %s (%s)\n",
versionToString(latestLogEndVersion).c_str(),
timeStampToString(latestLogEndVersionTimestamp).c_str());
statusText += format("\nSnapshot interval is %lld seconds.\n", snapshotInterval);
}
// Append the errors, if requested
if (showErrors) {
KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
co_await config.lastErrorPerType().getRange(
tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY);
std::string recentErrors;
std::string pastErrors;
for (auto& e : errors.results) {
Version v = e.second.second;
std::string msg = format(
"%s ago : %s\n",
secondsToTimeFormat((recentReadVersion - v) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND)
.c_str(),
e.second.first.c_str());
// If error version is at or more recent than the latest restorable version then it
// could be inhibiting progress
if (v >= latestRestorableVersion.orDefault(0)) {
recentErrors += msg;
} else {
pastErrors += msg;
}
}
if (!recentErrors.empty()) {
if (latestRestorableVersion.present()) {
statusText +=
format("Recent Errors (since latest restorable point %s ago)\n",
secondsToTimeFormat((recentReadVersion - latestRestorableVersion.get()) /
CLIENT_KNOBS->CORE_VERSIONSPERSECOND)
.c_str()) +
recentErrors;
} else {
statusText += "Recent Errors (since initialization)\n" + recentErrors;
}
}
if (!pastErrors.empty())
statusText += "Older Errors\n" + pastErrors;
}
}
Optional<Value> paused = co_await fPaused;
if (paused.present()) {
statusText += format("\nAll backup agents have been paused.\n");
}
break;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
co_return statusText;
}
static Future<Optional<Version>> getLastRestorable(FileBackupAgent* backupAgent,
Reference<ReadYourWritesTransaction> tr,
Key tagName,
Snapshot snapshot) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> version = co_await tr->get(backupAgent->lastRestorable.pack(tagName), snapshot);
co_return (version.present())
? Optional<Version>(BinaryReader::fromStringRef<Version>(version.get(), Unversioned()))
: Optional<Version>();
}
static StringRef read(StringRef& data, int bytes) {
if (bytes > data.size())
throw restore_error();
StringRef r = data.substr(0, bytes);
data = data.substr(bytes);
return r;
}
// Submits the restore request to the database and throws "restore_invalid_version" error if
// restore is not possible. Parameters:
// cx: the database to be restored to
// cxOrig: if present, is used to resolve the restore timestamp into a version.
// tagName: restore tag
// url: the backup container's URL that contains all backup files
// ranges: the restored key ranges; if empty, restore all key ranges in the backup
// waitForComplete: if set, wait until the restore is completed before returning; otherwise,
// return when the request is submitted to the database.
// targetVersion: the version to be restored.
// verbose: print verbose information.
// addPrefix: each key is added this prefix during restore.
// removePrefix: for each key to be restored, remove this prefix first.
// lockDB: if set lock the database with randomUid before performing restore;
// otherwise, check database is locked with the randomUid
// onlyApplyMutationLogs: only perform incremental restore, by only applying mutation logs
// inconsistentSnapshotOnly: Ignore mutation log files during the restore to speedup the process.
// When set to true, gives an inconsistent snapshot, thus not recommended
// beginVersions: restore's begin version for each range
// randomUid: the UID for lock the database
static Future<Version> restore(FileBackupAgent* backupAgent,
Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Optional<std::string> proxy,
Standalone<VectorRef<KeyRangeRef>> ranges,
Standalone<VectorRef<Version>> beginVersions,
WaitForComplete waitForComplete,
Version targetVersion,
Verbose verbose,
Key addPrefix,
Key removePrefix,
LockDB lockDB,
UnlockDB unlockDB,
OnlyApplyMutationLogs onlyApplyMutationLogs,
InconsistentSnapshotOnly inconsistentSnapshotOnly,
Optional<std::string> encryptionKeyFileName,
UID randomUid,
bool useRangeFileRestore = true) {
// The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might.
if (ranges.empty()) {
throw restore_error();
}
std::string urlStr = url.toString();
Reference<IBackupContainer> bc =
IBackupContainer::openContainer(urlStr, proxy, /*encryptionKeyFileName=*/{}, /*encryptionBlockSize=*/0);
// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
// This is needed for S3 where metadata may not be immediately consistent
BackupDescription desc = co_await bc->describeBackup(true, isBlobstoreUrl(urlStr) ? invalidVersion : 0);
if (desc.fileLevelEncryption && !encryptionKeyFileName.present()) {
fprintf(stderr, "ERROR: Backup is encrypted, please provide the encryption key file path.\n");
throw restore_error();
} else if (!desc.fileLevelEncryption && encryptionKeyFileName.present()) {
fprintf(stderr, "ERROR: Backup is not encrypted, please remove the encryption key file path.\n");
throw restore_error();
}
if (desc.fileLevelEncryption) {
bc = IBackupContainer::openContainer(urlStr, proxy, encryptionKeyFileName, desc.encryptionBlockSize);
// openContainer may return a cached container that has blockSize=0 (seeded earlier without blockSize).
// Set it explicitly to ensure the correct value is used.
bc->setEncryptionBlockSize(desc.encryptionBlockSize);
}
if (cxOrig.present()) {
co_await desc.resolveVersionTimes(cxOrig.get());
}
printf("Backup Description\n%s", desc.toString().c_str());
if (targetVersion == invalidVersion && desc.maxRestorableVersion.present())
targetVersion = desc.maxRestorableVersion.get();
if (targetVersion == invalidVersion && onlyApplyMutationLogs && desc.contiguousLogEnd.present()) {
targetVersion = desc.contiguousLogEnd.get() - 1;
}
Version beginVersion = invalidVersion; // min begin version for all ranges
if (!beginVersions.empty()) {
beginVersion = *std::min_element(beginVersions.begin(), beginVersions.end());
}
Optional<RestorableFileSet> restoreSet =
co_await bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion);
if (!restoreSet.present()) {
TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
.detail("BackupContainer", bc->getURL())
.detail("BeginVersion", beginVersion)
.detail("TargetVersion", targetVersion);
throw restore_invalid_version();
}
if (verbose) {
printf("Restoring backup to version: %lld\n", (long long)targetVersion);
}
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
while (true) {
Error err;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
co_await submitRestore(backupAgent,
tr,
tagName,
url,
proxy,
ranges,
targetVersion,
addPrefix,
removePrefix,
lockDB,
unlockDB,
onlyApplyMutationLogs,
inconsistentSnapshotOnly,
beginVersion,
randomUid,
desc.mutationLogType,
encryptionKeyFileName,
desc.encryptionBlockSize,
useRangeFileRestore);
co_await tr->commit();
break;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_restore_duplicate_tag) {
throw err;
}
co_await tr->onError(err);
}
if (waitForComplete) {
ERestoreState finalState = co_await waitRestore(cx, tagName, verbose);
if (finalState != ERestoreState::COMPLETED)
throw restore_error();
}
co_return targetVersion;
}
// used for correctness only, locks the database before discontinuing the backup and that same lock is then used
// while doing the restore. the tagname of the backup must be the same as the restore.
static Future<Version> atomicRestore(FileBackupAgent* backupAgent,
Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix) {
auto ryw_tr = makeReference<ReadYourWritesTransaction>(cx);
BackupConfig backupConfig;
DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
while (true) {
Error err;
try {
ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedTag tag = makeBackupTag(tagName.toString());
UidAndAbortedFlagT uidFlag = co_await tag.getOrThrow(ryw_tr);
backupConfig = BackupConfig(uidFlag.first);
EBackupState status = co_await backupConfig.stateEnum().getOrThrow(ryw_tr);
if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL) {
throw backup_duplicate();
}
break;
} catch (Error& e) {
err = e;
}
co_await ryw_tr->onError(err);
}
// Lock src, record commit version
Transaction tr(cx);
Version commitVersion{ 0 };
UID randomUid = deterministicRandom()->randomUniqueID();
while (true) {
Error err;
try {
// We must get a commit version so add a conflict range that won't likely cause conflicts
// but will ensure that the transaction is actually submitted.
tr.addWriteConflictRange(backupConfig.snapshotRangeDispatchMap().subspace);
co_await lockDatabase(&tr, randomUid);
co_await tr.commit();
commitVersion = tr.getCommittedVersion();
TraceEvent("AS_Locked").detail("CommitVer", commitVersion);
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
ryw_tr->reset();
while (true) {
Error err;
try {
Optional<Version> restoreVersion = co_await backupConfig.getLatestRestorableVersion(ryw_tr);
if (restoreVersion.present() && restoreVersion.get() >= commitVersion) {
TraceEvent("AS_RestoreVersion").detail("RestoreVer", restoreVersion.get());
break;
} else {
ryw_tr->reset();
co_await delay(0.2);
continue;
}
} catch (Error& e) {
err = e;
}
co_await ryw_tr->onError(err);
}
ryw_tr->reset();
while (true) {
Error err;
try {
co_await discontinueBackup(backupAgent, ryw_tr, tagName);
co_await ryw_tr->commit();
TraceEvent("AS_DiscontinuedBackup").log();
break;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_backup_unneeded || err.code() == error_code_backup_duplicate) {
break;
}
co_await ryw_tr->onError(err);
}
co_await waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::True);
TraceEvent("AS_BackupStopped").log();
ryw_tr->reset();
while (true) {
Error err;
try {
ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
for (auto& range : ranges) {
ryw_tr->addReadConflictRange(range);
ryw_tr->clear(range);
}
co_await ryw_tr->commit();
TraceEvent("AS_ClearedRange").log();
break;
} catch (Error& e) {
err = e;
}
co_await ryw_tr->onError(err);
}
Reference<IBackupContainer> bc = co_await backupConfig.backupContainer().getOrThrow(cx.getReference());
bc = fileBackup::getBackupContainerWithProxy(bc);
TraceEvent("AS_StartRestore").log();
Standalone<VectorRef<KeyRangeRef>> restoreRange;
Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
for (auto r : ranges) {
restoreRange.push_back_deep(restoreRange.arena(), r);
}
if (!systemRestoreRange.empty()) {
// restore system keys
co_await restore(backupAgent,
cx,
cx,
"system_restore"_sr,
KeyRef(bc->getURL()),
bc->getProxy(),
systemRestoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::False,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
auto rywTransaction = makeReference<ReadYourWritesTransaction>(cx);
// clear old restore config associated with system keys
while (true) {
Error err;
try {
rywTransaction->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
rywTransaction->setOption(FDBTransactionOptions::LOCK_AWARE);
RestoreConfig oldRestore(randomUid);
oldRestore.clear(rywTransaction);
co_await rywTransaction->commit();
break;
} catch (Error& e) {
err = e;
}
co_await rywTransaction->onError(err);
}
}
// restore user data
Version ver = co_await restore(backupAgent,
cx,
cx,
tagName,
KeyRef(bc->getURL()),
bc->getProxy(),
restoreRange,
{},
WaitForComplete::True,
::invalidVersion,
Verbose::True,
addPrefix,
removePrefix,
LockDB::True,
UnlockDB::True,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
{},
randomUid);
co_return ver;
}
};
const int FileBackupAgent::dataFooterSize = 20;
Future<Version> FileBackupAgent::restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Optional<std::string> proxy,
Standalone<VectorRef<KeyRangeRef>> ranges,
Standalone<VectorRef<Version>> versions,
WaitForComplete waitForComplete,
Version targetVersion,
Verbose verbose,
Key addPrefix,
Key removePrefix,
LockDB lockDB,
UnlockDB unlockDB,
OnlyApplyMutationLogs onlyApplyMutationLogs,
InconsistentSnapshotOnly inconsistentSnapshotOnly,
Optional<std::string> const& encryptionKeyFileName,
Optional<UID> lockUID,
bool useRangeFileRestore) {
return FileBackupAgentImpl::restore(this,
cx,
cxOrig,
tagName,
url,
proxy,
ranges,
versions,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
unlockDB,
onlyApplyMutationLogs,
inconsistentSnapshotOnly,
encryptionKeyFileName,
lockUID.present() ? lockUID.get() : deterministicRandom()->randomUniqueID(),
useRangeFileRestore);
}
Future<Version> FileBackupAgent::restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Optional<std::string> proxy,
Standalone<VectorRef<KeyRangeRef>> ranges,
WaitForComplete waitForComplete,
Version targetVersion,
Verbose verbose,
Key addPrefix,
Key removePrefix,
LockDB lockDB,
UnlockDB unlockDB,
OnlyApplyMutationLogs onlyApplyMutationLogs,
InconsistentSnapshotOnly inconsistentSnapshotOnly,
Version beginVersion,
Optional<std::string> const& encryptionKeyFileName,
Optional<UID> lockUID,
bool useRangeFileRestore) {
Standalone<VectorRef<Version>> beginVersions;
for (auto i = 0; i < ranges.size(); ++i) {
beginVersions.push_back(beginVersions.arena(), beginVersion);
}
return restore(cx,
cxOrig,
tagName,
url,
proxy,
ranges,
beginVersions,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
unlockDB,
onlyApplyMutationLogs,
inconsistentSnapshotOnly,
encryptionKeyFileName,
lockUID,
useRangeFileRestore);
}
Future<Version> FileBackupAgent::restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Optional<std::string> proxy,
WaitForComplete waitForComplete,
Version targetVersion,
Verbose verbose,
KeyRange range,
Key addPrefix,
Key removePrefix,
LockDB lockDB,
OnlyApplyMutationLogs onlyApplyMutationLogs,
InconsistentSnapshotOnly inconsistentSnapshotOnly,
Version beginVersion,
Optional<std::string> const& encryptionKeyFileName,
bool useRangeFileRestore) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
if (range.begin.empty() && range.end.empty()) {
addDefaultBackupRanges(rangeRef);
} else {
rangeRef.push_back_deep(rangeRef.arena(), range);
}
Standalone<VectorRef<Version>> versionRef;
versionRef.push_back(versionRef.arena(), beginVersion);
return restore(cx,
cxOrig,
tagName,
url,
proxy,
rangeRef,
versionRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
UnlockDB::True,
onlyApplyMutationLogs,
inconsistentSnapshotOnly,
encryptionKeyFileName,
{},
useRangeFileRestore);
}
Future<Version> FileBackupAgent::atomicRestore(Database cx,
Key tagName,
KeyRange range,
Key addPrefix,
Key removePrefix) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
if (range.begin.empty() && range.end.empty()) {
addDefaultBackupRanges(rangeRef);
} else {
rangeRef.push_back_deep(rangeRef.arena(), range);
}
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
}
Future<Version> FileBackupAgent::atomicRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix) {
return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
}
Future<ERestoreState> FileBackupAgent::abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
return fileBackup::abortRestore(tr, tagName);
}
Future<ERestoreState> FileBackupAgent::abortRestore(Database cx, Key tagName) {
return fileBackup::abortRestore(cx, tagName);
}
Future<std::string> FileBackupAgent::restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
return fileBackup::restoreStatus(tr, tagName);
}
Future<ERestoreState> FileBackupAgent::waitRestore(Database cx, Key tagName, Verbose verbose) {
return FileBackupAgentImpl::waitRestore(cx, tagName, verbose);
};
Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
Optional<std::string> proxy,
int initialSnapshotIntervalSeconds,
int snapshotIntervalSeconds,
std::string const& tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
StopWhenDone stopWhenDone,
MutationLogType mutationLogType,
IncrementalBackupOnly incrementalBackupOnly,
Optional<std::string> const& encryptionKeyFileName,
int encryptionBlockSize,
int snapshotMode) {
return FileBackupAgentImpl::submitBackup(this,
tr,
outContainer,
proxy,
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
tagName,
backupRanges,
stopWhenDone,
mutationLogType,
incrementalBackupOnly,
encryptionKeyFileName,
encryptionBlockSize,
snapshotMode);
}
Future<Void> FileBackupAgent::discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName) {
return FileBackupAgentImpl::discontinueBackup(this, tr, tagName);
}
Future<Void> FileBackupAgent::abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName) {
return FileBackupAgentImpl::abortBackup(this, tr, tagName);
}
Future<Void> FileBackupAgent::checkAndDisableBackupWorkers(Database cx) {
return FileBackupAgentImpl::checkAndDisableBackupWorkers(cx);
}
Future<Void> FileBackupAgent::checkAndDisableRangePartitionedBackupWorkers(Database cx) {
return FileBackupAgentImpl::checkAndDisableRangePartitionedBackupWorkers(cx);
}
Future<std::string> FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) {
return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName);
}
Future<std::string> FileBackupAgent::getStatusJSON(Database cx, std::string tagName) {
return FileBackupAgentImpl::getStatusJSON(this, cx, tagName);
}
Future<Optional<Version>> FileBackupAgent::getLastRestorable(Reference<ReadYourWritesTransaction> tr,
Key tagName,
Snapshot snapshot) {
return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot);
}
void FileBackupAgent::setLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, Version version) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->set(lastRestorable.pack(tagName), BinaryWriter::toValue<Version>(version, Unversioned()));
}
Future<EBackupState> FileBackupAgent::waitBackup(Database cx,
std::string tagName,
StopWhenDone stopWhenDone,
Reference<IBackupContainer>* pContainer,
UID* pUID) {
return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID);
}
Future<Void> FileBackupAgent::changePause(Database db, bool pause) {
return FileBackupAgentImpl::changePause(this, db, pause);
}
Reference in New Issue View Git Blame Copy Permalink