From db8def68db2549b7ee7ba247311cb9c93c7ead7c Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Tue, 1 Feb 2022 23:57:17 -0800 Subject: [PATCH 01/90] Use std::unique_ptr for ISimulator::extraDB --- fdbrpc/simulator.h | 11 +++----- fdbserver/SimulatedCluster.actor.cpp | 39 ++++++++++++++++------------ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index e03f16cfef..f82ba5842a 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -37,12 +37,6 @@ enum ClogMode { ClogDefault, ClogAll, ClogSend, ClogReceive }; class ISimulator : public INetwork { public: - ISimulator() - : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), - extraDB(nullptr), usableRegions(1), allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false), - lastConnectionFailure(0), connectionFailuresDisableDuration(0), speedUpSimulation(false), - backupAgents(BackupAgentType::WaitForType), drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {} - // Order matters! enum KillType { KillInstantly, @@ -393,7 +387,7 @@ public: int listenersPerProcess; std::set protectedAddresses; std::map currentlyRebootingProcesses; - class ClusterConnectionString* extraDB; + std::unique_ptr extraDB; Reference storagePolicy; Reference tLogPolicy; int32_t tLogWriteAntiQuorum; @@ -455,6 +449,9 @@ public: return false; } + ISimulator(); + virtual ~ISimulator(); + protected: Mutex mutex; diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index e7cb21e74a..be39ee49e8 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -53,6 +53,13 @@ extern "C" int g_expect_full_pointermap; extern const char* getSourceVersion(); +ISimulator::ISimulator() + : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), usableRegions(1), + allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false), lastConnectionFailure(0), + connectionFailuresDisableDuration(0), speedUpSimulation(false), backupAgents(BackupAgentType::WaitForType), + drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {} +ISimulator::~ISimulator() = default; + using namespace std::literals; // TODO: Defining these here is just asking for ODR violations. @@ -1045,7 +1052,7 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor bool enableExtraDB = (testConfig.extraDB == 3); ClusterConnectionString conn(ini.GetValue("META", "connectionString")); if (enableExtraDB) { - g_simulator.extraDB = new ClusterConnectionString(ini.GetValue("META", "connectionString")); + g_simulator.extraDB = std::make_unique(ini.GetValue("META", "connectionString")); } if (!testConfig.disableHostname) { auto mockDNSStr = ini.GetValue("META", "mockDNS"); @@ -1128,7 +1135,7 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor } LocalityData localities(Optional>(), zoneId, machineId, dcUID); - localities.set(LiteralStringRef("data_hall"), dcUID); + localities.set("data_hall"_sr, dcUID); // SOMEDAY: parse backup agent from test file systemActors->push_back(reportErrors( @@ -2020,9 +2027,9 @@ void setupSimulatedSystem(std::vector>* systemActors, deterministicRandom()->randomShuffle(coordinatorAddresses); ASSERT_EQ(coordinatorAddresses.size(), coordinatorCount); - ClusterConnectionString conn(coordinatorAddresses, LiteralStringRef("TestCluster:0")); + ClusterConnectionString conn(coordinatorAddresses, "TestCluster:0"_sr); if (useHostname) { - conn = ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0")); + conn = ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr); } // If extraDB==0, leave g_simulator.extraDB as null because the test does not use DR. @@ -2030,21 +2037,21 @@ void setupSimulatedSystem(std::vector>* systemActors, // The DR database can be either a new database or itself g_simulator.extraDB = BUGGIFY - ? (useHostname ? new ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0")) - : new ClusterConnectionString(coordinatorAddresses, LiteralStringRef("TestCluster:0"))) + ? (useHostname ? std::make_unique(coordinatorHostnames, "TestCluster:0"_sr) + : std::make_unique(coordinatorAddresses, "TestCluster:0"_sr)) : (useHostname - ? new ClusterConnectionString(extraCoordinatorHostnames, LiteralStringRef("ExtraCluster:0")) - : new ClusterConnectionString(extraCoordinatorAddresses, LiteralStringRef("ExtraCluster:0"))); + ? std::make_unique(extraCoordinatorHostnames, "ExtraCluster:0"_sr) + : std::make_unique(extraCoordinatorAddresses, "ExtraCluster:0"_sr)); } else if (testConfig.extraDB == 2) { // The DR database is a new database g_simulator.extraDB = - useHostname ? new ClusterConnectionString(extraCoordinatorHostnames, LiteralStringRef("ExtraCluster:0")) - : new ClusterConnectionString(extraCoordinatorAddresses, LiteralStringRef("ExtraCluster:0")); + useHostname ? std::make_unique(extraCoordinatorHostnames, "ExtraCluster:0"_sr) + : std::make_unique(extraCoordinatorAddresses, "ExtraCluster:0"_sr); } else if (testConfig.extraDB == 3) { // The DR database is the same database - g_simulator.extraDB = - useHostname ? new ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0")) - : new ClusterConnectionString(coordinatorAddresses, LiteralStringRef("TestCluster:0")); + g_simulator.extraDB = useHostname + ? std::make_unique(coordinatorHostnames, "TestCluster:0"_sr) + : std::make_unique(coordinatorAddresses, "TestCluster:0"_sr); } *pConnString = conn; @@ -2132,7 +2139,7 @@ void setupSimulatedSystem(std::vector>* systemActors, // check the sslEnablementMap using only one ip LocalityData localities(Optional>(), zoneId, machineId, dcUID); - localities.set(LiteralStringRef("data_hall"), dcUID); + localities.set("data_hall"_sr, dcUID); systemActors->push_back(reportErrors(simulatedMachine(conn, ips, sslEnabled, @@ -2159,7 +2166,7 @@ void setupSimulatedSystem(std::vector>* systemActors, Standalone newMachineId(deterministicRandom()->randomUniqueID().toString()); LocalityData localities(Optional>(), newZoneId, newMachineId, dcUID); - localities.set(LiteralStringRef("data_hall"), dcUID); + localities.set("data_hall"_sr, dcUID); systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB, extraIps, sslEnabled, @@ -2331,7 +2338,7 @@ ACTOR void setupAndRun(std::string dataFolder, 100.0)); // FIXME: snapshot restore does not support multi-region restore, hence restore it as single region always if (restoring) { - startingConfiguration = LiteralStringRef("usable_regions=1"); + startingConfiguration = "usable_regions=1"_sr; } } else { g_expect_full_pointermap = 1; From 7acb633db88539138c5cb5a8ab700f81795dd7a7 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Tue, 18 Jan 2022 22:17:51 -0800 Subject: [PATCH 02/90] Clean up SaveAndKillWorkload --- fdbserver/workloads/SaveAndKill.actor.cpp | 29 ++++++++++------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.actor.cpp index 43d3e0d2b3..7c775b931d 100644 --- a/fdbserver/workloads/SaveAndKill.actor.cpp +++ b/fdbserver/workloads/SaveAndKill.actor.cpp @@ -37,11 +37,9 @@ struct SaveAndKillWorkload : TestWorkload { int isRestoring; SaveAndKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - restartInfo = - getOption(options, LiteralStringRef("restartInfoLocation"), LiteralStringRef("simfdb/restartInfo.ini")) - .toString(); - testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); - isRestoring = getOption(options, LiteralStringRef("isRestoring"), 0); + restartInfo = getOption(options, "restartInfoLocation"_sr, "simfdb/restartInfo.ini"_sr).toString(); + testDuration = getOption(options, "testDuration"_sr, 10.0); + isRestoring = getOption(options, "isRestoring"_sr, 0); } std::string description() const override { return "SaveAndKillWorkload"; } @@ -70,23 +68,22 @@ struct SaveAndKillWorkload : TestWorkload { std::vector processes = g_simulator.getAllProcesses(); std::map rebootingProcesses = g_simulator.currentlyRebootingProcesses; - std::map allProcessesMap = - std::map(); - for (auto it = rebootingProcesses.begin(); it != rebootingProcesses.end(); it++) { - if (allProcessesMap.find(it->second->dataFolder) == allProcessesMap.end()) - allProcessesMap[it->second->dataFolder] = it->second; + std::map allProcessesMap; + for (const auto& [_, process] : rebootingProcesses) { + if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) { + allProcessesMap[process->dataFolder] = process; + } } - for (auto it = processes.begin(); it != processes.end(); it++) { - if (allProcessesMap.find((*it)->dataFolder) == allProcessesMap.end()) - allProcessesMap[(*it)->dataFolder] = *it; + for (const auto& process : processes) { + if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) { + allProcessesMap[process->dataFolder] = process; + } } ini.SetValue("META", "processCount", format("%d", allProcessesMap.size() - 1).c_str()); std::map machines; int j = 0; - for (auto processIterator = allProcessesMap.begin(); processIterator != allProcessesMap.end(); - processIterator++) { - ISimulator::ProcessInfo* process = processIterator->second; + for (const auto& [_, process] : allProcessesMap) { std::string machineId = printable(process->locality.machineId()); const char* machineIdString = machineId.c_str(); if (strcmp(process->name, "TestSystem") != 0) { From 59794ffcb2b6bab0d8e4de220cc72cc9e528192b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 28 Feb 2022 13:12:08 -0800 Subject: [PATCH 03/90] Add randomlyRenameZoneId test parameter --- fdbserver/SimulatedCluster.actor.cpp | 18 ++++++++++++++++-- .../from_7.1.0/ConfigureTestRestart-2.toml | 3 +++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index be39ee49e8..87481d76d6 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -269,6 +269,9 @@ class TestConfig { configDBType = configDBTypeFromString(value); } } + if (attrib == "randomlyRenameZoneId") { + randomlyRenameZoneId = strcmp(value.c_str(), "true") == 0; + } } ifs.close(); @@ -306,6 +309,7 @@ public: Optional datacenters, desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType, stderrSeverity, machineCount, processesPerMachine, coordinators; Optional config; + bool randomlyRenameZoneId = false; ConfigDBType getConfigDBType() const { return configDBType; } @@ -357,7 +361,8 @@ public: .add("processesPerMachine", &processesPerMachine) .add("coordinators", &coordinators) .add("configDB", &configDBType) - .add("extraMachineCountDC", &extraMachineCountDC); + .add("extraMachineCountDC", &extraMachineCountDC) + .add("randomlyRenameZoneId", &randomlyRenameZoneId); try { auto file = toml::parse(testFile); if (file.contains("configuration") && toml::find(file, "configuration").is_table()) { @@ -1032,6 +1037,11 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor auto configDBType = testConfig.getConfigDBType(); + // Randomly change data center id names to test that localities + // can be modified on cluster restart + bool renameZoneIds = testConfig.randomlyRenameZoneId ? deterministicRandom()->random01() < 0.1 : false; + TEST(renameZoneIds); // Zone ID names altered in restart test + // allows multiple ipAddr entries ini.SetMultiKey(); @@ -1080,7 +1090,11 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor if (zoneIDini == nullptr) { zoneId = machineId; } else { - zoneId = StringRef(zoneIDini); + auto zoneIdStr = std::string(zoneIDini); + if (renameZoneIds) { + zoneIdStr = "modified/" + zoneIdStr; + } + zoneId = Standalone(zoneIdStr); } ProcessClass::ClassType cType = diff --git a/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml b/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml index a181ce821e..8230bec479 100644 --- a/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml +++ b/tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml @@ -1,3 +1,6 @@ +[[configuration]] +randomlyRenameZoneId=true + [[test]] testTitle='CloggedConfigureDatabaseTest' runSetup=false From 55c98a42870e5dc518bcffe246fd11fa0730b961 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 7 Mar 2022 22:30:20 -0800 Subject: [PATCH 04/90] Large refactor of redwood debug output to improve context and readability. --- fdbserver/VersionedBTree.actor.cpp | 211 ++++++++++++++++------------- 1 file changed, 116 insertions(+), 95 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index f77c7aa8de..158df44a75 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -62,7 +62,7 @@ { \ std::string prefix = format("%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); \ std::string msg = format(__VA_ARGS__); \ - writePrefixedLines(debug_printf_stream, prefix, msg); \ + fputs(addPrefix(prefix, msg).c_str(), debug_printf_stream); \ fflush(debug_printf_stream); \ } @@ -73,11 +73,13 @@ std::string prefix = \ format("%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); \ std::string msg = format(__VA_ARGS__); \ - writePrefixedLines(debug_printf_stream, prefix, msg); \ + fputs(addPrefix(prefix, msg).c_str(), debug_printf_stream); \ fflush(debug_printf_stream); \ } \ } +#define debug_print(str) debug_printf("%s\n", str.c_str()) +#define debug_print_always(str) debug_printf_always("%s\n", str.c_str()) #define debug_printf_noop(...) #if defined(NO_INTELLISENSE) @@ -97,13 +99,18 @@ #define TRACE \ debug_printf_always("%s: %s line %d %s\n", __FUNCTION__, __FILE__, __LINE__, platform::get_backtrace().c_str()); -// Writes prefix:line for each line in msg to fout -void writePrefixedLines(FILE* fout, std::string prefix, std::string msg) { - StringRef m = msg; +// Returns a string where every line in lines is prefixed with prefix +std::string addPrefix(std::string prefix, std::string lines) { + StringRef m = lines; + std::string s; while (m.size() != 0) { StringRef line = m.eat("\n"); - fprintf(fout, "%s %s\n", prefix.c_str(), line.toString().c_str()); + s += prefix; + s += ' '; + s += line.toString(); + s += '\n'; } + return s; } #define PRIORITYMULTILOCK_DEBUG 0 @@ -4550,7 +4557,7 @@ struct BTreePage { ValueTree* valueTree() const { return (ValueTree*)(this + 1); } - std::string toString(bool write, + std::string toString(const char* context, BTreePageIDRef id, Version ver, const RedwoodRecordRef& lowerBound, @@ -4558,7 +4565,7 @@ struct BTreePage { std::string r; r += format("BTreePage op=%s %s @%" PRId64 " ptr=%p height=%d count=%d kvBytes=%d\n lowerBound: %s\n upperBound: %s\n", - write ? "write" : "read", + context, ::toString(id).c_str(), ver, this, @@ -4692,11 +4699,12 @@ struct DecodeBoundaryVerifier { --b; if (b->second.lower != lowerBound || b->second.upper != upperBound) { fprintf(stderr, - "Boundary mismatch on %s %s\nFound :%s %s\nExpected:%s %s\n", + "Boundary mismatch on %s %s\nUsing:\n\t'%s'\n\t'%s'\nWritten %s:\n\t'%s'\n\t'%s'\n", ::toString(id).c_str(), ::toString(v).c_str(), lowerBound.toString().c_str(), upperBound.toString().c_str(), + ::toString(b->first).c_str(), b->second.lower.toString().c_str(), b->second.upper.toString().c_str()); return false; @@ -4705,15 +4713,17 @@ struct DecodeBoundaryVerifier { } void update(Version v, LogicalPageID oldID, LogicalPageID newID) { - debug_printf("decodeBoundariesUpdate copy %s %s to %s\n", - ::toString(v).c_str(), - ::toString(oldID).c_str(), - ::toString(newID).c_str()); auto& old = boundariesByPageID[oldID]; ASSERT(!old.empty()); auto i = old.end(); --i; boundariesByPageID[newID][v] = i->second; + debug_printf("decodeBoundariesUpdate copy %s %s to %s '%s' to '%s'\n", + ::toString(v).c_str(), + ::toString(oldID).c_str(), + ::toString(newID).c_str(), + i->second.lower.toString().c_str(), + i->second.upper.toString().c_str()); } }; @@ -5800,10 +5810,17 @@ private: const RedwoodRecordRef& lowerBound, const RedwoodRecordRef& upperBound) { if (page->userData == nullptr) { - debug_printf("Creating DecodeCache for ptr=%p lower=%s upper=%s\n", + debug_printf("Creating DecodeCache for ptr=%p lower=%s upper=%s %s\n", page->begin(), lowerBound.toString(false).c_str(), - upperBound.toString(false).c_str()); + upperBound.toString(false).c_str(), + ((BTreePage*)page->begin()) + ->toString("cursor", + lowerBound.value.present() ? lowerBound.getChildPage() : BTreePageIDRef(), + -1, + lowerBound, + upperBound) + .c_str()); BTreePage::BinaryTree::DecodeCache* cache = new BTreePage::BinaryTree::DecodeCache(lowerBound, upperBound, m_pDecodeCacheMemory); @@ -5870,7 +5887,8 @@ private: ::toString(writeVersion).c_str(), cache == nullptr ? "" - : btPage->toString(true, oldID, writeVersion, cache->lowerBound, cache->upperBound).c_str()); + : btPage->toString("updateBTreePage", oldID, writeVersion, cache->lowerBound, cache->upperBound) + .c_str()); } state unsigned int height = (unsigned int)((BTreePage*)page->begin())->height; @@ -6045,6 +6063,7 @@ private: s += format("SubtreeUpper: %s\n", subtreeUpperBound.toString(false).c_str()); s += format("expectedUpperBound: %s\n", expectedUpperBound.present() ? expectedUpperBound.get().toString(false).c_str() : "(null)"); + s += format("newLinks:\n"); for (int i = 0; i < newLinks.size(); ++i) { s += format(" %i: %s\n", i, newLinks[i].toString(false).c_str()); } @@ -6153,10 +6172,10 @@ private: // This must be called for each of the InternalPageSliceUpdates in sorted order. void applyUpdate(InternalPageSliceUpdate& u, const RedwoodRecordRef* nextBoundary) { - debug_printf("applyUpdate nextBoundary=(%p) %s %s\n", + debug_printf("applyUpdate nextBoundary=(%p) %s\n", nextBoundary, - (nextBoundary != nullptr) ? nextBoundary->toString(false).c_str() : "", - u.toString().c_str()); + (nextBoundary != nullptr) ? nextBoundary->toString(false).c_str() : ""); + debug_print(addPrefix("applyUpdate", u.toString())); // If the children changed, replace [cBegin, cEnd) with newLinks if (u.childrenChanged) { @@ -6170,7 +6189,7 @@ private: } while (c != u.cEnd) { - debug_printf("internal page (updating) erasing: %s\n", c.get().toString(false).c_str()); + debug_printf("applyUpdate (updating) erasing: %s\n", c.get().toString(false).c_str()); btPage()->kvBytes -= c.get().kvBytes(); c.erase(); } @@ -6201,7 +6220,7 @@ private: keep(u.cBegin, u.cEnd); } - // If there is an expected upper boundary for the next range after u + // If there is an expected upper boundary for the next range start after u if (u.expectedUpperBound.present()) { // Then if it does not match the next boundary then insert a dummy record if (nextBoundary == nullptr || (nextBoundary != &u.expectedUpperBound.get() && @@ -6228,23 +6247,28 @@ private: state std::string context; if (REDWOOD_DEBUG) { - context = format("CommitSubtree(root=%s): ", toString(rootID).c_str()); + context = format("CommitSubtree(root=%s+%d %s): ", + toString(rootID.front()).c_str(), + rootID.size() - 1, + ::toString(batch->writeVersion).c_str()); } - debug_printf("%s %s\n", context.c_str(), update->toString().c_str()); + debug_printf("%s rootID=%s\n", context.c_str(), toString(rootID).c_str()); + debug_print(addPrefix(context, update->toString())); + if (REDWOOD_DEBUG) { - debug_printf("%s ---------MUTATION BUFFER SLICE ---------------------\n", context.c_str()); - auto begin = mBegin; + int c = 0; + auto i = mBegin; while (1) { - debug_printf("%s Mutation: '%s': %s\n", + debug_printf("%s Mutation %4d '%s': %s\n", context.c_str(), - printable(begin.key()).c_str(), - begin.mutation().toString().c_str()); - if (begin == mEnd) { + c++, + printable(i.key()).c_str(), + i.mutation().toString().c_str()); + if (i == mEnd) { break; } - ++begin; + ++i; } - debug_printf("%s -------------------------------------\n", context.c_str()); } state Reference page = @@ -6266,13 +6290,13 @@ private: // TryToUpdate indicates insert and erase operations should be tried on the existing page first state bool tryToUpdate = btPage->tree()->numItems > 0 && update->boundariesNormal(); - debug_printf( - "%s commitSubtree(): %s\n", - context.c_str(), - btPage - ->toString( - false, rootID, batch->snapshot->getVersion(), update->decodeLowerBound, update->decodeUpperBound) - .c_str()); + debug_printf("%s tryToUpdate=%d\n", context.c_str(), tryToUpdate); + debug_print(addPrefix(context, + btPage->toString("commitSubtreeStart", + rootID, + batch->snapshot->getVersion(), + update->decodeLowerBound, + update->decodeUpperBound))); state BTreePage::BinaryTree::Cursor cursor = update->cBegin.valid() ? self->getCursor(page.getPtr(), update->cBegin) @@ -6287,22 +6311,6 @@ private: } } - if (REDWOOD_DEBUG) { - debug_printf("%s ---------MUTATION BUFFER SLICE ---------------------\n", context.c_str()); - auto begin = mBegin; - while (1) { - debug_printf("%s Mutation: '%s': %s\n", - context.c_str(), - printable(begin.key()).c_str(), - begin.mutation().toString().c_str()); - if (begin == mEnd) { - break; - } - ++begin; - } - debug_printf("%s -------------------------------------\n", context.c_str()); - } - // Leaf Page if (btPage->isLeaf()) { // When true, we are modifying the existing DeltaTree @@ -6541,9 +6549,8 @@ private: // No changes were actually made. This could happen if the only mutations are clear ranges which do not // match any records. if (!changesMade) { - debug_printf("%s No changes were made during mutation merge, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf("%s No changes were made during mutation merge, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); return Void(); } else { debug_printf( @@ -6556,17 +6563,26 @@ private: if (cursor.tree->numItems == 0) { update->cleared(); self->freeBTreePage(height, rootID, batch->writeVersion); - debug_printf("%s Page updates cleared all entries, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf("%s Page updates cleared all entries, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); } else { // Otherwise update it. BTreePageIDRef newID = wait(self->updateBTreePage( self, rootID, &update->newLinks.arena(), pageCopy.castTo(), batch->writeVersion)); + debug_printf("%s Leaf node updated in-place at version %s, new contents:\n", + context.c_str(), + toString(batch->writeVersion).c_str()); + debug_print(addPrefix(context, + btPage->toString("updateLeafNode", + newID, + batch->snapshot->getVersion(), + update->decodeLowerBound, + update->decodeUpperBound))); + update->updatedInPlace(newID, btPage, newID.size() * self->m_blockSize); - debug_printf( - "%s Page updated in-place, returning %s\n", context.c_str(), toString(*update).c_str()); + debug_printf("%s Leaf node updated in-place, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); } return Void(); } @@ -6576,9 +6592,8 @@ private: update->cleared(); self->freeBTreePage(height, rootID, batch->writeVersion); - debug_printf("%s All leaf page contents were cleared, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf("%s All leaf page contents were cleared, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); return Void(); } @@ -6594,7 +6609,8 @@ private: // Put new links into update and tell update that pages were rebuilt update->rebuilt(entries); - debug_printf("%s Merge complete, returning %s\n", context.c_str(), toString(*update).c_str()); + debug_printf("%s Merge complete, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); return Void(); } else { // Internal Page @@ -6731,12 +6747,12 @@ private: RedwoodRecordRef rec = c.get(); if (rec.value.present()) { if (height == 2) { - debug_printf("%s: freeing child page in cleared subtree range: %s\n", + debug_printf("%s freeing child page in cleared subtree range: %s\n", context.c_str(), ::toString(rec.getChildPage()).c_str()); self->freeBTreePage(height, rec.getChildPage(), batch->writeVersion); } else { - debug_printf("%s: queuing subtree deletion cleared subtree range: %s\n", + debug_printf("%s queuing subtree deletion cleared subtree range: %s\n", context.c_str(), ::toString(rec.getChildPage()).c_str()); self->m_lazyClearQueue.pushBack(LazyClearQueueEntry{ @@ -6749,9 +6765,8 @@ private: // Subtree range unchanged } - debug_printf("%s: MutationBuffer covers this range in a single mutation, not recursing: %s\n", - context.c_str(), - u.toString().c_str()); + debug_printf("%s Not recursing, one mutation range covers this slice:\n", context.c_str()); + debug_print(addPrefix(context, u.toString())); // u has already been initialized with the correct result, no recursion needed, so restart the // loop. @@ -6760,6 +6775,9 @@ private: } // If this page has height of 2 then its children are leaf nodes + debug_printf("%s Recursing for %s\n", context.c_str(), toString(pageID).c_str()); + debug_print(addPrefix(context, u.toString())); + recursions.push_back(self->commitSubtree(self, batch, pageID, height - 1, mBegin, mEnd, &u)); } @@ -6798,10 +6816,11 @@ private: // passed, so in the event a different upper boundary is needed it will be added to the already-modified // page. Otherwise, the decode boundary is used which will prevent this page from being modified for the // sole purpose of adding a dummy upper bound record. - debug_printf("%s Applying final child range update. changesMade=%d Parent update is: %s\n", + debug_printf("%s Applying final child range update. changesMade=%d\nSubtree Root Update:\n", context.c_str(), - modifier.changesMade, - update->toString().c_str()); + modifier.changesMade); + debug_print(addPrefix(context, update->toString())); + modifier.applyUpdate(*slices.back(), modifier.changesMade ? &update->subtreeUpperBound : &update->decodeUpperBound); @@ -6834,9 +6853,11 @@ private: if (modifier.changesMade || forceUpdate) { if (modifier.empty()) { update->cleared(); - debug_printf("%s All internal page children were deleted so deleting this page too, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf( + "%s All internal page children were deleted so deleting this page too. Returning slice:\n", + context.c_str()); + debug_print(addPrefix(context, update->toString())); + self->freeBTreePage(height, rootID, batch->writeVersion); } else { if (modifier.updating) { @@ -6874,9 +6895,10 @@ private: } parentInfo->clear(); if (forceUpdate && detached == 0) { - debug_printf("%s No children detached during forced update, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf("%s No children detached during forced update, returning slice:\n", + context.c_str()); + debug_print(addPrefix(context, update->toString())); + return Void(); } } @@ -6887,21 +6909,19 @@ private: pageCopy.castTo(), batch->writeVersion)); debug_printf( - "%s commitSubtree(): Internal page updated in-place at version %s, new contents: %s\n", + "%s commitSubtree(): Internal node updated in-place at version %s, new contents:\n", context.c_str(), - toString(batch->writeVersion).c_str(), - btPage - ->toString(false, - newID, - batch->snapshot->getVersion(), - update->decodeLowerBound, - update->decodeUpperBound) - .c_str()); + toString(batch->writeVersion).c_str()); + debug_print(addPrefix(context, + btPage->toString("updateInternalNode", + newID, + batch->snapshot->getVersion(), + update->decodeLowerBound, + update->decodeUpperBound))); update->updatedInPlace(newID, btPage, newID.size() * self->m_blockSize); - debug_printf("%s Internal page updated in-place, returning %s\n", - context.c_str(), - toString(*update).c_str()); + debug_printf("%s Internal node updated in-place, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); } else { // Page was rebuilt, possibly split. debug_printf("%s Internal page could not be modified, rebuilding replacement(s).\n", @@ -6948,12 +6968,13 @@ private: rootID)); update->rebuilt(newChildEntries); - debug_printf( - "%s Internal page rebuilt, returning %s\n", context.c_str(), toString(*update).c_str()); + debug_printf("%s Internal page rebuilt, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); } } } else { - debug_printf("%s Page has no changes, returning %s\n", context.c_str(), toString(*update).c_str()); + debug_printf("%s Page has no changes, returning slice:\n", context.c_str()); + debug_print(addPrefix(context, update->toString())); } return Void(); } From 35df00eba2ff9f7f6d0928784bc0d5edbf6dd54c Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 7 Mar 2022 22:31:29 -0800 Subject: [PATCH 05/90] Re-enable simulation-only boundary verification. --- fdbserver/VersionedBTree.actor.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 158df44a75..9b7aee8c47 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -4669,12 +4669,10 @@ struct DecodeBoundaryVerifier { static DecodeBoundaryVerifier* getVerifier(std::string name) { static std::map verifiers; - // Verifier disabled due to not being finished - // // Only use verifier in a non-restarted simulation so that all page writes are captured - // if (g_network->isSimulated() && !g_simulator.restarted) { - // return &verifiers[name]; - // } + if (g_network->isSimulated() && !g_simulator.restarted) { + return &verifiers[name]; + } return nullptr; } From 9f690d5bd5c70c84e77c60783851904f6df196e7 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 7 Mar 2022 22:44:24 -0800 Subject: [PATCH 06/90] Added StringRef::same() which checks data pointers and lengths for match. Fixed a false negative (but not a bug) in boundariesNormal() and used same() to avoid string comparisons. --- fdbserver/VersionedBTree.actor.cpp | 14 ++++++++------ flow/Arena.h | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 9b7aee8c47..db5626d521 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -5947,13 +5947,15 @@ private: RedwoodRecordRef decodeLowerBound; RedwoodRecordRef decodeUpperBound; + // Returns true of BTree logical boundaries and DeltaTree decoding boundaries are the same. bool boundariesNormal() const { - // If the decode upper boundary is the subtree upper boundary the pointers will be the same - // For the lower boundary, if the pointers are not the same there is still a possibility - // that the keys are the same. This happens for the first remaining subtree of an internal page - // after the prior subtree(s) were cleared. - return (decodeUpperBound == subtreeUpperBound) && - (decodeLowerBound == subtreeLowerBound || decodeLowerBound.sameExceptValue(subtreeLowerBound)); + // Often these strings will refer to the same memory so same() is used as a faster way of determining + // equality in thec common case, but if it does not match a string comparison is needed as they can + // still be the same. This can happen for the first remaining subtree of an internal page + // after all prior subtree(s) were cleared. + return ( + (decodeUpperBound.key.same(subtreeUpperBound.key) || decodeUpperBound.key == subtreeUpperBound.key) && + (decodeLowerBound.key.same(subtreeLowerBound.key) || decodeLowerBound.key == subtreeLowerBound.key)); } // The record range of the subtree slice is cBegin to cEnd diff --git a/flow/Arena.h b/flow/Arena.h index a9448f364b..69dabbc005 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -633,6 +633,9 @@ public: return tokens; } + // True if both StringRefs reference exactly the same memory + bool same(const StringRef& s) const { return data == s.data && length == s.length; } + private: // Unimplemented; blocks conversion through std::string StringRef(char*); From 8f844437da0ce2a330f7218333067df875191c49 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 7 Mar 2022 23:12:32 -0800 Subject: [PATCH 07/90] Avoid unused variable warning when debug output is not on. --- fdbserver/VersionedBTree.actor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index db5626d521..0359eca951 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6261,12 +6261,13 @@ private: while (1) { debug_printf("%s Mutation %4d '%s': %s\n", context.c_str(), - c++, + c, printable(i.key()).c_str(), i.mutation().toString().c_str()); if (i == mEnd) { break; } + ++c; ++i; } } From 77f06eedfd727327fe813bb46895fe6d5b17008a Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 7 Mar 2022 23:21:07 -0800 Subject: [PATCH 08/90] Use printable() on page boundary debug output. --- fdbserver/VersionedBTree.actor.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 0359eca951..9e412607ae 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -4680,8 +4680,8 @@ struct DecodeBoundaryVerifier { debug_printf("decodeBoundariesUpdate %s %s '%s' to '%s'\n", ::toString(id).c_str(), ::toString(v).c_str(), - lowerBound.toString().c_str(), - upperBound.toString().c_str()); + lowerBound.printable().c_str(), + upperBound.printable().c_str()); auto& b = boundariesByPageID[id.front()][v]; ASSERT(b.empty()); @@ -4700,11 +4700,11 @@ struct DecodeBoundaryVerifier { "Boundary mismatch on %s %s\nUsing:\n\t'%s'\n\t'%s'\nWritten %s:\n\t'%s'\n\t'%s'\n", ::toString(id).c_str(), ::toString(v).c_str(), - lowerBound.toString().c_str(), - upperBound.toString().c_str(), + lowerBound.printable().c_str(), + upperBound.printable().c_str(), ::toString(b->first).c_str(), - b->second.lower.toString().c_str(), - b->second.upper.toString().c_str()); + b->second.lower.printable().c_str(), + b->second.upper.printable().c_str()); return false; } return true; @@ -4720,8 +4720,8 @@ struct DecodeBoundaryVerifier { ::toString(v).c_str(), ::toString(oldID).c_str(), ::toString(newID).c_str(), - i->second.lower.toString().c_str(), - i->second.upper.toString().c_str()); + i->second.lower.printable().c_str(), + i->second.upper.printable().c_str()); } }; From d034d0c30f7ff4bf45e5bc5624dd9c506ef71e82 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 8 Mar 2022 03:33:22 -0800 Subject: [PATCH 09/90] Bug fix in boundary verifier which caused false failures after a process restart when a commit was in progress because the verification map would contain changes that were rolled back. --- fdbserver/VersionedBTree.actor.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 9e412607ae..506c866a8e 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -4723,6 +4723,28 @@ struct DecodeBoundaryVerifier { i->second.lower.printable().c_str(), i->second.upper.printable().c_str()); } + + void removeAfterVersion(Version version) { + auto i = boundariesByPageID.begin(); + while (i != boundariesByPageID.end()) { + auto v = i->second.upper_bound(version); + while (v != i->second.end()) { + debug_printf("decodeBoundariesUpdate remove %s %s '%s' to '%s'\n", + ::toString(v->first).c_str(), + ::toString(i->first).c_str(), + v->second.lower.printable().c_str(), + v->second.upper.printable().c_str()); + v = i->second.erase(v); + } + + if (i->second.empty()) { + debug_printf("decodeBoundariesUpdate remove empty map for %s\n", ::toString(i->first).c_str()); + i = boundariesByPageID.erase(i); + } else { + ++i; + } + } + } }; class VersionedBTree { @@ -5007,8 +5029,14 @@ public: self->m_newOldestVersion = self->m_pager->getOldestReadableVersion(); debug_printf("Recovered pager to version %" PRId64 ", oldest version is %" PRId64 "\n", + self->getLastCommittedVersion(), self->m_newOldestVersion); + // Clear any changes that occurred after the latest committed version + if (self->m_pBoundaryVerifier != nullptr) { + self->m_pBoundaryVerifier->removeAfterVersion(self->getLastCommittedVersion()); + } + state Key meta = self->m_pager->getMetaKey(); if (meta.size() == 0) { // Create new BTree From e96dc76aad0705af9ca85ba508f3537a5c223732 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 8 Mar 2022 03:56:29 -0800 Subject: [PATCH 10/90] Bug fix: When a BTree node is updated and is to the left of a completely removed sibling subtree the placeholder record providing its upper decode boundary could be lost because expectedUpperBound was not being set. --- fdbserver/VersionedBTree.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 506c866a8e..025594b6f9 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6047,6 +6047,7 @@ private: // Set the child page ID, which has already been allocated in result.arena() newLinks.back().setChildPage(maybeNewID); childrenChanged = true; + expectedUpperBound = decodeUpperBound; } else { childrenChanged = false; } From 2ec15965ba228280aa53e2ddfbea406167af6ab0 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 9 Mar 2022 18:45:03 -0800 Subject: [PATCH 11/90] No logic changes, just debugging output changes and variable renames for clarity. --- fdbserver/VersionedBTree.actor.cpp | 44 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 025594b6f9..01c956585c 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -929,7 +929,7 @@ public: // The next page will be waited for if load is true // Only mutex holders will wait on the page read. ACTOR static Future> waitThenReadNext(Cursor* self, - Optional upperBound, + Optional upperLimit, FlowMutex::Lock* lock, bool load) { state FlowMutex::Lock localLock; @@ -947,7 +947,7 @@ public: wait(success(self->nextPageReader)); } - state Optional result = wait(self->readNext(upperBound, &localLock)); + state Optional result = wait(self->readNext(upperLimit, &localLock)); // If a lock was not passed in, so this actor locked the mutex above, then unlock it if (lock == nullptr) { @@ -966,10 +966,10 @@ public: return result; } - // Read the next item at the cursor (if < upperBound), moving to a new page first if the current page is - // exhausted If locked is true, this call owns the mutex, which would have been locked by readNext() before a + // Read the next item at the cursor (if <= upperLimit), moving to a new page first if the current page is + // exhausted. If locked is true, this call owns the mutex, which would have been locked by readNext() before a // recursive call - Future> readNext(const Optional& upperBound = {}, FlowMutex::Lock* lock = nullptr) { + Future> readNext(const Optional& upperLimit = {}, FlowMutex::Lock* lock = nullptr) { if ((mode != POP && mode != READONLY) || pageID == invalidLogicalPageID || pageID == endPageID) { debug_printf("FIFOQueue::Cursor(%s) readNext returning nothing\n", toString().c_str()); return Optional(); @@ -977,7 +977,7 @@ public: // If we don't have a lock and the mutex isn't available then acquire it if (lock == nullptr && isBusy()) { - return waitThenReadNext(this, upperBound, lock, false); + return waitThenReadNext(this, upperLimit, lock, false); } // We now know pageID is valid and should be used, but page might not point to it yet @@ -993,7 +993,7 @@ public: } if (!nextPageReader.isReady()) { - return waitThenReadNext(this, upperBound, lock, true); + return waitThenReadNext(this, upperLimit, lock, true); } page = nextPageReader.get(); @@ -1014,11 +1014,11 @@ public: int bytesRead; const T result = Codec::readFromBytes(p->begin() + offset, bytesRead); - if (upperBound.present() && upperBound.get() < result) { + if (upperLimit.present() && upperLimit.get() < result) { debug_printf("FIFOQueue::Cursor(%s) not popping %s, exceeds upper bound %s\n", toString().c_str(), ::toString(result).c_str(), - ::toString(upperBound.get()).c_str()); + ::toString(upperLimit.get()).c_str()); return Optional(); } @@ -1066,10 +1066,10 @@ public: } } - debug_printf("FIFOQueue(%s) %s(upperBound=%s) -> %s\n", + debug_printf("FIFOQueue(%s) %s(upperLimit=%s) -> %s\n", queue->name.c_str(), (mode == POP ? "pop" : "peek"), - ::toString(upperBound).c_str(), + ::toString(upperLimit).c_str(), ::toString(result).c_str()); return Optional(result); } @@ -1297,8 +1297,8 @@ public: Future> peek() { return peek_impl(this); } - // Pop the next item on front of queue if it is <= upperBound or if upperBound is not present - Future> pop(Optional upperBound = {}) { return headReader.readNext(upperBound); } + // Pop the next item on front of queue if it is <= upperLimit or if upperLimit is not present + Future> pop(Optional upperLimit = {}) { return headReader.readNext(upperLimit); } QueueState getState() const { QueueState s; @@ -2764,6 +2764,8 @@ public: return f; } + // Free pageID as of version v. This means that once the oldest readable pager snapshot is at version v, pageID is + // not longer in use by any structure so it can be used to write new data. void freeUnmappedPage(PhysicalPageID pageID, Version v) { // If v is older than the oldest version still readable then mark pageID as free as of the next commit if (v < effectiveOldestVersion()) { @@ -2829,7 +2831,7 @@ public: void freePage(LogicalPageID pageID, Version v) override { // If pageID has been remapped, then it can't be freed until all existing remaps for that page have been undone, - // so queue it for later deletion + // so queue it for later deletion during remap cleanup auto i = remappedPages.find(pageID); if (i != remappedPages.end()) { debug_printf("DWALPager(%s) op=freeRemapped %s @%" PRId64 " oldestVersion=%" PRId64 "\n", @@ -3337,7 +3339,11 @@ public: // Since the next item can be arbitrarily ahead in the queue, secondType is determined by // looking at the remappedPages structure. // - // R == Remap F == Free D == Detach | == oldestRetaineedVersion + // R == Remap F == Free D == Detach | == oldestRetainedVersion + // + // oldestRetainedVersion is the oldest version being maintained as readable, either because it is explicitly the + // oldest readable version set or because there is an active snapshot for the version even though it is older + // than the explicitly set oldest readable version. // // R R | free new ID // R F | free new ID if R and D are at different versions @@ -5908,7 +5914,7 @@ private: BTreePage* btPage = (BTreePage*)page->begin(); BTreePage::BinaryTree::DecodeCache* cache = (BTreePage::BinaryTree::DecodeCache*)page->userData; debug_printf_always( - "updateBTreePage(%s, %s) %s\n", + "updateBTreePage(%s, %s) start, page:\n%s\n", ::toString(oldID).c_str(), ::toString(writeVersion).c_str(), cache == nullptr @@ -5931,7 +5937,11 @@ private: LogicalPageID id = wait(self->m_pager->newPageID()); emptyPages[i] = id; } - debug_printf("updateBTreePage: newPages %s", toString(emptyPages).c_str()); + debug_printf("updateBTreePage(%s, %s): newPages %s", + ::toString(oldID).c_str(), + ::toString(writeVersion).c_str(), + toString(emptyPages).c_str()); + self->m_pager->updatePage(PagerEventReasons::Commit, height, emptyPages, page); i = 0; for (const LogicalPageID id : emptyPages) { From 81d1e704352e0bb4967af095c3feb541443a0e8e Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 9 Mar 2022 19:08:00 -0800 Subject: [PATCH 12/90] Bug fix: Remap cleanup must delay freeing of unmapped destination pages until the oldest readable version passes the current latest readable version to prevent a very slow reader from reading a page after it has been recycled and reused. This is a very rare bug, seen in about 1 in 1 million runs of the Redwood unit test. --- fdbserver/VersionedBTree.actor.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 01c956585c..1e6aa16ea0 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -3423,13 +3423,32 @@ public: } if (freeNewID) { - debug_printf("DWALPager(%s) remapCleanup freeNew %s\n", self->filename.c_str(), p.toString().c_str()); - self->freeUnmappedPage(p.newPageID, 0); + debug_printf("DWALPager(%s) remapCleanup freeNew %s %s\n", + self->filename.c_str(), + p.toString().c_str(), + toString(self->getLastCommittedVersion()).c_str()); + + // newID must be freed at the latest committed version to avoid a read race between caching and non-caching + // readers. It is possible that there are readers of newID in flight right now that either + // - Did not read through the page cache + // - Did read through the page cache but there was no entry for the page at the time, so one was created + // and the read future is still pending + // In either case the physical read of newID from disk can happen at some time after right now and after the + // current commit is finished. + // + // If newID is freed immediately, meaning as of the end of the current commit, then it could be reused in + // the next commit which could be before any reads fitting the above description have completed, causing + // those reads to the new write which is incorrect. Since such readers could be using pager snapshots at + // versions up to and including the latest committed version, newID must be freed *after* that version is no + // longer readable. + self->freeUnmappedPage(p.newPageID, self->getLastCommittedVersion() + 1); ++g_redwoodMetrics.metric.pagerRemapFree; } if (freeOriginalID) { debug_printf("DWALPager(%s) remapCleanup freeOriginal %s\n", self->filename.c_str(), p.toString().c_str()); + // originalID can be freed immediately because it is already the case that there are no readers at a version + // prior to oldestRetainedVersion so no reader will need originalID. self->freeUnmappedPage(p.originalPageID, 0); ++g_redwoodMetrics.metric.pagerRemapFree; } From c76ba149b2aa02867e98b9f40ebd3a7851139803 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 9 Mar 2022 20:28:14 -0800 Subject: [PATCH 13/90] Removed unnecessary line as it does nothing, updated comment. --- fdbserver/VersionedBTree.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 1e6aa16ea0..14680feac2 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6718,8 +6718,8 @@ private: if (!cursor.get().value.present()) { // If the upper bound is provided by a dummy record in [cBegin, cEnd) then there is no // requirement on the next subtree range or the parent page to have a specific upper boundary - // for decoding the subtree. - u.expectedUpperBound.reset(); + // for decoding the subtree. The expected upper bound has not yet been set so it can remain + // empty. cursor.moveNext(); // If there is another record after the null child record, it must have a child page value ASSERT(!cursor.valid() || cursor.get().value.present()); From 6cb5f86994eb095e1e436cd02111b8b15a25e9ef Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 9 Mar 2022 23:17:55 -0800 Subject: [PATCH 14/90] Added a boundary sample to the boundary verifier, which clear operations in the Redwood unit test will randomly make use of. Added cold start limit in BTree unit test to prevent test running too long, and a few other parameter changes. --- fdbserver/VersionedBTree.actor.cpp | 57 ++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 14680feac2..7e8e2ca13c 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -4691,6 +4691,9 @@ struct DecodeBoundaryVerifier { typedef std::map BoundariesByVersion; std::unordered_map boundariesByPageID; + std::vector boundarySamples; + int boundarySampleSize = 1000; + int boundaryPopulation = 0; static DecodeBoundaryVerifier* getVerifier(std::string name) { static std::map verifiers; @@ -4701,7 +4704,25 @@ struct DecodeBoundaryVerifier { return nullptr; } + void sampleBoundary(Key b) { + if (boundaryPopulation <= boundarySampleSize) { + boundarySamples.push_back(b); + } else if (deterministicRandom()->random01() < ((double)boundarySampleSize / boundaryPopulation)) { + boundarySamples[deterministicRandom()->randomInt(0, boundarySampleSize)] = b; + } + ++boundaryPopulation; + } + + Key getSample() const { + if (boundarySamples.empty()) { + return Key(); + } + return boundarySamples[deterministicRandom()->randomInt(0, boundarySamples.size())]; + } + void update(BTreePageIDRef id, Version v, Key lowerBound, Key upperBound) { + sampleBoundary(lowerBound); + sampleBoundary(upperBound); debug_printf("decodeBoundariesUpdate %s %s '%s' to '%s'\n", ::toString(id).c_str(), ::toString(v).c_str(), @@ -9521,9 +9542,11 @@ TEST_CASE("Lredwood/correctness/btree") { state double clearProbability = params.getDouble("clearProbability").orDefault(deterministicRandom()->random01() * .1); state double clearExistingBoundaryProbability = - params.getDouble("clearProbability").orDefault(deterministicRandom()->random01() * .5); + params.getDouble("clearExistingBoundaryProbability").orDefault(deterministicRandom()->random01() * .5); state double clearSingleKeyProbability = - params.getDouble("clearSingleKeyProbability").orDefault(deterministicRandom()->random01()); + params.getDouble("clearSingleKeyProbability").orDefault(deterministicRandom()->random01() * .1); + state double clearKnownNodeBoundaryProbability = + params.getDouble("clearKnownNodeBoundaryProbability").orDefault(deterministicRandom()->random01() * .1); state double clearPostSetProbability = params.getDouble("clearPostSetProbability").orDefault(deterministicRandom()->random01() * .1); state double coldStartProbability = @@ -9544,10 +9567,11 @@ TEST_CASE("Lredwood/correctness/btree") { // These settings are an attempt to keep the test execution real reasonably short state int64_t maxPageOps = params.getInt("maxPageOps").orDefault((shortTest || serialTest) ? 50e3 : 1e6); - state int maxVerificationMapEntries = - params.getInt("maxVerificationMapEntries").orDefault((1.0 - coldStartProbability) * 300e3); + state int maxVerificationMapEntries = params.getInt("maxVerificationMapEntries").orDefault(300e3); + state int maxColdStarts = params.getInt("maxColdStarts").orDefault(300); + // Max number of records in the BTree or the versioned written map to visit - state int64_t maxRecordsRead = 300e6; + state int64_t maxRecordsRead = params.getInt("maxRecordsRead").orDefault(300e6); printf("\n"); printf("file: %s\n", file.c_str()); @@ -9565,9 +9589,11 @@ TEST_CASE("Lredwood/correctness/btree") { printf("setExistingKeyProbability: %f\n", setExistingKeyProbability); printf("clearProbability: %f\n", clearProbability); printf("clearExistingBoundaryProbability: %f\n", clearExistingBoundaryProbability); + printf("clearKnownNodeBoundaryProbability: %f\n", clearKnownNodeBoundaryProbability); printf("clearSingleKeyProbability: %f\n", clearSingleKeyProbability); printf("clearPostSetProbability: %f\n", clearPostSetProbability); printf("coldStartProbability: %f\n", coldStartProbability); + printf("maxColdStarts: %d\n", maxColdStarts); printf("advanceOldVersionProbability: %f\n", advanceOldVersionProbability); printf("pageCacheBytes: %s\n", pageCacheBytes == 0 ? "default" : format("%" PRId64, pageCacheBytes).c_str()); printf("versionIncrement: %" PRId64 "\n", versionIncrement); @@ -9583,9 +9609,11 @@ TEST_CASE("Lredwood/correctness/btree") { state VersionedBTree* btree = new VersionedBTree(pager, file); wait(btree->init()); + state DecodeBoundaryVerifier* pBoundaries = DecodeBoundaryVerifier::getVerifier(file); state std::map, Optional> written; state int64_t totalRecordsRead = 0; state std::set keys; + state int coldStarts = 0; state Version lastVer = btree->getLastCommittedVersion(); printf("Starting from version: %" PRId64 "\n", lastVer); @@ -9644,6 +9672,21 @@ TEST_CASE("Lredwood/correctness/btree") { end = *i; } + if (!pBoundaries->boundarySamples.empty() && + deterministicRandom()->random01() < clearKnownNodeBoundaryProbability) { + start = pBoundaries->getSample(); + + // Can't allow the end boundary to be a start, so just convert to empty string. + if (start == VersionedBTree::dbEnd.key) { + start = Key(); + } + } + + if (!pBoundaries->boundarySamples.empty() && + deterministicRandom()->random01() < clearKnownNodeBoundaryProbability) { + end = pBoundaries->getSample(); + } + // Do a single key clear based on probability or end being randomly chosen to be the same as begin // (unlikely) if (deterministicRandom()->random01() < clearSingleKeyProbability || end == start) { @@ -9779,7 +9822,9 @@ TEST_CASE("Lredwood/correctness/btree") { mutationBytesTargetThisCommit = randomSize(maxCommitSize); // Recover from disk at random - if (!pagerMemoryOnly && deterministicRandom()->random01() < coldStartProbability) { + if (!pagerMemoryOnly && coldStarts < maxColdStarts && + deterministicRandom()->random01() < coldStartProbability) { + ++coldStarts; printf("Recovering from disk after next commit.\n"); // Wait for outstanding commit From bade9a3ec3f7352d4857e7abb09c701b2ab30d9d Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Thu, 10 Mar 2022 00:07:22 -0800 Subject: [PATCH 15/90] Added toString() methods for DeltaTree::DecodeCache. --- fdbserver/DeltaTree.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fdbserver/DeltaTree.h b/fdbserver/DeltaTree.h index 9cd2e69b4c..746b96dc09 100644 --- a/fdbserver/DeltaTree.h +++ b/fdbserver/DeltaTree.h @@ -1077,7 +1077,7 @@ public: Node* node(DeltaTree2* tree) const { return tree->nodeAt(nodeOffset); } - std::string toString() { + std::string toString() const { return format("DecodedNode{nodeOffset=%d leftChildIndex=%d rightChildIndex=%d leftParentIndex=%d " "rightParentIndex=%d}", (int)nodeOffset, @@ -1154,6 +1154,19 @@ public: arena = a; updateUsedMemory(); } + + std::string toString() const { + std::string s = format("DecodeCache{%p\n", this); + s += format("upperBound %s\n", upperBound.toString().c_str()); + s += format("lowerBound %s\n", lowerBound.toString().c_str()); + s += format("arenaSize %d\n", arena.getSize()); + s += format("decodedNodes %d {\n", decodedNodes.size()); + for (auto const& n : decodedNodes) { + s += format(" %s\n", n.toString().c_str()); + } + s += format("}}\n"); + return s; + } }; // Cursor provides a way to seek into a DeltaTree and iterate over its contents From e496f3efb766ded2ff53a37436f111145289bfeb Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Thu, 10 Mar 2022 14:35:46 -0800 Subject: [PATCH 16/90] Improved comments and readability of FIFOQueue::Cursor read methods. --- fdbserver/VersionedBTree.actor.cpp | 39 +++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 8754b269df..115490b815 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -924,12 +924,15 @@ public: } } - // If readNext() cannot complete immediately, it will route to here - // The mutex will be taken if locked is false - // The next page will be waited for if load is true + // If readNext() cannot complete immediately because it must wait for IO, it will route to here. + // The purpose of this function is to serialize simultaneous readers on self while letting the + // common case (>99.8% of the time) be handled with low overhead by the non-actor readNext() function. + // + // The mutex will be taken if locked is false. + // The next page will be waited for if load is true. // Only mutex holders will wait on the page read. ACTOR static Future> waitThenReadNext(Cursor* self, - Optional upperLimit, + Optional inclusiveMaximum, FlowMutex::Lock* lock, bool load) { state FlowMutex::Lock localLock; @@ -947,7 +950,7 @@ public: wait(success(self->nextPageReader)); } - state Optional result = wait(self->readNext(upperLimit, &localLock)); + state Optional result = wait(self->readNext(inclusiveMaximum, &localLock)); // If a lock was not passed in, so this actor locked the mutex above, then unlock it if (lock == nullptr) { @@ -966,10 +969,12 @@ public: return result; } - // Read the next item at the cursor (if <= upperLimit), moving to a new page first if the current page is - // exhausted. If locked is true, this call owns the mutex, which would have been locked by readNext() before a - // recursive call - Future> readNext(const Optional& upperLimit = {}, FlowMutex::Lock* lock = nullptr) { + // Read the next item from the cursor, possibly moving to and waiting for a new page if the prior page was + // exhausted. If the item is <= inclusiveMaximum, then return it after advancing the cursor to the next item. + // Otherwise, return nothing and do not advance the cursor. + // If locked is true, this call owns the mutex, which would have been locked by readNext() before a recursive + // call. See waitThenReadNext() for more detail. + Future> readNext(const Optional& inclusiveMaximum = {}, FlowMutex::Lock* lock = nullptr) { if ((mode != POP && mode != READONLY) || pageID == invalidLogicalPageID || pageID == endPageID) { debug_printf("FIFOQueue::Cursor(%s) readNext returning nothing\n", toString().c_str()); return Optional(); @@ -977,7 +982,7 @@ public: // If we don't have a lock and the mutex isn't available then acquire it if (lock == nullptr && isBusy()) { - return waitThenReadNext(this, upperLimit, lock, false); + return waitThenReadNext(this, inclusiveMaximum, lock, false); } // We now know pageID is valid and should be used, but page might not point to it yet @@ -993,7 +998,7 @@ public: } if (!nextPageReader.isReady()) { - return waitThenReadNext(this, upperLimit, lock, true); + return waitThenReadNext(this, inclusiveMaximum, lock, true); } page = nextPageReader.get(); @@ -1014,11 +1019,11 @@ public: int bytesRead; const T result = Codec::readFromBytes(p->begin() + offset, bytesRead); - if (upperLimit.present() && upperLimit.get() < result) { + if (inclusiveMaximum.present() && inclusiveMaximum.get() < result) { debug_printf("FIFOQueue::Cursor(%s) not popping %s, exceeds upper bound %s\n", toString().c_str(), ::toString(result).c_str(), - ::toString(upperLimit.get()).c_str()); + ::toString(inclusiveMaximum.get()).c_str()); return Optional(); } @@ -1066,10 +1071,10 @@ public: } } - debug_printf("FIFOQueue(%s) %s(upperLimit=%s) -> %s\n", + debug_printf("FIFOQueue(%s) %s(inclusiveMaximum=%s) -> %s\n", queue->name.c_str(), (mode == POP ? "pop" : "peek"), - ::toString(upperLimit).c_str(), + ::toString(inclusiveMaximum).c_str(), ::toString(result).c_str()); return Optional(result); } @@ -1297,8 +1302,8 @@ public: Future> peek() { return peek_impl(this); } - // Pop the next item on front of queue if it is <= upperLimit or if upperLimit is not present - Future> pop(Optional upperLimit = {}) { return headReader.readNext(upperLimit); } + // Pop the next item on front of queue if it is <= inclusiveMaximum or if inclusiveMaximum is not present + Future> pop(Optional inclusiveMaximum = {}) { return headReader.readNext(inclusiveMaximum); } QueueState getState() const { QueueState s; From 56613bcde53196997e04179a09495874ecb7681f Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 17 Mar 2022 15:59:41 -0700 Subject: [PATCH 17/90] Create a boolean state indicating whether an SSI is open for traffic --- fdbclient/StorageServerInterface.h | 11 ++++++++--- fdbserver/CommitProxyServer.actor.cpp | 18 ++++++++++++------ fdbserver/Ratekeeper.actor.cpp | 2 +- fdbserver/storageserver.actor.cpp | 5 +++++ 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 592d2dd167..58f5e2f349 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -89,12 +89,16 @@ struct StorageServerInterface { RequestStream checkpoint; RequestStream fetchCheckpoint; + bool acceptingRequests; + explicit StorageServerInterface(UID uid) : uniqueID(uid) {} StorageServerInterface() : uniqueID(deterministicRandom()->randomUniqueID()) {} NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } NetworkAddress stableAddress() const { return getValue.getEndpoint().getStableAddress(); } Optional secondaryAddress() const { return getValue.getEndpoint().addresses.secondaryAddress; } UID id() const { return uniqueID; } + bool isAcceptingRequests() const { return acceptingRequests; } + void startAcceptingRequests() { acceptingRequests = true; } bool isTss() const { return tssPairID.present(); } std::string toString() const { return id().shortString(); } template @@ -105,9 +109,9 @@ struct StorageServerInterface { if (ar.protocolVersion().hasSmallEndpoints()) { if (ar.protocolVersion().hasTSS()) { - serializer(ar, uniqueID, locality, getValue, tssPairID); + serializer(ar, uniqueID, locality, getValue, tssPairID, acceptingRequests); } else { - serializer(ar, uniqueID, locality, getValue); + serializer(ar, uniqueID, locality, getValue, acceptingRequests); } if (Ar::isDeserializing) { getKey = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(1)); @@ -161,7 +165,8 @@ struct StorageServerInterface { getStorageMetrics, waitFailure, getQueuingMetrics, - getKeyValueStoreType); + getKeyValueStoreType, + acceptingRequests); if (ar.protocolVersion().hasWatches()) { serializer(ar, watchValue); } diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 13f3729ef0..b6b218ee76 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1584,8 +1584,10 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + if (it->interf.isAcceptingRequests()) { + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + } } rep.results.emplace_back(r.range(), ssis); } else if (!req.reverse) { @@ -1596,8 +1598,10 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + if (it->interf.isAcceptingRequests()) { + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + } } rep.results.emplace_back(r.range(), ssis); count++; @@ -1609,8 +1613,10 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + if (it->interf.isAcceptingRequests()) { + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); + } } rep.results.emplace_back(r.range(), ssis); if (r == commitData->keyInfo.ranges().begin()) { diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index 9b8b62e5ac..e5fb6b0bbe 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -255,7 +255,7 @@ public: when(state std::pair> change = waitNext(serverChanges)) { wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack if (change.second.present()) { - if (!change.second.get().isTss()) { + if (!change.second.get().isTss() && change.second.get().isAcceptingRequests()) { auto& a = actors[change.first]; a = Future(); a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 90fc288bf5..bb5644900b 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -32,6 +32,7 @@ #include "flow/IRandom.h" #include "flow/IndexedSet.h" #include "flow/SystemMonitor.h" +#include "flow/Trace.h" #include "flow/Tracing.h" #include "flow/Util.h" #include "fdbclient/Atomic.h" @@ -7607,6 +7608,8 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, wait(self.storage.commit()); ++self.counters.kvCommits; + ssi.startAcceptingRequests(); + TraceEvent("StorageServerInit", ssi.id()) .detail("Version", self.version.get()) .detail("SeedTag", seedTag.toString()) @@ -7833,6 +7836,8 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (recovered.canBeSet()) recovered.send(Void()); + ssi.startAcceptingRequests(); + try { if (self.isTss()) { wait(replaceTSSInterface(&self, ssi)); From 564c016da574d9101a24ba514a07abb9b3457700 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 17 Mar 2022 16:11:06 -0700 Subject: [PATCH 18/90] Create actor for storage interface registration --- fdbserver/storageserver.actor.cpp | 61 +++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index bb5644900b..5e09ba7dfc 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -7781,6 +7781,38 @@ ACTOR Future replaceTSSInterface(StorageServer* self, StorageServerInterfa return Void(); } +ACTOR Future storageInterfaceRegistration(StorageServer* self, StorageServerInterface ssi) { + try { + if (self->isTss()) { + wait(replaceTSSInterface(self, ssi)); + } else { + wait(replaceInterface(self, ssi)); + } + } catch (Error& e) { + if (e.code() != error_code_worker_removed) { + throw; + } + state UID clusterId = wait(getClusterId(self)); + ASSERT(self->clusterId.isValid()); + UID durableClusterId = wait(self->clusterId.getFuture()); + ASSERT(durableClusterId.isValid()); + if (clusterId == durableClusterId) { + throw worker_removed(); + } + // When a storage server connects to a new cluster, it deletes its + // old data and creates a new, empty data file for the new cluster. + // We want to avoid this and force a manual removal of the storage + // servers' old data when being assigned to a new cluster to avoid + // accidental data loss. + TraceEvent(SevError, "StorageServerBelongsToExistingCluster") + .detail("ClusterID", durableClusterId) + .detail("NewClusterID", clusterId); + wait(Future(Never())); + } + + return Void(); +} + // for recovering an existing storage server ACTOR Future storageServer(IKeyValueStore* persistentData, StorageServerInterface ssi, @@ -7838,33 +7870,8 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, ssi.startAcceptingRequests(); - try { - if (self.isTss()) { - wait(replaceTSSInterface(&self, ssi)); - } else { - wait(replaceInterface(&self, ssi)); - } - } catch (Error& e) { - if (e.code() != error_code_worker_removed) { - throw; - } - state UID clusterId = wait(getClusterId(&self)); - ASSERT(self.clusterId.isValid()); - UID durableClusterId = wait(self.clusterId.getFuture()); - ASSERT(durableClusterId.isValid()); - if (clusterId == durableClusterId) { - throw worker_removed(); - } - // When a storage server connects to a new cluster, it deletes its - // old data and creates a new, empty data file for the new cluster. - // We want to avoid this and force a manual removal of the storage - // servers' old data when being assigned to a new cluster to avoid - // accidental data loss. - TraceEvent(SevError, "StorageServerBelongsToExistingCluster") - .detail("ClusterID", durableClusterId) - .detail("NewClusterID", clusterId); - wait(Future(Never())); - } + auto f = storageInterfaceRegistration(&self, ssi); + wait(f); TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); From d27757146352cb63287ae6ca4b3f3d2b935b4899 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 17 Mar 2022 17:16:27 -0700 Subject: [PATCH 19/90] Re-register SS interface when update() runs upon recovery --- fdbserver/storageserver.actor.cpp | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 5e09ba7dfc..b295e5feb7 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -779,6 +779,9 @@ public: Promise coreStarted; bool shuttingDown; + Promise registerInterfaceAcceptingRequests; + Future interfaceRegistered; + bool behind; bool versionBehind; @@ -5565,6 +5568,12 @@ ACTOR Future tssDelayForever() { ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { state double start; try { + + if (data->registerInterfaceAcceptingRequests.canBeSet()) { + data->registerInterfaceAcceptingRequests.send(true); + wait(data->interfaceRegistered); + } + // If we are disk bound and durableVersion is very old, we need to block updates or we could run out of // memory. This is often referred to as the storage server e-brake (emergency brake) @@ -7582,6 +7591,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, self.sk = serverKeysPrefixFor(self.tssPairID.present() ? self.tssPairID.get() : self.thisServerID) .withPrefix(systemKeys.begin); // FFFF/serverKeys/[this server]/ self.folder = folder; + self.registerInterfaceAcceptingRequests.send(false); try { wait(self.storage.init()); @@ -7781,7 +7791,14 @@ ACTOR Future replaceTSSInterface(StorageServer* self, StorageServerInterfa return Void(); } -ACTOR Future storageInterfaceRegistration(StorageServer* self, StorageServerInterface ssi) { +ACTOR Future storageInterfaceRegistration(StorageServer* self, + StorageServerInterface ssi, + Future interfaceAcceptingRequests) { + bool acceptingRequests = wait(interfaceAcceptingRequests); + + if (acceptingRequests) + ssi.startAcceptingRequests(); + try { if (self->isTss()) { wait(replaceTSSInterface(self, ssi)); @@ -7868,11 +7885,14 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (recovered.canBeSet()) recovered.send(Void()); - ssi.startAcceptingRequests(); - - auto f = storageInterfaceRegistration(&self, ssi); + Promise acceptingRequests; + auto f = storageInterfaceRegistration(&self, ssi, acceptingRequests.getFuture()); + acceptingRequests.send(false); wait(f); + self.interfaceRegistered = + storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); + TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); // wait( delay(0) ); // To make sure self->zkMasterInfo.onChanged is available to wait on From 2a8d39d5e572ed98b899af84bbb5c919cda01e24 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Mon, 21 Mar 2022 10:41:28 -0700 Subject: [PATCH 20/90] Update ser-des unit test for the SSI boolean --- fdbclient/SystemData.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 9d1329f98b..8a177ccb95 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1368,28 +1368,31 @@ const KeyRef tenantDataPrefixKey = "\xff/tenantDataPrefix"_sr; // for tests void testSSISerdes(StorageServerInterface const& ssi, bool useFB) { - printf("ssi=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\naddress=%s\ngetValue=%s\n\n\n", + printf("ssi=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\nacceptingRequests=%s\naddress=%s\ngetValue=%s\n\n\n", ssi.id().toString().c_str(), ssi.locality.toString().c_str(), ssi.isTss() ? "true" : "false", ssi.isTss() ? ssi.tssPairID.get().toString().c_str() : "", + ssi.acceptingRequests ? "true" : "false", ssi.address().toString().c_str(), ssi.getValue.getEndpoint().token.toString().c_str()); StorageServerInterface ssi2 = (useFB) ? decodeServerListValueFB(serverListValueFB(ssi)) : decodeServerListValue(serverListValue(ssi)); - printf("ssi2=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\naddress=%s\ngetValue=%s\n\n\n", + printf("ssi2=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\nacceptingRequests=%s\naddress=%s\ngetValue=%s\n\n\n", ssi2.id().toString().c_str(), ssi2.locality.toString().c_str(), ssi2.isTss() ? "true" : "false", ssi2.isTss() ? ssi2.tssPairID.get().toString().c_str() : "", + ssi2.acceptingRequests ? "true" : "false", ssi2.address().toString().c_str(), ssi2.getValue.getEndpoint().token.toString().c_str()); ASSERT(ssi.id() == ssi2.id()); ASSERT(ssi.locality == ssi2.locality); ASSERT(ssi.isTss() == ssi2.isTss()); + ASSERT(ssi.acceptingRequests == ssi2.acceptingRequests); if (ssi.isTss()) { ASSERT(ssi2.tssPairID.get() == ssi2.tssPairID.get()); } From b4bf80c01d6959af59bcbd73699fb3ff89fbf57d Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Mon, 21 Mar 2022 14:47:34 -0700 Subject: [PATCH 21/90] Make sure SSI accepting request state is set before adding new storage servers --- fdbserver/storageserver.actor.cpp | 32 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index b295e5feb7..3722497de3 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -779,7 +779,7 @@ public: Promise coreStarted; bool shuttingDown; - Promise registerInterfaceAcceptingRequests; + Promise registerInterfaceAcceptingRequests; Future interfaceRegistered; bool behind; @@ -5569,10 +5569,10 @@ ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { state double start; try { - if (data->registerInterfaceAcceptingRequests.canBeSet()) { - data->registerInterfaceAcceptingRequests.send(true); - wait(data->interfaceRegistered); - } + // if (data->registerInterfaceAcceptingRequests.canBeSet()) { + // data->registerInterfaceAcceptingRequests.send(true); + // wait(data->interfaceRegistered); + // } // If we are disk bound and durableVersion is very old, we need to block updates or we could run out of // memory. This is often referred to as the storage server e-brake (emergency brake) @@ -7591,13 +7591,15 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, self.sk = serverKeysPrefixFor(self.tssPairID.present() ? self.tssPairID.get() : self.thisServerID) .withPrefix(systemKeys.begin); // FFFF/serverKeys/[this server]/ self.folder = folder; - self.registerInterfaceAcceptingRequests.send(false); + self.registerInterfaceAcceptingRequests.send(Void()); try { wait(self.storage.init()); wait(self.storage.commit()); ++self.counters.kvCommits; + ssi.startAcceptingRequests(); + if (seedTag == invalidTag) { // Might throw recruitment_failed in case of simultaneous master failure std::pair verAndTag = wait(addStorageServer(self.cx, ssi)); @@ -7618,8 +7620,6 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, wait(self.storage.commit()); ++self.counters.kvCommits; - ssi.startAcceptingRequests(); - TraceEvent("StorageServerInit", ssi.id()) .detail("Version", self.version.get()) .detail("SeedTag", seedTag.toString()) @@ -7793,11 +7793,8 @@ ACTOR Future replaceTSSInterface(StorageServer* self, StorageServerInterfa ACTOR Future storageInterfaceRegistration(StorageServer* self, StorageServerInterface ssi, - Future interfaceAcceptingRequests) { - bool acceptingRequests = wait(interfaceAcceptingRequests); - - if (acceptingRequests) - ssi.startAcceptingRequests(); + Future interfaceAcceptingRequests) { + wait(interfaceAcceptingRequests); try { if (self->isTss()) { @@ -7885,13 +7882,12 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (recovered.canBeSet()) recovered.send(Void()); - Promise acceptingRequests; - auto f = storageInterfaceRegistration(&self, ssi, acceptingRequests.getFuture()); - acceptingRequests.send(false); - wait(f); - + ssi.startAcceptingRequests(); self.interfaceRegistered = storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); + wait(delay(0)); + self.registerInterfaceAcceptingRequests.send(Void()); + wait(self.interfaceRegistered); TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); From 13233ca46d66d08aadc473d93e2f0e28c69ef30e Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Mon, 21 Mar 2022 16:28:16 -0700 Subject: [PATCH 22/90] Init the acceptingRequests state for SSIs --- fdbclient/StorageServerInterface.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 58f5e2f349..0d62bdc8cc 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -91,8 +91,8 @@ struct StorageServerInterface { bool acceptingRequests; - explicit StorageServerInterface(UID uid) : uniqueID(uid) {} - StorageServerInterface() : uniqueID(deterministicRandom()->randomUniqueID()) {} + explicit StorageServerInterface(UID uid) : uniqueID(uid) { acceptingRequests = false; } + StorageServerInterface() : uniqueID(deterministicRandom()->randomUniqueID()) { acceptingRequests = false; } NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } NetworkAddress stableAddress() const { return getValue.getEndpoint().getStableAddress(); } Optional secondaryAddress() const { return getValue.getEndpoint().addresses.secondaryAddress; } From 3a67faca7a70890af9e93d3fb7a526c6284e9268 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Tue, 22 Mar 2022 13:41:06 -0700 Subject: [PATCH 23/90] Re-register SSI as ready to accept requests --- fdbclient/StorageServerInterface.h | 1 + fdbserver/ApplyMetadataMutation.cpp | 8 ++++++- fdbserver/storageserver.actor.cpp | 36 ++++++++++++++++++++++------- flow/ProtocolVersion.h | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 0d62bdc8cc..69746f53a9 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -99,6 +99,7 @@ struct StorageServerInterface { UID id() const { return uniqueID; } bool isAcceptingRequests() const { return acceptingRequests; } void startAcceptingRequests() { acceptingRequests = true; } + void stopAcceptingRequests() { acceptingRequests = false; } bool isTss() const { return tssPairID.present(); } std::string toString() const { return id().shortString(); } template diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 5afc862b92..728b69ea01 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -40,6 +40,12 @@ Reference getStorageInfo(UID id, (*storageCache)[id] = storageInfo; } else { storageInfo = cacheItr->second; + if (!storageInfo->interf.isAcceptingRequests()) { + storageInfo->interf = decodeServerListValue(txnStateStore->readValue(serverListKeyFor(id)).get().get()); + if (storageInfo->interf.isAcceptingRequests()) { + TraceEvent(SevInfo, "StorageInfoUpdatedAcceptingRequests", storageInfo->interf.id()).log(); + } + } } return storageInfo; } @@ -232,7 +238,7 @@ private: txnStateStore->set(KeyValueRef(m.param1, m.param2)); if (storageCache) { auto cacheItr = storageCache->find(id); - if (cacheItr == storageCache->end()) { + if (cacheItr == storageCache->end() || !cacheItr->second->interf.isAcceptingRequests()) { Reference storageInfo = makeReference(); storageInfo->tag = tag; Optional interfKey = txnStateStore->readValue(serverListKeyFor(id)).get(); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 3722497de3..9e693ab62f 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -779,7 +779,7 @@ public: Promise coreStarted; bool shuttingDown; - Promise registerInterfaceAcceptingRequests; + Promise registerInterfaceAcceptingRequests; Future interfaceRegistered; bool behind; @@ -7382,6 +7382,11 @@ ACTOR Future storageServerCore(StorageServer* self, StorageServerInterface self->coreStarted.send(Void()); + // if (self->registerInterfaceAcceptingRequests.canBeSet()) { + // self->registerInterfaceAcceptingRequests.send(true); + // wait(self->interfaceRegistered); + // } + loop { ++self->counters.loops; @@ -7591,7 +7596,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, self.sk = serverKeysPrefixFor(self.tssPairID.present() ? self.tssPairID.get() : self.thisServerID) .withPrefix(systemKeys.begin); // FFFF/serverKeys/[this server]/ self.folder = folder; - self.registerInterfaceAcceptingRequests.send(Void()); + self.registerInterfaceAcceptingRequests.send(false); try { wait(self.storage.init()); @@ -7793,8 +7798,14 @@ ACTOR Future replaceTSSInterface(StorageServer* self, StorageServerInterfa ACTOR Future storageInterfaceRegistration(StorageServer* self, StorageServerInterface ssi, - Future interfaceAcceptingRequests) { - wait(interfaceAcceptingRequests); + Future interfaceAcceptingRequests) { + + bool acceptingRequests = wait(interfaceAcceptingRequests); + if (acceptingRequests) { + ssi.startAcceptingRequests(); + } else { + ssi.stopAcceptingRequests(); + } try { if (self->isTss()) { @@ -7882,16 +7893,25 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (recovered.canBeSet()) recovered.send(Void()); - ssi.startAcceptingRequests(); + state Promise registerInterface; + state Future f = storageInterfaceRegistration(&self, ssi, registerInterface.getFuture()); + wait(delay(0)); + registerInterface.send(false); + wait(f); + self.interfaceRegistered = storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); wait(delay(0)); - self.registerInterfaceAcceptingRequests.send(Void()); - wait(self.interfaceRegistered); + + ASSERT(self.registerInterfaceAcceptingRequests.canBeSet()); + + if (self.registerInterfaceAcceptingRequests.canBeSet()) { + self.registerInterfaceAcceptingRequests.send(true); + wait(self.interfaceRegistered); + } TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); - // wait( delay(0) ); // To make sure self->zkMasterInfo.onChanged is available to wait on ssCore = storageServerCore(&self, ssi); wait(ssCore); diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index 81deef5a4e..2b23239dfa 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -162,6 +162,7 @@ public: // introduced features PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageMetadata); PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, PerpetualWiggleMetadata); PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, Tenants); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness); }; template <> From faff94ed2bca66320d9c58e08c4c72c8eb063a4d Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 23 Mar 2022 08:54:20 -0700 Subject: [PATCH 24/90] Retract changes to limit which servers are published to clients as ready for requests --- fdbserver/CommitProxyServer.actor.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index b6b218ee76..13f3729ef0 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1584,10 +1584,8 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - if (it->interf.isAcceptingRequests()) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); - } + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } rep.results.emplace_back(r.range(), ssis); } else if (!req.reverse) { @@ -1598,10 +1596,8 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - if (it->interf.isAcceptingRequests()) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); - } + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } rep.results.emplace_back(r.range(), ssis); count++; @@ -1613,10 +1609,8 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques std::vector ssis; ssis.reserve(r.value().src_info.size()); for (auto& it : r.value().src_info) { - if (it->interf.isAcceptingRequests()) { - ssis.push_back(it->interf); - maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); - } + ssis.push_back(it->interf); + maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } rep.results.emplace_back(r.range(), ssis); if (r == commitData->keyInfo.ranges().begin()) { From abce71c14683598535d005d6958cf1c11eb56438 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 24 Mar 2022 07:48:17 -0700 Subject: [PATCH 25/90] Properly handle protocol-version-based ser-des for SSI --- fdbclient/StorageServerInterface.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 69746f53a9..342ee536b6 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -110,9 +110,13 @@ struct StorageServerInterface { if (ar.protocolVersion().hasSmallEndpoints()) { if (ar.protocolVersion().hasTSS()) { - serializer(ar, uniqueID, locality, getValue, tssPairID, acceptingRequests); + if (ar.protocolVersion().hasStorageInterfaceReadiness()) { + serializer(ar, uniqueID, locality, getValue, tssPairID, acceptingRequests); + } else { + serializer(ar, uniqueID, locality, getValue, tssPairID); + } } else { - serializer(ar, uniqueID, locality, getValue, acceptingRequests); + serializer(ar, uniqueID, locality, getValue); } if (Ar::isDeserializing) { getKey = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(1)); @@ -166,8 +170,7 @@ struct StorageServerInterface { getStorageMetrics, waitFailure, getQueuingMetrics, - getKeyValueStoreType, - acceptingRequests); + getKeyValueStoreType); if (ar.protocolVersion().hasWatches()) { serializer(ar, watchValue); } From 961e4ae7fd7f82789da3f027d6e7c8f5bced2bd0 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 24 Mar 2022 17:25:07 -0700 Subject: [PATCH 26/90] ratekeeper and ser-des fixes --- fdbclient/StorageServerInterface.h | 145 +++++++++++++++++++++++ fdbclient/SystemData.cpp | 118 +++++++++++++++++-- fdbclient/SystemData.h | 1 + fdbserver/Ratekeeper.actor.cpp | 8 +- fdbserver/Ratekeeper.h | 6 +- fdbserver/storageserver.actor.cpp | 182 +++++++++++++++-------------- 6 files changed, 355 insertions(+), 105 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 342ee536b6..8045ed0a74 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -51,6 +51,151 @@ struct VersionReply { } }; +struct StorageServerInterfaceOld { + constexpr static FileIdentifier file_identifier = 15302073; + enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 }; + + enum { LocationAwareLoadBalance = 1 }; + enum { AlwaysFresh = 0 }; + + LocalityData locality; + UID uniqueID; + Optional tssPairID; + + RequestStream getValue; + RequestStream getKey; + + // Throws a wrong_shard_server if the keys in the request or result depend on data outside this server OR if a large + // selector offset prevents all data from being read in one range read + RequestStream getKeyValues; + RequestStream getMappedKeyValues; + + RequestStream getShardState; + RequestStream waitMetrics; + RequestStream splitMetrics; + RequestStream getStorageMetrics; + RequestStream> waitFailure; + RequestStream getQueuingMetrics; + + RequestStream> getKeyValueStoreType; + RequestStream watchValue; + RequestStream getReadHotRanges; + RequestStream getRangeSplitPoints; + RequestStream getKeyValuesStream; + RequestStream changeFeedStream; + RequestStream overlappingChangeFeeds; + RequestStream changeFeedPop; + RequestStream changeFeedVersionUpdate; + RequestStream checkpoint; + RequestStream fetchCheckpoint; + + explicit StorageServerInterfaceOld(UID uid) : uniqueID(uid) {} + StorageServerInterfaceOld() : uniqueID(deterministicRandom()->randomUniqueID()) {} + NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } + NetworkAddress stableAddress() const { return getValue.getEndpoint().getStableAddress(); } + Optional secondaryAddress() const { return getValue.getEndpoint().addresses.secondaryAddress; } + UID id() const { return uniqueID; } + bool isTss() const { return tssPairID.present(); } + std::string toString() const { return id().shortString(); } + template + void serialize(Ar& ar) { + // StorageServerInterface is persisted in the database, so changes here have to be versioned carefully! + // To change this serialization, ProtocolVersion::ServerListValue must be updated, and downgrades need to be + // considered + + if (ar.protocolVersion().hasSmallEndpoints()) { + if (ar.protocolVersion().hasTSS()) { + serializer(ar, uniqueID, locality, getValue, tssPairID); + } else { + serializer(ar, uniqueID, locality, getValue); + } + if (Ar::isDeserializing) { + getKey = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(1)); + getKeyValues = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(2)); + getShardState = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(3)); + waitMetrics = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(4)); + splitMetrics = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(5)); + getStorageMetrics = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(6)); + waitFailure = RequestStream>(getValue.getEndpoint().getAdjustedEndpoint(7)); + getQueuingMetrics = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(8)); + getKeyValueStoreType = + RequestStream>(getValue.getEndpoint().getAdjustedEndpoint(9)); + watchValue = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(10)); + getReadHotRanges = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(11)); + getRangeSplitPoints = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); + getKeyValuesStream = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(13)); + getMappedKeyValues = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(14)); + changeFeedStream = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(15)); + overlappingChangeFeeds = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(16)); + changeFeedPop = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(17)); + changeFeedVersionUpdate = RequestStream( + getValue.getEndpoint().getAdjustedEndpoint(18)); + checkpoint = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(19)); + fetchCheckpoint = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(20)); + } + } else { + ASSERT(Ar::isDeserializing); + if constexpr (is_fb_function) { + ASSERT(false); + } + serializer(ar, + uniqueID, + locality, + getValue, + getKey, + getKeyValues, + getShardState, + waitMetrics, + splitMetrics, + getStorageMetrics, + waitFailure, + getQueuingMetrics, + getKeyValueStoreType); + if (ar.protocolVersion().hasWatches()) { + serializer(ar, watchValue); + } + } + } + bool operator==(StorageServerInterfaceOld const& s) const { return uniqueID == s.uniqueID; } + bool operator<(StorageServerInterfaceOld const& s) const { return uniqueID < s.uniqueID; } + void initEndpoints() { + std::vector> streams; + streams.push_back(getValue.getReceiver(TaskPriority::LoadBalancedEndpoint)); + streams.push_back(getKey.getReceiver(TaskPriority::LoadBalancedEndpoint)); + streams.push_back(getKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint)); + streams.push_back(getShardState.getReceiver()); + streams.push_back(waitMetrics.getReceiver()); + streams.push_back(splitMetrics.getReceiver()); + streams.push_back(getStorageMetrics.getReceiver()); + streams.push_back(waitFailure.getReceiver()); + streams.push_back(getQueuingMetrics.getReceiver()); + streams.push_back(getKeyValueStoreType.getReceiver()); + streams.push_back(watchValue.getReceiver()); + streams.push_back(getReadHotRanges.getReceiver()); + streams.push_back(getRangeSplitPoints.getReceiver()); + streams.push_back(getKeyValuesStream.getReceiver(TaskPriority::LoadBalancedEndpoint)); + streams.push_back(getMappedKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint)); + streams.push_back(changeFeedStream.getReceiver()); + streams.push_back(overlappingChangeFeeds.getReceiver()); + streams.push_back(changeFeedPop.getReceiver()); + streams.push_back(changeFeedVersionUpdate.getReceiver()); + streams.push_back(checkpoint.getReceiver()); + streams.push_back(fetchCheckpoint.getReceiver()); + FlowTransport::transport().addEndpoints(streams); + } +}; + struct StorageServerInterface { constexpr static FileIdentifier file_identifier = 15302073; enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 8a177ccb95..4fa3ba219f 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -587,29 +587,30 @@ const Key serverListKeyFor(UID serverID) { return wr.toValue(); } -// TODO use flatbuffers depending on version -const Value serverListValue(StorageServerInterface const& server) { - BinaryWriter wr(IncludeVersion(ProtocolVersion::withServerListValue())); +const Value serverListValueOld(StorageServerInterfaceOld const& server) { + BinaryWriter wr(IncludeVersion(ProtocolVersion::withTSS())); wr << server; return wr.toValue(); } + +const Value serverListValue(StorageServerInterface const& server) { + return serverListValueFB(server); +} + UID decodeServerListKey(KeyRef const& key) { UID serverID; BinaryReader rd(key.removePrefix(serverListKeys.begin), Unversioned()); rd >> serverID; return serverID; } -StorageServerInterface decodeServerListValue(ValueRef const& value) { - StorageServerInterface s; - BinaryReader reader(value, IncludeVersion()); + +StorageServerInterfaceOld decodeServerListValueOld(ValueRef const& value) { + StorageServerInterfaceOld s; + BinaryReader reader(value, IncludeVersion(ProtocolVersion::withTSS())); reader >> s; return s; } -const Value serverListValueFB(StorageServerInterface const& server) { - return ObjectWriter::toValue(server, IncludeVersion()); -} - StorageServerInterface decodeServerListValueFB(ValueRef const& value) { StorageServerInterface s; ObjectReader reader(value.begin(), IncludeVersion()); @@ -617,6 +618,24 @@ StorageServerInterface decodeServerListValueFB(ValueRef const& value) { return s; } +StorageServerInterface decodeServerListValue(ValueRef const& value) { + StorageServerInterface s; + BinaryReader reader(value, IncludeVersion()); + + if (!reader.protocolVersion().hasStorageInterfaceReadiness()) { + reader >> s; + return s; + } + + return decodeServerListValueFB(value); +} + +const Value serverListValueFB(StorageServerInterface const& server) { + auto protocolVersion = currentProtocolVersion; + protocolVersion.addObjectSerializerFlag(); + return ObjectWriter::toValue(server, IncludeVersion(protocolVersion)); +} + // processClassKeys.contains(k) iff k.startsWith( processClassKeys.begin ) because '/'+1 == '0' const KeyRangeRef processClassKeys(LiteralStringRef("\xff/processClass/"), LiteralStringRef("\xff/processClass0")); const KeyRef processClassPrefix = processClassKeys.begin; @@ -1401,7 +1420,7 @@ void testSSISerdes(StorageServerInterface const& ssi, bool useFB) { } // unit test for serialization since tss stuff had bugs -TEST_CASE("/SystemData/SerDes/SSI") { +TEST_CASE("/SystemData/SSI/SerDes") { printf("testing ssi serdes\n"); LocalityData localityData(Optional>(), Standalone(deterministicRandom()->randomUniqueID().toString()), @@ -1425,3 +1444,80 @@ TEST_CASE("/SystemData/SerDes/SSI") { return Void(); } + +TEST_CASE("/SystemData/SSI/Downgrade") { + std::vector newssis; + constexpr int num_ssis = 10; + + LocalityData localityData(Optional>(), + Standalone(deterministicRandom()->randomUniqueID().toString()), + Standalone(deterministicRandom()->randomUniqueID().toString()), + Optional>()); + + for (int i = 0; i < num_ssis; i++) { + StorageServerInterface ssi; + ssi.locality = localityData; + ssi.uniqueID = UID(0x1234123412341234 + i, 0x5678567856785678 + i); + ssi.acceptingRequests = i % 2; + ssi.initEndpoints(); + newssis.push_back(ssi); + } + + for (int i = 0; i < num_ssis; i++) { + StorageServerInterfaceOld oldssi; + StorageServerInterface newssi; + + auto value = serverListValueFB(newssis[i]); + oldssi = decodeServerListValueOld(value); + newssi = decodeServerListValue(value); + + ASSERT(oldssi.locality == newssis[i].locality); + ASSERT(oldssi.id() == newssis[i].id()); + ASSERT(oldssi.getValue.getEndpoint().token == newssis[i].getValue.getEndpoint().token); + + ASSERT(newssi.locality == newssis[i].locality); + ASSERT(newssi.id() == newssis[i].id()); + ASSERT(newssi.isAcceptingRequests() == newssis[i].isAcceptingRequests()); + ASSERT(newssi.getValue.getEndpoint().token == newssis[i].getValue.getEndpoint().token); + } + + return Void(); +} + +TEST_CASE("/SystemData/SSI/Upgrade") { + std::vector oldssis; + constexpr int num_ssis = 10; + + LocalityData localityData(Optional>(), + Standalone(deterministicRandom()->randomUniqueID().toString()), + Standalone(deterministicRandom()->randomUniqueID().toString()), + Optional>()); + + for (int i = 0; i < num_ssis; i++) { + StorageServerInterfaceOld ssi; + ssi.locality = localityData; + ssi.uniqueID = UID(0x1234123412341234 + i, 0x5678567856785678 + i); + ssi.initEndpoints(); + oldssis.push_back(ssi); + } + + for (int i = 0; i < num_ssis; i++) { + StorageServerInterfaceOld oldssi; + StorageServerInterface newssi; + + auto value = serverListValueOld(oldssis[i]); + oldssi = decodeServerListValueOld(value); + newssi = decodeServerListValue(value); + + ASSERT(oldssi.locality == oldssis[i].locality); + ASSERT(oldssi.id() == oldssis[i].id()); + ASSERT(oldssi.getValue.getEndpoint().token == oldssis[i].getValue.getEndpoint().token); + + ASSERT(newssi.locality == oldssis[i].locality); + ASSERT(newssi.id() == oldssis[i].id()); + ASSERT(newssi.isAcceptingRequests() == 0); + ASSERT(newssi.getValue.getEndpoint().token == oldssis[i].getValue.getEndpoint().token); + } + + return Void(); +} \ No newline at end of file diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 228c058d77..33c426add3 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -202,6 +202,7 @@ extern const KeyRangeRef serverListKeys; extern const KeyRef serverListPrefix; const Key serverListKeyFor(UID serverID); const Value serverListValue(StorageServerInterface const&); +const Value serverListValueFB(StorageServerInterface const&); UID decodeServerListKey(KeyRef const&); StorageServerInterface decodeServerListValue(ValueRef const&); diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index e5fb6b0bbe..bdc55219b3 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -121,7 +121,8 @@ public: newServers[serverId] = ssi; if (oldServers.count(serverId)) { - if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) { + if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint() || + ssi.isAcceptingRequests() != oldServers[serverId].isAcceptingRequests()) { serverChanges.send(std::make_pair(serverId, Optional(ssi))); } oldServers.erase(serverId); @@ -183,6 +184,7 @@ public: myQueueInfo->value.busiestReadTag = reply.get().busiestTag; myQueueInfo->value.busiestReadTagFractionalBusyness = reply.get().busiestTagFractionalBusyness; myQueueInfo->value.busiestReadTagRate = reply.get().busiestTagRate; + myQueueInfo->value.acceptingRequests = ssi.isAcceptingRequests(); } else { if (myQueueInfo->value.valid) { TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id()); @@ -255,7 +257,7 @@ public: when(state std::pair> change = waitNext(serverChanges)) { wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack if (change.second.present()) { - if (!change.second.get().isTss() && change.second.get().isAcceptingRequests()) { + if (!change.second.get().isTss()) { auto& a = actors[change.first]; a = Future(); a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err); @@ -523,7 +525,7 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) { // Look at each storage server's write queue and local rate, compute and store the desired rate ratio for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) { auto const& ss = i->value; - if (!ss.valid || (remoteDC.present() && ss.locality.dcId() == remoteDC)) + if (!ss.valid || !ss.acceptingRequests || (remoteDC.present() && ss.locality.dcId() == remoteDC)) continue; ++sscount; diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h index 8552eeb521..25f4140447 100644 --- a/fdbserver/Ratekeeper.h +++ b/fdbserver/Ratekeeper.h @@ -52,6 +52,7 @@ struct StorageQueueInfo { LocalityData locality; StorageQueuingMetricsReply lastReply; StorageQueuingMetricsReply prevReply; + bool acceptingRequests; Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes; Smoother smoothDurableVersion, smoothLatestVersion; Smoother smoothFreeSpace; @@ -70,8 +71,9 @@ struct StorageQueueInfo { int totalWriteOps = 0; StorageQueueInfo(UID id, LocalityData locality) - : valid(false), id(id), locality(locality), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), - smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), + : valid(false), id(id), locality(locality), acceptingRequests(false), + smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), + verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), limitReason(limitReason_t::unlimited), diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 9e693ab62f..ad9c235aca 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -7382,10 +7382,10 @@ ACTOR Future storageServerCore(StorageServer* self, StorageServerInterface self->coreStarted.send(Void()); - // if (self->registerInterfaceAcceptingRequests.canBeSet()) { - // self->registerInterfaceAcceptingRequests.send(true); - // wait(self->interfaceRegistered); - // } + if (self->registerInterfaceAcceptingRequests.canBeSet()) { + self->registerInterfaceAcceptingRequests.send(true); + wait(self->interfaceRegistered); + } loop { ++self->counters.loops; @@ -7576,91 +7576,6 @@ ACTOR Future initTenantMap(StorageServer* self) { return Void(); } -// for creating a new storage server -ACTOR Future storageServer(IKeyValueStore* persistentData, - StorageServerInterface ssi, - Tag seedTag, - UID clusterId, - Version tssSeedVersion, - ReplyPromise recruitReply, - Reference const> db, - std::string folder) { - state StorageServer self(persistentData, db, ssi); - state Future ssCore; - self.clusterId.send(clusterId); - if (ssi.isTss()) { - self.setTssPair(ssi.tssPairID.get()); - ASSERT(self.isTss()); - } - - self.sk = serverKeysPrefixFor(self.tssPairID.present() ? self.tssPairID.get() : self.thisServerID) - .withPrefix(systemKeys.begin); // FFFF/serverKeys/[this server]/ - self.folder = folder; - self.registerInterfaceAcceptingRequests.send(false); - - try { - wait(self.storage.init()); - wait(self.storage.commit()); - ++self.counters.kvCommits; - - ssi.startAcceptingRequests(); - - if (seedTag == invalidTag) { - // Might throw recruitment_failed in case of simultaneous master failure - std::pair verAndTag = wait(addStorageServer(self.cx, ssi)); - - self.tag = verAndTag.second; - if (ssi.isTss()) { - self.setInitialVersion(tssSeedVersion); - } else { - self.setInitialVersion(verAndTag.first - 1); - } - - wait(initTenantMap(&self)); - } else { - self.tag = seedTag; - } - - self.storage.makeNewStorageServerDurable(); - wait(self.storage.commit()); - ++self.counters.kvCommits; - - TraceEvent("StorageServerInit", ssi.id()) - .detail("Version", self.version.get()) - .detail("SeedTag", seedTag.toString()) - .detail("TssPair", ssi.isTss() ? ssi.tssPairID.get().toString() : ""); - InitializeStorageReply rep; - rep.interf = ssi; - rep.addedVersion = self.version.get(); - recruitReply.send(rep); - self.byteSampleRecovery = Void(); - - ssCore = storageServerCore(&self, ssi); - wait(ssCore); - - throw internal_error(); - } catch (Error& e) { - // If we die with an error before replying to the recruitment request, send the error to the recruiter - // (ClusterController, and from there to the DataDistributionTeamCollection) - if (!recruitReply.isSet()) - recruitReply.sendError(recruitment_failed()); - - // If the storage server dies while something that uses self is still on the stack, - // we want that actor to complete before we terminate and that memory goes out of scope - state Error err = e; - if (storageServerTerminated(self, persistentData, err)) { - ssCore.cancel(); - self.actors.clear(true); - wait(delay(0)); - return Void(); - } - ssCore.cancel(); - self.actors.clear(true); - wait(delay(0)); - throw err; - } -} - ACTOR Future replaceInterface(StorageServer* self, StorageServerInterface ssi) { ASSERT(!ssi.isTss()); state Transaction tr(self->cx); @@ -7838,6 +7753,95 @@ ACTOR Future storageInterfaceRegistration(StorageServer* self, return Void(); } +// for creating a new storage server +ACTOR Future storageServer(IKeyValueStore* persistentData, + StorageServerInterface ssi, + Tag seedTag, + UID clusterId, + Version tssSeedVersion, + ReplyPromise recruitReply, + Reference const> db, + std::string folder) { + state StorageServer self(persistentData, db, ssi); + state Future ssCore; + self.clusterId.send(clusterId); + if (ssi.isTss()) { + self.setTssPair(ssi.tssPairID.get()); + ASSERT(self.isTss()); + } + + self.sk = serverKeysPrefixFor(self.tssPairID.present() ? self.tssPairID.get() : self.thisServerID) + .withPrefix(systemKeys.begin); // FFFF/serverKeys/[this server]/ + self.folder = folder; + + try { + wait(self.storage.init()); + wait(self.storage.commit()); + ++self.counters.kvCommits; + + if (seedTag == invalidTag) { + ssi.startAcceptingRequests(); + self.registerInterfaceAcceptingRequests.send(false); + + // Might throw recruitment_failed in case of simultaneous master failure + std::pair verAndTag = wait(addStorageServer(self.cx, ssi)); + + self.tag = verAndTag.second; + if (ssi.isTss()) { + self.setInitialVersion(tssSeedVersion); + } else { + self.setInitialVersion(verAndTag.first - 1); + } + + wait(initTenantMap(&self)); + } else { + self.tag = seedTag; + } + + self.interfaceRegistered = + storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); + wait(delay(0)); + + self.storage.makeNewStorageServerDurable(); + wait(self.storage.commit()); + ++self.counters.kvCommits; + + TraceEvent("StorageServerInit", ssi.id()) + .detail("Version", self.version.get()) + .detail("SeedTag", seedTag.toString()) + .detail("TssPair", ssi.isTss() ? ssi.tssPairID.get().toString() : ""); + InitializeStorageReply rep; + rep.interf = ssi; + rep.addedVersion = self.version.get(); + recruitReply.send(rep); + self.byteSampleRecovery = Void(); + + ssCore = storageServerCore(&self, ssi); + wait(ssCore); + + throw internal_error(); + } catch (Error& e) { + // If we die with an error before replying to the recruitment request, send the error to the recruiter + // (ClusterController, and from there to the DataDistributionTeamCollection) + if (!recruitReply.isSet()) + recruitReply.sendError(recruitment_failed()); + + // If the storage server dies while something that uses self is still on the stack, + // we want that actor to complete before we terminate and that memory goes out of scope + state Error err = e; + if (storageServerTerminated(self, persistentData, err)) { + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); + return Void(); + } + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); + throw err; + } +} + // for recovering an existing storage server ACTOR Future storageServer(IKeyValueStore* persistentData, StorageServerInterface ssi, From b4cfcc10d3d31590c9f5b6db5a93e418b06db04f Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Fri, 25 Mar 2022 11:36:35 -0700 Subject: [PATCH 27/90] Move Python tenant management to its own module --- bindings/python/fdb/__init__.py | 3 + bindings/python/fdb/impl.py | 46 ----------- bindings/python/fdb/tenant_management.py | 95 +++++++++++++++++++++ bindings/python/tests/tenant_tests.py | 96 ++++++++++++++++++---- bindings/python/tests/tester.py | 4 +- documentation/sphinx/source/api-python.rst | 35 ++++++-- 6 files changed, 209 insertions(+), 70 deletions(-) create mode 100644 bindings/python/fdb/tenant_management.py diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 413c81249a..a9d5f01810 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -100,6 +100,9 @@ def api_version(ver): _add_symbols(fdb.impl, list) + if ver >= 710: + import fdb.tenant_management + if ver < 610: globals()["init"] = getattr(fdb.impl, "init") globals()["open"] = getattr(fdb.impl, "open_v609") diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 023e85ae95..80a468a1fc 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -1178,52 +1178,6 @@ class Database(_TransactionCreator): self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer)) return Transaction(pointer.value, self) - def allocate_tenant(self, name): - Database.__database_allocate_tenant(self, process_tenant_name(name), []) - - def delete_tenant(self, name): - Database.__database_delete_tenant(self, process_tenant_name(name), []) - - # Attempt to allocate a tenant in the cluster. If the tenant already exists, - # this function will return a tenant_already_exists error. If the tenant is created - # concurrently, then this function may return success even if another caller creates - # it. - # - # The existence_check_marker is expected to be an empty list. This function will - # modify the list after completing the existence check to avoid checking for existence - # on retries. This allows the operation to be idempotent. - @staticmethod - @transactional - def __database_allocate_tenant(tr, name, existence_check_marker): - tr.options.set_special_key_space_enable_writes() - key = b'\xff\xff/management/tenant_map/%s' % name - if not existence_check_marker: - existing_tenant = tr[key].wait() - existence_check_marker.append(None) - if existing_tenant != None: - raise fdb.FDBError(2132) # tenant_already_exists - tr[key] = b'' - - # Attempt to remove a tenant in the cluster. If the tenant doesn't exist, this - # function will return a tenant_not_found error. If the tenant is deleted - # concurrently, then this function may return success even if another caller deletes - # it. - # - # The existence_check_marker is expected to be an empty list. This function will - # modify the list after completing the existence check to avoid checking for existence - # on retries. This allows the operation to be idempotent. - @staticmethod - @transactional - def __database_delete_tenant(tr, name, existence_check_marker): - tr.options.set_special_key_space_enable_writes() - key = b'\xff\xff/management/tenant_map/%s' % name - if not existence_check_marker: - existing_tenant = tr[key].wait() - existence_check_marker.append(None) - if existing_tenant == None: - raise fdb.FDBError(2131) # tenant_not_found - del tr[key] - class Tenant(_TransactionCreator): def __init__(self, tpointer): diff --git a/bindings/python/fdb/tenant_management.py b/bindings/python/fdb/tenant_management.py new file mode 100644 index 0000000000..b371a34226 --- /dev/null +++ b/bindings/python/fdb/tenant_management.py @@ -0,0 +1,95 @@ +# +# tenant_management.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# FoundationDB Python API + +"""Documentation for this API can be found at +https://apple.github.io/foundationdb/api-python.html""" + +from fdb import impl as _impl + +_tenant_map_prefix = b'\xff\xff/management/tenant_map/' + +# If the existence_check_marker is an empty list, then check whether the tenant exists. +# After the check, append an item to the existence_check_marker list so that subsequent +# calls to this function will not perform the existence check. +# +# If the existence_check_marker is a non-empty list, return None. +def _check_tenant_existence(tr, key, existence_check_marker, force_maybe_commited): + if not existence_check_marker: + existing_tenant = tr[key].wait() + existence_check_marker.append(None) + if force_maybe_commited: + raise _impl.FDBError(1021) # maybe_committed + return existing_tenant != None + + return None + +# Attempt to create a tenant in the cluster. If existence_check_marker is an empty +# list, then this function will check if the tenant already exists and fail if it does. +# Once the existence check is completed, it will not be done again if this function +# retries. As a result, this function may return successfully if the tenant is created +# by someone else concurrently. This behavior allows the operation to be idempotent with +# respect to retries. +# +# If the existence_check_marker is a non-empty list, then the existence check is skipped. +@_impl.transactional +def _create_tenant_impl(tr, tenant_name, existence_check_marker, force_existence_check_maybe_committed=False): + tr.options.set_special_key_space_enable_writes() + key = b'%s%s' % (_tenant_map_prefix, tenant_name) + + if _check_tenant_existence(tr, key, existence_check_marker, force_existence_check_maybe_committed) is True: + raise _impl.FDBError(2132) # tenant_already_exists + + tr[key] = b'' + +# Attempt to delete a tenant from the cluster. If existence_check_marker is an empty +# list, then this function will check if the tenant already exists and fail if it does +# not. Once the existence check is completed, it will not be done again if this function +# retries. As a result, this function may return successfully if the tenant is deleted +# by someone else concurrently. This behavior allows the operation to be idempotent with +# respect to retries. +# +# If the existence_check_marker is a non-empty list, then the existence check is skipped. +@_impl.transactional +def _delete_tenant_impl(tr, tenant_name, existence_check_marker, force_existence_check_maybe_committed=False): + tr.options.set_special_key_space_enable_writes() + key = b'%s%s' % (_tenant_map_prefix, tenant_name) + + if _check_tenant_existence(tr, key, existence_check_marker, force_existence_check_maybe_committed) is False: + raise _impl.FDBError(2131) # tenant_not_found + + del tr[key] + +def create_tenant(db_or_tr, tenant_name): + tenant_name = _impl.process_tenant_name(tenant_name) + + # Only perform the existence check when run using a database + # Callers using a transaction are expected to check existence themselves if required + existence_check_marker = [] if not isinstance(db_or_tr, _impl.TransactionRead) else [None] + _create_tenant_impl(db_or_tr, tenant_name, existence_check_marker) + +def delete_tenant(db_or_tr, tenant_name): + tenant_name = _impl.process_tenant_name(tenant_name) + + # Only perform the existence check when run using a database + # Callers using a transaction are expected to check existence themselves if required + existence_check_marker = [] if not isinstance(db_or_tr, _impl.TransactionRead) else [None] + _delete_tenant_impl(db_or_tr, tenant_name, existence_check_marker) diff --git a/bindings/python/tests/tenant_tests.py b/bindings/python/tests/tenant_tests.py index 9f35620b6a..81c00fbdce 100755 --- a/bindings/python/tests/tenant_tests.py +++ b/bindings/python/tests/tenant_tests.py @@ -26,9 +26,22 @@ from fdb.tuple import pack if __name__ == '__main__': fdb.api_version(710) +def cleanup_tenant(db, tenant_name): + try: + tenant = db.open_tenant(tenant_name) + del tenant[:] + fdb.tenant_management.delete_tenant(db, tenant_name) + except fdb.FDBError as e: + if e.code == 2131: # tenant not found + pass + else: + raise + def test_tenant_tuple_name(db): tuplename=(b'test', b'level', b'hierarchy', 3, 1.24, 'str') - db.allocate_tenant(tuplename) + cleanup_tenant(db, tuplename) + + fdb.tenant_management.create_tenant(db, tuplename) tenant=db.open_tenant(tuplename) tenant[b'foo'] = b'bar' @@ -36,25 +49,15 @@ def test_tenant_tuple_name(db): assert tenant[b'foo'] == b'bar' del tenant[b'foo'] - db.delete_tenant(tuplename) + fdb.tenant_management.delete_tenant(db, tuplename) -def cleanup_tenant(db, tenant_name): - try: - tenant = db.open_tenant(tenant_name) - del tenant[:] - db.delete_tenant(tenant_name) - except fdb.FDBError as e: - if e.code == 2131: # tenant not found - pass - else: - raise def test_tenant_operations(db): cleanup_tenant(db, b'tenant1') cleanup_tenant(db, b'tenant2') - db.allocate_tenant(b'tenant1') - db.allocate_tenant(b'tenant2') + fdb.tenant_management.create_tenant(db, b'tenant1') + fdb.tenant_management.create_tenant(db, b'tenant2') tenant1 = db.open_tenant(b'tenant1') tenant2 = db.open_tenant(b'tenant2') @@ -90,7 +93,7 @@ def test_tenant_operations(db): assert db[prefix2 + b'tenant_test_key'] == b'tenant2' assert db[b'tenant_test_key'] == b'no_tenant' - db.delete_tenant(b'tenant1') + fdb.tenant_management.delete_tenant(db, b'tenant1') try: tenant1[b'tenant_test_key'] assert False @@ -98,7 +101,7 @@ def test_tenant_operations(db): assert e.code == 2131 # tenant not found del tenant2[:] - db.delete_tenant(b'tenant2') + fdb.tenant_management.delete_tenant(db, b'tenant2') assert db[prefix1 + b'tenant_test_key'] == None assert db[prefix2 + b'tenant_test_key'] == None @@ -108,9 +111,70 @@ def test_tenant_operations(db): assert db[b'tenant_test_key'] == None +def test_tenant_operation_retries(db): + cleanup_tenant(db, b'tenant1') + cleanup_tenant(db, b'tenant2') + + # Test that the tenant creation only performs the existence check once + fdb.tenant_management._create_tenant_impl(db, b'tenant1', [], force_existence_check_maybe_committed=True) + + # An attempt to create the tenant again should fail + try: + fdb.tenant_management.create_tenant(db, b'tenant1') + assert False + except fdb.FDBError as e: + assert e.code == 2132 # tenant already exists + + # Using a transaction skips the existence check + tr = db.create_transaction() + fdb.tenant_management.create_tenant(tr, b'tenant1') + + # Test that a concurrent tenant creation doesn't interfere with the existence check logic + tr = db.create_transaction() + existence_check_marker = [] + fdb.tenant_management._create_tenant_impl(tr, b'tenant2', existence_check_marker) + + fdb.tenant_management.create_tenant(db, b'tenant2') + + tr = db.create_transaction() + try: + fdb.tenant_management._create_tenant_impl(tr, b'tenant2', existence_check_marker) + tr.commit().wait() + except fdb.FDBError as e: + tr.on_error(e).wait() + + # Test that tenant deletion only performs the existence check once + fdb.tenant_management._delete_tenant_impl(db, b'tenant1', [], force_existence_check_maybe_committed=True) + + # An attempt to delete the tenant again should fail + try: + fdb.tenant_management.delete_tenant(db, b'tenant1') + assert False + except fdb.FDBError as e: + assert e.code == 2131 # tenant not found + + # Using a transaction skips the existence check + tr = db.create_transaction() + fdb.tenant_management.delete_tenant(tr, b'tenant1') + + # Test that a concurrent tenant deletion doesn't interfere with the existence check logic + tr = db.create_transaction() + existence_check_marker = [] + fdb.tenant_management._delete_tenant_impl(tr, b'tenant2', existence_check_marker) + + fdb.tenant_management.delete_tenant(db, b'tenant2') + + tr = db.create_transaction() + try: + fdb.tenant_management._delete_tenant_impl(tr, b'tenant2', existence_check_marker) + tr.commit().wait() + except fdb.FDBError as e: + tr.on_error(e).wait() + def test_tenants(db): test_tenant_tuple_name(db) test_tenant_operations(db) + test_tenant_operation_retries(db) # Expect a cluster file as input. This test will write to the FDB cluster, so # be aware of potential side effects. diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index 7f8d794207..c514d3a948 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -593,11 +593,11 @@ class Tester: inst.push(b"WAITED_FOR_EMPTY") elif inst.op == six.u("TENANT_CREATE"): name = inst.pop() - self.db.allocate_tenant(name) + fdb.tenant_management.create_tenant(self.db, name) inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("TENANT_DELETE"): name = inst.pop() - self.db.delete_tenant(name) + fdb.tenant_management.delete_tenant(self.db, name) inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("TENANT_SET_ACTIVE"): name = inst.pop() diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 5dab3e49c6..406735aad5 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -325,12 +325,6 @@ A |database-blurb1| |database-blurb2| .. |sync-read| replace:: This read is fully synchronous. .. |sync-write| replace:: This change will be committed immediately, and is fully synchronous. -.. method:: Database.allocate_tenant(tenant_name): - - Creates a new tenant in the cluster. |sync-write| - - The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. - .. method:: Database.delete_tenant(tenant_name): Delete a tenant from the cluster. |sync-write| @@ -1590,3 +1584,32 @@ Locality information .. method:: fdb.locality.get_addresses_for_key(tr, key) Returns a :class:`fdb.FutureStringArray`. You must call the :meth:`fdb.Future.wait()` method on this object to retrieve a list of public network addresses as strings, one for each of the storage servers responsible for storing ``key`` and its associated value. + +Tenant management +================= + +.. module:: fdb.tenant_management + +The FoundationDB API includes function to manage the set of tenants in a cluster. + +.. method:: fdb.tenant_management.create_tenant(db_or_tr, tenant_name) + + Creates a new tenant in the cluster. + + The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + + If a database is provided to this function for the ``db_or_tr`` parameter, then this function will first check if the tenant already exists. If it does, it will fail with a ``tenant_already_exists`` error. Otherwise, it will create a transaction and attempt to create the tenant in a retry loop. If the tenant is created concurrently by another transaction, this function may still return successfully. + + If a transaction is provided to this function for the ``db_or_tr`` parameter, then this function will not check if the tenant already exists. It is up to the user to perform that check if required. The user must also successfully commit the transaction in order for the creation to take effect. + +.. method:: fdb.tenant_management.delete_tenant(db_or_tr, tenant_name) + + Delete a tenant from the cluster. + + The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + + It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant. + + If a database is provided to this function for the ``db_or_tr`` parameter, then this function will first check if the tenant already exists. If it does not, it will fail with a ``tenant_not_found`` error. Otherwise, it will create a transaction and attempt to delete the tenant in a retry loop. If the tenant is deleted concurrently by another transaction, this function may still return successfully. + + If a transaction is provided to this function for the ``db_or_tr`` parameter, then this function will not check if the tenant already exists. It is up to the user to perform that check if required. The user must also successfully commit the transaction in order for the deletion to take effect. From 301e64a1b6f4773cc024185a6b2a58ac01c56849 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 25 Mar 2022 13:27:55 -0700 Subject: [PATCH 28/90] Remove unit tests added for SSI upgrade/downgrade --- fdbclient/StorageServerInterface.h | 145 ----------------------------- fdbclient/SystemData.cpp | 90 ------------------ 2 files changed, 235 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 8045ed0a74..342ee536b6 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -51,151 +51,6 @@ struct VersionReply { } }; -struct StorageServerInterfaceOld { - constexpr static FileIdentifier file_identifier = 15302073; - enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 }; - - enum { LocationAwareLoadBalance = 1 }; - enum { AlwaysFresh = 0 }; - - LocalityData locality; - UID uniqueID; - Optional tssPairID; - - RequestStream getValue; - RequestStream getKey; - - // Throws a wrong_shard_server if the keys in the request or result depend on data outside this server OR if a large - // selector offset prevents all data from being read in one range read - RequestStream getKeyValues; - RequestStream getMappedKeyValues; - - RequestStream getShardState; - RequestStream waitMetrics; - RequestStream splitMetrics; - RequestStream getStorageMetrics; - RequestStream> waitFailure; - RequestStream getQueuingMetrics; - - RequestStream> getKeyValueStoreType; - RequestStream watchValue; - RequestStream getReadHotRanges; - RequestStream getRangeSplitPoints; - RequestStream getKeyValuesStream; - RequestStream changeFeedStream; - RequestStream overlappingChangeFeeds; - RequestStream changeFeedPop; - RequestStream changeFeedVersionUpdate; - RequestStream checkpoint; - RequestStream fetchCheckpoint; - - explicit StorageServerInterfaceOld(UID uid) : uniqueID(uid) {} - StorageServerInterfaceOld() : uniqueID(deterministicRandom()->randomUniqueID()) {} - NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } - NetworkAddress stableAddress() const { return getValue.getEndpoint().getStableAddress(); } - Optional secondaryAddress() const { return getValue.getEndpoint().addresses.secondaryAddress; } - UID id() const { return uniqueID; } - bool isTss() const { return tssPairID.present(); } - std::string toString() const { return id().shortString(); } - template - void serialize(Ar& ar) { - // StorageServerInterface is persisted in the database, so changes here have to be versioned carefully! - // To change this serialization, ProtocolVersion::ServerListValue must be updated, and downgrades need to be - // considered - - if (ar.protocolVersion().hasSmallEndpoints()) { - if (ar.protocolVersion().hasTSS()) { - serializer(ar, uniqueID, locality, getValue, tssPairID); - } else { - serializer(ar, uniqueID, locality, getValue); - } - if (Ar::isDeserializing) { - getKey = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(1)); - getKeyValues = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(2)); - getShardState = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(3)); - waitMetrics = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(4)); - splitMetrics = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(5)); - getStorageMetrics = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(6)); - waitFailure = RequestStream>(getValue.getEndpoint().getAdjustedEndpoint(7)); - getQueuingMetrics = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(8)); - getKeyValueStoreType = - RequestStream>(getValue.getEndpoint().getAdjustedEndpoint(9)); - watchValue = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(10)); - getReadHotRanges = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(11)); - getRangeSplitPoints = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); - getKeyValuesStream = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(13)); - getMappedKeyValues = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(14)); - changeFeedStream = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(15)); - overlappingChangeFeeds = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(16)); - changeFeedPop = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(17)); - changeFeedVersionUpdate = RequestStream( - getValue.getEndpoint().getAdjustedEndpoint(18)); - checkpoint = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(19)); - fetchCheckpoint = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(20)); - } - } else { - ASSERT(Ar::isDeserializing); - if constexpr (is_fb_function) { - ASSERT(false); - } - serializer(ar, - uniqueID, - locality, - getValue, - getKey, - getKeyValues, - getShardState, - waitMetrics, - splitMetrics, - getStorageMetrics, - waitFailure, - getQueuingMetrics, - getKeyValueStoreType); - if (ar.protocolVersion().hasWatches()) { - serializer(ar, watchValue); - } - } - } - bool operator==(StorageServerInterfaceOld const& s) const { return uniqueID == s.uniqueID; } - bool operator<(StorageServerInterfaceOld const& s) const { return uniqueID < s.uniqueID; } - void initEndpoints() { - std::vector> streams; - streams.push_back(getValue.getReceiver(TaskPriority::LoadBalancedEndpoint)); - streams.push_back(getKey.getReceiver(TaskPriority::LoadBalancedEndpoint)); - streams.push_back(getKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint)); - streams.push_back(getShardState.getReceiver()); - streams.push_back(waitMetrics.getReceiver()); - streams.push_back(splitMetrics.getReceiver()); - streams.push_back(getStorageMetrics.getReceiver()); - streams.push_back(waitFailure.getReceiver()); - streams.push_back(getQueuingMetrics.getReceiver()); - streams.push_back(getKeyValueStoreType.getReceiver()); - streams.push_back(watchValue.getReceiver()); - streams.push_back(getReadHotRanges.getReceiver()); - streams.push_back(getRangeSplitPoints.getReceiver()); - streams.push_back(getKeyValuesStream.getReceiver(TaskPriority::LoadBalancedEndpoint)); - streams.push_back(getMappedKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint)); - streams.push_back(changeFeedStream.getReceiver()); - streams.push_back(overlappingChangeFeeds.getReceiver()); - streams.push_back(changeFeedPop.getReceiver()); - streams.push_back(changeFeedVersionUpdate.getReceiver()); - streams.push_back(checkpoint.getReceiver()); - streams.push_back(fetchCheckpoint.getReceiver()); - FlowTransport::transport().addEndpoints(streams); - } -}; - struct StorageServerInterface { constexpr static FileIdentifier file_identifier = 15302073; enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 4fa3ba219f..bca9c37b0f 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -587,12 +587,6 @@ const Key serverListKeyFor(UID serverID) { return wr.toValue(); } -const Value serverListValueOld(StorageServerInterfaceOld const& server) { - BinaryWriter wr(IncludeVersion(ProtocolVersion::withTSS())); - wr << server; - return wr.toValue(); -} - const Value serverListValue(StorageServerInterface const& server) { return serverListValueFB(server); } @@ -604,13 +598,6 @@ UID decodeServerListKey(KeyRef const& key) { return serverID; } -StorageServerInterfaceOld decodeServerListValueOld(ValueRef const& value) { - StorageServerInterfaceOld s; - BinaryReader reader(value, IncludeVersion(ProtocolVersion::withTSS())); - reader >> s; - return s; -} - StorageServerInterface decodeServerListValueFB(ValueRef const& value) { StorageServerInterface s; ObjectReader reader(value.begin(), IncludeVersion()); @@ -1444,80 +1431,3 @@ TEST_CASE("/SystemData/SSI/SerDes") { return Void(); } - -TEST_CASE("/SystemData/SSI/Downgrade") { - std::vector newssis; - constexpr int num_ssis = 10; - - LocalityData localityData(Optional>(), - Standalone(deterministicRandom()->randomUniqueID().toString()), - Standalone(deterministicRandom()->randomUniqueID().toString()), - Optional>()); - - for (int i = 0; i < num_ssis; i++) { - StorageServerInterface ssi; - ssi.locality = localityData; - ssi.uniqueID = UID(0x1234123412341234 + i, 0x5678567856785678 + i); - ssi.acceptingRequests = i % 2; - ssi.initEndpoints(); - newssis.push_back(ssi); - } - - for (int i = 0; i < num_ssis; i++) { - StorageServerInterfaceOld oldssi; - StorageServerInterface newssi; - - auto value = serverListValueFB(newssis[i]); - oldssi = decodeServerListValueOld(value); - newssi = decodeServerListValue(value); - - ASSERT(oldssi.locality == newssis[i].locality); - ASSERT(oldssi.id() == newssis[i].id()); - ASSERT(oldssi.getValue.getEndpoint().token == newssis[i].getValue.getEndpoint().token); - - ASSERT(newssi.locality == newssis[i].locality); - ASSERT(newssi.id() == newssis[i].id()); - ASSERT(newssi.isAcceptingRequests() == newssis[i].isAcceptingRequests()); - ASSERT(newssi.getValue.getEndpoint().token == newssis[i].getValue.getEndpoint().token); - } - - return Void(); -} - -TEST_CASE("/SystemData/SSI/Upgrade") { - std::vector oldssis; - constexpr int num_ssis = 10; - - LocalityData localityData(Optional>(), - Standalone(deterministicRandom()->randomUniqueID().toString()), - Standalone(deterministicRandom()->randomUniqueID().toString()), - Optional>()); - - for (int i = 0; i < num_ssis; i++) { - StorageServerInterfaceOld ssi; - ssi.locality = localityData; - ssi.uniqueID = UID(0x1234123412341234 + i, 0x5678567856785678 + i); - ssi.initEndpoints(); - oldssis.push_back(ssi); - } - - for (int i = 0; i < num_ssis; i++) { - StorageServerInterfaceOld oldssi; - StorageServerInterface newssi; - - auto value = serverListValueOld(oldssis[i]); - oldssi = decodeServerListValueOld(value); - newssi = decodeServerListValue(value); - - ASSERT(oldssi.locality == oldssis[i].locality); - ASSERT(oldssi.id() == oldssis[i].id()); - ASSERT(oldssi.getValue.getEndpoint().token == oldssis[i].getValue.getEndpoint().token); - - ASSERT(newssi.locality == oldssis[i].locality); - ASSERT(newssi.id() == oldssis[i].id()); - ASSERT(newssi.isAcceptingRequests() == 0); - ASSERT(newssi.getValue.getEndpoint().token == oldssis[i].getValue.getEndpoint().token); - } - - return Void(); -} \ No newline at end of file From 62b7e79482224f42c6c8413da477d69f32953e2a Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 25 Mar 2022 13:39:08 -0700 Subject: [PATCH 29/90] Retract changes to apply-metadata-mutations; the change to the storagecache in commit proxy data is not required unless the get request path is to be made sensitive to the SSI state --- fdbserver/ApplyMetadataMutation.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 728b69ea01..5afc862b92 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -40,12 +40,6 @@ Reference getStorageInfo(UID id, (*storageCache)[id] = storageInfo; } else { storageInfo = cacheItr->second; - if (!storageInfo->interf.isAcceptingRequests()) { - storageInfo->interf = decodeServerListValue(txnStateStore->readValue(serverListKeyFor(id)).get().get()); - if (storageInfo->interf.isAcceptingRequests()) { - TraceEvent(SevInfo, "StorageInfoUpdatedAcceptingRequests", storageInfo->interf.id()).log(); - } - } } return storageInfo; } @@ -238,7 +232,7 @@ private: txnStateStore->set(KeyValueRef(m.param1, m.param2)); if (storageCache) { auto cacheItr = storageCache->find(id); - if (cacheItr == storageCache->end() || !cacheItr->second->interf.isAcceptingRequests()) { + if (cacheItr == storageCache->end()) { Reference storageInfo = makeReference(); storageInfo->tag = tag; Optional interfKey = txnStateStore->readValue(serverListKeyFor(id)).get(); From 48447c2788f5b7ddb69c4e8b89a530d68548e8a7 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Fri, 25 Mar 2022 13:32:15 -0700 Subject: [PATCH 30/90] Add the tenant management module to CMakeLists. Don't test tenants before API version 710. --- bindings/bindingtester/bindingtester.py | 2 +- bindings/python/CMakeLists.txt | 1 + bindings/python/tests/tester.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index d914e9d9dc..2856d35855 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -202,7 +202,7 @@ class TestRunner(object): self.args.types = list(reduce(lambda t1, t2: filter(t1.__contains__, t2), map(lambda tester: tester.types, self.testers))) self.args.no_directory_snapshot_ops = self.args.no_directory_snapshot_ops or any([not tester.directory_snapshot_ops_enabled for tester in self.testers]) - self.args.no_tenants = self.args.no_tenants or any([not tester.tenants_enabled for tester in self.testers]) + self.args.no_tenants = self.args.no_tenants or any([not tester.tenants_enabled for tester in self.testers]) or self.args.api_version < 710 def print_test(self): test_instructions = self._generate_test() diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 2174050712..0f871d8c87 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -5,6 +5,7 @@ set(SRCS fdb/locality.py fdb/six.py fdb/subspace_impl.py + fdb/tenant_management.py fdb/tuple.py README.rst MANIFEST.in) diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index c514d3a948..936f7015c0 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -621,7 +621,8 @@ class Tester: test_size_limit_option(db) test_get_approximate_size(db) - test_tenants(db) + if fdb.get_api_version() >= 710: + test_tenants(db) except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) From aa515524eb900b40091f81ca4e4a514f52185123 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 25 Mar 2022 14:35:48 -0700 Subject: [PATCH 31/90] Fix initialization of interface registration promise --- fdbserver/storageserver.actor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index ad9c235aca..7485d0a1df 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -7781,7 +7781,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (seedTag == invalidTag) { ssi.startAcceptingRequests(); - self.registerInterfaceAcceptingRequests.send(false); + self.registerInterfaceAcceptingRequests.send(true); // Might throw recruitment_failed in case of simultaneous master failure std::pair verAndTag = wait(addStorageServer(self.cx, ssi)); @@ -7798,14 +7798,14 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, self.tag = seedTag; } - self.interfaceRegistered = - storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); - wait(delay(0)); - self.storage.makeNewStorageServerDurable(); wait(self.storage.commit()); ++self.counters.kvCommits; + self.interfaceRegistered = + storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); + wait(delay(0)); + TraceEvent("StorageServerInit", ssi.id()) .detail("Version", self.version.get()) .detail("SeedTag", seedTag.toString()) From f13c09eec704201625188929f028bf3d057ed862 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Sat, 26 Mar 2022 14:20:15 -0700 Subject: [PATCH 32/90] Refactor SSI registration actor for error handling --- fdbserver/storageserver.actor.cpp | 62 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 7485d0a1df..8fa6da021b 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -27,6 +27,7 @@ #include "fdbrpc/LoadBalance.h" #include "flow/ActorCollection.h" #include "flow/Arena.h" +#include "flow/Error.h" #include "flow/Hash3.h" #include "flow/Histogram.h" #include "flow/IRandom.h" @@ -5568,7 +5569,6 @@ ACTOR Future tssDelayForever() { ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { state double start; try { - // if (data->registerInterfaceAcceptingRequests.canBeSet()) { // data->registerInterfaceAcceptingRequests.send(true); // wait(data->interfaceRegistered); @@ -7384,7 +7384,12 @@ ACTOR Future storageServerCore(StorageServer* self, StorageServerInterface if (self->registerInterfaceAcceptingRequests.canBeSet()) { self->registerInterfaceAcceptingRequests.send(true); - wait(self->interfaceRegistered); + ErrorOr e = wait(errorOr(self->interfaceRegistered)); + if (e.isError()) { + TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed") + .detail("ServerID", ssi.id()) + .detail("Error", e.getError().code()); + } } loop { @@ -7729,25 +7734,7 @@ ACTOR Future storageInterfaceRegistration(StorageServer* self, wait(replaceInterface(self, ssi)); } } catch (Error& e) { - if (e.code() != error_code_worker_removed) { - throw; - } - state UID clusterId = wait(getClusterId(self)); - ASSERT(self->clusterId.isValid()); - UID durableClusterId = wait(self->clusterId.getFuture()); - ASSERT(durableClusterId.isValid()); - if (clusterId == durableClusterId) { - throw worker_removed(); - } - // When a storage server connects to a new cluster, it deletes its - // old data and creates a new, empty data file for the new cluster. - // We want to avoid this and force a manual removal of the storage - // servers' old data when being assigned to a new cluster to avoid - // accidental data loss. - TraceEvent(SevError, "StorageServerBelongsToExistingCluster") - .detail("ClusterID", durableClusterId) - .detail("NewClusterID", clusterId); - wait(Future(Never())); + throw; } return Void(); @@ -7901,19 +7888,36 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, state Future f = storageInterfaceRegistration(&self, ssi, registerInterface.getFuture()); wait(delay(0)); registerInterface.send(false); - wait(f); + ErrorOr e = wait(errorOr(f)); + if (e.isError()) { + Error e = f.getError(); + + if (e.code() != error_code_worker_removed) { + throw e; + } + state UID clusterId = wait(getClusterId(&self)); + ASSERT(self.clusterId.isValid()); + UID durableClusterId = wait(self.clusterId.getFuture()); + ASSERT(durableClusterId.isValid()); + if (clusterId == durableClusterId) { + throw worker_removed(); + } + // When a storage server connects to a new cluster, it deletes its + // old data and creates a new, empty data file for the new cluster. + // We want to avoid this and force a manual removal of the storage + // servers' old data when being assigned to a new cluster to avoid + // accidental data loss. + TraceEvent(SevWarn, "StorageServerBelongsToExistingCluster") + .detail("ServerID", ssi.id()) + .detail("ClusterID", durableClusterId) + .detail("NewClusterID", clusterId); + wait(Future(Never())); + } self.interfaceRegistered = storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture()); wait(delay(0)); - ASSERT(self.registerInterfaceAcceptingRequests.canBeSet()); - - if (self.registerInterfaceAcceptingRequests.canBeSet()) { - self.registerInterfaceAcceptingRequests.send(true); - wait(self.interfaceRegistered); - } - TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); ssCore = storageServerCore(&self, ssi); From aa8ab494a2882a8117430e6a8498211c87782725 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sat, 26 Mar 2022 19:37:09 -0700 Subject: [PATCH 33/90] Fix undefined behavior. --- fdbserver/DeltaTree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/DeltaTree.h b/fdbserver/DeltaTree.h index a00c2e4e79..e1bb0edfd0 100644 --- a/fdbserver/DeltaTree.h +++ b/fdbserver/DeltaTree.h @@ -1699,7 +1699,7 @@ public: int count = end - begin; numItems = count; nodeBytesDeleted = 0; - initialHeight = (uint8_t)log2(count) + 1; + initialHeight = count == 0 ? 0 : (uint8_t)log2(count) + 1; maxHeight = 0; // The boundary leading to the new page acts as the last time we branched right From f09bdc840c00d712487500b9e752d87cedb1964a Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sat, 26 Mar 2022 19:38:59 -0700 Subject: [PATCH 34/90] Fix undefined behavior where struct members are written to disk and restored later in a situation where they are unused but can contain random values that are not proper booleans, which ubsan complains about. --- fdbserver/VersionedBTree.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index c10f628a52..6fb18b0a79 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1496,8 +1496,8 @@ public: int64_t numEntries; int dataBytesPerPage; int pagesPerExtent; - bool usesExtents; - bool tailPageNewExtent; + bool usesExtents = false; + bool tailPageNewExtent = false; LogicalPageID prevExtentEndPageID; Cursor headReader; From 6d7a4b91c878a595e1e1e2b1f984de42b4688078 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Sun, 27 Mar 2022 12:37:38 -0700 Subject: [PATCH 35/90] Create a server knob to control server version convergence threshold before SSI registration. Watch version lag in SS update loop and register when within lag limit --- fdbclient/ServerKnobs.cpp | 1 + fdbclient/ServerKnobs.h | 1 + fdbserver/storageserver.actor.cpp | 26 ++++++++++++-------------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 0f0f0de89e..f6eb5642f4 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -650,6 +650,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( FETCH_KEYS_PARALLELISM, 2 ); init( FETCH_KEYS_LOWER_PRIORITY, 0 ); init( BUGGIFY_BLOCK_BYTES, 10000 ); + init( STORAGE_RECOVERY_VERSION_LAG_LIMIT, 2 * MAX_READ_TRANSACTION_LIFE_VERSIONS ); init( STORAGE_COMMIT_BYTES, 10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000; init( STORAGE_FETCH_BYTES, 2500000 ); if( randomize && BUGGIFY ) STORAGE_FETCH_BYTES = 500000; init( STORAGE_DURABILITY_LAG_REJECT_THRESHOLD, 0.25 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 4c2d708274..cbb8eacb04 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -586,6 +586,7 @@ public: int FETCH_KEYS_PARALLELISM; int FETCH_KEYS_LOWER_PRIORITY; int BUGGIFY_BLOCK_BYTES; + int64_t STORAGE_RECOVERY_VERSION_LAG_LIMIT; double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD; double STORAGE_DURABILITY_LAG_MIN_RATE; int STORAGE_COMMIT_BYTES; diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 8fa6da021b..b7747c24b5 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -5569,10 +5569,6 @@ ACTOR Future tssDelayForever() { ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { state double start; try { - // if (data->registerInterfaceAcceptingRequests.canBeSet()) { - // data->registerInterfaceAcceptingRequests.send(true); - // wait(data->interfaceRegistered); - // } // If we are disk bound and durableVersion is very old, we need to block updates or we could run out of // memory. This is often referred to as the storage server e-brake (emergency brake) @@ -5970,6 +5966,18 @@ ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { validate(data); + if ((data->lastTLogVersion - data->version.get()) < SERVER_KNOBS->STORAGE_RECOVERY_VERSION_LAG_LIMIT) { + if (data->registerInterfaceAcceptingRequests.canBeSet()) { + data->registerInterfaceAcceptingRequests.send(true); + ErrorOr e = wait(errorOr(data->interfaceRegistered)); + if (e.isError()) { + TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed") + .detail("ServerID", data->thisServerID) + .detail("Error", e.getError().code()); + } + } + } + data->logCursor->advanceTo(cloneCursor2->version()); if (cursor->version().version >= data->lastTLogVersion) { if (data->behind) { @@ -7382,16 +7390,6 @@ ACTOR Future storageServerCore(StorageServer* self, StorageServerInterface self->coreStarted.send(Void()); - if (self->registerInterfaceAcceptingRequests.canBeSet()) { - self->registerInterfaceAcceptingRequests.send(true); - ErrorOr e = wait(errorOr(self->interfaceRegistered)); - if (e.isError()) { - TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed") - .detail("ServerID", ssi.id()) - .detail("Error", e.getError().code()); - } - } - loop { ++self->counters.loops; From 31812d7ad3f96325843a2a508aebd1993ef163a2 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 23 Mar 2022 23:37:32 -0700 Subject: [PATCH 36/90] Fix aggressive storage migration mode to behave as documented and migrates all storages at once. --- fdbserver/DDTeamCollection.actor.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index d83ded0edc..bdc6259a98 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -1381,7 +1381,10 @@ public: bool foundSSToRemove = false; for (auto& server : self->server_info) { - if (!server.second->isCorrectStoreType(self->configuration.storageServerStoreType)) { + // If this server isn't the right storage type and its wrong-type trigger has not yet been set + // then set it if we're in aggressive mode and log its presence either way. + if (!server.second->isCorrectStoreType(self->configuration.storageServerStoreType) && + !server.second->wrongStoreTypeToRemove.get()) { // Server may be removed due to failure while the wrongStoreTypeToRemove is sent to the // storageServerTracker. This race may cause the server to be removed before react to // wrongStoreTypeToRemove @@ -1394,12 +1397,16 @@ public: TraceEvent("WrongStoreTypeRemover", self->distributorId) .detail("Server", server.first) .detail("StoreType", server.second->getStoreType()) - .detail("ConfiguredStoreType", self->configuration.storageServerStoreType); - break; + .detail("ConfiguredStoreType", self->configuration.storageServerStoreType) + .detail("RemovingNow", + self->configuration.storageMigrationType == StorageMigrationType::AGGRESSIVE); } } - if (!foundSSToRemove) { + // Stop if no incorrect storage types were found, or if we're not in aggressive mode and can't act on any + // found. Aggressive mode is checked at this location so that in non-aggressive mode the loop will execute + // once and log any incorrect storage types found. + if (!foundSSToRemove || self->configuration.storageMigrationType != StorageMigrationType::AGGRESSIVE) { break; } } From 4d277fe19ab5a5e9805f7f935545ceca4c112aef Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 28 Mar 2022 13:06:17 -0700 Subject: [PATCH 37/90] Fix typo --- documentation/sphinx/source/api-python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 406735aad5..91a5f0da5a 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -1590,7 +1590,7 @@ Tenant management .. module:: fdb.tenant_management -The FoundationDB API includes function to manage the set of tenants in a cluster. +The FoundationDB API includes functions to manage the set of tenants in a cluster. .. method:: fdb.tenant_management.create_tenant(db_or_tr, tenant_name) From 643c0004c752733bb4065cd5d7922aacfbdb78a2 Mon Sep 17 00:00:00 2001 From: Ata E Husain Bohra Date: Mon, 28 Mar 2022 16:37:00 -0700 Subject: [PATCH 38/90] FDB Encryption data at-rest design documentation (#6629) * FDB Encryption data at-rest design documentation Patch details FDB Encryption data at-rest design documentation --- design/encryption-data-at-rest.md | 237 ++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 design/encryption-data-at-rest.md diff --git a/design/encryption-data-at-rest.md b/design/encryption-data-at-rest.md new file mode 100644 index 0000000000..135a5e5105 --- /dev/null +++ b/design/encryption-data-at-rest.md @@ -0,0 +1,237 @@ +# FDB Encryption **data at-rest** + +## Threat Model + +The proposed solution is `able to handle` the following attacks: + +* An attacker, if able to get access to any FDB cluster host or attached disk, would not be able to read the persisted data. Further, for cloud deployments, returning a cloud instance back to the cloud provider will prevent the cloud provider from reading the contents of data stored on the disk. + +* Data stored on a lost or stolen FDB host persistent disk storage device can’t be recovered. + +The proposed solution `will not be able` to handle the following attacks: + +* Encryption is enabled for data at-rest only, generating a memory dump of FDB processes could enable an attacker to read in-memory data contents. +* An FDB cluster host access, if compromised, would allow an attacker to read/write data managed by the FDB cluster. + +## Goals + +FoundationDB being a multi-model, easily scalable and fault-tolerant, with an ability to provide great performance even with commodity hardware, plays a critical role enabling enterprises to deploy, manage and run mission critical applications. + +Data encryption support is a table-stake feature for modern day enterprise service offerings in the cloud. Customers expect, and at times warrant, that their data and metadata be fully encrypted using the latest security standards. The goal of this document includes: + +* Discuss detailed design to support data at-rest encryption support for data stored in FDB clusters. Encrypting data in-transit and/or in-memory caches at various layers in the query execution pipeline (inside and external to FDB) is out of the scope of this feature. + +* Isolation guarantees: the encryption domain matches with `tenant` partition semantics supported by FDB clusters. Tenants are discrete namespaces in FDB that serve as transaction domains. A tenant is a `identifier` that maps to a `prefix` within the data-FDB cluster, and all operations within a tenant are implicitly bound within a `tenant-prefix`. Refer to `Multi-Tenant FoundationDB API` documentation more details. However, it is possible to use a single encryption key for the whole cluster, in case `tenant partitioning` isn’t available. + +* Ease of integration with external Key Management Services enabling persisting, caching, and lookup of encryption keys. + +## Config Knobs + +* `ServerKnob::ENABLE_ENCRYPION` allows enable/disable encryption feature. +* `ServerKnob::ENCRYPTION_MODE` controls the encryption mode supported. The current scheme supports `AES-256-CTR` encryption mode. + +## Encryption Mode + +The proposal is to use strong AES-256 CTR encryption mode. Salient properties are: + +* HMAC_SHA256 key hashing technique is used to derive encryption keys using a base encryption key and locally generated random number. The formula used is as follows: + +``` + DEK = HMAC SHA256(BEK || UID) + +Where +DEK = Derived Encryption Key +BEK = Base Encryption key +UID = Host local random generated number +``` + +UID is an 8 byte host-local random number. Another option would have been a simple host-local incrementing counter, however, the scheme runs the risk of repeated encryption-key generation on cluster/process restarts. + +* An encryption key derived using the above formula will be cached (in-memory) for a short time interval (10 mins, for instance). The encryption-key is immutable, but, the TTL approach allows refreshing encryption key by reaching out to External Encryption KeyManagement solutions, hence, supporting “restricting lifetime of an encryption” feature if implemented by Encryption Key Management solution. + +* Initialization Vector (IV) selection would be random. + +## Architecture + +The encryption responsibilities are split across multiple modules to ensure data and metadata stored in the cluster is never persisted in plain text on any durable storages (temporary and/or long-term durable storage). + +## Encryption Request Workflow + +### **Write Request** + +* An FDB client initiates a write transaction providing {key, value} in plaintext format. +* An FDB cluster host as part of processing a write transaction would do the following: + 1. Obtain required encryption key based on the transaction request tenant information. + 2. Encrypt mutations before persisting them on Transaction Logs (TLogs). As a background process, the mutations are moved to a long-term durable storage by the Storage Server processes. + +Refer to the sections below for more details. + +### **Read Request** + +* An FDB client initiates a read transaction request. +* An FDB cluster host as part of processing request would do the following: + 1. StorageServer would read desired data blocks from the persistent storage. + 2. Regenerate the encryption key required to decrypt the data. + 3. Decrypt data and pass results as plaintext to the caller. + + +Below diagram depicts the end-to-end encryption workflow detailing various modules involved and their interactions. The following section discusses detailed design for involved components. + +``` + _______________________________________________________ + | FDB CLUSER HOST | + | | + _____________________ | ________________________ _________________ | + | | (proprietary) | | | | | + | |<---------- |--| KMS CONNECTOR | | COMMIT PROXIES | | + | ENCRYPTION KEY | | | | | | | + | MANAGEMENT SOLUTION | | |(non FDB - proprietary) | | | | + | | | |________________________| |_________________| | + | | | ^ | | + |_____________________| | | (REST API) | (Encrypt | + | | V Mutation) | + | _________________________________________ | __________________ + | | | | | | + | | ENCRYPT KEYPROXY SERVER |<------|-----------| | + | |_________________________________________| | | | + | | | | BACKUP FILES | + | | (Encrypt Node) | | | + | V | | | + | _________________________________________ | | (Encrypt file) | + | | |<------|-----------| | + | | REDWOOD STORAGE SERVER | | |__________________| + | |_________________________________________| | + |_______________________________________________________| +``` + +## FDB Encryption + +An FDB client would insert data i.e. plaintext {key, value} in a FDB cluster for persistence. + +### KMS-Connector + +A non-FDB process running on FDB cluster hosts enables an FDB cluster to interact with external Encryption Key Managements services. Salient features includes: + +* An external (non-FDB) standalone process implementing a REST server. + +* Abstracts organization specific KeyManagementService integration details. The proposed design ensures ease of integration given limited infrastructure needed to implement a local/remote REST server. + +* Ensure organization specific code is implemented outside the FDB codebase. + +* The KMS-Connector process is launched and maintained by the FDBMonitor. The process needs to handle the following REST endpoint: + 1. GET - http://localhost/getEncryptionKey + + Define a single interface returning “encryption key string in plaintext” and accepting an + JSON input which can be customized as needed: + +```json + json_input_payload + { + “Version” : int // version + “KeyId” : keyId // string + } +``` + +Few benefits of the above proposed schemes are: +* JSON input format is extensible (adding new fields is backward compatible). + +* Popular Cloud KMS “getPublicKey” API accepts “keyId” as a string, hence, API should be easy to integrate. + + 1. AWS: https://docs.aws.amazon.com/cli/latest/reference/kms/get-public-key.html + 2. GCP: https://cloud.google.com/kms/docs/retrieve-public-key + +`Future improvements`: FDBMonitor at present will launch one KMS-Connector process per FDB cluster host. Though multiple KMS-Connector processes are launched, only one process (collocated with EncryptKeyServer) would consume cluster resources. In future, possible enhancements could be: + +* Enable FDBMonitor to launch “N” (configurable) processes per cluster. +* Enable the FDB cluster to manage external processes as well. + +### Encrypt KeyServer + +Salient features include: + +* New FDB role/process to allow fetching of encryption keys from external KeyManagementService interfaces. The process connects to the KMS-Connector REST interface to fetch desired encryption keys. + +* On an encryption-key fetch from KMS-Connector, it applies HMAC derivative function to generate a new encryption key and cache it in-memory. The in-memory cache is used to serve encryption key fetch requests from other FDB processes. + + +Given encryption keys will be needed as part of cluster-recovery, this process/role needs to be recruited at the start of the cluster-recovery process (just after the “master/sequencer” process/role recruitment). All other FDB processes will interact with this process to obtain encryption keys needed to encrypt and/or decrypt the data payload. + +`Note`: An alternative would be to incorporate the functionality into the ClusterController process itself, however, having clear responsibility separation would make design more flexible and extensible in future if needed. + +### Commit Proxies (CPs) + +When a FDB client initiates a write transaction to insert/update data stored in a FDB cluster, the transaction is received by a CP, which then resolves the transaction by checking if the transaction is allowed. If allowed, it commits the transaction to TLogs. The proposal is to extend CP responsibilities by encrypting mutations using the desired encryption key before mutations get persisted into TLogs (durable storage). The encryption key derivation is achieved using the following formula: + +``` + DEK = HMAC SHA256(BEK || UID) + +Where: + +DEK = Derived Encryption Key +BEK = Base Encryption Key +UID = Host local random generated number +``` + +The Transaction State Store (commonly referred as TxnStateStore) is a Key-Value datastore used by FDB to store metadata about the database itself for bootstrap purposes. The data stored in this store plays a critical role in: guiding the transaction system to persist writes (storage tags to mutations at CPs), and managing FDB internal data movement. The TxnStateStore data gets encrypted with the desired encryption key before getting persisted on the disk queues. + +As part of encryption, every Mutation would be appended by a plaintext `BlobCipherEncryptHeader` to assist decrypting the information for reads. + +CPs would cache (in-memory) recently used encryption-keys to optimize network traffic due to encryption related operations. Further, the caching would improve overall performance, avoiding frequent RPC calls to EncryptKeyServer which may eventually become a scalability bottleneck. Each encryption-key in the cache has a short Time-To-Live (10 mins) and on expiry the process will interact with the EncryptKeyServer to fetch the required encryption-keys. The same caching policy is followed by the Redwood Storage Server and the Backup File processes too. + +### **Caveats** + +The encryption is done inline in the transaction path, which will increase the total commit latencies. Few possible ways to minimize this impact are: + +* Overlap encryption operations with the CP::resolution phase, which would minimize the latency penalty per transaction at the cost of spending more CPU cycles. If needed, for production deployments, we may need to increase the number of CPs per FDB cluster. +* Implement an external process to offload encryption. If done, encryption would appear no different than the CP::resolution phase, where the process would invoke RPC calls to encrypt the buffer and wait for operation completion. + +### Storage Servers + +The encryption design only supports Redwood Storage Server integration, support for other storage engines is yet to be planned. + +### Redwood Storage Nodes + +Redwood at heart is a B+ tree and stores data in two types of nodes: + +* `Non-leaf` nodes: Nodes will only store keys and not values(prefix compression is applied). +* `Leaf` Nodes: Will store `{key, value}` tuples for a given key-range. + +Both above-mentioned nodes will be converted into one or more fixed size pages (likely 4K or 8K) before being persisted on a durable storage. The encryption will be performed at the node level instead of “page level”, i.e. all pages constituting a given Redwood node will be encrypted using the same encryption key generated using the following formula: + +``` + DEK = HMAC SHA256(BEK || UID) + +Where: + +DEK = Derived Encryption Key +BEK = Base Encryption Key +UID = Host local random generated number +``` + +### Backup Files + +Backup Files are designed to pull committed mutations from StorageServers and persist them as “files” stored on cloud backed BlobStorage such as Amazon S3. Each persisted file stores mutations for a given key-range and will be encrypted by generating an encryption key using below formula: + +``` + DEK = HMAC SHA256(BEK || FID) + +Where: + +DEK = Derived Encryption Key +BEK = Base Encryption Key +FID = File Identifier (unique) +``` + +## Decryption on Reads + +To assist reads, FDB processes (StorageServers, Backup Files workers) will be modified to read/parse the encryption header. The data decryption will be done as follows: + +* The FDB process will interact with Encrypt KeyServer to fetch the desired base encryption key corresponding to the key-id persisted in the encryption header. +* Reconstruct the encryption key and decrypt the data block. + +## Future Work + +* Extend the TLog API to allow clients to read “plaintext mutations” directly from a TLogServer. In current implementations there are two consumers of TLogs: + + 1. Storage Server: At present the plan is for StorageServer to decrypt the mutations. + 2. BackupWorker (Apple implementation) which is currently not used in the code. From 0a332ee1c17975552784a5689cbf12367fa05302 Mon Sep 17 00:00:00 2001 From: Renxuan Wang Date: Mon, 28 Mar 2022 17:10:49 -0700 Subject: [PATCH 39/90] Add proxy option to backup and restore params. --- fdbbackup/FileConverter.actor.cpp | 8 ++- fdbbackup/FileDecoder.actor.cpp | 8 ++- fdbbackup/backup.actor.cpp | 72 ++++++++++++++----- fdbclient/BackupAgent.actor.h | 28 ++++++-- fdbclient/BackupContainer.actor.cpp | 16 +++-- fdbclient/BackupContainer.h | 9 ++- fdbclient/BackupContainerFileSystem.actor.cpp | 19 +++-- fdbclient/BackupContainerFileSystem.h | 6 +- fdbclient/FileBackupAgent.actor.cpp | 24 +++++-- fdbclient/RestoreInterface.h | 15 ++-- fdbclient/S3BlobStore.actor.cpp | 23 ++++-- fdbclient/S3BlobStore.h | 23 ++++-- fdbserver/BlobManager.actor.cpp | 2 +- fdbserver/BlobWorker.actor.cpp | 2 +- fdbserver/RestoreController.actor.cpp | 12 ++-- fdbserver/RestoreController.actor.h | 8 ++- fdbserver/RestoreLoader.actor.cpp | 2 +- fdbserver/RestoreLoader.actor.h | 4 +- fdbserver/RestoreWorkerInterface.actor.h | 4 +- fdbserver/workloads/AtomicRestore.actor.cpp | 1 + ...kupAndParallelRestoreCorrectness.actor.cpp | 7 +- .../workloads/BackupCorrectness.actor.cpp | 11 ++- fdbserver/workloads/BackupToBlob.actor.cpp | 1 + .../BlobGranuleCorrectnessWorkload.actor.cpp | 4 +- .../workloads/BlobGranuleVerifier.actor.cpp | 4 +- .../workloads/IncrementalBackup.actor.cpp | 8 ++- fdbserver/workloads/RestoreBackup.actor.cpp | 1 + fdbserver/workloads/RestoreFromBlob.actor.cpp | 4 +- fdbserver/workloads/SubmitBackup.actor.cpp | 1 + 29 files changed, 232 insertions(+), 95 deletions(-) diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp index 1e48bd523d..8aeea5017f 100644 --- a/fdbbackup/FileConverter.actor.cpp +++ b/fdbbackup/FileConverter.actor.cpp @@ -101,6 +101,7 @@ std::vector getRelevantLogFiles(const std::vector& files, Vers struct ConvertParams { std::string container_url; + Optional proxy; Version begin = invalidVersion; Version end = invalidVersion; bool log_enabled = false; @@ -112,6 +113,10 @@ struct ConvertParams { std::string s; s.append("ContainerURL:"); s.append(container_url); + if (proxy.present()) { + s.append(" Proxy:"); + s.append(proxy.get()); + } s.append(" Begin:"); s.append(format("%" PRId64, begin)); s.append(" End:"); @@ -448,7 +453,8 @@ private: }; ACTOR Future convert(ConvertParams params) { - state Reference container = IBackupContainer::openContainer(params.container_url); + state Reference container = + IBackupContainer::openContainer(params.container_url, params.proxy, {}); state BackupFileList listing = wait(container->dumpFileList()); std::sort(listing.logs.begin(), listing.logs.end()); TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index 7e851bf6e0..7d9e27dcb1 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -94,6 +94,7 @@ void printBuildInformation() { struct DecodeParams { std::string container_url; + Optional proxy; std::string fileFilter; // only files match the filter will be decoded bool log_enabled = true; std::string log_dir, trace_format, trace_log_group; @@ -115,6 +116,10 @@ struct DecodeParams { std::string s; s.append("ContainerURL: "); s.append(container_url); + if (proxy.present()) { + s.append(", Proxy: "); + s.append(proxy.get()); + } s.append(", FileFilter: "); s.append(fileFilter); if (log_enabled) { @@ -526,7 +531,8 @@ ACTOR Future process_file(Reference container, LogFile f } ACTOR Future decode_logs(DecodeParams params) { - state Reference container = IBackupContainer::openContainer(params.container_url); + state Reference container = + IBackupContainer::openContainer(params.container_url, params.proxy, {}); state UID uid = deterministicRandom()->randomUniqueID(); state BackupFileList listing = wait(container->dumpFileList()); // remove partitioned logs diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 219d9ab820..431bc7798d 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -130,6 +130,7 @@ enum { OPT_USE_PARTITIONED_LOG, // Backup and Restore constants + OPT_PROXY, OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, @@ -234,6 +235,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, // Enable "-p" option after GA // { OPT_USE_PARTITIONED_LOG, "-p", SO_NONE }, { OPT_USE_PARTITIONED_LOG, "--partitioned-log-experimental", SO_NONE }, @@ -294,6 +296,7 @@ CSimpleOpt::SOption g_rgBackupModifyOptions[] = { { OPT_MOD_VERIFY_UID, "--verify-uid", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, { OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP }, { OPT_MOD_ACTIVE_INTERVAL, "--active-snapshot-interval", SO_REQ_SEP }, @@ -482,6 +485,7 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = { { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, @@ -517,6 +521,7 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { #endif { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, @@ -546,6 +551,7 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, @@ -578,6 +584,7 @@ CSimpleOpt::SOption g_rgBackupDumpOptions[] = { { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, @@ -652,6 +659,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { { OPT_RESTORE_TIMESTAMP, "--query-restore-timestamp", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "--query-restore-version", SO_REQ_SEP }, { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, @@ -689,6 +697,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, { OPT_KNOB, "--knob-", SO_REQ_SEP }, { OPT_RESTORECONTAINER, "-r", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, { OPT_PREFIX_ADD, "--add-prefix", SO_REQ_SEP }, { OPT_PREFIX_REMOVE, "--remove-prefix", SO_REQ_SEP }, { OPT_TAGNAME, "-t", SO_REQ_SEP }, @@ -1920,6 +1929,7 @@ ACTOR Future submitDBBackup(Database src, ACTOR Future submitBackup(Database db, std::string url, + Optional proxy, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, Standalone> backupRanges, @@ -1977,6 +1987,7 @@ ACTOR Future submitBackup(Database db, else { wait(backupAgent.submitBackup(db, KeyRef(url), + proxy, initialSnapshotIntervalSeconds, snapshotIntervalSeconds, tagName, @@ -2260,8 +2271,9 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause } Reference openBackupContainer(const char* name, - std::string destinationContainer, - Optional const& encryptionKeyFile = {}) { + const std::string& destinationContainer, + const Optional& proxy, + const Optional& encryptionKeyFile) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2271,7 +2283,7 @@ Reference openBackupContainer(const char* name, Reference c; try { - c = IBackupContainer::openContainer(destinationContainer, encryptionKeyFile); + c = IBackupContainer::openContainer(destinationContainer, proxy, encryptionKeyFile); } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { @@ -2291,6 +2303,7 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std::string tagName, std::string container, + Optional proxy, Standalone> ranges, Version beginVersion, Version targetVersion, @@ -2339,7 +2352,7 @@ ACTOR Future runRestore(Database db, state FileBackupAgent backupAgent; state Reference bc = - openBackupContainer(exeRestore.toString().c_str(), container, encryptionKeyFile); + openBackupContainer(exeRestore.toString().c_str(), container, proxy, encryptionKeyFile); // If targetVersion is unset then use the maximum restorable version from the backup description if (targetVersion == invalidVersion) { @@ -2368,6 +2381,7 @@ ACTOR Future runRestore(Database db, origDb, KeyRef(tagName), KeyRef(container), + proxy, ranges, waitForDone, targetVersion, @@ -2411,6 +2425,7 @@ ACTOR Future runRestore(Database db, ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::string container, + Optional proxy, Standalone> ranges, Version dbVersion, bool performRestore, @@ -2440,7 +2455,7 @@ ACTOR Future runFastRestoreTool(Database db, if (performRestore) { if (dbVersion == invalidVersion) { TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", "Largest restorable version"); - BackupDescription desc = wait(IBackupContainer::openContainer(container)->describeBackup()); + BackupDescription desc = wait(IBackupContainer::openContainer(container, proxy, {})->describeBackup()); if (!desc.maxRestorableVersion.present()) { fprintf(stderr, "The specified backup is not restorable to any version.\n"); throw restore_error(); @@ -2457,6 +2472,7 @@ ACTOR Future runFastRestoreTool(Database db, KeyRef(tagName), ranges, KeyRef(container), + proxy, dbVersion, LockDB::True, randomUID, @@ -2478,7 +2494,7 @@ ACTOR Future runFastRestoreTool(Database db, restoreVersion = dbVersion; } else { - state Reference bc = IBackupContainer::openContainer(container); + state Reference bc = IBackupContainer::openContainer(container, proxy, {}); state BackupDescription description = wait(bc->describeBackup()); if (dbVersion <= 0) { @@ -2522,9 +2538,10 @@ ACTOR Future runFastRestoreTool(Database db, ACTOR Future dumpBackupData(const char* name, std::string destinationContainer, + Optional proxy, Version beginVersion, Version endVersion) { - state Reference c = openBackupContainer(name, destinationContainer); + state Reference c = openBackupContainer(name, destinationContainer, proxy, {}); if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); @@ -2552,6 +2569,7 @@ ACTOR Future dumpBackupData(const char* name, ACTOR Future expireBackupData(const char* name, std::string destinationContainer, + Optional proxy, Version endVersion, std::string endDatetime, Database db, @@ -2577,7 +2595,7 @@ ACTOR Future expireBackupData(const char* name, } try { - Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); + Reference c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile); state IBackupContainer::ExpireProgress progress; state std::string lastProgress; @@ -2623,9 +2641,11 @@ ACTOR Future expireBackupData(const char* name, return Void(); } -ACTOR Future deleteBackupContainer(const char* name, std::string destinationContainer) { +ACTOR Future deleteBackupContainer(const char* name, + std::string destinationContainer, + Optional proxy) { try { - state Reference c = openBackupContainer(name, destinationContainer); + state Reference c = openBackupContainer(name, destinationContainer, proxy, {}); state int numDeleted = 0; state Future done = c->deleteContainer(&numDeleted); @@ -2657,12 +2677,13 @@ ACTOR Future deleteBackupContainer(const char* name, std::string destinati ACTOR Future describeBackup(const char* name, std::string destinationContainer, + Optional proxy, bool deep, Optional cx, bool json, Optional encryptionKeyFile) { try { - Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); + Reference c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile); state BackupDescription desc = wait(c->describeBackup(deep)); if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); @@ -2688,6 +2709,7 @@ static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, s // resolved to that timestamp. ACTOR Future queryBackup(const char* name, std::string destinationContainer, + Optional proxy, Standalone> keyRangesFilter, Version restoreVersion, std::string originalClusterFile, @@ -2734,7 +2756,7 @@ ACTOR Future queryBackup(const char* name, } try { - state Reference bc = openBackupContainer(name, destinationContainer); + state Reference bc = openBackupContainer(name, destinationContainer, proxy, {}); if (restoreVersion == invalidVersion) { BackupDescription desc = wait(bc->describeBackup()); if (desc.maxRestorableVersion.present()) { @@ -2814,9 +2836,9 @@ ACTOR Future queryBackup(const char* name, return Void(); } -ACTOR Future listBackup(std::string baseUrl) { +ACTOR Future listBackup(std::string baseUrl, Optional proxy) { try { - std::vector containers = wait(IBackupContainer::listContainers(baseUrl)); + std::vector containers = wait(IBackupContainer::listContainers(baseUrl, proxy)); for (std::string container : containers) { printf("%s\n", container.c_str()); } @@ -2852,6 +2874,7 @@ ACTOR Future listBackupTags(Database cx) { struct BackupModifyOptions { Optional verifyUID; Optional destURL; + Optional proxy; Optional snapshotIntervalSeconds; Optional activeSnapshotIntervalSeconds; bool hasChanges() const { @@ -2869,7 +2892,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp state Reference bc; if (options.destURL.present()) { - bc = openBackupContainer(exeBackup.toString().c_str(), options.destURL.get()); + bc = openBackupContainer(exeBackup.toString().c_str(), options.destURL.get(), options.proxy, {}); try { wait(timeoutError(bc->create(), 30)); } catch (Error& e) { @@ -3342,6 +3365,7 @@ int main(int argc, char* argv[]) { break; } + Optional proxy; std::string destinationContainer; bool describeDeep = false; bool describeTimestamps = false; @@ -3595,6 +3619,10 @@ int main(int argc, char* argv[]) { return FDB_EXIT_ERROR; } break; + case OPT_PROXY: + proxy = args->OptionArg(); + modifyOptions.proxy = proxy; + break; case OPT_DESTCONTAINER: destinationContainer = args->OptionArg(); // If the url starts with '/' then prepend "file://" for backwards compatibility @@ -3962,9 +3990,10 @@ int main(int argc, char* argv[]) { if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. - openBackupContainer(argv[0], destinationContainer, encryptionKeyFile); + openBackupContainer(argv[0], destinationContainer, proxy, encryptionKeyFile); f = stopAfter(submitBackup(db, destinationContainer, + proxy, initialSnapshotIntervalSeconds, snapshotIntervalSeconds, backupKeys, @@ -4036,6 +4065,7 @@ int main(int argc, char* argv[]) { } f = stopAfter(expireBackupData(argv[0], destinationContainer, + proxy, expireVersion, expireDatetime, db, @@ -4047,7 +4077,7 @@ int main(int argc, char* argv[]) { case BackupType::DELETE_BACKUP: initTraceFile(); - f = stopAfter(deleteBackupContainer(argv[0], destinationContainer)); + f = stopAfter(deleteBackupContainer(argv[0], destinationContainer, proxy)); break; case BackupType::DESCRIBE: @@ -4060,6 +4090,7 @@ int main(int argc, char* argv[]) { // given, but quietly skip them if not. f = stopAfter(describeBackup(argv[0], destinationContainer, + proxy, describeDeep, describeTimestamps ? Optional(db) : Optional(), jsonOutput, @@ -4068,7 +4099,7 @@ int main(int argc, char* argv[]) { case BackupType::LIST: initTraceFile(); - f = stopAfter(listBackup(baseUrl)); + f = stopAfter(listBackup(baseUrl, proxy)); break; case BackupType::TAGS: @@ -4081,6 +4112,7 @@ int main(int argc, char* argv[]) { initTraceFile(); f = stopAfter(queryBackup(argv[0], destinationContainer, + proxy, backupKeysFilter, restoreVersion, restoreClusterFileOrig, @@ -4090,7 +4122,7 @@ int main(int argc, char* argv[]) { case BackupType::DUMP: initTraceFile(); - f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, proxy, dumpBegin, dumpEnd)); break; case BackupType::UNDEFINED: @@ -4141,6 +4173,7 @@ int main(int argc, char* argv[]) { restoreClusterFileOrig, tagName, restoreContainer, + proxy, backupKeys, beginVersion, restoreVersion, @@ -4218,6 +4251,7 @@ int main(int argc, char* argv[]) { f = stopAfter(runFastRestoreTool(db, tagName, restoreContainer, + proxy, backupKeys, restoreVersion, !dryRun, diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index 94cb10d290..a938dcd51f 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -165,6 +165,7 @@ public: Key backupTag, Standalone> backupRanges, Key bcUrl, + Optional proxy, Version targetVersion, LockDB lockDB, UID randomUID, @@ -187,6 +188,7 @@ public: Optional cxOrig, Key tagName, Key url, + Optional proxy, Standalone> ranges, WaitForComplete = WaitForComplete::True, Version targetVersion = ::invalidVersion, @@ -202,6 +204,7 @@ public: Optional cxOrig, Key tagName, Key url, + Optional proxy, WaitForComplete waitForComplete = WaitForComplete::True, Version targetVersion = ::invalidVersion, Verbose verbose = Verbose::True, @@ -219,6 +222,7 @@ public: cxOrig, tagName, url, + proxy, rangeRef, waitForComplete, targetVersion, @@ -263,6 +267,7 @@ public: Future submitBackup(Reference tr, Key outContainer, + Optional proxy, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, std::string const& tagName, @@ -273,6 +278,7 @@ public: Optional const& encryptionKeyFileName = {}); Future submitBackup(Database cx, Key outContainer, + Optional proxy, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, std::string const& tagName, @@ -284,6 +290,7 @@ public: return runRYWTransactionFailIfLocked(cx, [=](Reference tr) { return submitBackup(tr, outContainer, + proxy, initialSnapshotIntervalSeconds, snapshotIntervalSeconds, tagName, @@ -720,20 +727,31 @@ template <> inline Tuple Codec>::pack(Reference const& bc) { Tuple tuple; tuple.append(StringRef(bc->getURL())); + if (bc->getProxy().present()) { + tuple.append(StringRef(bc->getProxy().get())); + } else { + tuple.append(StringRef()); + } if (bc->getEncryptionKeyFileName().present()) { tuple.append(bc->getEncryptionKeyFileName().get()); + } else { + tuple.append(StringRef()); } return tuple; } template <> inline Reference Codec>::unpack(Tuple const& val) { - ASSERT(val.size() == 1 || val.size() == 2); + ASSERT(val.size() == 3); auto url = val.getString(0).toString(); - Optional encryptionKeyFileName; - if (val.size() == 2) { - encryptionKeyFileName = val.getString(1).toString(); + Optional proxy; + if (!val.getString(1).empty()) { + proxy = val.getString(1).toString(); } - return IBackupContainer::openContainer(url, encryptionKeyFileName); + Optional encryptionKeyFileName; + if (!val.getString(2).empty()) { + encryptionKeyFileName = val.getString(2).toString(); + } + return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName); } class BackupConfig : public KeyBackedConfig { diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 37b2eae015..416d15c548 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -256,7 +256,8 @@ std::vector IBackupContainer::getURLFormats() { // Get an IBackupContainer based on a container URL string Reference IBackupContainer::openContainer(const std::string& url, - Optional const& encryptionKeyFileName) { + const Optional& proxy, + const Optional& encryptionKeyFileName) { static std::map> m_cache; Reference& r = m_cache[url]; @@ -273,7 +274,7 @@ Reference IBackupContainer::openContainer(const std::string& u // The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options. S3BlobStoreEndpoint::ParametersT backupParams; Reference bstore = - S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams); + S3BlobStoreEndpoint::fromString(url, proxy, &resource, &lastOpenError, &backupParams); if (resource.empty()) throw backup_invalid_url(); @@ -317,7 +318,7 @@ Reference IBackupContainer::openContainer(const std::string& u // Get a list of URLS to backup containers based on some a shorter URL. This function knows about some set of supported // URL types which support this sort of backup discovery. -ACTOR Future> listContainers_impl(std::string baseURL) { +ACTOR Future> listContainers_impl(std::string baseURL, Optional proxy) { try { StringRef u(baseURL); if (u.startsWith("file://"_sr)) { @@ -327,8 +328,8 @@ ACTOR Future> listContainers_impl(std::string baseURL) std::string resource; S3BlobStoreEndpoint::ParametersT backupParams; - Reference bstore = - S3BlobStoreEndpoint::fromString(baseURL, &resource, &IBackupContainer::lastOpenError, &backupParams); + Reference bstore = S3BlobStoreEndpoint::fromString( + baseURL, proxy, &resource, &IBackupContainer::lastOpenError, &backupParams); if (!resource.empty()) { TraceEvent(SevWarn, "BackupContainer") @@ -370,8 +371,9 @@ ACTOR Future> listContainers_impl(std::string baseURL) } } -Future> IBackupContainer::listContainers(const std::string& baseURL) { - return listContainers_impl(baseURL); +Future> IBackupContainer::listContainers(const std::string& baseURL, + const Optional& proxy) { + return listContainers_impl(baseURL, proxy); } ACTOR Future timeKeeperVersionFromDatetime(std::string datetime, Database db) { diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 312e3b8ac7..36c9ff7cfa 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -156,6 +156,7 @@ struct BackupFileList { struct BackupDescription { BackupDescription() : snapshotBytes(0) {} std::string url; + Optional proxy; std::vector snapshots; int64_t snapshotBytes; // The version before which everything has been deleted by an expire @@ -294,11 +295,14 @@ public: // Get an IBackupContainer based on a container spec string static Reference openContainer(const std::string& url, - const Optional& encryptionKeyFileName = {}); + const Optional& proxy, + const Optional& encryptionKeyFileName); static std::vector getURLFormats(); - static Future> listContainers(const std::string& baseURL); + static Future> listContainers(const std::string& baseURL, + const Optional& proxy); std::string const& getURL() const { return URL; } + Optional const& getProxy() const { return proxy; } Optional const& getEncryptionKeyFileName() const { return encryptionKeyFileName; } static std::string lastOpenError; @@ -306,6 +310,7 @@ public: // TODO: change the following back to `private` once blob obj access is refactored protected: std::string URL; + Optional proxy; Optional encryptionKeyFileName; }; diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 7acbd227f2..a4778ecc10 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -409,6 +409,7 @@ public: Version logStartVersionOverride) { state BackupDescription desc; desc.url = bc->getURL(); + desc.proxy = bc->getProxy(); TraceEvent("BackupContainerDescribe1") .detail("URL", bc->getURL()) @@ -1500,7 +1501,8 @@ Future BackupContainerFileSystem::createTestEncryptionKeyFile(std::string // code but returning a different template type because you can't cast between them Reference BackupContainerFileSystem::openContainerFS( const std::string& url, - Optional const& encryptionKeyFileName) { + const Optional& proxy, + const Optional& encryptionKeyFileName) { static std::map> m_cache; Reference& r = m_cache[url]; @@ -1517,7 +1519,7 @@ Reference BackupContainerFileSystem::openContainerFS( // The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options. S3BlobStoreEndpoint::ParametersT backupParams; Reference bstore = - S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams); + S3BlobStoreEndpoint::fromString(url, proxy, &resource, &lastOpenError, &backupParams); if (resource.empty()) throw backup_invalid_url(); @@ -1635,7 +1637,9 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { +ACTOR Future testBackupContainer(std::string url, + Optional proxy, + Optional encryptionKeyFileName) { state FlowLock lock(100e6); if (encryptionKeyFileName.present()) { @@ -1644,7 +1648,7 @@ ACTOR Future testBackupContainer(std::string url, Optional en printf("BackupContainerTest URL %s\n", url.c_str()); - state Reference c = IBackupContainer::openContainer(url, encryptionKeyFileName); + state Reference c = IBackupContainer::openContainer(url, proxy, encryptionKeyFileName); // Make sure container doesn't exist, then create it. try { @@ -1789,12 +1793,13 @@ ACTOR Future testBackupContainer(std::string url, Optional en } TEST_CASE("/backup/containers/localdir/unencrypted") { - wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {})); + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}, {})); return Void(); } TEST_CASE("/backup/containers/localdir/encrypted") { wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), + {}, format("%s/test_encryption_key", params.getDataDir().c_str()))); return Void(); } @@ -1803,7 +1808,7 @@ TEST_CASE("/backup/containers/url") { if (!g_network->isSimulated()) { const char* url = getenv("FDB_TEST_BACKUP_URL"); ASSERT(url != nullptr); - wait(testBackupContainer(url, {})); + wait(testBackupContainer(url, {}, {})); } return Void(); } @@ -1813,7 +1818,7 @@ TEST_CASE("/backup/containers_list") { state const char* url = getenv("FDB_TEST_BACKUP_URL"); ASSERT(url != nullptr); printf("Listing %s\n", url); - std::vector urls = wait(IBackupContainer::listContainers(url)); + std::vector urls = wait(IBackupContainer::listContainers(url, {})); for (auto& u : urls) { printf("%s\n", u.c_str()); } diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index 52c5d3fc54..784b113395 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -81,9 +81,9 @@ public: Future exists() override = 0; // TODO: refactor this to separate out the "deal with blob store" stuff from the backup business logic - static Reference openContainerFS( - const std::string& url, - const Optional& encryptionKeyFileName = {}); + static Reference openContainerFS(const std::string& url, + const Optional& proxy, + const Optional& encryptionKeyFileName); // Get a list of fileNames and their sizes in the container under the given path // Although not required, an implementation can avoid traversing unwanted subfolders diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index fc1dc558c7..b451747f08 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -4363,13 +4363,14 @@ public: Key backupTag, Standalone> backupRanges, Key bcUrl, + Optional proxy, Version targetVersion, LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { // Sanity check backup is valid - state Reference bc = IBackupContainer::openContainer(bcUrl.toString()); + state Reference bc = IBackupContainer::openContainer(bcUrl.toString(), proxy, {}); state BackupDescription desc = wait(bc->describeBackup()); wait(desc.resolveVersionTimes(cx)); @@ -4430,6 +4431,7 @@ public: struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, + proxy, targetVersion, range, deterministicRandom()->randomUniqueID(), @@ -4510,6 +4512,7 @@ public: ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, Key outContainer, + Optional proxy, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, std::string tagName, @@ -4555,7 +4558,8 @@ public: backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString()); } - state Reference bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName); + state Reference bc = + IBackupContainer::openContainer(backupContainer, proxy, encryptionKeyFileName); try { wait(timeoutError(bc->create(), 30)); } catch (Error& e) { @@ -4642,6 +4646,7 @@ public: Reference tr, Key tagName, Key backupURL, + Optional proxy, Standalone> ranges, Version restoreVersion, Key addPrefix, @@ -4710,7 +4715,7 @@ public: // Point the tag to the new uid tag.set(tr, { uid, false }); - Reference bc = IBackupContainer::openContainer(backupURL.toString()); + Reference bc = IBackupContainer::openContainer(backupURL.toString(), proxy, {}); // Configure the new restore restore.tag().set(tr, tagName.toString()); @@ -5303,6 +5308,7 @@ public: Optional cxOrig, Key tagName, Key url, + Optional proxy, Standalone> ranges, WaitForComplete waitForComplete, Version targetVersion, @@ -5320,7 +5326,7 @@ public: throw restore_error(); } - state Reference bc = IBackupContainer::openContainer(url.toString()); + state Reference bc = IBackupContainer::openContainer(url.toString(), proxy, {}); state BackupDescription desc = wait(bc->describeBackup(true)); if (cxOrig.present()) { @@ -5360,6 +5366,7 @@ public: tr, tagName, url, + proxy, ranges, targetVersion, addPrefix, @@ -5499,6 +5506,7 @@ public: tagName, ranges, KeyRef(bc->getURL()), + bc->getProxy(), targetVersion, LockDB::True, randomUid, @@ -5520,6 +5528,7 @@ public: cx, tagName, KeyRef(bc->getURL()), + bc->getProxy(), ranges, WaitForComplete::True, ::invalidVersion, @@ -5561,13 +5570,14 @@ Future FileBackupAgent::submitParallelRestore(Database cx, Key backupTag, Standalone> backupRanges, Key bcUrl, + Optional proxy, Version targetVersion, LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { return FileBackupAgentImpl::submitParallelRestore( - cx, backupTag, backupRanges, bcUrl, targetVersion, lockDB, randomUID, addPrefix, removePrefix); + cx, backupTag, backupRanges, bcUrl, proxy, targetVersion, lockDB, randomUID, addPrefix, removePrefix); } Future FileBackupAgent::atomicParallelRestore(Database cx, @@ -5582,6 +5592,7 @@ Future FileBackupAgent::restore(Database cx, Optional cxOrig, Key tagName, Key url, + Optional proxy, Standalone> ranges, WaitForComplete waitForComplete, Version targetVersion, @@ -5598,6 +5609,7 @@ Future FileBackupAgent::restore(Database cx, cxOrig, tagName, url, + proxy, ranges, waitForComplete, targetVersion, @@ -5639,6 +5651,7 @@ Future FileBackupAgent::waitRestore(Database cx, Key tagName, Ver Future FileBackupAgent::submitBackup(Reference tr, Key outContainer, + Optional proxy, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, std::string const& tagName, @@ -5650,6 +5663,7 @@ Future FileBackupAgent::submitBackup(Reference return FileBackupAgentImpl::submitBackup(this, tr, outContainer, + proxy, initialSnapshotIntervalSeconds, snapshotIntervalSeconds, tagName, diff --git a/fdbclient/RestoreInterface.h b/fdbclient/RestoreInterface.h index bdb2499298..b7f3b04bcc 100644 --- a/fdbclient/RestoreInterface.h +++ b/fdbclient/RestoreInterface.h @@ -49,6 +49,7 @@ struct RestoreRequest { int index; Key tagName; Key url; + Optional proxy; Version targetVersion; KeyRange range; UID randomUid; @@ -64,27 +65,29 @@ struct RestoreRequest { explicit RestoreRequest(const int index, const Key& tagName, const Key& url, + const Optional& proxy, Version targetVersion, const KeyRange& range, const UID& randomUid, Key& addPrefix, Key removePrefix) - : index(index), tagName(tagName), url(url), targetVersion(targetVersion), range(range), randomUid(randomUid), - addPrefix(addPrefix), removePrefix(removePrefix) {} + : index(index), tagName(tagName), url(url), proxy(proxy), targetVersion(targetVersion), range(range), + randomUid(randomUid), addPrefix(addPrefix), removePrefix(removePrefix) {} // To change this serialization, ProtocolVersion::RestoreRequestValue must be updated, and downgrades need to be // considered template void serialize(Ar& ar) { - serializer(ar, index, tagName, url, targetVersion, range, randomUid, addPrefix, removePrefix, reply); + serializer(ar, index, tagName, url, proxy, targetVersion, range, randomUid, addPrefix, removePrefix, reply); } std::string toString() const { std::stringstream ss; ss << "index:" << std::to_string(index) << " tagName:" << tagName.contents().toString() - << " url:" << url.contents().toString() << " targetVersion:" << std::to_string(targetVersion) - << " range:" << range.toString() << " randomUid:" << randomUid.toString() - << " addPrefix:" << addPrefix.toString() << " removePrefix:" << removePrefix.toString(); + << " url:" << url.contents().toString() << " proxy:" << (proxy.present() ? proxy.get() : "") + << " targetVersion:" << std::to_string(targetVersion) << " range:" << range.toString() + << " randomUid:" << randomUid.toString() << " addPrefix:" << addPrefix.toString() + << " removePrefix:" << removePrefix.toString(); return ss.str(); } }; diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp index a4fa95616a..799f631c6e 100644 --- a/fdbclient/S3BlobStore.actor.cpp +++ b/fdbclient/S3BlobStore.actor.cpp @@ -162,7 +162,8 @@ std::string S3BlobStoreEndpoint::BlobKnobs::getURLParameters() const { return r; } -Reference S3BlobStoreEndpoint::fromString(std::string const& url, +Reference S3BlobStoreEndpoint::fromString(const std::string& url, + const Optional& proxy, std::string* resourceFromURL, std::string* error, ParametersT* ignored_parameters) { @@ -175,6 +176,13 @@ Reference S3BlobStoreEndpoint::fromString(std::string const if (prefix != LiteralStringRef("blobstore")) throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str()); + Optional proxyHost, proxyPort; + if (proxy.present()) { + StringRef p(proxy.get()); + proxyHost = p.eat(":").toString(); + proxyPort = p.eat().toString(); + } + Optional cred; if (url.find("@") != std::string::npos) { cred = t.eat("@"); @@ -261,7 +269,8 @@ Reference S3BlobStoreEndpoint::fromString(std::string const creds = S3BlobStoreEndpoint::Credentials{ key.toString(), secret.toString(), securityToken.toString() }; } - return makeReference(host.toString(), service.toString(), creds, knobs, extraHeaders); + return makeReference( + host.toString(), service.toString(), proxyHost, proxyPort, creds, knobs, extraHeaders); } catch (std::string& err) { if (error != nullptr) @@ -624,11 +633,11 @@ ACTOR Future connect_impl(Referenceservice; + std::string host = b->host, service = b->service; if (service.empty()) service = b->knobs.secure_connection ? "https" : "http"; state Reference conn = - wait(INetworkConnections::net()->connect(b->host, service, b->knobs.secure_connection ? true : false)); + wait(INetworkConnections::net()->connect(host, service, b->knobs.secure_connection ? true : false)); wait(conn->connectHandshake()); TraceEvent("S3BlobStoreEndpointNewConnection") @@ -1609,7 +1618,7 @@ TEST_CASE("/backup/s3/v4headers") { S3BlobStoreEndpoint::Credentials creds{ "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "" } // GET without query parameters { - S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", creds); + S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds); std::string verb("GET"); std::string resource("/test.txt"); HTTP::Headers headers; @@ -1624,7 +1633,7 @@ TEST_CASE("/backup/s3/v4headers") { // GET with query parameters { - S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", creds); + S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds); std::string verb("GET"); std::string resource("/test/examplebucket?Action=DescribeRegions&Version=2013-10-15"); HTTP::Headers headers; @@ -1639,7 +1648,7 @@ TEST_CASE("/backup/s3/v4headers") { // POST { - S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "s3", creds); + S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "s3", "proxy", "port", creds); std::string verb("POST"); std::string resource("/simple.json"); HTTP::Headers headers; diff --git a/fdbclient/S3BlobStore.h b/fdbclient/S3BlobStore.h index 21f39e1d0e..bd29675bae 100644 --- a/fdbclient/S3BlobStore.h +++ b/fdbclient/S3BlobStore.h @@ -99,11 +99,15 @@ public: }; S3BlobStoreEndpoint(std::string const& host, - std::string service, + std::string const& service, + Optional const& proxyHost, + Optional const& proxyPort, Optional const& creds, BlobKnobs const& knobs = BlobKnobs(), HTTP::Headers extraHeaders = HTTP::Headers()) - : host(host), service(service), credentials(creds), lookupKey(creds.present() && creds.get().key.empty()), + : host(host), service(service), proxyHost(proxyHost), proxyPort(proxyPort), + useProxy(proxyHost.present() && proxyPort.present()), credentials(creds), + lookupKey(creds.present() && creds.get().key.empty()), lookupSecret(creds.present() && creds.get().secret.empty()), knobs(knobs), extraHeaders(extraHeaders), requestRate(new SpeedLimit(knobs.requests_per_second, 1)), requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)), @@ -114,7 +118,7 @@ public: recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests), concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) { - if (host.empty()) + if (host.empty() || (proxyHost.present() != proxyPort.present())) throw connection_string_invalid(); } @@ -132,10 +136,11 @@ public: // Parse url and return a S3BlobStoreEndpoint // If the url has parameters that S3BlobStoreEndpoint can't consume then an error will be thrown unless // ignored_parameters is given in which case the unconsumed parameters will be added to it. - static Reference fromString(std::string const& url, - std::string* resourceFromURL = nullptr, - std::string* error = nullptr, - ParametersT* ignored_parameters = nullptr); + static Reference fromString(const std::string& url, + const Optional& proxy, + std::string* resourceFromURL, + std::string* error, + ParametersT* ignored_parameters); // Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL // parameters in addition to the passed params string @@ -151,6 +156,10 @@ public: std::string host; std::string service; + Optional proxyHost; + Optional proxyPort; + bool useProxy; + Optional credentials; bool lookupKey; bool lookupSecret; diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index b81fee7d70..736d5b7a59 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -2506,7 +2506,7 @@ ACTOR Future monitorPruneKeys(Reference self) { if (BM_DEBUG) { fmt::print("BM constructing backup container from {}\n", SERVER_KNOBS->BG_URL.c_str()); } - self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); if (BM_DEBUG) { printf("BM constructed backup container\n"); } diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index 791ee7a05a..906994922b 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -3021,7 +3021,7 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, if (BW_DEBUG) { fmt::print("BW constructing backup container from {0}\n", SERVER_KNOBS->BG_URL); } - self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); if (BW_DEBUG) { printf("BW constructed backup container\n"); } diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index 64d7d3d785..8092ebe39a 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -47,7 +47,8 @@ ACTOR static Future collectBackupFiles(Reference bc, RestoreRequest request); ACTOR static Future buildRangeVersions(KeyRangeMap* pRangeVersions, std::vector* pRangeFiles, - Key url); + Key url, + Optional proxy); ACTOR static Future processRestoreRequest(Reference self, Database cx, @@ -317,7 +318,7 @@ ACTOR static Future processRestoreRequest(Reference allFiles; state Version minRangeVersion = MAX_VERSION; - self->initBackupContainer(request.url); + self->initBackupContainer(request.url, request.proxy); // Get all backup files' description and save them to files state Version targetVersion = @@ -334,7 +335,7 @@ ACTOR static Future processRestoreRequest(Reference rangeVersions(minRangeVersion, allKeys.end); if (SERVER_KNOBS->FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE) { - wait(buildRangeVersions(&rangeVersions, &rangeFiles, request.url)); + wait(buildRangeVersions(&rangeVersions, &rangeFiles, request.url, request.proxy)); } else { // Debug purpose, dump range versions auto ranges = rangeVersions.ranges(); @@ -881,13 +882,14 @@ ACTOR static Future insertRangeVersion(KeyRangeMap* pRangeVersion // Expensive and slow operation that should not run in real prod. ACTOR static Future buildRangeVersions(KeyRangeMap* pRangeVersions, std::vector* pRangeFiles, - Key url) { + Key url, + Optional proxy) { if (!g_network->isSimulated()) { TraceEvent(SevError, "ExpensiveBuildRangeVersions") .detail("Reason", "Parsing all range files is slow and memory intensive"); return Void(); } - Reference bc = IBackupContainer::openContainer(url.toString()); + Reference bc = IBackupContainer::openContainer(url.toString(), proxy, {}); // Key ranges not in range files are empty; // Assign highest version to avoid applying any mutation in these ranges diff --git a/fdbserver/RestoreController.actor.h b/fdbserver/RestoreController.actor.h index 5c9a271f7a..77aa5e6494 100644 --- a/fdbserver/RestoreController.actor.h +++ b/fdbserver/RestoreController.actor.h @@ -446,13 +446,15 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted proxy) { if (bcUrl == url && bc.isValid()) { return; } - TraceEvent("FastRestoreControllerInitBackupContainer").detail("URL", url); + TraceEvent("FastRestoreControllerInitBackupContainer") + .detail("URL", url) + .detail("Proxy", proxy.present() ? proxy.get() : ""); bcUrl = url; - bc = IBackupContainer::openContainer(url.toString()); + bc = IBackupContainer::openContainer(url.toString(), proxy, {}); } }; diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index 9aa1aadee3..1afabdcb95 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -262,7 +262,7 @@ ACTOR Future restoreLoaderCore(RestoreLoaderInterface loaderInterf, when(RestoreLoadFileRequest req = waitNext(loaderInterf.loadFile.getFuture())) { requestTypeStr = "loadFile"; hasQueuedRequests = !self->loadingQueue.empty() || !self->sendingQueue.empty(); - self->initBackupContainer(req.param.url); + self->initBackupContainer(req.param.url, req.param.proxy); self->loadingQueue.push(req); if (!hasQueuedRequests) { self->hasPendingRequests->set(true); diff --git a/fdbserver/RestoreLoader.actor.h b/fdbserver/RestoreLoader.actor.h index b16e4c11fa..92b11a5a1c 100644 --- a/fdbserver/RestoreLoader.actor.h +++ b/fdbserver/RestoreLoader.actor.h @@ -226,12 +226,12 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted proxy) { if (bcUrl == url && bc.isValid()) { return; } bcUrl = url; - bc = IBackupContainer::openContainer(url.toString()); + bc = IBackupContainer::openContainer(url.toString(), proxy, {}); } }; diff --git a/fdbserver/RestoreWorkerInterface.actor.h b/fdbserver/RestoreWorkerInterface.actor.h index 065b22c468..3c2830514a 100644 --- a/fdbserver/RestoreWorkerInterface.actor.h +++ b/fdbserver/RestoreWorkerInterface.actor.h @@ -368,6 +368,7 @@ struct LoadingParam { bool isRangeFile; Key url; + Optional proxy; Optional rangeVersion; // range file's version int64_t blockSize; @@ -386,12 +387,13 @@ struct LoadingParam { template void serialize(Ar& ar) { - serializer(ar, isRangeFile, url, rangeVersion, blockSize, asset); + serializer(ar, isRangeFile, url, proxy, rangeVersion, blockSize, asset); } std::string toString() const { std::stringstream str; str << "isRangeFile:" << isRangeFile << " url:" << url.toString() + << " proxy:" << (proxy.present() ? proxy.get() : "") << " rangeVersion:" << (rangeVersion.present() ? rangeVersion.get() : -1) << " blockSize:" << blockSize << " RestoreAsset:" << asset.toString(); return str.str(); diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp index 4c3c2703f9..86d90e1093 100644 --- a/fdbserver/workloads/AtomicRestore.actor.cpp +++ b/fdbserver/workloads/AtomicRestore.actor.cpp @@ -93,6 +93,7 @@ struct AtomicRestoreWorkload : TestWorkload { try { wait(backupAgent.submitBackup(cx, StringRef(backupContainer), + {}, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), BackupAgentBase::getDefaultTagName(), diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp index 890bdf6a3a..650ca6f2c6 100644 --- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp @@ -222,6 +222,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { try { wait(backupAgent->submitBackup(cx, StringRef(backupContainer), + {}, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), tag.toString(), @@ -377,6 +378,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { cx, self->backupTag, KeyRef(lastBackupContainer), + {}, WaitForComplete::True, ::invalidVersion, Verbose::True, @@ -478,6 +480,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { // the configuration to disable backup workers before restore. extraBackup = backupAgent.submitBackup(cx, LiteralStringRef("file://simfdb/backups/"), + {}, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), @@ -523,7 +526,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { .detail("BackupTag", printable(self->backupTag)); // start restoring - auto container = IBackupContainer::openContainer(lastBackupContainer->getURL()); + auto container = + IBackupContainer::openContainer(lastBackupContainer->getURL(), lastBackupContainer->getProxy(), {}); BackupDescription desc = wait(container->describeBackup()); ASSERT(self->usePartitionedLogs == desc.partitioned); ASSERT(desc.minRestorableVersion.present()); // We must have a valid backup now. @@ -566,6 +570,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { self->backupTag, self->backupRanges, KeyRef(lastBackupContainer->getURL()), + lastBackupContainer->getProxy(), targetVersion, self->locked, randomID, diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index 92550a23bf..4c82762764 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -266,6 +266,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { try { wait(backupAgent->submitBackup(cx, StringRef(backupContainer), + {}, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), tag.toString(), @@ -423,6 +424,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, self->backupTag, KeyRef(lastBackupContainer), + {}, WaitForComplete::True, ::invalidVersion, Verbose::True, @@ -523,6 +525,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { try { extraBackup = backupAgent.submitBackup(cx, "file://simfdb/backups/"_sr, + {}, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), @@ -557,7 +560,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { .detail("RestoreAfter", self->restoreAfter) .detail("BackupTag", printable(self->backupTag)); - auto container = IBackupContainer::openContainer(lastBackupContainer->getURL()); + auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(), + lastBackupContainer->getProxy(), + lastBackupContainer->getEncryptionKeyFileName()); BackupDescription desc = wait(container->describeBackup()); Version targetVersion = -1; @@ -593,6 +598,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTag, KeyRef(lastBackupContainer->getURL()), + lastBackupContainer->getProxy(), WaitForComplete::True, targetVersion, Verbose::True, @@ -616,6 +622,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTag, KeyRef(lastBackupContainer->getURL()), + lastBackupContainer->getProxy(), self->restoreRanges, WaitForComplete::True, targetVersion, @@ -646,6 +653,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTags[restoreIndex], KeyRef(lastBackupContainer->getURL()), + lastBackupContainer->getProxy(), self->restoreRanges, WaitForComplete::True, ::invalidVersion, @@ -675,6 +683,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTags[restoreIndex], KeyRef(lastBackupContainer->getURL()), + lastBackupContainer->getProxy(), WaitForComplete::True, ::invalidVersion, Verbose::True, diff --git a/fdbserver/workloads/BackupToBlob.actor.cpp b/fdbserver/workloads/BackupToBlob.actor.cpp index ee27e1a480..480ae62466 100644 --- a/fdbserver/workloads/BackupToBlob.actor.cpp +++ b/fdbserver/workloads/BackupToBlob.actor.cpp @@ -62,6 +62,7 @@ struct BackupToBlobWorkload : TestWorkload { wait(delay(self->backupAfter)); wait(backupAgent.submitBackup(cx, self->backupURL, + {}, self->initSnapshotInterval, self->snapshotInterval, self->backupTag.toString(), diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp index e0d309954e..fc6d3035ae 100644 --- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp +++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp @@ -250,13 +250,13 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload { if (BGW_DEBUG) { printf("Blob Granule Correctness constructing simulated backup container\n"); } - self->bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/"); + self->bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/", {}, {}); } else { if (BGW_DEBUG) { printf("Blob Granule Correctness constructing backup container from %s\n", SERVER_KNOBS->BG_URL.c_str()); } - self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); if (BGW_DEBUG) { printf("Blob Granule Correctness constructed backup container\n"); } diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index cd97b1960b..ba49923bf1 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -90,13 +90,13 @@ struct BlobGranuleVerifierWorkload : TestWorkload { if (BGV_DEBUG) { printf("Blob Granule Verifier constructing simulated backup container\n"); } - bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/"); + bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/", {}, {}); } else { if (BGV_DEBUG) { printf("Blob Granule Verifier constructing backup container from %s\n", SERVER_KNOBS->BG_URL.c_str()); } - bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); if (BGV_DEBUG) { printf("Blob Granule Verifier constructed backup container\n"); } diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.actor.cpp index 688387023e..e40133ffd0 100644 --- a/fdbserver/workloads/IncrementalBackup.actor.cpp +++ b/fdbserver/workloads/IncrementalBackup.actor.cpp @@ -98,12 +98,12 @@ struct IncrementalBackupWorkload : TestWorkload { if (!backupContainer.isValid()) { TraceEvent("IBackupCheckListContainersAttempt").log(); state std::vector containers = - wait(IBackupContainer::listContainers(self->backupDir.toString())); + wait(IBackupContainer::listContainers(self->backupDir.toString(), {})); TraceEvent("IBackupCheckListContainersSuccess") .detail("Size", containers.size()) .detail("First", containers.front()); if (containers.size()) { - backupContainer = IBackupContainer::openContainer(containers.front()); + backupContainer = IBackupContainer::openContainer(containers.front(), {}, {}); } } state bool e = wait(backupContainer->exists()); @@ -152,6 +152,7 @@ struct IncrementalBackupWorkload : TestWorkload { try { wait(self->backupAgent.submitBackup(cx, self->backupDir, + {}, 0, 1e8, self->tag.toString(), @@ -219,7 +220,7 @@ struct IncrementalBackupWorkload : TestWorkload { } TraceEvent("IBackupStartListContainersAttempt").log(); state std::vector containers = - wait(IBackupContainer::listContainers(self->backupDir.toString())); + wait(IBackupContainer::listContainers(self->backupDir.toString(), {})); TraceEvent("IBackupStartListContainersSuccess") .detail("Size", containers.size()) .detail("First", containers.front()); @@ -229,6 +230,7 @@ struct IncrementalBackupWorkload : TestWorkload { cx, Key(self->tag.toString()), backupURL, + {}, WaitForComplete::True, invalidVersion, Verbose::True, diff --git a/fdbserver/workloads/RestoreBackup.actor.cpp b/fdbserver/workloads/RestoreBackup.actor.cpp index c7122bc107..c08fc7de70 100644 --- a/fdbserver/workloads/RestoreBackup.actor.cpp +++ b/fdbserver/workloads/RestoreBackup.actor.cpp @@ -114,6 +114,7 @@ struct RestoreBackupWorkload final : TestWorkload { cx, self->tag, Key(self->backupContainer->getURL()), + self->backupContainer->getProxy(), WaitForComplete::True, ::invalidVersion, Verbose::True))); diff --git a/fdbserver/workloads/RestoreFromBlob.actor.cpp b/fdbserver/workloads/RestoreFromBlob.actor.cpp index 482f22ded4..9d072bb731 100644 --- a/fdbserver/workloads/RestoreFromBlob.actor.cpp +++ b/fdbserver/workloads/RestoreFromBlob.actor.cpp @@ -61,8 +61,8 @@ struct RestoreFromBlobWorkload : TestWorkload { restoreRanges.push_back_deep(restoreRanges.arena(), normalKeys); wait(delay(self->restoreAfter)); - Version v = - wait(backupAgent.restore(cx, {}, self->backupTag, self->backupURL, restoreRanges, self->waitForComplete)); + Version v = wait( + backupAgent.restore(cx, {}, self->backupTag, self->backupURL, {}, restoreRanges, self->waitForComplete)); return Void(); } diff --git a/fdbserver/workloads/SubmitBackup.actor.cpp b/fdbserver/workloads/SubmitBackup.actor.cpp index 50759bf014..aa4dd13d9b 100644 --- a/fdbserver/workloads/SubmitBackup.actor.cpp +++ b/fdbserver/workloads/SubmitBackup.actor.cpp @@ -57,6 +57,7 @@ struct SubmitBackupWorkload final : TestWorkload { try { wait(self->backupAgent.submitBackup(cx, self->backupDir, + {}, self->initSnapshotInterval, self->snapshotInterval, self->tag.toString(), From 1e9c8b36849a6c18fd3e9713d2dad7769b2ec79e Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 28 Mar 2022 18:14:05 -0700 Subject: [PATCH 40/90] Shutdown bug fix, extent cache should be cleared on shutdown as if recovery never completed it wouldn't have been cleared yet. --- fdbserver/VersionedBTree.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 6fb18b0a79..724256a353 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -3691,6 +3691,7 @@ public: self->operations.clear(); debug_printf("DWALPager(%s) shutdown destroy page cache\n", self->filename.c_str()); + wait(self->extentCache.clear()); wait(self->pageCache.clear()); wait(delay(0)); From 16afeb43fa5888c898e11b6883c0977a03263a60 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 28 Mar 2022 20:00:03 -0700 Subject: [PATCH 41/90] Avoid false positive for determinism check in DEBUG_DETERMINISM by avoiding use of shared memory. --- fdbserver/fdbserver.actor.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 17f1bcf9d2..5355d36546 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -285,6 +285,13 @@ private: }; UID getSharedMemoryMachineId() { + // new UID to use if an existing one is not found + UID newUID = deterministicRandom()->randomUniqueID(); + +#if DEBUG_DETERMINISM + // Don't use shared memory if DEBUG_DETERMINISM is set + return newUID; +#else UID* machineId = nullptr; int numTries = 0; @@ -297,7 +304,7 @@ UID getSharedMemoryMachineId() { // "0" is the default parameter "addr" boost::interprocess::managed_shared_memory segment( boost::interprocess::open_or_create, sharedMemoryIdentifier.c_str(), 1000, 0, p.permission); - machineId = segment.find_or_construct("machineId")(deterministicRandom()->randomUniqueID()); + machineId = segment.find_or_construct("machineId")(newUID); if (!machineId) criticalError( FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate or create shared memory - 'machineId'"); @@ -321,6 +328,7 @@ UID getSharedMemoryMachineId() { } } } +#endif } ACTOR void failAfter(Future trigger, ISimulator::ProcessInfo* m = g_simulator.getCurrentProcess()) { From 2348c46dac903ce85aa45a744e46193dbedf42ab Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Mon, 28 Mar 2022 22:54:00 -0700 Subject: [PATCH 42/90] Resolve merge conflict --- fdbserver/Ratekeeper.h | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/fdbserver/Ratekeeper.h b/fdbserver/Ratekeeper.h index 4a978f0395..c0b1769c90 100644 --- a/fdbserver/Ratekeeper.h +++ b/fdbserver/Ratekeeper.h @@ -59,11 +59,6 @@ public: UID id; LocalityData locality; StorageQueuingMetricsReply lastReply; -<<<<<<< HEAD - StorageQueuingMetricsReply prevReply; - -======= ->>>>>>> ad98d6479992d2fcf1f89ff59d20945479a54cf1 bool acceptingRequests; Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes; Smoother smoothDurableVersion, smoothLatestVersion; @@ -72,37 +67,12 @@ public: limitReason_t limitReason; std::vector busiestReadTags, busiestWriteTags; -<<<<<<< HEAD - Optional busiestReadTag, busiestWriteTag; - double busiestReadTagFractionalBusyness = 0, busiestWriteTagFractionalBusyness = 0; - double busiestReadTagRate = 0, busiestWriteTagRate = 0; - - Reference busiestWriteTagEventHolder; - - // refresh periodically - TransactionTagMap tagCostEst; - uint64_t totalWriteCosts = 0; - int totalWriteOps = 0; - - StorageQueueInfo(UID id, LocalityData locality) - : valid(false), id(id), locality(locality), acceptingRequests(false), - smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), - verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), - smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), - smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), - limitReason(limitReason_t::unlimited), - busiestWriteTagEventHolder(makeReference(id.toString() + "/BusiestWriteTag")) { - // FIXME: this is a tacky workaround for a potential uninitialized use in trackStorageServerQueueInfo - lastReply.instanceID = -1; - } -======= StorageQueueInfo(UID id, LocalityData locality); void refreshCommitCost(double elapsed); int64_t getStorageQueueBytes() const { return lastReply.bytesInput - smoothDurableBytes.smoothTotal(); } int64_t getDurabilityLag() const { return smoothLatestVersion.smoothTotal() - smoothDurableVersion.smoothTotal(); } void update(StorageQueuingMetricsReply const&, Smoother& smoothTotalDurableBytes); void addCommitCost(TransactionTagRef tagName, TransactionCommitCostEstimation const& cost); ->>>>>>> ad98d6479992d2fcf1f89ff59d20945479a54cf1 }; struct TLogQueueInfo { From dd3a453f5b5a96f90cb3e336f7153cde4a752556 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Mon, 28 Mar 2022 23:52:26 -0700 Subject: [PATCH 43/90] Address suggestions to make new SSI member private, and reduce the number of serialization methods for serverList value --- fdbclient/StorageServerInterface.h | 2 ++ fdbclient/SystemData.cpp | 29 +++++++++++------------------ fdbclient/SystemData.h | 1 - 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 4750c7ff4a..b83101ab46 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -89,8 +89,10 @@ struct StorageServerInterface { RequestStream checkpoint; RequestStream fetchCheckpoint; +private: bool acceptingRequests; +public: explicit StorageServerInterface(UID uid) : uniqueID(uid) { acceptingRequests = false; } StorageServerInterface() : uniqueID(deterministicRandom()->randomUniqueID()) { acceptingRequests = false; } NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index ff93a89aed..a346a7e0f2 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -588,7 +588,9 @@ const Key serverListKeyFor(UID serverID) { } const Value serverListValue(StorageServerInterface const& server) { - return serverListValueFB(server); + auto protocolVersion = currentProtocolVersion; + protocolVersion.addObjectSerializerFlag(); + return ObjectWriter::toValue(server, IncludeVersion(protocolVersion)); } UID decodeServerListKey(KeyRef const& key) { @@ -617,12 +619,6 @@ StorageServerInterface decodeServerListValue(ValueRef const& value) { return decodeServerListValueFB(value); } -const Value serverListValueFB(StorageServerInterface const& server) { - auto protocolVersion = currentProtocolVersion; - protocolVersion.addObjectSerializerFlag(); - return ObjectWriter::toValue(server, IncludeVersion(protocolVersion)); -} - // processClassKeys.contains(k) iff k.startsWith( processClassKeys.begin ) because '/'+1 == '0' const KeyRangeRef processClassKeys(LiteralStringRef("\xff/processClass/"), LiteralStringRef("\xff/processClass0")); const KeyRef processClassPrefix = processClassKeys.begin; @@ -1396,32 +1392,31 @@ const KeyRef tenantLastIdKey = "\xff/tenantLastId/"_sr; const KeyRef tenantDataPrefixKey = "\xff/tenantDataPrefix"_sr; // for tests -void testSSISerdes(StorageServerInterface const& ssi, bool useFB) { +void testSSISerdes(StorageServerInterface const& ssi) { printf("ssi=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\nacceptingRequests=%s\naddress=%s\ngetValue=%s\n\n\n", ssi.id().toString().c_str(), ssi.locality.toString().c_str(), ssi.isTss() ? "true" : "false", ssi.isTss() ? ssi.tssPairID.get().toString().c_str() : "", - ssi.acceptingRequests ? "true" : "false", + ssi.isAcceptingRequests() ? "true" : "false", ssi.address().toString().c_str(), ssi.getValue.getEndpoint().token.toString().c_str()); - StorageServerInterface ssi2 = - (useFB) ? decodeServerListValueFB(serverListValueFB(ssi)) : decodeServerListValue(serverListValue(ssi)); + StorageServerInterface ssi2 = decodeServerListValue(serverListValue(ssi)); printf("ssi2=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\nacceptingRequests=%s\naddress=%s\ngetValue=%s\n\n\n", ssi2.id().toString().c_str(), ssi2.locality.toString().c_str(), ssi2.isTss() ? "true" : "false", ssi2.isTss() ? ssi2.tssPairID.get().toString().c_str() : "", - ssi2.acceptingRequests ? "true" : "false", + ssi2.isAcceptingRequests() ? "true" : "false", ssi2.address().toString().c_str(), ssi2.getValue.getEndpoint().token.toString().c_str()); ASSERT(ssi.id() == ssi2.id()); ASSERT(ssi.locality == ssi2.locality); ASSERT(ssi.isTss() == ssi2.isTss()); - ASSERT(ssi.acceptingRequests == ssi2.acceptingRequests); + ASSERT(ssi.isAcceptingRequests() == ssi2.isAcceptingRequests()); if (ssi.isTss()) { ASSERT(ssi2.tssPairID.get() == ssi2.tssPairID.get()); } @@ -1430,7 +1425,7 @@ void testSSISerdes(StorageServerInterface const& ssi, bool useFB) { } // unit test for serialization since tss stuff had bugs -TEST_CASE("/SystemData/SSI/SerDes") { +TEST_CASE("/SystemData/SerDes/SSI") { printf("testing ssi serdes\n"); LocalityData localityData(Optional>(), Standalone(deterministicRandom()->randomUniqueID().toString()), @@ -1443,13 +1438,11 @@ TEST_CASE("/SystemData/SSI/SerDes") { ssi.locality = localityData; ssi.initEndpoints(); - testSSISerdes(ssi, false); - testSSISerdes(ssi, true); + testSSISerdes(ssi); ssi.tssPairID = UID(0x2345234523452345, 0x1238123812381238); - testSSISerdes(ssi, false); - testSSISerdes(ssi, true); + testSSISerdes(ssi); printf("ssi serdes test complete\n"); return Void(); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index bf5a12ee3d..171130559e 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -202,7 +202,6 @@ extern const KeyRangeRef serverListKeys; extern const KeyRef serverListPrefix; const Key serverListKeyFor(UID serverID); const Value serverListValue(StorageServerInterface const&); -const Value serverListValueFB(StorageServerInterface const&); UID decodeServerListKey(KeyRef const&); StorageServerInterface decodeServerListValue(ValueRef const&); From 2f8e9d9de036290ff6d99590b2f883595b9c7fa6 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Mon, 28 Mar 2022 13:48:25 -0500 Subject: [PATCH 44/90] misc bg fixes --- fdbclient/StorageServerInterface.h | 4 ++-- fdbserver/BlobManager.actor.cpp | 2 ++ fdbserver/storageserver.actor.cpp | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 6dce9351cb..18c2d1044d 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -924,7 +924,7 @@ struct OverlappingChangeFeedsReply { }; struct OverlappingChangeFeedsRequest { - constexpr static FileIdentifier file_identifier = 10726174; + constexpr static FileIdentifier file_identifier = 7228462; KeyRange range; Version minVersion; ReplyPromise reply; @@ -939,7 +939,7 @@ struct OverlappingChangeFeedsRequest { }; struct ChangeFeedVersionUpdateReply { - constexpr static FileIdentifier file_identifier = 11815134; + constexpr static FileIdentifier file_identifier = 4246160; Version version = 0; ChangeFeedVersionUpdateReply() {} diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index b81fee7d70..b096056784 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -1581,6 +1581,7 @@ static void addAssignment(KeyRangeMap>& map, } ACTOR Future recoverBlobManager(Reference bmData) { + state double recoveryStartTime = now(); state Promise workerListReady; bmData->addActor.send(checkBlobWorkerList(bmData, workerListReady)); wait(workerListReady.getFuture()); @@ -1836,6 +1837,7 @@ ACTOR Future recoverBlobManager(Reference bmData) { TraceEvent("BlobManagerRecovered", bmData->id) .detail("Epoch", bmData->epoch) + .detail("Duration", now() - recoveryStartTime) .detail("Granules", bmData->workerAssignments.size()) .detail("Assigned", explicitAssignments) .detail("Revoked", outOfDateAssignments.size()); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index aaf33f3ee7..a56632c1e6 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -858,7 +858,7 @@ public: CounterCollection cc; Counter allQueries, getKeyQueries, getValueQueries, getRangeQueries, getMappedRangeQueries, getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried, bytesQueried, watchQueries, - emptyQueries, feedRowsQueried, feedBytesQueried; + emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries, feedVersionQueries; // Bytes of the mutations that have been added to the memory of the storage server. When the data is durable // and cleared from the memory, we do not subtract it but add it to bytesDurable. @@ -930,6 +930,7 @@ public: lowPriorityQueries("LowPriorityQueries", cc), rowsQueried("RowsQueried", cc), bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc), emptyQueries("EmptyQueries", cc), feedRowsQueried("FeedRowsQueried", cc), feedBytesQueried("FeedBytesQueried", cc), + feedStreamQueries("FeedStreamQueries", cc), feedVersionQueries("FeedVersionQueries", cc), bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc), logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc), kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc), @@ -2436,6 +2437,8 @@ ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques req.reply.setByteLimit(std::min((int64_t)req.replyBufferSize, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES)); } + ++data->counters.feedStreamQueries; + wait(delay(0, TaskPriority::DefaultEndpoint)); try { @@ -2587,6 +2590,7 @@ ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques } ACTOR Future changeFeedVersionUpdateQ(StorageServer* data, ChangeFeedVersionUpdateRequest req) { + ++data->counters.feedVersionQueries; wait(data->version.whenAtLeast(req.minVersion)); wait(delay(0)); Version minVersion = data->minFeedVersionForAddress(req.reply.getEndpoint().getPrimaryAddress()); From 61474d5d548071183684bddf5e1d47b8f97424dd Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Mon, 28 Mar 2022 14:48:12 -0500 Subject: [PATCH 45/90] Future-proof blob granules with full file size --- fdbclient/BlobGranuleCommon.h | 9 +++++---- fdbclient/BlobGranuleFiles.cpp | 11 ++++++++--- fdbclient/SystemData.cpp | 9 ++++++--- fdbclient/SystemData.h | 4 ++-- fdbserver/BlobGranuleServerCommon.actor.cpp | 16 ++++++++++------ fdbserver/BlobGranuleServerCommon.actor.h | 5 +++-- fdbserver/BlobWorker.actor.cpp | 18 ++++++++++++------ 7 files changed, 46 insertions(+), 26 deletions(-) diff --git a/fdbclient/BlobGranuleCommon.h b/fdbclient/BlobGranuleCommon.h index c76e72342d..97074326d9 100644 --- a/fdbclient/BlobGranuleCommon.h +++ b/fdbclient/BlobGranuleCommon.h @@ -52,19 +52,20 @@ struct BlobFilePointerRef { StringRef filename; int64_t offset; int64_t length; + int64_t fullFileLength; BlobFilePointerRef() {} - BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length) - : filename(to, filename), offset(offset), length(length) {} + BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength) + : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {} template void serialize(Ar& ar) { - serializer(ar, filename, offset, length); + serializer(ar, filename, offset, length, fullFileLength); } std::string toString() const { std::stringstream ss; - ss << filename.toString() << ":" << offset << ":" << length; + ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength; return std::move(ss).str(); } }; diff --git a/fdbclient/BlobGranuleFiles.cpp b/fdbclient/BlobGranuleFiles.cpp index 1697722fb7..469573e3d9 100644 --- a/fdbclient/BlobGranuleFiles.cpp +++ b/fdbclient/BlobGranuleFiles.cpp @@ -240,22 +240,27 @@ static void startLoad(const ReadBlobGranuleContext granuleContext, // Start load process for all files in chunk if (chunk.snapshotFile.present()) { std::string snapshotFname = chunk.snapshotFile.get().filename.toString(); - // FIXME: full file length won't always be length of read + // FIXME: remove when we implement file multiplexing + ASSERT(chunk.snapshotFile.get().offset == 0); + ASSERT(chunk.snapshotFile.get().length == chunk.snapshotFile.get().fullFileLength); loadIds.snapshotId = granuleContext.start_load_f(snapshotFname.c_str(), snapshotFname.size(), chunk.snapshotFile.get().offset, chunk.snapshotFile.get().length, - chunk.snapshotFile.get().length, + chunk.snapshotFile.get().fullFileLength, granuleContext.userContext); } loadIds.deltaIds.reserve(chunk.deltaFiles.size()); for (int deltaFileIdx = 0; deltaFileIdx < chunk.deltaFiles.size(); deltaFileIdx++) { std::string deltaFName = chunk.deltaFiles[deltaFileIdx].filename.toString(); + // FIXME: remove when we implement file multiplexing + ASSERT(chunk.deltaFiles[deltaFileIdx].offset == 0); + ASSERT(chunk.deltaFiles[deltaFileIdx].length == chunk.deltaFiles[deltaFileIdx].fullFileLength); int64_t deltaLoadId = granuleContext.start_load_f(deltaFName.c_str(), deltaFName.size(), chunk.deltaFiles[deltaFileIdx].offset, chunk.deltaFiles[deltaFileIdx].length, - chunk.deltaFiles[deltaFileIdx].length, + chunk.deltaFiles[deltaFileIdx].fullFileLength, granuleContext.userContext); loadIds.deltaIds.push_back(deltaLoadId); } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 5c24441f32..42e0f83a26 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1190,23 +1190,26 @@ const KeyRange blobGranuleFileKeyRangeFor(UID granuleID) { return KeyRangeRef(startKey, strinc(startKey)); } -const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length) { +const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length, int64_t fullFileLength) { BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule())); wr << filename; wr << offset; wr << length; + wr << fullFileLength; return wr.toValue(); } -std::tuple, int64_t, int64_t> decodeBlobGranuleFileValue(ValueRef const& value) { +std::tuple, int64_t, int64_t, int64_t> decodeBlobGranuleFileValue(ValueRef const& value) { StringRef filename; int64_t offset; int64_t length; + int64_t fullFileLength; BinaryReader reader(value, IncludeVersion()); reader >> filename; reader >> offset; reader >> length; - return std::tuple(filename, offset, length); + reader >> fullFileLength; + return std::tuple(filename, offset, length, fullFileLength); } const Value blobGranulePruneValueFor(Version version, KeyRange range, bool force) { diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 171130559e..fcbc20bf97 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -572,8 +572,8 @@ const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t file std::tuple decodeBlobGranuleFileKey(KeyRef const& key); const KeyRange blobGranuleFileKeyRangeFor(UID granuleID); -const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length); -std::tuple, int64_t, int64_t> decodeBlobGranuleFileValue(ValueRef const& value); +const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length, int64_t fullFileLength); +std::tuple, int64_t, int64_t, int64_t> decodeBlobGranuleFileValue(ValueRef const& value); const Value blobGranulePruneValueFor(Version version, KeyRange range, bool force); std::tuple decodeBlobGranulePruneValue(ValueRef const& value); diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp index 4792984d62..35b8d2e22f 100644 --- a/fdbserver/BlobGranuleServerCommon.actor.cpp +++ b/fdbserver/BlobGranuleServerCommon.actor.cpp @@ -60,13 +60,14 @@ ACTOR Future readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, Standalone filename; int64_t offset; int64_t length; + int64_t fullFileLength; std::tie(gid, version, fileType) = decodeBlobGranuleFileKey(it.key); ASSERT(gid == granuleID); - std::tie(filename, offset, length) = decodeBlobGranuleFileValue(it.value); + std::tie(filename, offset, length, fullFileLength) = decodeBlobGranuleFileValue(it.value); - BlobFileIndex idx(version, filename.toString(), offset, length); + BlobFileIndex idx(version, filename.toString(), offset, length, fullFileLength); if (fileType == 'S') { ASSERT(files->snapshotFiles.empty() || files->snapshotFiles.back().version < idx.version); files->snapshotFiles.push_back(idx); @@ -168,14 +169,16 @@ void GranuleFiles::getFiles(Version beginVersion, Version lastIncluded = invalidVersion; if (snapshotF != snapshotFiles.end()) { chunk.snapshotVersion = snapshotF->version; - chunk.snapshotFile = BlobFilePointerRef(replyArena, snapshotF->filename, snapshotF->offset, snapshotF->length); + chunk.snapshotFile = BlobFilePointerRef( + replyArena, snapshotF->filename, snapshotF->offset, snapshotF->length, snapshotF->fullFileLength); lastIncluded = chunk.snapshotVersion; } else { chunk.snapshotVersion = invalidVersion; } while (deltaF != deltaFiles.end() && deltaF->version < readVersion) { - chunk.deltaFiles.emplace_back_deep(replyArena, deltaF->filename, deltaF->offset, deltaF->length); + chunk.deltaFiles.emplace_back_deep( + replyArena, deltaF->filename, deltaF->offset, deltaF->length, deltaF->fullFileLength); deltaBytesCounter += deltaF->length; ASSERT(lastIncluded < deltaF->version); lastIncluded = deltaF->version; @@ -183,7 +186,8 @@ void GranuleFiles::getFiles(Version beginVersion, } // include last delta file that passes readVersion, if it exists if (deltaF != deltaFiles.end() && lastIncluded < readVersion) { - chunk.deltaFiles.emplace_back_deep(replyArena, deltaF->filename, deltaF->offset, deltaF->length); + chunk.deltaFiles.emplace_back_deep( + replyArena, deltaF->filename, deltaF->offset, deltaF->length, deltaF->fullFileLength); deltaBytesCounter += deltaF->length; lastIncluded = deltaF->version; } @@ -194,7 +198,7 @@ static std::string makeTestFileName(Version v) { } static BlobFileIndex makeTestFile(Version v, int64_t len) { - return BlobFileIndex(v, makeTestFileName(v), 0, len); + return BlobFileIndex(v, makeTestFileName(v), 0, len, len); } static void checkFile(int expectedVersion, const BlobFilePointerRef& actualFile) { diff --git a/fdbserver/BlobGranuleServerCommon.actor.h b/fdbserver/BlobGranuleServerCommon.actor.h index 399ae5b7b0..ea3f8c1e3b 100644 --- a/fdbserver/BlobGranuleServerCommon.actor.h +++ b/fdbserver/BlobGranuleServerCommon.actor.h @@ -49,11 +49,12 @@ struct BlobFileIndex { std::string filename; int64_t offset; int64_t length; + int64_t fullFileLength; BlobFileIndex() {} - BlobFileIndex(Version version, std::string filename, int64_t offset, int64_t length) - : version(version), filename(filename), offset(offset), length(length) {} + BlobFileIndex(Version version, std::string filename, int64_t offset, int64_t length, int64_t fullFileLength) + : version(version), filename(filename), offset(offset), length(length), fullFileLength(fullFileLength) {} // compare on version bool operator<(const BlobFileIndex& r) const { return version < r.version; } diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index 791ee7a05a..79d0981e70 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -511,7 +511,8 @@ ACTOR Future writeDeltaFile(Reference bwData, numIterations++; Key dfKey = blobGranuleFileKeyFor(granuleID, currentDeltaVersion, 'D'); - Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize); + // TODO change once we support file multiplexing + Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize, serializedSize); tr->set(dfKey, dfValue); if (oldGranuleComplete.present()) { @@ -538,7 +539,8 @@ ACTOR Future writeDeltaFile(Reference bwData, if (BUGGIFY_WITH_PROB(0.01)) { wait(delay(deterministicRandom()->random01())); } - return BlobFileIndex(currentDeltaVersion, fname, 0, serializedSize); + // FIXME: change when we implement multiplexing + return BlobFileIndex(currentDeltaVersion, fname, 0, serializedSize, serializedSize); } catch (Error& e) { wait(tr->onError(e)); } @@ -648,7 +650,8 @@ ACTOR Future writeSnapshot(Reference bwData, wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno)); numIterations++; Key snapshotFileKey = blobGranuleFileKeyFor(granuleID, version, 'S'); - Key snapshotFileValue = blobGranuleFileValueFor(fname, 0, serializedSize); + // TODO change once we support file multiplexing + Key snapshotFileValue = blobGranuleFileValueFor(fname, 0, serializedSize, serializedSize); tr->set(snapshotFileKey, snapshotFileValue); // create granule history at version if this is a new granule with the initial dump from FDB if (createGranuleHistory) { @@ -692,7 +695,8 @@ ACTOR Future writeSnapshot(Reference bwData, wait(delay(deterministicRandom()->random01())); } - return BlobFileIndex(version, fname, 0, serializedSize); + // FIXME: change when we implement multiplexing + return BlobFileIndex(version, fname, 0, serializedSize, serializedSize); } ACTOR Future dumpInitialSnapshotFromFDB(Reference bwData, @@ -797,7 +801,8 @@ ACTOR Future compactFromBlob(Reference bwData, ASSERT(snapshotVersion < version); - chunk.snapshotFile = BlobFilePointerRef(filenameArena, snapshotF.filename, snapshotF.offset, snapshotF.length); + chunk.snapshotFile = BlobFilePointerRef( + filenameArena, snapshotF.filename, snapshotF.offset, snapshotF.length, snapshotF.fullFileLength); compactBytesRead += snapshotF.length; int deltaIdx = files.deltaFiles.size() - 1; while (deltaIdx >= 0 && files.deltaFiles[deltaIdx].version > snapshotVersion) { @@ -807,7 +812,8 @@ ACTOR Future compactFromBlob(Reference bwData, Version lastDeltaVersion = invalidVersion; while (deltaIdx < files.deltaFiles.size() && files.deltaFiles[deltaIdx].version <= version) { BlobFileIndex deltaF = files.deltaFiles[deltaIdx]; - chunk.deltaFiles.emplace_back_deep(filenameArena, deltaF.filename, deltaF.offset, deltaF.length); + chunk.deltaFiles.emplace_back_deep( + filenameArena, deltaF.filename, deltaF.offset, deltaF.length, deltaF.fullFileLength); compactBytesRead += deltaF.length; lastDeltaVersion = files.deltaFiles[deltaIdx].version; deltaIdx++; From 23fcd8c076c9a1e47c9d92212a74597556d0d3ee Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Tue, 29 Mar 2022 09:07:33 -0700 Subject: [PATCH 46/90] Fix issues with command completion for exclude and storage_migration_type. Add missing documentation for tenant_mode in one spot. --- documentation/sphinx/source/command-line-interface.rst | 2 +- fdbcli/fdbcli.actor.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index 72d7da54ef..a51e4f10da 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu configure --------- -The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [storage_migration_type={disabled|aggressive|gradual}]``. +The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [storage_migration_type={disabled|aggressive|gradual}] [tenant_mode={disabled|optional_experimental|required_experimental}]``. The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified. diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index c956af3dd4..b89ecaab2a 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -354,10 +354,13 @@ static std::vector> parseLine(std::string& line, bool& er forcetoken = true; break; case ' ': + case '\n': + case '\t': + case '\r': if (!quoted) { if (i > offset || (forcetoken && i == offset)) buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - offset = i = line.find_first_not_of(' ', i); + offset = i = line.find_first_not_of(" \n\t\r", i); forcetoken = false; } else i++; @@ -788,7 +791,7 @@ void configureGenerator(const char* text, const char* line, std::vectorROCKSDB_BLOCK_CACHE_SIZE > 0) { - bbOpts.block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE); + if (rocksdb_block_cache == nullptr) { + rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE); + } + bbOpts.block_cache = rocksdb_block_cache; } options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts)); From be0f0ce90393168e9b6c56a8fed0f5b35668606a Mon Sep 17 00:00:00 2001 From: akashhansda <99724223+akashhansda@users.noreply.github.com> Date: Mon, 28 Mar 2022 23:43:10 -0700 Subject: [PATCH 48/90] Update README.md Use https --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fbcd4d3ef6..e40bf6ae23 100755 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The official docker image for building is [`foundationdb/build`](https://hub.doc To build outside the official docker image you'll need at least these dependencies: 1. Install cmake Version 3.13 or higher [CMake](https://cmake.org/) -1. Install [Mono](http://www.mono-project.com/download/stable/) +1. Install [Mono](https://www.mono-project.com/download/stable/) 1. Install [Ninja](https://ninja-build.org/) (optional, but recommended) If compiling for local development, please set `-DUSE_WERROR=ON` in @@ -177,7 +177,7 @@ Under Windows, only Visual Studio with ClangCl is supported 1. Install [Python](https://www.python.org/downloads/) if is not already installed by Visual Studio 1. (Optional) Install [OpenJDK 11](https://developers.redhat.com/products/openjdk/download) to build Java bindings 1. (Optional) Install [OpenSSL 3.x](https://slproweb.com/products/Win32OpenSSL.html) to build with TLS support -1. (Optional) Install [WIX Toolset](http://wixtoolset.org/) to build Windows installer +1. (Optional) Install [WIX Toolset](https://wixtoolset.org/) to build Windows installer 1. `mkdir build && cd build` 1. `cmake -G "Visual Studio 16 2019" -A x64 -T ClangCl ` 1. `msbuild /p:Configuration=Release foundationdb.sln` From e1775627ab7793e23920a74c191b748d6a51bd08 Mon Sep 17 00:00:00 2001 From: Renxuan Wang Date: Mon, 28 Mar 2022 22:15:24 -0700 Subject: [PATCH 49/90] Add a check on proxy format. --- fdbbackup/backup.actor.cpp | 4 ++++ fdbclient/S3BlobStore.actor.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 431bc7798d..a8b4218569 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -3621,6 +3621,10 @@ int main(int argc, char* argv[]) { break; case OPT_PROXY: proxy = args->OptionArg(); + if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) { + fprintf(stderr, "ERROR: Proxy format should be either IP:port or host:port\n"); + return FDB_EXIT_ERROR; + } modifyOptions.proxy = proxy; break; case OPT_DESTCONTAINER: diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp index 799f631c6e..edfc1d1bc0 100644 --- a/fdbclient/S3BlobStore.actor.cpp +++ b/fdbclient/S3BlobStore.actor.cpp @@ -178,6 +178,10 @@ Reference S3BlobStoreEndpoint::fromString(const std::string Optional proxyHost, proxyPort; if (proxy.present()) { + if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) { + throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.", + proxy.get().c_str()); + } StringRef p(proxy.get()); proxyHost = p.eat(":").toString(); proxyPort = p.eat().toString(); From dd15489605586dd99d6ab911f53f3d4e3895c31c Mon Sep 17 00:00:00 2001 From: He Liu Date: Mon, 28 Mar 2022 21:06:41 -0700 Subject: [PATCH 50/90] rename ssd-rocksdb-experimental as ssd-rocksdb-v1. --- documentation/sphinx/source/mr-status-json-schemas.rst.inc | 4 ++-- fdbclient/DatabaseConfiguration.cpp | 4 ++-- fdbclient/FDBTypes.h | 2 +- fdbclient/ManagementAPI.actor.cpp | 2 +- fdbclient/Schemas.cpp | 4 ++-- fdbserver/SimulatedCluster.actor.cpp | 4 ++-- fdbserver/workloads/KVStoreTest.actor.cpp | 2 +- tests/RocksDBTest.txt | 6 +++--- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index c9e42402a1..be1bbebb29 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -701,7 +701,7 @@ "ssd-1", "ssd-2", "ssd-redwood-1-experimental", - "ssd-rocksdb-experimental", + "ssd-rocksdb-v1", "memory", "memory-1", "memory-2", @@ -714,7 +714,7 @@ "ssd-1", "ssd-2", "ssd-redwood-1-experimental", - "ssd-rocksdb-experimental", + "ssd-rocksdb-v1", "memory", "memory-1", "memory-2", diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index e915950361..7978c14fbb 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -302,7 +302,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { result["storage_engine"] = "ssd-redwood-1-experimental"; } else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) { - result["storage_engine"] = "ssd-rocksdb-experimental"; + result["storage_engine"] = "ssd-rocksdb-v1"; } else if (tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY) { result["storage_engine"] = "memory-1"; } else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && @@ -324,7 +324,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { } else if (testingStorageServerStoreType == KeyValueStoreType::SSD_REDWOOD_V1) { result["tss_storage_engine"] = "ssd-redwood-1-experimental"; } else if (testingStorageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) { - result["tss_storage_engine"] = "ssd-rocksdb-experimental"; + result["tss_storage_engine"] = "ssd-rocksdb-v1"; } else if (testingStorageServerStoreType == KeyValueStoreType::MEMORY_RADIXTREE) { result["tss_storage_engine"] = "memory-radixtree-beta"; } else if (testingStorageServerStoreType == KeyValueStoreType::MEMORY) { diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 9b0c4ca46f..bbc2cb0adf 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -831,7 +831,7 @@ struct KeyValueStoreType { case SSD_REDWOOD_V1: return "ssd-redwood-1-experimental"; case SSD_ROCKSDB_V1: - return "ssd-rocksdb-experimental"; + return "ssd-rocksdb-v1"; case MEMORY: return "memory"; case MEMORY_RADIXTREE: diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index 02f8882072..d4633807b8 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -214,7 +214,7 @@ std::map configForToken(std::string const& mode) { } else if (mode == "ssd-redwood-1-experimental") { logType = KeyValueStoreType::SSD_BTREE_V2; storeType = KeyValueStoreType::SSD_REDWOOD_V1; - } else if (mode == "ssd-rocksdb-experimental") { + } else if (mode == "ssd-rocksdb-v1") { logType = KeyValueStoreType::SSD_BTREE_V2; storeType = KeyValueStoreType::SSD_ROCKSDB_V1; } else if (mode == "memory" || mode == "memory-2") { diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index ab2b18ec57..18ffac2fa2 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -768,7 +768,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "ssd-1", "ssd-2", "ssd-redwood-1-experimental", - "ssd-rocksdb-experimental", + "ssd-rocksdb-v1", "memory", "memory-1", "memory-2", @@ -781,7 +781,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "ssd-1", "ssd-2", "ssd-redwood-1-experimental", - "ssd-rocksdb-experimental", + "ssd-rocksdb-v1", "memory", "memory-1", "memory-2", diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index ef00a0f001..b369f5d3df 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -303,7 +303,7 @@ public: // 1 = "memory" // 2 = "memory-radixtree-beta" // 3 = "ssd-redwood-1-experimental" - // 4 = "ssd-rocksdb-experimental" + // 4 = "ssd-rocksdb-v1" // Requires a comma-separated list of numbers WITHOUT whitespaces std::vector storageEngineExcludeTypes; // Set the maximum TLog version that can be selected for a test @@ -1395,7 +1395,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) { } case 4: { TEST(true); // Simulated cluster using RocksDB storage engine - set_config("ssd-rocksdb-experimental"); + set_config("ssd-rocksdb-v1"); // Tests using the RocksDB engine are necessarily non-deterministic because of RocksDB // background threads. TraceEvent(SevWarnAlways, "RocksDBNonDeterminism") diff --git a/fdbserver/workloads/KVStoreTest.actor.cpp b/fdbserver/workloads/KVStoreTest.actor.cpp index 85eb2f0931..177c95da6d 100644 --- a/fdbserver/workloads/KVStoreTest.actor.cpp +++ b/fdbserver/workloads/KVStoreTest.actor.cpp @@ -386,7 +386,7 @@ ACTOR Future testKVStore(KVStoreTestWorkload* workload) { test.store = keyValueStoreSQLite(fn, id, KeyValueStoreType::SSD_REDWOOD_V1); else if (workload->storeType == "ssd-redwood-1-experimental") test.store = keyValueStoreRedwoodV1(fn, id); - else if (workload->storeType == "ssd-rocksdb-experimental") + else if (workload->storeType == "ssd-rocksdb-v1") test.store = keyValueStoreRocksDB(fn, id, KeyValueStoreType::SSD_ROCKSDB_V1); else if (workload->storeType == "memory") test.store = keyValueStoreMemory(fn, id, 500e6); diff --git a/tests/RocksDBTest.txt b/tests/RocksDBTest.txt index a1aeb2d32b..6e4abe0051 100644 --- a/tests/RocksDBTest.txt +++ b/tests/RocksDBTest.txt @@ -8,7 +8,7 @@ nodeCount=20000000 keyBytes=16 valueBytes=96 filename=bttest -storeType=ssd-rocksdb-experimental +storeType=ssd-rocksdb-v1 setup=true clear=false count=false @@ -25,7 +25,7 @@ nodeCount=20000000 keyBytes=16 valueBytes=96 filename=bttest -storeType=ssd-rocksdb-experimental +storeType=ssd-rocksdb-v1 setup=false clear=false count=false @@ -41,7 +41,7 @@ nodeCount=20000000 keyBytes=16 valueBytes=96 filename=bttest -storeType=ssd-rocksdb-experimental +storeType=ssd-rocksdb-v1 setup=false clear=false count=true From 2f7b68d06ff6a97f4d5d9d181afe870786af3877 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Tue, 29 Mar 2022 11:50:46 -0700 Subject: [PATCH 51/90] Switch to signalling storageIntefaceReg actor with an Optional> --- fdbserver/storageserver.actor.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 177ffa3980..acfe09c5d1 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -836,7 +836,7 @@ public: Promise coreStarted; bool shuttingDown; - Promise registerInterfaceAcceptingRequests; + Promise registerInterfaceAcceptingRequests; Future interfaceRegistered; bool behind; @@ -6803,12 +6803,10 @@ ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { if ((data->lastTLogVersion - data->version.get()) < SERVER_KNOBS->STORAGE_RECOVERY_VERSION_LAG_LIMIT) { if (data->registerInterfaceAcceptingRequests.canBeSet()) { - data->registerInterfaceAcceptingRequests.send(true); + data->registerInterfaceAcceptingRequests.send(Void()); ErrorOr e = wait(errorOr(data->interfaceRegistered)); if (e.isError()) { - TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed") - .detail("ServerID", data->thisServerID) - .detail("Error", e.getError().code()); + TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed", data->thisServerID).error(e.getError()); } } } @@ -8623,10 +8621,10 @@ ACTOR Future replaceTSSInterface(StorageServer* self, StorageServerInterfa ACTOR Future storageInterfaceRegistration(StorageServer* self, StorageServerInterface ssi, - Future interfaceAcceptingRequests) { + Optional> readyToAcceptRequests) { - bool acceptingRequests = wait(interfaceAcceptingRequests); - if (acceptingRequests) { + if (readyToAcceptRequests.present()) { + wait(readyToAcceptRequests.get()); ssi.startAcceptingRequests(); } else { ssi.stopAcceptingRequests(); @@ -8673,7 +8671,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (seedTag == invalidTag) { ssi.startAcceptingRequests(); - self.registerInterfaceAcceptingRequests.send(true); + self.registerInterfaceAcceptingRequests.send(Void()); // Might throw recruitment_failed in case of simultaneous master failure std::pair verAndTag = wait(addStorageServer(self.cx, ssi)); @@ -8789,10 +8787,8 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, if (recovered.canBeSet()) recovered.send(Void()); - state Promise registerInterface; - state Future f = storageInterfaceRegistration(&self, ssi, registerInterface.getFuture()); + state Future f = storageInterfaceRegistration(&self, ssi, {}); wait(delay(0)); - registerInterface.send(false); ErrorOr e = wait(errorOr(f)); if (e.isError()) { Error e = f.getError(); From bc3e5cdaa17a6986fd9709cb5ca09dfb35c02fda Mon Sep 17 00:00:00 2001 From: Xiaoxi Wang Date: Tue, 29 Mar 2022 12:02:56 -0700 Subject: [PATCH 52/90] fix cmake error when OPEN_FOR_IDE=ON --- bindings/c/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index 62d369f4dc..c049e6f0fc 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -124,6 +124,7 @@ if(NOT WIN32) add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h) add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h) add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h) + add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp) add_library(mako OBJECT ${MAKO_SRCS}) add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp) add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS}) From d727e7648ecd23010d2561eed064ea83f5d33b9a Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Mon, 28 Mar 2022 14:33:59 -0700 Subject: [PATCH 53/90] Fix a few memory issues found by ASAN --- bindings/c/test/unit/fdb_api.cpp | 8 +++++++- bindings/c/test/unit/fdb_api.hpp | 7 ++++++- fdbcli/TenantCommands.actor.cpp | 18 +++++++++++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/bindings/c/test/unit/fdb_api.cpp b/bindings/c/test/unit/fdb_api.cpp index 4fc715dbc5..b26d7bdf82 100644 --- a/bindings/c/test/unit/fdb_api.cpp +++ b/bindings/c/test/unit/fdb_api.cpp @@ -138,6 +138,12 @@ Tenant::Tenant(FDBDatabase* db, const uint8_t* name, int name_length) { } } +Tenant::~Tenant() { + if (tenant != nullptr) { + fdb_tenant_destroy(tenant); + } +} + // Transaction Transaction::Transaction(FDBDatabase* db) { if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) { @@ -146,7 +152,7 @@ Transaction::Transaction(FDBDatabase* db) { } } -Transaction::Transaction(Tenant tenant) { +Transaction::Transaction(Tenant& tenant) { if (fdb_error_t err = fdb_tenant_create_transaction(tenant.tenant, &tr_)) { std::cerr << fdb_get_error(err) << std::endl; std::abort(); diff --git a/bindings/c/test/unit/fdb_api.hpp b/bindings/c/test/unit/fdb_api.hpp index 5653d6e7cb..fcf1c7e5ca 100644 --- a/bindings/c/test/unit/fdb_api.hpp +++ b/bindings/c/test/unit/fdb_api.hpp @@ -206,6 +206,11 @@ public: class Tenant final { public: Tenant(FDBDatabase* db, const uint8_t* name, int name_length); + ~Tenant(); + Tenant(const Tenant&) = delete; + Tenant& operator=(const Tenant&) = delete; + Tenant(Tenant&&) = delete; + Tenant& operator=(Tenant&&) = delete; private: friend class Transaction; @@ -219,7 +224,7 @@ class Transaction final { public: // Given an FDBDatabase, initializes a new transaction. Transaction(FDBDatabase* db); - Transaction(Tenant tenant); + Transaction(Tenant& tenant); ~Transaction(); // Wrapper around fdb_transaction_reset. diff --git a/fdbcli/TenantCommands.actor.cpp b/fdbcli/TenantCommands.actor.cpp index c03bb17c88..6660893a1f 100644 --- a/fdbcli/TenantCommands.actor.cpp +++ b/fdbcli/TenantCommands.actor.cpp @@ -51,7 +51,9 @@ ACTOR Future createTenantCommandActor(Reference db, std::vector tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); try { if (!doneExistenceCheck) { - Optional existingTenant = wait(safeThreadFutureToFuture(tr->get(tenantNameKey))); + // Hold the reference to the standalone's memory + state ThreadFuture> existingTenantFuture = tr->get(tenantNameKey); + Optional existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture)); if (existingTenant.present()) { throw tenant_already_exists(); } @@ -96,7 +98,9 @@ ACTOR Future deleteTenantCommandActor(Reference db, std::vector tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); try { if (!doneExistenceCheck) { - Optional existingTenant = wait(safeThreadFutureToFuture(tr->get(tenantNameKey))); + // Hold the reference to the standalone's memory + state ThreadFuture> existingTenantFuture = tr->get(tenantNameKey); + Optional existingTenant = wait(safeThreadFutureToFuture(existingTenantFuture)); if (!existingTenant.present()) { throw tenant_not_found(); } @@ -163,8 +167,10 @@ ACTOR Future listTenantsCommandActor(Reference db, std::vector< loop { try { - RangeResult tenants = wait(safeThreadFutureToFuture( - tr->getRange(firstGreaterOrEqual(beginTenantKey), firstGreaterOrEqual(endTenantKey), limit))); + // Hold the reference to the standalone's memory + state ThreadFuture kvsFuture = + tr->getRange(firstGreaterOrEqual(beginTenantKey), firstGreaterOrEqual(endTenantKey), limit); + RangeResult tenants = wait(safeThreadFutureToFuture(kvsFuture)); if (tenants.empty()) { if (tokens.size() == 1) { @@ -213,7 +219,9 @@ ACTOR Future getTenantCommandActor(Reference db, std::vector tenant = wait(safeThreadFutureToFuture(tr->get(tenantNameKey))); + // Hold the reference to the standalone's memory + state ThreadFuture> tenantFuture = tr->get(tenantNameKey); + Optional tenant = wait(safeThreadFutureToFuture(tenantFuture)); if (!tenant.present()) { throw tenant_not_found(); } From 7fc6dfa6c5abe914e58ba0c56a794fc217848a21 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Tue, 29 Mar 2022 13:16:41 -0500 Subject: [PATCH 54/90] Adding useful debugging trace events --- fdbclient/NativeAPI.actor.cpp | 1 + fdbserver/BlobManager.actor.cpp | 38 ++++++++++++++++++++++++++++++--- fdbserver/BlobWorker.actor.cpp | 31 +++++++++++++++++++-------- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 4b21429737..7a041e1cd9 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -7372,6 +7372,7 @@ ACTOR Future>> readBlobGranulesActor( fmt::print( "BG Mapping for [{0} - %{1}) too large!\n", keyRange.begin.printable(), keyRange.end.printable()); } + TraceEvent(SevWarn, "BGMappingTooLarge").detail("Range", range).detail("Max", 1000); throw unsupported_operation(); } ASSERT(!blobGranuleMapping.more && blobGranuleMapping.size() < CLIENT_KNOBS->TOO_MANY); diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index 79b89f54dc..192475f4dd 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -211,6 +211,24 @@ struct SplitEvaluation { : epoch(epoch), seqno(seqno), inProgress(inProgress) {} }; +struct BlobManagerStats { + CounterCollection cc; + + // FIXME: pruning stats + + Counter granuleSplits; + Counter granuleWriteHotSplits; + Future logger; + + // Current stats maintained for a given blob worker process + explicit BlobManagerStats(UID id, double interval, std::unordered_map* workers) + : cc("BlobManagerStats", id.toString()), granuleSplits("GranuleSplits", cc), + granuleWriteHotSplits("GranuleWriteHotSplits", cc) { + specialCounter(cc, "WorkerCount", [workers]() { return workers->size(); }); + logger = traceCounters("BlobManagerMetrics", id, interval, &cc, "BlobManagerMetrics"); + } +}; + struct BlobManagerData : NonCopyable, ReferenceCounted { UID id; Database db; @@ -218,6 +236,8 @@ struct BlobManagerData : NonCopyable, ReferenceCounted { PromiseStream> addActor; Promise doLockCheck; + BlobManagerStats stats; + Reference bstore; std::unordered_map workersById; @@ -246,8 +266,9 @@ struct BlobManagerData : NonCopyable, ReferenceCounted { PromiseStream rangesToAssign; BlobManagerData(UID id, Database db, Optional dcId) - : id(id), db(db), dcId(dcId), knownBlobRanges(false, normalKeys.end), - restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), recruitingStream(0) {} + : id(id), db(db), dcId(dcId), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &workersById), + knownBlobRanges(false, normalKeys.end), restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), + recruitingStream(0) {} }; ACTOR Future>> splitRange(Reference bmData, @@ -753,6 +774,7 @@ ACTOR Future monitorClientRanges(Reference bmData) { } for (KeyRangeRef range : rangesToRemove) { + TraceEvent("ClientBlobRangeRemoved", bmData->id).detail("Range", range); if (BM_DEBUG) { fmt::print( "BM Got range to revoke [{0} - {1})\n", range.begin.printable(), range.end.printable()); @@ -768,6 +790,7 @@ ACTOR Future monitorClientRanges(Reference bmData) { state std::vector>>> splitFutures; // Divide new ranges up into equal chunks by using SS byte sample for (KeyRangeRef range : rangesToAdd) { + TraceEvent("ClientBlobRangeAdded", bmData->id).detail("Range", range); splitFutures.push_back(splitRange(bmData, range, false)); } @@ -1096,6 +1119,11 @@ ACTOR Future maybeSplitRange(Reference bmData, splitVersion); } + ++bmData->stats.granuleSplits; + if (writeHot) { + ++bmData->stats.granuleWriteHotSplits; + } + // transaction committed, send range assignments // range could have been moved since split eval started, so just revoke from whoever has it RangeAssignment raRevoke; @@ -1182,6 +1210,8 @@ ACTOR Future killBlobWorker(Reference bmData, BlobWorkerI // Remove it from workersById also since otherwise that worker addr will remain excluded // when we try to recruit new blob workers. + TraceEvent("KillBlobWorker", bmData->id).detail("WorkerId", bwId); + if (registered) { bmData->deadWorkers.insert(bwId); bmData->workerStats.erase(bwId); @@ -1838,7 +1868,7 @@ ACTOR Future recoverBlobManager(Reference bmData) { TraceEvent("BlobManagerRecovered", bmData->id) .detail("Epoch", bmData->epoch) .detail("Duration", now() - recoveryStartTime) - .detail("Granules", bmData->workerAssignments.size()) + .detail("Granules", bmData->workerAssignments.size()) // TODO this includes un-set ranges, so it is inaccurate .detail("Assigned", explicitAssignments) .detail("Revoked", outOfDateAssignments.size()); @@ -2089,6 +2119,8 @@ ACTOR Future loadHistoryFiles(Reference bmData, U } } +// FIXME: trace events for pruning + /* * Deletes all files pertaining to the granule with id granuleId and * also removes the history entry for this granule from the system keyspace diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index 7b1eab8477..f44bbf6a2d 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -206,6 +206,7 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { if (BW_DEBUG) { fmt::print("BW {0} found new manager epoch {1}\n", id.toString(), currentManagerEpoch); } + TraceEvent(SevDebug, "BlobWorkerFoundNewManager", id).detail("Epoch", epoch); } return true; @@ -735,7 +736,7 @@ ACTOR Future dumpInitialSnapshotFromFDB(Reference Future streamFuture = tr->getTransaction().getRangeStream(rowsStream, metadata->keyRange, GetRangeLimits(), Snapshot::True); wait(streamFuture && success(snapshotWriter)); - TraceEvent("BlobGranuleSnapshotFile", bwData->id) + TraceEvent(SevDebug, "BlobGranuleSnapshotFile", bwData->id) .detail("Granule", metadata->keyRange) .detail("Version", readVersion); DEBUG_KEY_RANGE("BlobWorkerFDBSnapshot", readVersion, metadata->keyRange, bwData->id); @@ -759,7 +760,8 @@ ACTOR Future dumpInitialSnapshotFromFDB(Reference wait(tr->onError(e)); retries++; TEST(true); // Granule initial snapshot failed - TraceEvent(SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id) + // FIXME: why can't we supress error event? + TraceEvent(retries < 10 ? SevDebug : SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id) .error(err) .detail("Granule", metadata->keyRange) .detail("Count", retries); @@ -883,7 +885,7 @@ ACTOR Future checkSplitAndReSnapshot(Reference bw metadata->bytesInNewDeltaFiles); } - TraceEvent("BlobGranuleSnapshotCheck", bwData->id) + TraceEvent(SevDebug, "BlobGranuleSnapshotCheck", bwData->id) .detail("Granule", metadata->keyRange) .detail("Version", reSnapshotVersion); @@ -960,7 +962,7 @@ ACTOR Future checkSplitAndReSnapshot(Reference bw metadata->keyRange.end.printable(), bytesInNewDeltaFiles); } - TraceEvent("BlobGranuleSnapshotFile", bwData->id) + TraceEvent(SevDebug, "BlobGranuleSnapshotFile", bwData->id) .detail("Granule", metadata->keyRange) .detail("Version", metadata->durableDeltaVersion.get()); @@ -1540,7 +1542,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, bwData->id.toString().substr(0, 5).c_str(), deltas.version, rollbackVersion); - TraceEvent(SevWarn, "GranuleRollback", bwData->id) + TraceEvent(SevDebug, "GranuleRollback", bwData->id) .detail("Granule", metadata->keyRange) .detail("Version", deltas.version) .detail("RollbackVersion", rollbackVersion); @@ -1654,7 +1656,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, lastDeltaVersion, oldChangeFeedDataComplete.present() ? ". Finalizing " : ""); } - TraceEvent("BlobGranuleDeltaFile", bwData->id) + TraceEvent(SevDebug, "BlobGranuleDeltaFile", bwData->id) .detail("Granule", metadata->keyRange) .detail("Version", lastDeltaVersion); @@ -1831,13 +1833,13 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, } if (e.code() == error_code_granule_assignment_conflict) { - TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id) + TraceEvent("GranuleAssignmentConflict", bwData->id) .detail("Granule", metadata->keyRange) .detail("GranuleID", startState.granuleID); return Void(); } if (e.code() == error_code_change_feed_popped) { - TraceEvent(SevInfo, "GranuleGotChangeFeedPopped", bwData->id) + TraceEvent("GranuleChangeFeedPopped", bwData->id) .detail("Granule", metadata->keyRange) .detail("GranuleID", startState.granuleID); return Void(); @@ -2579,7 +2581,16 @@ ACTOR Future openGranule(Reference bwData, As info.changeFeedStartVersion = tr.getCommittedVersion(); } - TraceEvent("GranuleOpen", bwData->id).detail("Granule", req.keyRange); + TraceEvent openEv("GranuleOpen", bwData->id); + openEv.detail("GranuleID", info.granuleID) + .detail("Granule", req.keyRange) + .detail("Epoch", req.managerEpoch) + .detail("Seqno", req.managerSeqno) + .detail("CFStartVersion", info.changeFeedStartVersion) + .detail("PreviousDurableVersion", info.previousDurableVersion); + if (info.parentGranule.present()) { + openEv.detail("ParentGranuleID", info.parentGranule.get().second); + } return info; } catch (Error& e) { @@ -2900,6 +2911,7 @@ ACTOR Future handleRangeRevoke(Reference bwData, RevokeBlo ACTOR Future registerBlobWorker(Reference bwData, BlobWorkerInterface interf) { state Reference tr = makeReference(bwData->db); + TraceEvent("BlobWorkerRegister", bwData->id); loop { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); @@ -2920,6 +2932,7 @@ ACTOR Future registerBlobWorker(Reference bwData, BlobWork if (BW_DEBUG) { fmt::print("Registered blob worker {}\n", interf.id().toString()); } + TraceEvent("BlobWorkerRegistered", bwData->id); return Void(); } catch (Error& e) { if (BW_DEBUG) { From 971aa2dc4ed961c5496c48e7e81a50e455237796 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 20:53:40 -0700 Subject: [PATCH 55/90] Refactored callback tracking in ThreadCallback and ThreadMultiCallback to not use an unordered_map of pointers to prevent it from falsely triggering the DEBUG_DETERMINISM check, plus it is lower overhead, saving about 6% CPU in the AbortableSingleAssignmentVar unit test. --- flow/ThreadHelper.actor.h | 95 ++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 6627b9e25e..5aa9baebc7 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -22,6 +22,8 @@ // When actually compiled (NO_INTELLISENSE), include the generated // version of this file. In intellisense use the source version. +#include "flow/Error.h" +#include #if defined(NO_INTELLISENSE) && !defined(FLOW_THREADHELPER_ACTOR_G_H) #define FLOW_THREADHELPER_ACTOR_G_H #include "flow/ThreadHelper.actor.g.h" @@ -69,20 +71,78 @@ void onMainThreadVoid(F f, Error* err = nullptr, TaskPriority taskID = TaskPrior g_network->onMainThread(std::move(signal), taskID); } +class ThreadMultiCallback; + struct ThreadCallback { virtual bool canFire(int notMadeActive) const = 0; virtual void fire(const Void& unused, int& userParam) = 0; virtual void error(const Error&, int& userParam) = 0; virtual ThreadCallback* addCallback(ThreadCallback* cb); - virtual bool contains(ThreadCallback* cb) const { return false; } - virtual void clearCallback(ThreadCallback* cb) { // If this is the only registered callback this will be called with (possibly) arbitrary pointers } virtual void destroy() { UNSTOPPABLE_ASSERT(false); } virtual bool isMultiCallback() const { return false; } + + // MultiCallbackHolder is a helper object for ThreadMultiCallback which allows it to store its index + // within the callback vector inside the ThreadCallback rather than having a map of pointers or + // some other scheme to store the indices by callback. + // MultiCallbackHolder objects can form a doubly linked list. + struct MultiCallbackHolder : public FastAllocated { + MultiCallbackHolder(ThreadMultiCallback* holder = nullptr, + MultiCallbackHolder* prev = nullptr, + MultiCallbackHolder* next = nullptr) + : holder(holder), previous(prev), next(next) {} + + ThreadMultiCallback* holder; + int index; + MultiCallbackHolder* previous; + MultiCallbackHolder* next; + }; + + // firstHolder is both the inline first record of a MultiCallbackHolder and the head of the + // doubly linked list of MultiCallbackHolder entries. + MultiCallbackHolder firstHolder; + + // Return a MultiCallbackHolder for the given holder, using the firstHolder if free or allocating + // a new one. No check for an existing record for holder is done. + MultiCallbackHolder* addHolder(ThreadMultiCallback* holder) { + if (firstHolder.holder == nullptr) { + firstHolder.holder = holder; + return &firstHolder; + } + firstHolder.next = new MultiCallbackHolder(holder, &firstHolder, firstHolder.next); + return firstHolder.next; + } + + // Get the MultiCallbackHolder for holder if it exists, or nullptr. + MultiCallbackHolder* getHolder(ThreadMultiCallback* holder) { + MultiCallbackHolder* h = &firstHolder; + while (h != nullptr && h->holder != holder) { + h = h->next; + } + return h; + } + + // Destroy the given MultiCallbackHolder, freeing it if it is not firstHolder. + void destroyHolder(MultiCallbackHolder* h) { + UNSTOPPABLE_ASSERT(h != nullptr); + + // If h is the firstHolder just clear its holder pointer to indicate unusedness + if (h == &firstHolder) { + h->holder = nullptr; + } else { + // Otherwise unlink h from the doubly linked list and free it + // h->previous is definitely valid + h->previous->next = h->next; + if (h->next) { + h->next->previous = h->previous; + } + delete h; + } + } }; class ThreadMultiCallback final : public ThreadCallback, public FastAllocated { @@ -90,29 +150,31 @@ public: ThreadMultiCallback() {} ThreadCallback* addCallback(ThreadCallback* callback) override { - UNSTOPPABLE_ASSERT(callbackMap.count(callback) == - 0); // May be triggered by a waitForAll on a vector with the same future in it more than once - callbackMap[callback] = callbacks.size(); + UNSTOPPABLE_ASSERT( + callback->getHolder(this) == + nullptr); // May be triggered by a waitForAll on a vector with the same future in it more than once + callback->addHolder(this)->index = callbacks.size(); callbacks.push_back(callback); return (ThreadCallback*)this; } - bool contains(ThreadCallback* cb) const override { return callbackMap.count(cb) != 0; } - void clearCallback(ThreadCallback* callback) override { - auto it = callbackMap.find(callback); - if (it == callbackMap.end()) + MultiCallbackHolder* h = callback->getHolder(this); + if (h == nullptr) { return; + } - UNSTOPPABLE_ASSERT(it->second < callbacks.size() && it->second >= 0); + UNSTOPPABLE_ASSERT(h->index < callbacks.size() && h->index >= 0); - if (it->second != callbacks.size() - 1) { - callbacks[it->second] = callbacks.back(); - callbackMap[callbacks[it->second]] = it->second; + // Swap callback with last callback if it isn't the last + if (h->index != callbacks.size() - 1) { + callbacks[h->index] = callbacks.back(); + // Update the index of the Holder entry for the moved callback + callbacks[h->index]->getHolder(this)->index = h->index; } callbacks.pop_back(); - callbackMap.erase(it); + callback->destroyHolder(h); } bool canFire(int notMadeActive) const override { return true; } @@ -126,7 +188,7 @@ public: while (callbacks.size()) { auto cb = callbacks.back(); callbacks.pop_back(); - callbackMap.erase(cb); + cb->destroyHolder(cb->getHolder(this)); if (cb->canFire(0)) { int ld = 0; cb->fire(value, ld); @@ -143,7 +205,7 @@ public: while (callbacks.size()) { auto cb = callbacks.back(); callbacks.pop_back(); - callbackMap.erase(cb); + cb->destroyHolder(cb->getHolder(this)); if (cb->canFire(0)) { int ld = 0; cb->error(err, ld); @@ -160,7 +222,6 @@ public: private: std::vector callbacks; - std::unordered_map callbackMap; }; struct SetCallbackResult { From 01facc8dfa3ae9d83a6e459cb49be877b3020322 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 20:53:40 -0700 Subject: [PATCH 56/90] Refactored callback tracking in ThreadCallback and ThreadMultiCallback to not use an unordered_map of pointers to prevent it from falsely triggering the DEBUG_DETERMINISM check, plus it is lower overhead, saving about 6% CPU in the AbortableSingleAssignmentVar unit test. --- flow/ThreadHelper.actor.h | 95 ++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 6627b9e25e..5aa9baebc7 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -22,6 +22,8 @@ // When actually compiled (NO_INTELLISENSE), include the generated // version of this file. In intellisense use the source version. +#include "flow/Error.h" +#include #if defined(NO_INTELLISENSE) && !defined(FLOW_THREADHELPER_ACTOR_G_H) #define FLOW_THREADHELPER_ACTOR_G_H #include "flow/ThreadHelper.actor.g.h" @@ -69,20 +71,78 @@ void onMainThreadVoid(F f, Error* err = nullptr, TaskPriority taskID = TaskPrior g_network->onMainThread(std::move(signal), taskID); } +class ThreadMultiCallback; + struct ThreadCallback { virtual bool canFire(int notMadeActive) const = 0; virtual void fire(const Void& unused, int& userParam) = 0; virtual void error(const Error&, int& userParam) = 0; virtual ThreadCallback* addCallback(ThreadCallback* cb); - virtual bool contains(ThreadCallback* cb) const { return false; } - virtual void clearCallback(ThreadCallback* cb) { // If this is the only registered callback this will be called with (possibly) arbitrary pointers } virtual void destroy() { UNSTOPPABLE_ASSERT(false); } virtual bool isMultiCallback() const { return false; } + + // MultiCallbackHolder is a helper object for ThreadMultiCallback which allows it to store its index + // within the callback vector inside the ThreadCallback rather than having a map of pointers or + // some other scheme to store the indices by callback. + // MultiCallbackHolder objects can form a doubly linked list. + struct MultiCallbackHolder : public FastAllocated { + MultiCallbackHolder(ThreadMultiCallback* holder = nullptr, + MultiCallbackHolder* prev = nullptr, + MultiCallbackHolder* next = nullptr) + : holder(holder), previous(prev), next(next) {} + + ThreadMultiCallback* holder; + int index; + MultiCallbackHolder* previous; + MultiCallbackHolder* next; + }; + + // firstHolder is both the inline first record of a MultiCallbackHolder and the head of the + // doubly linked list of MultiCallbackHolder entries. + MultiCallbackHolder firstHolder; + + // Return a MultiCallbackHolder for the given holder, using the firstHolder if free or allocating + // a new one. No check for an existing record for holder is done. + MultiCallbackHolder* addHolder(ThreadMultiCallback* holder) { + if (firstHolder.holder == nullptr) { + firstHolder.holder = holder; + return &firstHolder; + } + firstHolder.next = new MultiCallbackHolder(holder, &firstHolder, firstHolder.next); + return firstHolder.next; + } + + // Get the MultiCallbackHolder for holder if it exists, or nullptr. + MultiCallbackHolder* getHolder(ThreadMultiCallback* holder) { + MultiCallbackHolder* h = &firstHolder; + while (h != nullptr && h->holder != holder) { + h = h->next; + } + return h; + } + + // Destroy the given MultiCallbackHolder, freeing it if it is not firstHolder. + void destroyHolder(MultiCallbackHolder* h) { + UNSTOPPABLE_ASSERT(h != nullptr); + + // If h is the firstHolder just clear its holder pointer to indicate unusedness + if (h == &firstHolder) { + h->holder = nullptr; + } else { + // Otherwise unlink h from the doubly linked list and free it + // h->previous is definitely valid + h->previous->next = h->next; + if (h->next) { + h->next->previous = h->previous; + } + delete h; + } + } }; class ThreadMultiCallback final : public ThreadCallback, public FastAllocated { @@ -90,29 +150,31 @@ public: ThreadMultiCallback() {} ThreadCallback* addCallback(ThreadCallback* callback) override { - UNSTOPPABLE_ASSERT(callbackMap.count(callback) == - 0); // May be triggered by a waitForAll on a vector with the same future in it more than once - callbackMap[callback] = callbacks.size(); + UNSTOPPABLE_ASSERT( + callback->getHolder(this) == + nullptr); // May be triggered by a waitForAll on a vector with the same future in it more than once + callback->addHolder(this)->index = callbacks.size(); callbacks.push_back(callback); return (ThreadCallback*)this; } - bool contains(ThreadCallback* cb) const override { return callbackMap.count(cb) != 0; } - void clearCallback(ThreadCallback* callback) override { - auto it = callbackMap.find(callback); - if (it == callbackMap.end()) + MultiCallbackHolder* h = callback->getHolder(this); + if (h == nullptr) { return; + } - UNSTOPPABLE_ASSERT(it->second < callbacks.size() && it->second >= 0); + UNSTOPPABLE_ASSERT(h->index < callbacks.size() && h->index >= 0); - if (it->second != callbacks.size() - 1) { - callbacks[it->second] = callbacks.back(); - callbackMap[callbacks[it->second]] = it->second; + // Swap callback with last callback if it isn't the last + if (h->index != callbacks.size() - 1) { + callbacks[h->index] = callbacks.back(); + // Update the index of the Holder entry for the moved callback + callbacks[h->index]->getHolder(this)->index = h->index; } callbacks.pop_back(); - callbackMap.erase(it); + callback->destroyHolder(h); } bool canFire(int notMadeActive) const override { return true; } @@ -126,7 +188,7 @@ public: while (callbacks.size()) { auto cb = callbacks.back(); callbacks.pop_back(); - callbackMap.erase(cb); + cb->destroyHolder(cb->getHolder(this)); if (cb->canFire(0)) { int ld = 0; cb->fire(value, ld); @@ -143,7 +205,7 @@ public: while (callbacks.size()) { auto cb = callbacks.back(); callbacks.pop_back(); - callbackMap.erase(cb); + cb->destroyHolder(cb->getHolder(this)); if (cb->canFire(0)) { int ld = 0; cb->error(err, ld); @@ -160,7 +222,6 @@ public: private: std::vector callbacks; - std::unordered_map callbackMap; }; struct SetCallbackResult { From 5a0274db653e77193dc1143ee8999fbd8febb6fd Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 22:59:28 -0700 Subject: [PATCH 57/90] Fixed Codec> backward compatibility bug recently introduced. --- fdbclient/BackupAgent.actor.h | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index a938dcd51f..7589d3485e 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -727,30 +727,40 @@ template <> inline Tuple Codec>::pack(Reference const& bc) { Tuple tuple; tuple.append(StringRef(bc->getURL())); - if (bc->getProxy().present()) { - tuple.append(StringRef(bc->getProxy().get())); - } else { - tuple.append(StringRef()); - } + if (bc->getEncryptionKeyFileName().present()) { tuple.append(bc->getEncryptionKeyFileName().get()); } else { tuple.append(StringRef()); } + + if (bc->getProxy().present()) { + tuple.append(StringRef(bc->getProxy().get())); + } else { + tuple.append(StringRef()); + } + return tuple; } template <> inline Reference Codec>::unpack(Tuple const& val) { - ASSERT(val.size() == 3); + ASSERT(val.size() >= 1 || val.size() <= 3); auto url = val.getString(0).toString(); - Optional proxy; - if (!val.getString(1).empty()) { - proxy = val.getString(1).toString(); - } + Optional encryptionKeyFileName; - if (!val.getString(2).empty()) { - encryptionKeyFileName = val.getString(2).toString(); + if (val.size() > 1) { + if (!val.getString(1).empty()) { + encryptionKeyFileName = val.getString(1).toString(); + } } + + Optional proxy; + if (val.size() > 2) { + if (!val.getString(2).empty()) { + proxy = val.getString(2).toString(); + }; + } + return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName); } From c00ae2fe85d37950d894770d843233ea001f9659 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 23:11:15 -0700 Subject: [PATCH 58/90] Fixed Codec> backward compatibility bug recently introduced. --- fdbclient/BackupAgent.actor.h | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index a938dcd51f..7589d3485e 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -727,30 +727,40 @@ template <> inline Tuple Codec>::pack(Reference const& bc) { Tuple tuple; tuple.append(StringRef(bc->getURL())); - if (bc->getProxy().present()) { - tuple.append(StringRef(bc->getProxy().get())); - } else { - tuple.append(StringRef()); - } + if (bc->getEncryptionKeyFileName().present()) { tuple.append(bc->getEncryptionKeyFileName().get()); } else { tuple.append(StringRef()); } + + if (bc->getProxy().present()) { + tuple.append(StringRef(bc->getProxy().get())); + } else { + tuple.append(StringRef()); + } + return tuple; } template <> inline Reference Codec>::unpack(Tuple const& val) { - ASSERT(val.size() == 3); + ASSERT(val.size() >= 1 || val.size() <= 3); auto url = val.getString(0).toString(); - Optional proxy; - if (!val.getString(1).empty()) { - proxy = val.getString(1).toString(); - } + Optional encryptionKeyFileName; - if (!val.getString(2).empty()) { - encryptionKeyFileName = val.getString(2).toString(); + if (val.size() > 1) { + if (!val.getString(1).empty()) { + encryptionKeyFileName = val.getString(1).toString(); + } } + + Optional proxy; + if (val.size() > 2) { + if (!val.getString(2).empty()) { + proxy = val.getString(2).toString(); + }; + } + return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName); } From b50886b10a23b2c73d0e847bd7433e692db146da Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 23:42:25 -0700 Subject: [PATCH 59/90] Fix size check Co-authored-by: Renxuan Wang --- fdbclient/BackupAgent.actor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index 7589d3485e..d803b38919 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -744,7 +744,7 @@ inline Tuple Codec>::pack(Reference inline Reference Codec>::unpack(Tuple const& val) { - ASSERT(val.size() >= 1 || val.size() <= 3); + ASSERT(val.size() >= 1 && val.size() <= 3); auto url = val.getString(0).toString(); Optional encryptionKeyFileName; From a1eca85b6686c4f70be3298824feef2f1f81ac25 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 23:43:07 -0700 Subject: [PATCH 60/90] Simplify conditional. Co-authored-by: Renxuan Wang --- fdbclient/BackupAgent.actor.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index d803b38919..777a3f2b7e 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -755,10 +755,8 @@ inline Reference Codec>::unpack(Tu } Optional proxy; - if (val.size() > 2) { - if (!val.getString(2).empty()) { - proxy = val.getString(2).toString(); - }; + if (val.size() > 2 && !val.getString(2).empty()) { + proxy = val.getString(2).toString(); } return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName); From bf505ed81641980e926322172a9632f0c6849d01 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 23:44:41 -0700 Subject: [PATCH 61/90] Simplify conditional. Co-authored-by: Renxuan Wang --- fdbclient/BackupAgent.actor.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index 777a3f2b7e..d0ddf548a3 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -748,10 +748,8 @@ inline Reference Codec>::unpack(Tu auto url = val.getString(0).toString(); Optional encryptionKeyFileName; - if (val.size() > 1) { - if (!val.getString(1).empty()) { - encryptionKeyFileName = val.getString(1).toString(); - } + if (val.size() > 1 && !val.getString(1).empty()) { + encryptionKeyFileName = val.getString(1).toString(); } Optional proxy; From c5995f2e274d3f4a9f821b9891a8833d2c978d0b Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 29 Mar 2022 23:53:45 -0700 Subject: [PATCH 62/90] Can't enforce size limit or downgrades from future versions could break. --- fdbclient/BackupAgent.actor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index d0ddf548a3..44fa7921d3 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -744,7 +744,7 @@ inline Tuple Codec>::pack(Reference inline Reference Codec>::unpack(Tuple const& val) { - ASSERT(val.size() >= 1 && val.size() <= 3); + ASSERT(val.size() >= 1); auto url = val.getString(0).toString(); Optional encryptionKeyFileName; From acfee48894ab3d125baa34aaa30dea592c2cbf20 Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Tue, 29 Mar 2022 22:28:22 -0700 Subject: [PATCH 63/90] AsyncFileKAIO: add latency histograms --- fdbrpc/AsyncFileKAIO.actor.h | 50 +++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h index 4294ee4724..7701097878 100644 --- a/fdbrpc/AsyncFileKAIO.actor.h +++ b/fdbrpc/AsyncFileKAIO.actor.h @@ -30,14 +30,17 @@ #define FLOW_ASYNCFILEKAIO_ACTOR_H #include "fdbrpc/IAsyncFile.h" + +#include #include #include #include #include #include "fdbrpc/linux_kaio.h" +#include "fdbserver/Knobs.h" #include "flow/Knobs.h" +#include "flow/Histogram.h" #include "flow/UnitTest.h" -#include #include "flow/crc32c.h" #include "flow/genericactors.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -46,6 +49,14 @@ // /data/v7/fdb/ #define KAIO_LOGGING 0 +struct AsyncFileKAIOMetrics { + Reference readLatencyDist; + Reference writeLatencyDist; + Reference syncLatencyDist; +} g_asyncFileKAIOMetrics; + +Future g_asyncFileKAIOHistogramLogger; + DESCR struct SlowAioSubmit { int64_t submitDuration; // ns int64_t truncateDuration; // ns @@ -343,6 +354,7 @@ public: #endif KAIOLogEvent(logFile, id, OpLogEntry::SYNC, OpLogEntry::START); + double start_time = now(); Future fsync = throwErrorIfFailed( Reference::addRef(this), @@ -352,12 +364,11 @@ public: submit(io, "write"); fsync=success(io->result.getFuture());*/ -#if KAIO_LOGGING fsync = map(fsync, [=](Void r) mutable { KAIOLogEvent(logFile, id, OpLogEntry::SYNC, OpLogEntry::COMPLETE); + g_asyncFileKAIOMetrics.syncLatencyDist->sampleSeconds(now() - start_time); return r; }); -#endif if (flags & OPEN_ATOMIC_WRITE_AND_CREATE) { flags &= ~OPEN_ATOMIC_WRITE_AND_CREATE; @@ -630,6 +641,16 @@ private: countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename); countLogicalWrites.init(LiteralStringRef("AsyncFile.CountLogicalWrites")); countLogicalReads.init(LiteralStringRef("AsyncFile.CountLogicalReads")); + if (!g_asyncFileKAIOHistogramLogger.isValid()) { + auto& metrics = g_asyncFileKAIOMetrics; + metrics.readLatencyDist = Reference(new Histogram( + Reference(), "AsyncFileKAIO", "ReadLatency", Histogram::Unit::microseconds)); + metrics.writeLatencyDist = Reference(new Histogram( + Reference(), "AsyncFileKAIO", "WriteLatency", Histogram::Unit::microseconds)); + metrics.syncLatencyDist = Reference(new Histogram( + Reference(), "AsyncFileKAIO", "SyncLatency", Histogram::Unit::microseconds)); + g_asyncFileKAIOHistogramLogger = histogramLogger(SERVER_KNOBS->DISK_METRIC_LOGGING_INTERVAL); + } } #if KAIO_LOGGING @@ -749,10 +770,33 @@ private: ctx.removeFromRequestList(iob); } + auto& metrics = g_asyncFileKAIOMetrics; + switch (iob->aio_lio_opcode) { + case IO_CMD_PREAD: + metrics.readLatencyDist->sampleSeconds(now() - iob->startTime); + break; + case IO_CMD_PWRITE: + metrics.writeLatencyDist->sampleSeconds(now() - iob->startTime); + break; + } + iob->setResult(ev[i].result); } } } + + ACTOR static Future histogramLogger(double interval) { + state double currentTime; + loop { + currentTime = now(); + wait(delay(interval)); + double elapsed = now() - currentTime; + auto& metrics = g_asyncFileKAIOMetrics; + metrics.readLatencyDist->writeToLog(elapsed); + metrics.writeLatencyDist->writeToLog(elapsed); + metrics.syncLatencyDist->writeToLog(elapsed); + } + } }; #if KAIO_LOGGING From 16cc74e91004d4b076b84724968d4fa042345eb4 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 30 Mar 2022 11:01:54 -0700 Subject: [PATCH 64/90] Remove documenation for Database.delete_tenant. --- documentation/sphinx/source/api-python.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 91a5f0da5a..eb8326654c 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -325,14 +325,6 @@ A |database-blurb1| |database-blurb2| .. |sync-read| replace:: This read is fully synchronous. .. |sync-write| replace:: This change will be committed immediately, and is fully synchronous. -.. method:: Database.delete_tenant(tenant_name): - - Delete a tenant from the cluster. |sync-write| - - The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. - - It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant. - .. method:: Database.get(key) Returns the value associated with the specified key in the database (or ``None`` if the key does not exist). |sync-read| From e6457b16561fd7583bfaddc8083292fdd80e5bd0 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 11:31:46 -0700 Subject: [PATCH 65/90] A few changes for clarity / readability. --- flow/ThreadHelper.actor.h | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 5aa9baebc7..ea6d6a3cd7 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -91,10 +91,13 @@ struct ThreadCallback { // some other scheme to store the indices by callback. // MultiCallbackHolder objects can form a doubly linked list. struct MultiCallbackHolder : public FastAllocated { - MultiCallbackHolder(ThreadMultiCallback* holder = nullptr, - MultiCallbackHolder* prev = nullptr, - MultiCallbackHolder* next = nullptr) - : holder(holder), previous(prev), next(next) {} + // Construction requires no arguments or all the arguments + MultiCallbackHolder() : holder(nullptr), index(0), previous(nullptr), next(nullptr) {} + MultiCallbackHolder(ThreadMultiCallback* multiCallback, + int index, + MultiCallbackHolder* prev, + MultiCallbackHolder* next) + : holder(multiCallback), index(0), previous(prev), next(next) {} ThreadMultiCallback* holder; int index; @@ -108,19 +111,20 @@ struct ThreadCallback { // Return a MultiCallbackHolder for the given holder, using the firstHolder if free or allocating // a new one. No check for an existing record for holder is done. - MultiCallbackHolder* addHolder(ThreadMultiCallback* holder) { + MultiCallbackHolder* addHolder(ThreadMultiCallback* multiCallback, int index) { if (firstHolder.holder == nullptr) { - firstHolder.holder = holder; + firstHolder.holder = multiCallback; + firstHolder.index = index; return &firstHolder; } - firstHolder.next = new MultiCallbackHolder(holder, &firstHolder, firstHolder.next); + firstHolder.next = new MultiCallbackHolder(multiCallback, index, &firstHolder, firstHolder.next); return firstHolder.next; } // Get the MultiCallbackHolder for holder if it exists, or nullptr. - MultiCallbackHolder* getHolder(ThreadMultiCallback* holder) { + MultiCallbackHolder* getHolder(ThreadMultiCallback* multiCallback) { MultiCallbackHolder* h = &firstHolder; - while (h != nullptr && h->holder != holder) { + while (h != nullptr && h->holder != multiCallback) { h = h->next; } return h; @@ -150,10 +154,10 @@ public: ThreadMultiCallback() {} ThreadCallback* addCallback(ThreadCallback* callback) override { - UNSTOPPABLE_ASSERT( - callback->getHolder(this) == - nullptr); // May be triggered by a waitForAll on a vector with the same future in it more than once - callback->addHolder(this)->index = callbacks.size(); + // May be triggered by a waitForAll on a vector with the same future in it more than once + UNSTOPPABLE_ASSERT(callback->getHolder(this) == nullptr); + + callback->addHolder(this, callbacks.size()); callbacks.push_back(callback); return (ThreadCallback*)this; } From 5d74e4d091bb3e793b4f09647935a93e6c0ff447 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 12:38:57 -0700 Subject: [PATCH 66/90] Added comments to explain some invariants with ThreadMultiCallback and ThreadCallback and how they are enforced. --- flow/ThreadHelper.actor.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index ea6d6a3cd7..39cdd345b1 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -83,6 +83,12 @@ struct ThreadCallback { // If this is the only registered callback this will be called with (possibly) arbitrary pointers } + // Note that when a ThreadCallback is destroyed it must have no MultiCallbackHolders, but this can't be + // asserted on destruction because throwing is not allowed in ~ThreadCallback() and the default destroy() + // implementation here is never called. + // However, ThreadMultiCallback::destroy() ensures that no ThreadMultiCallback will be destroyed while + // still holding a ThreadCallback so the invariant is effectively enforced there. See + // ThreadMultiCallback::destroy() for more details. virtual void destroy() { UNSTOPPABLE_ASSERT(false); } virtual bool isMultiCallback() const { return false; } @@ -218,6 +224,10 @@ public: } void destroy() override { + // This assert assures that all ThreadMultiCallbacks remove themselves as a holder from + // every ThreadCallback they hold prior to destruction, because if they do not then this + // assert will fire, so ThreadCallback does not attempt to destroy its MultiCallbackHolder + // linked list or verify that it is empty. UNSTOPPABLE_ASSERT(callbacks.empty()); delete this; } From c7d53b31ee1ddde4d9d84bc93669f7e0666c95e1 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 30 Mar 2022 12:52:27 -0700 Subject: [PATCH 67/90] Use a TenantState object in the MVC implementation to help manage tenant lifetime. --- fdbclient/MultiVersionTransaction.actor.cpp | 44 +++++++++++++++------ fdbclient/MultiVersionTransaction.h | 30 +++++++++----- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 82dc7768e2..75e252d4d7 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -780,7 +780,7 @@ void MultiVersionTransaction::updateTransaction() { TransactionInfo newTr; if (tenant.present()) { ASSERT(tenant.get()); - auto currentTenant = tenant.get()->tenantVar->get(); + auto currentTenant = tenant.get()->tenantState->tenantVar->get(); if (currentTenant.value) { newTr.transaction = currentTenant.value->createTransaction(); } @@ -1080,7 +1080,7 @@ ThreadFuture MultiVersionTransaction::onError(Error const& e) { Optional MultiVersionTransaction::getTenant() { if (tenant.present()) { - return tenant.get()->tenantName; + return tenant.get()->tenantState->tenantName; } else { return Optional(); } @@ -1214,20 +1214,31 @@ bool MultiVersionTransaction::isValid() { // MultiVersionTenant MultiVersionTenant::MultiVersionTenant(Reference db, StringRef tenantName) - : tenantVar(new ThreadSafeAsyncVar>(Reference(nullptr))), tenantName(tenantName), db(db) { - updateTenant(); + : tenantState(makeReference(db, tenantName)) {} + +MultiVersionTenant::~MultiVersionTenant() { + tenantState->close(); } -MultiVersionTenant::~MultiVersionTenant() {} - Reference MultiVersionTenant::createTransaction() { - return Reference(new MultiVersionTransaction( - db, Reference::addRef(this), db->dbState->transactionDefaultOptions)); + return Reference(new MultiVersionTransaction(tenantState->db, + Reference::addRef(this), + tenantState->db->dbState->transactionDefaultOptions)); +} + +MultiVersionTenant::TenantState::TenantState(Reference db, StringRef tenantName) + : tenantVar(new ThreadSafeAsyncVar>(Reference(nullptr))), tenantName(tenantName), db(db), + closed(false) { + updateTenant(); } // Creates a new underlying tenant object whenever the database connection changes. This change is signaled // to open transactions via an AsyncVar. -void MultiVersionTenant::updateTenant() { +void MultiVersionTenant::TenantState::updateTenant() { + if (closed) { + return; + } + Reference tenant; auto currentDb = db->dbState->dbVar->get(); if (currentDb.value) { @@ -1238,13 +1249,24 @@ void MultiVersionTenant::updateTenant() { tenantVar->set(tenant); + Reference self = Reference::addRef(this); + MutexHolder holder(tenantLock); - tenantUpdater = mapThreadFuture(currentDb.onChange, [this](ErrorOr result) { - updateTenant(); + tenantUpdater = mapThreadFuture(currentDb.onChange, [self](ErrorOr result) { + self->updateTenant(); return Void(); }); } +void MultiVersionTenant::TenantState::close() { + closed = true; + + MutexHolder holder(tenantLock); + if (tenantUpdater.isValid()) { + tenantUpdater.cancel(); + } +} + // MultiVersionDatabase MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api, int threadIdx, diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index e827335a2e..a8df462d88 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -646,18 +646,30 @@ public: void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } - Reference>> tenantVar; - const Standalone tenantName; + // A struct that manages the current connection state of the MultiVersionDatabase. This wraps the underlying + // IDatabase object that is currently interacting with the cluster. + struct TenantState : ThreadSafeReferenceCounted { + TenantState(Reference db, StringRef tenantName); -private: - Reference db; + // Creates a new underlying tenant object whenever the database connection changes. This change is signaled + // to open transactions via an AsyncVar. + void updateTenant(); - Mutex tenantLock; - ThreadFuture tenantUpdater; + // Cleans up local state to break reference cycles + void close(); - // Creates a new underlying tenant object whenever the database connection changes. This change is signaled - // to open transactions via an AsyncVar. - void updateTenant(); + Reference>> tenantVar; + const Standalone tenantName; + + Reference db; + + Mutex tenantLock; + ThreadFuture tenantUpdater; + + std::atomic_bool closed; + }; + + Reference tenantState; }; // An implementation of IDatabase that wraps a database created either locally or through a dynamically loaded From 75247affa3f9d4f3aecde4d982d6cf13915987cc Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 12:56:33 -0700 Subject: [PATCH 68/90] Renamed member for better readability. --- flow/ThreadHelper.actor.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 39cdd345b1..556fff89e4 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -98,14 +98,14 @@ struct ThreadCallback { // MultiCallbackHolder objects can form a doubly linked list. struct MultiCallbackHolder : public FastAllocated { // Construction requires no arguments or all the arguments - MultiCallbackHolder() : holder(nullptr), index(0), previous(nullptr), next(nullptr) {} + MultiCallbackHolder() : multiCallback(nullptr), index(0), previous(nullptr), next(nullptr) {} MultiCallbackHolder(ThreadMultiCallback* multiCallback, int index, MultiCallbackHolder* prev, MultiCallbackHolder* next) - : holder(multiCallback), index(0), previous(prev), next(next) {} + : multiCallback(multiCallback), index(0), previous(prev), next(next) {} - ThreadMultiCallback* holder; + ThreadMultiCallback* multiCallback; int index; MultiCallbackHolder* previous; MultiCallbackHolder* next; @@ -118,8 +118,8 @@ struct ThreadCallback { // Return a MultiCallbackHolder for the given holder, using the firstHolder if free or allocating // a new one. No check for an existing record for holder is done. MultiCallbackHolder* addHolder(ThreadMultiCallback* multiCallback, int index) { - if (firstHolder.holder == nullptr) { - firstHolder.holder = multiCallback; + if (firstHolder.multiCallback == nullptr) { + firstHolder.multiCallback = multiCallback; firstHolder.index = index; return &firstHolder; } @@ -130,7 +130,7 @@ struct ThreadCallback { // Get the MultiCallbackHolder for holder if it exists, or nullptr. MultiCallbackHolder* getHolder(ThreadMultiCallback* multiCallback) { MultiCallbackHolder* h = &firstHolder; - while (h != nullptr && h->holder != multiCallback) { + while (h != nullptr && h->multiCallback != multiCallback) { h = h->next; } return h; @@ -142,7 +142,7 @@ struct ThreadCallback { // If h is the firstHolder just clear its holder pointer to indicate unusedness if (h == &firstHolder) { - h->holder = nullptr; + h->multiCallback = nullptr; } else { // Otherwise unlink h from the doubly linked list and free it // h->previous is definitely valid From 88a439e156120c930ace53a1249c2facc3b7c152 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 30 Mar 2022 13:40:57 -0700 Subject: [PATCH 69/90] Fix typo from pull request #6698 (#6729) --- fdbmonitor/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbmonitor/CMakeLists.txt b/fdbmonitor/CMakeLists.txt index 622b0ec594..2c36c7bad3 100644 --- a/fdbmonitor/CMakeLists.txt +++ b/fdbmonitor/CMakeLists.txt @@ -15,10 +15,10 @@ target_link_libraries(fdbmonitor PUBLIC Threads::Threads) # processes). fdbmonitor is single-threaded anyway. get_target_property(fdbmonitor_options fdbmonitor COMPILE_OPTIONS) list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") -set_property(TARGET fdbmonitor PROPERTY COMPILE_OPTIONS ${target_options}) +set_property(TARGET fdbmonitor PROPERTY COMPILE_OPTIONS ${fdbmonitor_options}) get_target_property(fdbmonitor_options fdbmonitor LINK_OPTIONS) list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") -set_property(TARGET fdbmonitor PROPERTY LINK_OPTIONS ${target_options}) +set_property(TARGET fdbmonitor PROPERTY LINK_OPTIONS ${fdbmonitor_options}) if(GENERATE_DEBUG_PACKAGES) fdb_install(TARGETS fdbmonitor DESTINATION fdbmonitor COMPONENT server) From 2a52c76b7ad8595d023bb4fa7738968efd7d787f Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 14:47:24 -0700 Subject: [PATCH 70/90] Added INetwork::timer_int() for convenience. Clarified what timer_int() actually returns in header comments. --- fdbclient/FDBTypes.h | 2 +- flow/Platform.h | 2 +- flow/network.h | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 14fd1b023b..a32ab8d17b 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -1363,7 +1363,7 @@ struct StorageMetadataType { StorageMetadataType() : createdTime(0) {} StorageMetadataType(uint64_t t) : createdTime(t) {} - static uint64_t currentTime() { return g_network->timer() * 1e9; } + static uint64_t currentTime() { return g_network->timer_int(); } // To change this serialization, ProtocolVersion::StorageMetadata must be updated, and downgrades need // to be considered diff --git a/flow/Platform.h b/flow/Platform.h index dae2a63a08..6ccd8618a1 100644 --- a/flow/Platform.h +++ b/flow/Platform.h @@ -275,7 +275,7 @@ double timer(); // Returns the system real time clock with high precision. May jump around when system time is adjusted! double timer_monotonic(); // Returns a high precision monotonic clock which is adjusted to be kind of similar to timer() // at startup, but might not be a globally accurate time. -uint64_t timer_int(); // Return timer as uint64_t +uint64_t timer_int(); // Return timer as uint64_t representing epoch nanoseconds void getLocalTime(const time_t* timep, struct tm* result); diff --git a/flow/network.h b/flow/network.h index 967a145b7e..f3a3391288 100644 --- a/flow/network.h +++ b/flow/network.h @@ -567,6 +567,10 @@ public: // A wrapper for directly getting the system time. The time returned by now() only updates in the run loop, // so it cannot be used to measure times of functions that do not have wait statements. + // Simulation version of timer_int for convenience, based on timer() + // Returns epoch nanoseconds + uint64_t timer_int() { return (uint64_t)(g_network->timer() * 1e9); } + virtual double timer_monotonic() = 0; // Similar to timer, but monotonic From d6e2d2a1fe865c545eb4dcc718a0a67409b5a538 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 14:48:01 -0700 Subject: [PATCH 71/90] Fix nondeterminism in StorageWiggleMetrics caused by use of timer_int(). --- fdbserver/DataDistribution.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 9227cef3e4..de6d815453 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -296,7 +296,7 @@ Future StorageWiggler::restoreStats() { return map(readFuture, assignFunc); } Future StorageWiggler::startWiggle() { - metrics.last_wiggle_start = timer_int(); + metrics.last_wiggle_start = g_network->timer_int(); if (shouldStartNewRound()) { metrics.last_round_start = metrics.last_wiggle_start; } @@ -304,7 +304,7 @@ Future StorageWiggler::startWiggle() { } Future StorageWiggler::finishWiggle() { - metrics.last_wiggle_finish = timer_int(); + metrics.last_wiggle_finish = g_network->timer_int(); metrics.finished_wiggle += 1; auto duration = metrics.last_wiggle_finish - metrics.last_wiggle_start; metrics.smoothed_wiggle_duration.setTotal((double)duration); From 1b919f52e928e8a72d5acba9175eae32ed4b0c90 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 30 Mar 2022 16:29:35 -0700 Subject: [PATCH 72/90] Combine vector_like_traits::{insert,reserve} (#6689) * Combine vector_like_traits::{insert,reserve} and explain semantics better. This should make it more clear what implementers need to do when implementing the vector_like_traits concept. * Update std::unordered_set vector_like_traits impl --- flow/Arena.h | 14 ++++--- flow/ObjectSerializerTraits.h | 10 +++-- flow/flat_buffers.h | 77 ++++++++++++++++++++--------------- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/flow/Arena.h b/flow/Arena.h index 59923d67a8..88e6ee6a0c 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -1470,15 +1470,19 @@ struct vector_like_traits> : std::true static size_t num_entries(const VectorRef& v, Context&) { return v.size(); } - template - static void reserve(VectorRef& v, size_t s, Context& context) { - v.resize(context.arena(), s); - } + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t s, Context& context) { + // Logically v should be empty after this function returns, but since we're going to + // insert s times into the raw pointer insert_iterator it will end up + // with the correct size after deserialization finishes. + v.resize(context.arena(), s); return v.begin(); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h index 979db897e3..d396bd5808 100644 --- a/flow/ObjectSerializerTraits.h +++ b/flow/ObjectSerializerTraits.h @@ -108,13 +108,17 @@ struct vector_like_traits : std::false_type { using iterator = void; using insert_iterator = void; + // The number of entries in this vector template static size_t num_entries(VectorLike&, Context&); - template - static void reserve(VectorLike&, size_t, Context&); + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(VectorLike&, Context&); + static insert_iterator insert(VectorLike&, size_t size, Context&); + + // Return an iterator to read from this vector. template static iterator begin(const VectorLike&, Context&); }; diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h index 5b98f74536..c1445d92dd 100644 --- a/flow/flat_buffers.h +++ b/flow/flat_buffers.h @@ -115,16 +115,17 @@ struct vector_like_traits> : std::true_type { static size_t num_entries(const Vec& v, Context&) { return v.size(); } + + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static void reserve(Vec& v, size_t size, Context&) { + static insert_iterator insert(Vec& v, size_t size, Context&) { v.clear(); v.reserve(size); - } - - template - static insert_iterator insert(Vec& v, Context&) { return std::back_inserter(v); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -142,16 +143,16 @@ struct vector_like_traits> : std::true_type { static size_t num_entries(const Deq& v, Context&) { return v.size(); } - template - static void reserve(Deq& v, size_t size, Context&) { - v.resize(size); - v.clear(); - } + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Deq& v, Context&) { + static insert_iterator insert(Deq& v, size_t size, Context&) { + v.clear(); return std::back_inserter(v); } + template static iterator begin(const Deq& v, Context&) { return v.begin(); @@ -169,12 +170,15 @@ struct vector_like_traits> : std::true_type { static size_t num_entries(const Vec& v, Context&) { return N; } + + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static void reserve(Vec& v, size_t size, Context&) {} - template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t s, Context&) { return v.begin(); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -192,13 +196,16 @@ struct vector_like_traits> : std::true_type static size_t num_entries(const Vec& v, Context&) { return v.size(); } - template - static void reserve(Vec& v, size_t size, Context&) {} + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t s, Context&) { + v.clear(); return std::inserter(v, v.end()); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -215,13 +222,17 @@ struct vector_like_traits> : s static size_t num_entries(const Vec& v, Context&) { return v.size(); } - template - static void reserve(Vec& v, size_t size, Context&) {} + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t size, Context&) { + v.clear(); + v.reserve(size); return std::inserter(v, v.end()); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -239,13 +250,16 @@ struct vector_like_traits> : std::true_type { static size_t num_entries(const Vec& v, Context&) { return v.size(); } - template - static void reserve(Vec&, size_t, Context&) {} + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t size, Context&) { + v.clear(); return std::inserter(v, v.end()); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -262,15 +276,16 @@ struct vector_like_traits> : static size_t num_entries(const Vec& v, Context&) { return v.size(); } - template - static void reserve(Vec& v, size_t size, Context&) { - v.reserve(size); - } + // Return an insert_iterator starting with an empty vector. |size| is the + // number of elements to be inserted. Implementations may want to allocate + // enough memory up front to hold |size| elements. template - static insert_iterator insert(Vec& v, Context&) { + static insert_iterator insert(Vec& v, size_t size, Context&) { + v.reserve(size); return std::inserter(v, v.end()); } + template static iterator begin(const Vec& v, Context&) { return v.begin(); @@ -946,8 +961,7 @@ struct LoadMember { current += current_offset; uint32_t numEntries = interpret_as(current); current += sizeof(uint32_t); - VectorTraits::reserve(member, numEntries, context); - auto inserter = VectorTraits::insert(member, context); + auto inserter = VectorTraits::insert(member, numEntries, context); for (int i = 0; i < numEntries; ++i) { T value; if (types_current[i] > 0) { @@ -1082,8 +1096,7 @@ struct LoadSaveHelper : Context { current += current_offset; uint32_t numEntries = interpret_as(current); current += sizeof(uint32_t); - VectorTraits::reserve(member, numEntries, this->context()); - auto inserter = VectorTraits::insert(member, this->context()); + auto inserter = VectorTraits::insert(member, numEntries, this->context()); for (uint32_t i = 0; i < numEntries; ++i) { T value; load_helper(value, current, this->context()); From a2a97e71761145ce592c57e9e0fc8eb2d2c050e9 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Sat, 19 Feb 2022 15:48:49 -0800 Subject: [PATCH 73/90] Add tenant support to the Java bindings --- bindings/bindingtester/known_testers.py | 4 +- bindings/java/CMakeLists.txt | 2 + bindings/java/fdbJNI.cpp | 97 +++++++ .../main/com/apple/foundationdb/Database.java | 60 +++- .../com/apple/foundationdb/FDBDatabase.java | 46 ++++ .../com/apple/foundationdb/FDBTenant.java | 157 +++++++++++ .../main/com/apple/foundationdb/Tenant.java | 257 ++++++++++++++++++ .../foundationdb/test/AsyncStackTester.java | 31 ++- .../com/apple/foundationdb/test/Context.java | 118 +++++--- .../apple/foundationdb/test/Instruction.java | 33 ++- .../foundationdb/test/StackOperation.java | 6 + .../apple/foundationdb/test/StackTester.java | 24 +- 12 files changed, 777 insertions(+), 58 deletions(-) create mode 100644 bindings/java/src/main/com/apple/foundationdb/FDBTenant.java create mode 100644 bindings/java/src/main/com/apple/foundationdb/Tenant.java diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index fbae72d36c..70e1f81420 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -61,8 +61,8 @@ testers = { 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION), - 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), - 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), + 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), + 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 2040, 200, MAX_API_VERSION, types=ALL_TYPES), 'flow': Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION, directory_snapshot_ops_enabled=False), } diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index 9adf24a2f7..f3bb84a552 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -32,6 +32,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/DirectBufferPool.java src/main/com/apple/foundationdb/FDB.java src/main/com/apple/foundationdb/FDBDatabase.java + src/main/com/apple/foundationdb/FDBTenant.java src/main/com/apple/foundationdb/FDBTransaction.java src/main/com/apple/foundationdb/FutureInt64.java src/main/com/apple/foundationdb/FutureKey.java @@ -64,6 +65,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/ReadTransactionContext.java src/main/com/apple/foundationdb/subspace/package-info.java src/main/com/apple/foundationdb/subspace/Subspace.java + src/main/com/apple/foundationdb/Tenant.java src/main/com/apple/foundationdb/Transaction.java src/main/com/apple/foundationdb/TransactionContext.java src/main/com/apple/foundationdb/EventKeeper.java diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index d2164b9887..a516256e4f 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -663,6 +663,78 @@ JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1ge return result; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1allocateTenant(JNIEnv* jenv, + jobject, + jlong dbPtr, + jbyteArray tenantNameBytes) { + if (!dbPtr || !tenantNameBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBDatabase* database = (FDBDatabase*)dbPtr; + + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(tenantNameBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_database_allocate_tenant(database, barr, jenv->GetArrayLength(tenantNameBytes)); + jenv->ReleaseByteArrayElements(tenantNameBytes, (jbyte*)barr, JNI_ABORT); + return (jlong)f; +} + +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1deleteTenant(JNIEnv* jenv, + jobject, + jlong dbPtr, + jbyteArray tenantNameBytes) { + if (!dbPtr || !tenantNameBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBDatabase* database = (FDBDatabase*)dbPtr; + + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(tenantNameBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_database_remove_tenant(database, barr, jenv->GetArrayLength(tenantNameBytes)); + jenv->ReleaseByteArrayElements(tenantNameBytes, (jbyte*)barr, JNI_ABORT); + return (jlong)f; +} + +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1openTenant(JNIEnv* jenv, + jobject, + jlong dbPtr, + jbyteArray tenantNameBytes) { + if (!dbPtr || !tenantNameBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBDatabase* database = (FDBDatabase*)dbPtr; + FDBTenant* tenant; + + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(tenantNameBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + fdb_error_t err = fdb_database_open_tenant(database, barr, jenv->GetArrayLength(tenantNameBytes), &tenant); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); + return 0; + } + + jenv->ReleaseByteArrayElements(tenantNameBytes, (jbyte*)barr, JNI_ABORT); + return (jlong)tenant; +} + JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv* jenv, jobject, jlong dbPtr) { @@ -764,6 +836,31 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv* return (jlong)db; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1createTransaction(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { + throwParamNotNull(jenv); + return 0; + } + FDBTenant* tenant = (FDBTenant*)tPtr; + FDBTransaction* tr; + fdb_error_t err = fdb_tenant_create_transaction(tenant, &tr); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); + return 0; + } + return (jlong)tr; +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1dispose(JNIEnv* jenv, jobject, jlong tPtr) { + if (!tPtr) { + throwParamNotNull(jenv); + return; + } + fdb_tenant_destroy((FDBTenant*)tPtr); +} + JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv* jenv, jobject, jlong tPtr, diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index 741fa1c5eb..293d8d0b47 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -41,11 +41,67 @@ import java.util.function.Function; */ public interface Database extends AutoCloseable, TransactionContext { /** - * Creates a {@link Transaction} that operates on this {@code Database}.
+ * Creates a new tenant in the cluster. + * + * @param tenantName The name of the tenant. Can be any byte string that does not begin a 0xFF byte. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been created. + */ + CompletableFuture allocateTenant(byte[] tenantName); + + /** + * Deletes a tenant from the cluster.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must + * first use a clear operation to delete all of its keys. + * + * @param tenantName The name of the tenant being deleted. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been deleted. + */ + CompletableFuture deleteTenant(byte[] tenantName); + + /** + * Opens an existing tenant to be used for running transactions. + * + * @param tenantName The name of the tenant to open. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + default Tenant openTenant(byte[] tenantName) { + return openTenant(tenantName, getExecutor()); + } + + /** + * Opens an existing tenant to be used for running transactions. + * + * @param tenantName The name of the tenant to open. + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + Tenant openTenant(byte[] tenantName, Executor e); + + /** + * Opens an existing tenant to be used for running transactions. + * + * @param tenantName The name of the tenant to open. + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @param eventKeeper the {@link EventKeeper} to use when tracking instrumented calls for the tenant's transactions. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + Tenant openTenant(byte[] tenantName, Executor e, EventKeeper eventKeeper); + + /** + * Creates a {@link Transaction} that operates on this {@code Database}. Creating a transaction + * in this way does not associate it with a {@code Tenant}, and as a result the transaction will + * operate on the entire key-space for the database.
*
* Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable} * option. This is because the Java bindings disallow use of {@code Transaction} objects after - * {@link Transaction#onError} is called. + * {@link Transaction#onError} is called.
+ *
+ * Note: Transactions created directly on a {@code Database} object cannot be used in a cluster + * that requires tenant-based access. To run transactions in those clusters, you must first open a tenant + * with {@link #openTenant(byte[])}. * * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}. */ diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index 8df1fd75b6..52885be48f 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -116,6 +116,49 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public CompletableFuture allocateTenant(byte[] tenantName) { + pointerReadLock.lock(); + try { + return new FutureVoid(Database_allocateTenant(getPtr(), tenantName), executor); + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public CompletableFuture deleteTenant(byte[] tenantName) { + pointerReadLock.lock(); + try { + return new FutureVoid(Database_deleteTenant(getPtr(), tenantName), executor); + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public Tenant openTenant(byte[] tenantName, Executor e) { + return openTenant(tenantName, e, eventKeeper); + } + + @Override + public Tenant openTenant(byte[] tenantName, Executor e, EventKeeper eventKeeper) { + pointerReadLock.lock(); + Tenant tenant = null; + try { + tenant = new FDBTenant(Database_openTenant(getPtr(), tenantName), this, tenantName, e, eventKeeper); + return tenant; + } catch (RuntimeException err) { + if (tenant != null) { + tenant.close(); + } + + throw err; + } finally { + pointerReadLock.unlock(); + } + } + @Override public Transaction createTransaction(Executor e) { return createTransaction(e, eventKeeper); @@ -170,6 +213,9 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume Database_dispose(cPtr); } + private native long Database_allocateTenant(long cPtr, byte[] tenantName); + private native long Database_deleteTenant(long cPtr, byte[] tenantName); + private native long Database_openTenant(long cPtr, byte[] tenantName); private native long Database_createTransaction(long cPtr); private native void Database_dispose(long cPtr); private native void Database_setOption(long cPtr, int code, byte[] value) throws FDBException; diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java new file mode 100644 index 0000000000..029f671eb9 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java @@ -0,0 +1,157 @@ +/* + * FDBTenant.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import com.apple.foundationdb.async.AsyncUtil; + +class FDBTenant extends NativeObjectWrapper implements Tenant { + private final Database database; + private final byte[] name; + private final Executor executor; + private final EventKeeper eventKeeper; + + protected FDBTenant(long cPtr, Database database, byte[] name, Executor executor) { + this(cPtr, database, name, executor, null); + } + + protected FDBTenant(long cPtr, Database database, byte[] name, Executor executor, EventKeeper eventKeeper) { + super(cPtr); + this.database = database; + this.name = name; + this.executor = executor; + this.eventKeeper = eventKeeper; + } + + @Override + public T run(Function retryable, Executor e) { + Transaction t = this.createTransaction(e); + try { + while (true) { + try { + T returnVal = retryable.apply(t); + t.commit().join(); + return returnVal; + } catch (RuntimeException err) { + t = t.onError(err).join(); + } + } + } finally { + t.close(); + } + } + + @Override + public T read(Function retryable, Executor e) { + return this.run(retryable, e); + } + + @Override + public CompletableFuture runAsync(final Function> retryable, Executor e) { + final AtomicReference trRef = new AtomicReference<>(createTransaction(e)); + final AtomicReference returnValue = new AtomicReference<>(); + return AsyncUtil.whileTrue(() -> { + CompletableFuture process = AsyncUtil.applySafely(retryable, trRef.get()); + + return AsyncUtil.composeHandleAsync(process.thenComposeAsync(returnVal -> + trRef.get().commit().thenApply(o -> { + returnValue.set(returnVal); + return false; + }), e), + (value, t) -> { + if(t == null) + return CompletableFuture.completedFuture(value); + if(!(t instanceof RuntimeException)) + throw new CompletionException(t); + return trRef.get().onError(t).thenApply(newTr -> { + trRef.set(newTr); + return true; + }); + }, e); + }, e) + .thenApply(o -> returnValue.get()) + .whenComplete((v, t) -> trRef.get().close()); + } + + @Override + public CompletableFuture readAsync( + Function> retryable, Executor e) { + return this.runAsync(retryable, e); + } + + @Override + protected void finalize() throws Throwable { + try { + checkUnclosed("Tenant"); + close(); + } + finally { + super.finalize(); + } + } + + @Override + public Transaction createTransaction(Executor e) { + return createTransaction(e, eventKeeper); + } + + @Override + public Transaction createTransaction(Executor e, EventKeeper eventKeeper) { + pointerReadLock.lock(); + Transaction tr = null; + try { + tr = new FDBTransaction(Tenant_createTransaction(getPtr()), database, e, eventKeeper); + tr.options().setUsedDuringCommitProtectionDisable(); + return tr; + } catch (RuntimeException err) { + if (tr != null) { + tr.close(); + } + + throw err; + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public byte[] getName() { + return name; + } + + @Override + public Executor getExecutor() { + return executor; + } + + @Override + protected void closeInternal(long cPtr) { + Tenant_dispose(cPtr); + } + + private native long Tenant_createTransaction(long cPtr); + private native void Tenant_dispose(long cPtr); +} \ No newline at end of file diff --git a/bindings/java/src/main/com/apple/foundationdb/Tenant.java b/bindings/java/src/main/com/apple/foundationdb/Tenant.java new file mode 100644 index 0000000000..9aaadc57f1 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/Tenant.java @@ -0,0 +1,257 @@ +/* + * Tenant.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Function; + +/** + * A tenant represents a named key-space within a database that can be interacted with + * transactionally.
+ *
+ * The simplest correct programs using tenants will make use of the methods defined + * in the {@link TransactionContext} interface. When used on a {@code Tenant} these + * methods will call {@code Transaction#commit()} after user code has been + * executed. These methods will not return successfully until {@code commit()} has + * returned successfully.
+ *
+ * Note: {@code Tenant} objects must be {@link #close closed} when no longer + * in use in order to free any associated resources. + */ +public interface Tenant extends AutoCloseable, TransactionContext { + /** + * Creates a {@link Transaction} that operates on this {@code Tenant}.
+ *
+ * Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable} + * option. This is because the Java bindings disallow use of {@code Transaction} objects after + * {@link Transaction#onError} is called. + * + * @return a newly created {@code Transaction} that reads from and writes to this {@code Tenant}. + */ + default Transaction createTransaction() { + return createTransaction(getExecutor()); + } + + /** + * Creates a {@link Transaction} that operates on this {@code Tenant} with the given {@link Executor} + * for asynchronous callbacks. + * + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @return a newly created {@code Transaction} that reads from and writes to this {@code Tenant}. + */ + Transaction createTransaction(Executor e); + + /** + * Creates a {@link Transaction} that operates on this {@code Tenant} with the given {@link Executor} + * for asynchronous callbacks. + * + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @param eventKeeper the {@link EventKeeper} to use when tracking instrumented calls for the transaction. + * + * @return a newly created {@code Transaction} that reads from and writes to this {@code Tenant}. + */ + Transaction createTransaction(Executor e, EventKeeper eventKeeper); + + /** + * Returns the name of this {@code Tenant}. + * + * @return the name of this {@code Tenant} as a byte string. + */ + byte[] getName(); + + /** + * Runs a read-only transactional function against this {@code Tenant} with retry logic. + * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the + * supplied {@link Function} until a non-retryable + * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * is thrown. This call is blocking -- this + * method will not return until the {@code Function} has been called and completed without error.
+ * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param the return type of {@code retryable} + * + * @return the result of the last run of {@code retryable} + */ + @Override + default T read(Function retryable) { + return read(retryable, getExecutor()); + } + + /** + * Runs a read-only transactional function against this {@code Tenant} with retry logic. Use + * this formulation of {@link #read(Function)} if one wants to set a custom {@link Executor} + * for the transaction when run. + * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param e the {@link Executor} to use for asynchronous callbacks + * @param the return type of {@code retryable} + * @return the result of the last run of {@code retryable} + * + * @see #read(Function) + */ + T read(Function retryable, Executor e); + + /** + * Runs a read-only transactional function against this {@code Tenant} with retry logic. + * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the + * supplied {@link Function} until a non-retryable + * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * is thrown. This call is non-blocking -- this + * method will return immediately and with a {@link CompletableFuture} that will be + * set when the {@code Function} has been called and completed without error.
+ *
+ * Any errors encountered executing {@code retryable}, or received from the + * database, will be set on the returned {@code CompletableFuture}. + * + * @param retryable the block of logic to execute in a {@link ReadTransaction} against + * this tenant + * @param the return type of {@code retryable} + * + * @return a {@code CompletableFuture} that will be set to the value returned by the last call + * to {@code retryable} + */ + @Override + default CompletableFuture readAsync( + Function> retryable) { + return readAsync(retryable, getExecutor()); + } + + /** + * Runs a read-only transactional function against this {@code Tenant} with retry logic. + * Use this version of {@link #readAsync(Function)} if one wants to set a custom + * {@link Executor} for the transaction when run. + * + * @param retryable the block of logic to execute in a {@link ReadTransaction} against + * this tenant + * @param e the {@link Executor} to use for asynchronous callbacks + * @param the return type of {@code retryable} + * + * @return a {@code CompletableFuture} that will be set to the value returned by the last call + * to {@code retryable} + * + * @see #readAsync(Function) + */ + CompletableFuture readAsync( + Function> retryable, Executor e); + + /** + * Runs a transactional function against this {@code Tenant} with retry logic. + * {@link Function#apply(Object) apply(Transaction)} will be called on the + * supplied {@link Function} until a non-retryable + * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * is thrown or {@link Transaction#commit() commit()}, + * when called after {@code apply()}, returns success. This call is blocking -- this + * method will not return until {@code commit()} has been called and returned success.
+ *
+ * As with other client/server databases, in some failure scenarios a client may + * be unable to determine whether a transaction succeeded. In these cases, your + * transaction may be executed twice. For more information about how to reason + * about these situations see + * the FounationDB Developer Guide + * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param the return type of {@code retryable} + * + * @return the result of the last run of {@code retryable} + */ + @Override + default T run(Function retryable) { + return run(retryable, getExecutor()); + } + + /** + * Runs a transactional function against this {@code Tenant} with retry logic. + * Use this formulation of {@link #run(Function)} if one would like to set a + * custom {@link Executor} for the transaction when run. + * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param e the {@link Executor} to use for asynchronous callbacks + * @param the return type of {@code retryable} + * + * @return the result of the last run of {@code retryable} + */ + T run(Function retryable, Executor e); + + /** + * Runs a transactional function against this {@code Tenant} with retry logic. + * {@link Function#apply(Object) apply(Transaction)} will be called on the + * supplied {@link Function} until a non-retryable + * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * is thrown or {@link Transaction#commit() commit()}, + * when called after {@code apply()}, returns success. This call is non-blocking -- this + * method will return immediately and with a {@link CompletableFuture} that will be + * set when {@code commit()} has been called and returned success.
+ *
+ * As with other client/server databases, in some failure scenarios a client may + * be unable to determine whether a transaction succeeded. In these cases, your + * transaction may be executed twice. For more information about how to reason + * about these situations see + * the FounationDB Developer Guide
+ *
+ * Any errors encountered executing {@code retryable}, or received from the + * database, will be set on the returned {@code CompletableFuture}. + * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param the return type of {@code retryable} + * + * @return a {@code CompletableFuture} that will be set to the value returned by the last call + * to {@code retryable} + */ + @Override + default CompletableFuture runAsync( + Function> retryable) { + return runAsync(retryable, getExecutor()); + } + + /** + * Runs a transactional function against this {@code Tenant} with retry logic. Use + * this formulation of the non-blocking {@link #runAsync(Function)} if one wants + * to set a custom {@link Executor} for the transaction when run. + * + * @param retryable the block of logic to execute in a {@link Transaction} against + * this tenant + * @param e the {@link Executor} to use for asynchronous callbacks + * @param the return type of {@code retryable} + * + * @return a {@code CompletableFuture} that will be set to the value returned by the last call + * to {@code retryable} + * + * @see #run(Function) + */ + CompletableFuture runAsync( + Function> retryable, Executor e); + + /** + * Close the {@code Tenant} object and release any associated resources. This must be called at + * least once after the {@code Tenant} object is no longer in use. This can be called multiple + * times, but care should be taken that it is not in use in another thread at the time of the call. + */ + @Override + void close(); +} diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index 87ea5adfe0..b303ed3a3a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -184,7 +185,7 @@ public class AsyncStackTester { return AsyncUtil.DONE; } else if(op == StackOperation.RESET) { - inst.context.newTransaction(); + inst.context.resetTransaction(); return AsyncUtil.DONE; } else if(op == StackOperation.CANCEL) { @@ -332,9 +333,9 @@ public class AsyncStackTester { final Transaction oldTr = inst.tr; CompletableFuture f = oldTr.onError(err).whenComplete((tr, t) -> { if(t != null) { - inst.context.newTransaction(oldTr); // Other bindings allow reuse of non-retryable transactions, so we need to emulate that behavior. + inst.context.resetTransaction(oldTr); // Other bindings allow reuse of non-retryable transactions, so we need to emulate that behavior. } - else if(!inst.setTransaction(oldTr, tr)) { + else if(!inst.replaceTransaction(oldTr, tr)) { tr.close(); } }).thenApply(v -> null); @@ -469,6 +470,28 @@ public class AsyncStackTester { inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); }, FDB.DEFAULT_EXECUTOR); } + else if (op == StackOperation.TENANT_CREATE) { + return inst.popParam().thenAcceptAsync(param -> { + byte[] tenantName = (byte[])param; + inst.push(inst.context.db.allocateTenant(tenantName)); + }, FDB.DEFAULT_EXECUTOR); + } + else if (op == StackOperation.TENANT_DELETE) { + return inst.popParam().thenAcceptAsync(param -> { + byte[] tenantName = (byte[])param; + inst.push(inst.context.db.deleteTenant(tenantName)); + }, FDB.DEFAULT_EXECUTOR); + } + else if (op == StackOperation.TENANT_SET_ACTIVE) { + return inst.popParam().thenAcceptAsync(param -> { + byte[] tenantName = (byte[])param; + inst.context.setTenant(Optional.of(tenantName)); + }, FDB.DEFAULT_EXECUTOR); + } + else if (op == StackOperation.TENANT_CLEAR_ACTIVE) { + inst.context.setTenant(Optional.empty()); + return AsyncUtil.DONE; + } else if(op == StackOperation.UNIT_TESTS) { inst.context.db.options().setLocationCacheSize(100001); return inst.context.db.runAsync(tr -> { @@ -554,7 +577,7 @@ public class AsyncStackTester { private static CompletableFuture executeMutation(final Instruction inst, Function> r) { // run this with a retry loop return inst.tcx.runAsync(r).thenRunAsync(() -> { - if(inst.isDatabase) + if(inst.isDatabase || inst.isTenant) inst.push("RESULT_NOT_PRESENT".getBytes()); }, FDB.DEFAULT_EXECUTOR); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Context.java b/bindings/java/src/test/com/apple/foundationdb/test/Context.java index c71cb45f99..a594e088a1 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Context.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Context.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.CompletableFuture; @@ -35,6 +36,7 @@ import com.apple.foundationdb.FDBException; import com.apple.foundationdb.KeySelector; import com.apple.foundationdb.Range; import com.apple.foundationdb.StreamingMode; +import com.apple.foundationdb.Tenant; import com.apple.foundationdb.Transaction; import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; @@ -42,15 +44,27 @@ import com.apple.foundationdb.tuple.Tuple; abstract class Context implements Runnable, AutoCloseable { final Stack stack = new Stack(); final Database db; + Optional tenant = Optional.empty(); final String preStr; int instructionIndex = 0; KeySelector nextKey, endKey; Long lastVersion = null; + private static class TransactionState { + public Transaction transaction; + public Optional tenant; + + public TransactionState(Transaction transaction, Optional tenant) { + this.transaction = transaction; + this.tenant = tenant; + } + } + private String trName; private List children = new LinkedList<>(); - private static Map transactionMap = new HashMap<>(); + private static Map transactionMap = new HashMap<>(); private static Map transactionRefCounts = new HashMap<>(); + private static Map tenantMap = new HashMap<>(); Context(Database db, byte[] prefix) { this.db = db; @@ -86,15 +100,24 @@ abstract class Context implements Runnable, AutoCloseable { } } + public synchronized void setTenant(Optional tenantName) { + if (tenantName.isPresent()) { + tenant = Optional.of(tenantMap.computeIfAbsent(tenantName.get(), tn -> db.openTenant(tenantName.get()))); + } + else { + tenant = Optional.empty(); + } + } + public static synchronized void addTransactionReference(Transaction tr) { transactionRefCounts.computeIfAbsent(tr, x -> new AtomicInteger(0)).incrementAndGet(); } private static synchronized Transaction getTransaction(String trName) { - Transaction tr = transactionMap.get(trName); - assert tr != null : "Null transaction"; - addTransactionReference(tr); - return tr; + TransactionState state = transactionMap.get(trName); + assert state != null : "Null transaction"; + addTransactionReference(state.transaction); + return state.transaction; } public Transaction getCurrentTransaction() { @@ -105,59 +128,78 @@ abstract class Context implements Runnable, AutoCloseable { if(tr != null) { AtomicInteger count = transactionRefCounts.get(tr); if(count.decrementAndGet() == 0) { - assert !transactionMap.containsValue(tr); transactionRefCounts.remove(tr); tr.close(); } } } - private static synchronized void updateTransaction(String trName, Transaction tr) { - releaseTransaction(transactionMap.put(trName, tr)); - addTransactionReference(tr); - } - - private static synchronized boolean updateTransaction(String trName, Transaction oldTr, Transaction newTr) { - boolean added; - if(oldTr == null) { - added = (transactionMap.putIfAbsent(trName, newTr) == null); + private static Transaction createTransaction(Database db, Optional creatingTenant) { + if (creatingTenant.isPresent()) { + return creatingTenant.get().createTransaction(); } else { - added = transactionMap.replace(trName, oldTr, newTr); + return db.createTransaction(); + } + } + + private static synchronized boolean newTransaction(Database db, Optional tenant, String trName, boolean allowReplace) { + TransactionState oldState = transactionMap.get(trName); + if (oldState != null) { + releaseTransaction(oldState.transaction); + } + else if (!allowReplace) { + return false; } - if(added) { + TransactionState newState = new TransactionState(createTransaction(db, tenant), tenant); + + transactionMap.put(trName, newState); + addTransactionReference(newState.transaction); + + return true; + } + + private static synchronized boolean replaceTransaction(Database db, String trName, Transaction oldTr, Transaction newTr) { + TransactionState trState = transactionMap.get(trName); + assert trState != null : "Null transaction"; + + if(oldTr == null || trState.transaction == oldTr) { + if(newTr == null) { + newTr = createTransaction(db, trState.tenant); + } + releaseTransaction(trState.transaction); addTransactionReference(newTr); - releaseTransaction(oldTr); + trState.transaction = newTr; return true; } return false; } - public void updateCurrentTransaction(Transaction tr) { - updateTransaction(trName, tr); - } - - public boolean updateCurrentTransaction(Transaction oldTr, Transaction newTr) { - return updateTransaction(trName, oldTr, newTr); - } - public void newTransaction() { - Transaction tr = db.createTransaction(); - updateCurrentTransaction(tr); + newTransaction(db, tenant, trName, true); } - public void newTransaction(Transaction oldTr) { - Transaction newTr = db.createTransaction(); - if(!updateCurrentTransaction(oldTr, newTr)) { - newTr.close(); - } + public void replaceTransaction(Transaction tr) { + replaceTransaction(db, trName, null, tr); + } + + public boolean replaceTransaction(Transaction oldTr, Transaction newTr) { + return replaceTransaction(db, trName, oldTr, newTr); + } + + public void resetTransaction() { + replaceTransaction(db, trName, null, null); + } + + public boolean resetTransaction(Transaction oldTr) { + return replaceTransaction(db, trName, oldTr, null); } public void switchTransaction(byte[] rawTrName) { trName = ByteArrayUtil.printable(rawTrName); - newTransaction(null); + newTransaction(db, tenant, trName, false); } abstract void executeOperations() throws Throwable; @@ -224,8 +266,12 @@ abstract class Context implements Runnable, AutoCloseable { @Override public void close() { - for(Transaction tr : transactionMap.values()) { - tr.close(); + for(TransactionState tr : transactionMap.values()) { + tr.transaction.close(); + } + + for(Tenant tenant : tenantMap.values()) { + tenant.close(); } } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Instruction.java b/bindings/java/src/test/com/apple/foundationdb/test/Instruction.java index d991217a0a..2b41da3bc7 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Instruction.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Instruction.java @@ -33,11 +33,13 @@ import com.apple.foundationdb.tuple.Tuple; class Instruction extends Stack { private static final String SUFFIX_SNAPSHOT = "_SNAPSHOT"; private static final String SUFFIX_DATABASE = "_DATABASE"; + private static final String SUFFIX_TENANT = "_TENANT"; final String op; final Tuple tokens; final Context context; final boolean isDatabase; + final boolean isTenant; final boolean isSnapshot; final Transaction tr; final ReadTransaction readTr; @@ -49,14 +51,23 @@ class Instruction extends Stack { this.tokens = tokens; String fullOp = tokens.getString(0); - isDatabase = fullOp.endsWith(SUFFIX_DATABASE); + boolean isDatabaseLocal = fullOp.endsWith(SUFFIX_DATABASE); + isTenant = fullOp.endsWith(SUFFIX_TENANT); isSnapshot = fullOp.endsWith(SUFFIX_SNAPSHOT); - if(isDatabase) { + if(isDatabaseLocal) { tr = null; readTr = null; op = fullOp.substring(0, fullOp.length() - SUFFIX_DATABASE.length()); } + else if(isTenant) { + tr = null; + readTr = null; + op = fullOp.substring(0, fullOp.length() - SUFFIX_TENANT.length()); + if (!context.tenant.isPresent()) { + isDatabaseLocal = true; + } + } else if(isSnapshot) { tr = context.getCurrentTransaction(); readTr = tr.snapshot(); @@ -68,22 +79,24 @@ class Instruction extends Stack { op = fullOp; } - tcx = isDatabase ? context.db : tr; - readTcx = isDatabase ? context.db : readTr; + isDatabase = isDatabaseLocal; + + tcx = isDatabase ? context.db : isTenant ? context.tenant.get() : tr; + readTcx = isDatabase ? context.db : isTenant ? context.tenant.get() : readTr; } - boolean setTransaction(Transaction newTr) { - if(!isDatabase) { - context.updateCurrentTransaction(newTr); + boolean replaceTransaction(Transaction newTr) { + if(!isDatabase && !isTenant) { + context.replaceTransaction(newTr); return true; } return false; } - boolean setTransaction(Transaction oldTr, Transaction newTr) { - if(!isDatabase) { - return context.updateCurrentTransaction(oldTr, newTr); + boolean replaceTransaction(Transaction oldTr, Transaction newTr) { + if(!isDatabase && !isTenant) { + return context.replaceTransaction(oldTr, newTr); } return false; diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index bece744605..5cd013195d 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -73,5 +73,11 @@ enum StackOperation { DECODE_DOUBLE, UNIT_TESTS, /* Possibly unimplemented */ + // Tenants + TENANT_CREATE, + TENANT_DELETE, + TENANT_SET_ACTIVE, + TENANT_CLEAR_ACTIVE, + LOG_STACK } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index 0490e2a5fb..401afb6391 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.function.Function; @@ -197,7 +198,7 @@ public class StackTester { inst.tr.options().setNextWriteNoWriteConflictRange(); } else if(op == StackOperation.RESET) { - inst.context.newTransaction(); + inst.context.resetTransaction(); } else if(op == StackOperation.CANCEL) { inst.tr.cancel(); @@ -300,12 +301,12 @@ public class StackTester { try { Transaction tr = inst.tr.onError(err).join(); - if(!inst.setTransaction(tr)) { + if(!inst.replaceTransaction(tr)) { tr.close(); } } catch(Throwable t) { - inst.context.newTransaction(); // Other bindings allow reuse of non-retryable transactions, so we need to emulate that behavior. + inst.context.resetTransaction(); // Other bindings allow reuse of non-retryable transactions, so we need to emulate that behavior. throw t; } @@ -418,6 +419,21 @@ public class StackTester { double value = ((Number)param).doubleValue(); inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); } + else if (op == StackOperation.TENANT_CREATE) { + byte[] tenantName = (byte[])inst.popParam().join(); + inst.push(inst.context.db.allocateTenant(tenantName)); + } + else if (op == StackOperation.TENANT_DELETE) { + byte[] tenantName = (byte[])inst.popParam().join(); + inst.push(inst.context.db.deleteTenant(tenantName)); + } + else if (op == StackOperation.TENANT_SET_ACTIVE) { + byte[] tenantName = (byte[])inst.popParam().join(); + inst.context.setTenant(Optional.of(tenantName)); + } + else if (op == StackOperation.TENANT_CLEAR_ACTIVE) { + inst.context.setTenant(Optional.empty()); + } else if(op == StackOperation.UNIT_TESTS) { try { inst.context.db.options().setLocationCacheSize(100001); @@ -579,7 +595,7 @@ public class StackTester { private static void executeMutation(Instruction inst, Function r) { // run this with a retry loop (and commit) inst.tcx.run(r); - if(inst.isDatabase) + if(inst.isDatabase || inst.isTenant) inst.push("RESULT_NOT_PRESENT".getBytes()); } From 995f1b36010f577883befcbbad809b2aac309419 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Mon, 28 Feb 2022 18:15:10 -0500 Subject: [PATCH 74/90] Support tuples in Java tenants --- .../main/com/apple/foundationdb/Database.java | 54 +++++++++++++++++++ .../com/apple/foundationdb/FDBDatabase.java | 26 +++++++++ .../apple/foundationdb/test/StackTester.java | 31 +++++++++++ 3 files changed, 111 insertions(+) diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index 293d8d0b47..828ede6b5f 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -23,6 +23,7 @@ package com.apple.foundationdb; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.function.Function; +import com.apple.foundationdb.tuple.Tuple; /** * A mutable, lexicographically ordered mapping from binary keys to binary values. @@ -49,6 +50,16 @@ public interface Database extends AutoCloseable, TransactionContext { */ CompletableFuture allocateTenant(byte[] tenantName); + /** + * Creates a new tenant in the cluster. This is a convenience method that generates the tenant name by packing a + * {@code Tuple}. + * + * @param tenantName The name of the tenant, as a Tuple. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been created. + */ + CompletableFuture allocateTenant(Tuple tenantName); + /** * Deletes a tenant from the cluster.
*
@@ -61,6 +72,19 @@ public interface Database extends AutoCloseable, TransactionContext { */ CompletableFuture deleteTenant(byte[] tenantName); + /** + * Deletes a tenant from the cluster. This is a convenience method that generates the tenant name by packing a + * {@code Tuple}.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty + * tenant, you must first use a clear operation to delete all of its keys. + * + * @param tenantName The name of the tenant being deleted, as a Tuple. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been deleted. + */ + CompletableFuture deleteTenant(Tuple tenantName); + /** * Opens an existing tenant to be used for running transactions. * @@ -71,6 +95,15 @@ public interface Database extends AutoCloseable, TransactionContext { return openTenant(tenantName, getExecutor()); } + /** + * Opens an existing tenant to be used for running transactions. This is a convenience method that generates the + * tenant name by packing a {@code Tuple}. + * + * @param tenantName The name of the tenant to open, as a Tuple. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + Tenant openTenant(Tuple tenantName); + /** * Opens an existing tenant to be used for running transactions. * @@ -80,6 +113,16 @@ public interface Database extends AutoCloseable, TransactionContext { */ Tenant openTenant(byte[] tenantName, Executor e); + /** + * Opens an existing tenant to be used for running transactions. This is a convenience method that generates the + * tenant name by packing a {@code Tuple}. + * + * @param tenantName The name of the tenant to open, as a Tuple. + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + Tenant openTenant(Tuple tenantName, Executor e); + /** * Opens an existing tenant to be used for running transactions. * @@ -90,6 +133,17 @@ public interface Database extends AutoCloseable, TransactionContext { */ Tenant openTenant(byte[] tenantName, Executor e, EventKeeper eventKeeper); + /** + * Opens an existing tenant to be used for running transactions. This is a convenience method that generates the + * tenant name by packing a {@code Tuple}. + * + * @param tenantName The name of the tenant to open, as a Tuple. + * @param e the {@link Executor} to use when executing asynchronous callbacks. + * @param eventKeeper the {@link EventKeeper} to use when tracking instrumented calls for the tenant's transactions. + * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. + */ + Tenant openTenant(Tuple tenantName, Executor e, EventKeeper eventKeeper); + /** * Creates a {@link Transaction} that operates on this {@code Database}. Creating a transaction * in this way does not associate it with a {@code Tenant}, and as a result the transaction will diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index 52885be48f..d3200d6d6b 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.tuple.Tuple; class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsumer { private DatabaseOptions options; @@ -126,6 +127,11 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public CompletableFuture allocateTenant(Tuple tenantName) { + return allocateTenant(tenantName.pack()); + } + @Override public CompletableFuture deleteTenant(byte[] tenantName) { pointerReadLock.lock(); @@ -136,11 +142,26 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public CompletableFuture deleteTenant(Tuple tenantName) { + return deleteTenant(tenantName.pack()); + } + @Override public Tenant openTenant(byte[] tenantName, Executor e) { return openTenant(tenantName, e, eventKeeper); } + @Override + public Tenant openTenant(Tuple tenantName) { + return openTenant(tenantName.pack()); + } + + @Override + public Tenant openTenant(Tuple tenantName, Executor e) { + return openTenant(tenantName.pack(), e); + } + @Override public Tenant openTenant(byte[] tenantName, Executor e, EventKeeper eventKeeper) { pointerReadLock.lock(); @@ -159,6 +180,11 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public Tenant openTenant(Tuple tenantName, Executor e, EventKeeper eventKeeper) { + return openTenant(tenantName.pack(), e, eventKeeper); + } + @Override public Transaction createTransaction(Executor e) { return createTransaction(e, eventKeeper); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index 401afb6391..ef4c7ec6d6 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -50,6 +50,7 @@ import com.apple.foundationdb.async.AsyncIterable; import com.apple.foundationdb.async.AsyncUtil; import com.apple.foundationdb.async.CloseableAsyncIterator; import com.apple.foundationdb.tuple.ByteArrayUtil; +import com.apple.foundationdb.Tenant; import com.apple.foundationdb.tuple.Tuple; /** @@ -506,6 +507,7 @@ public class StackTester { testWatches(inst.context.db); testLocality(inst.context.db); + testTenantTupleNames(inst.context.db); } catch(Exception e) { throw new RuntimeException("Unit tests failed: " + e.getMessage()); @@ -757,6 +759,35 @@ public class StackTester { }); } + private static void testTenantTupleNames(Database db) { + try { + db.allocateTenant(Tuple.from("tenant")).join(); + Tenant tenant = db.openTenant(Tuple.from("tenant")); + + tenant.run(tr -> { + tr.set(Tuple.from("hello").pack(), Tuple.from("world").pack()); + return null; + }); + + String output = tenant.read(tr -> { + byte[] result = tr.get(Tuple.from("hello").pack()).join(); + return Tuple.fromBytes(result).getString(0); + }); + + assert output.equals("world"); + + tenant.run(tr -> { + tr.clear(Tuple.from("hello").pack()); + return null; + }); + + db.deleteTenant(Tuple.from("tenant")).join(); + } + catch(Exception e) { + e.printStackTrace(); + } + } + /** * Run a stack-machine based test. * From 6570ae44c63d733cd1382602ace6db44ca08243a Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Sun, 6 Mar 2022 21:14:25 -0800 Subject: [PATCH 75/90] Use special keys to create/delete tenants --- bindings/java/fdbJNI.cpp | 44 ---------------- .../com/apple/foundationdb/FDBDatabase.java | 50 ++++++++++++++----- .../com/apple/foundationdb/FDBTenant.java | 3 ++ .../main/com/apple/foundationdb/Tenant.java | 8 +-- 4 files changed, 45 insertions(+), 60 deletions(-) diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index a516256e4f..f44d52e169 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -663,50 +663,6 @@ JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1ge return result; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1allocateTenant(JNIEnv* jenv, - jobject, - jlong dbPtr, - jbyteArray tenantNameBytes) { - if (!dbPtr || !tenantNameBytes) { - throwParamNotNull(jenv); - return 0; - } - FDBDatabase* database = (FDBDatabase*)dbPtr; - - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(tenantNameBytes, JNI_NULL); - if (!barr) { - if (!jenv->ExceptionOccurred()) - throwRuntimeEx(jenv, "Error getting handle to native resources"); - return 0; - } - - FDBFuture* f = fdb_database_allocate_tenant(database, barr, jenv->GetArrayLength(tenantNameBytes)); - jenv->ReleaseByteArrayElements(tenantNameBytes, (jbyte*)barr, JNI_ABORT); - return (jlong)f; -} - -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1deleteTenant(JNIEnv* jenv, - jobject, - jlong dbPtr, - jbyteArray tenantNameBytes) { - if (!dbPtr || !tenantNameBytes) { - throwParamNotNull(jenv); - return 0; - } - FDBDatabase* database = (FDBDatabase*)dbPtr; - - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(tenantNameBytes, JNI_NULL); - if (!barr) { - if (!jenv->ExceptionOccurred()) - throwRuntimeEx(jenv, "Error getting handle to native resources"); - return 0; - } - - FDBFuture* f = fdb_database_remove_tenant(database, barr, jenv->GetArrayLength(tenantNameBytes)); - jenv->ReleaseByteArrayElements(tenantNameBytes, (jbyte*)barr, JNI_ABORT); - return (jlong)f; -} - JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1openTenant(JNIEnv* jenv, jobject, jlong dbPtr, diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index d3200d6d6b..5214dd62c7 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -23,10 +23,12 @@ package com.apple.foundationdb; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsumer { @@ -119,12 +121,24 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume @Override public CompletableFuture allocateTenant(byte[] tenantName) { - pointerReadLock.lock(); - try { - return new FutureVoid(Database_allocateTenant(getPtr(), tenantName), executor); - } finally { - pointerReadLock.unlock(); - } + final AtomicBoolean checkedExistence = new AtomicBoolean(false); + final byte[] key = ByteArrayUtil.join(FDBTenant.TENANT_MAP_PREFIX, tenantName); + return runAsync(tr -> { + tr.options().setSpecialKeySpaceEnableWrites(); + if(checkedExistence.get()) { + tr.set(key, new byte[0]); + return CompletableFuture.completedFuture(null); + } + else { + return tr.get(key).thenAcceptAsync(result -> { + checkedExistence.set(true); + if(result != null) { + throw new FDBException("A tenant with the given name already exists", 2132); + } + tr.set(key, new byte[0]); + }); + } + }); } @Override @@ -134,12 +148,24 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume @Override public CompletableFuture deleteTenant(byte[] tenantName) { - pointerReadLock.lock(); - try { - return new FutureVoid(Database_deleteTenant(getPtr(), tenantName), executor); - } finally { - pointerReadLock.unlock(); - } + final AtomicBoolean checkedExistence = new AtomicBoolean(false); + final byte[] key = ByteArrayUtil.join(FDBTenant.TENANT_MAP_PREFIX, tenantName); + return runAsync(tr -> { + tr.options().setSpecialKeySpaceEnableWrites(); + if(checkedExistence.get()) { + tr.clear(key); + return CompletableFuture.completedFuture(null); + } + else { + return tr.get(key).thenAcceptAsync(result -> { + checkedExistence.set(true); + if (result == null) { + throw new FDBException("Tenant does not exist", 2131); + } + tr.clear(key); + }); + } + }); } @Override diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java index 029f671eb9..2a5315fa4a 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.tuple.ByteArrayUtil; class FDBTenant extends NativeObjectWrapper implements Tenant { private final Database database; @@ -34,6 +35,8 @@ class FDBTenant extends NativeObjectWrapper implements Tenant { private final Executor executor; private final EventKeeper eventKeeper; + static final byte[] TENANT_MAP_PREFIX = ByteArrayUtil.join(new byte[] { (byte)255, (byte)255 }, "/management/tenant_map/".getBytes()); + protected FDBTenant(long cPtr, Database database, byte[] name, Executor executor) { this(cPtr, database, name, executor, null); } diff --git a/bindings/java/src/main/com/apple/foundationdb/Tenant.java b/bindings/java/src/main/com/apple/foundationdb/Tenant.java index 9aaadc57f1..c265cd29da 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Tenant.java +++ b/bindings/java/src/main/com/apple/foundationdb/Tenant.java @@ -82,7 +82,7 @@ public interface Tenant extends AutoCloseable, TransactionContext { * Runs a read-only transactional function against this {@code Tenant} with retry logic. * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the * supplied {@link Function} until a non-retryable - * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * FDBException (or any {@code Throwable} other than an {@code FDBException}) * is thrown. This call is blocking -- this * method will not return until the {@code Function} has been called and completed without error.
* @@ -116,7 +116,7 @@ public interface Tenant extends AutoCloseable, TransactionContext { * Runs a read-only transactional function against this {@code Tenant} with retry logic. * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the * supplied {@link Function} until a non-retryable - * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * FDBException (or any {@code Throwable} other than an {@code FDBException}) * is thrown. This call is non-blocking -- this * method will return immediately and with a {@link CompletableFuture} that will be * set when the {@code Function} has been called and completed without error.
@@ -159,7 +159,7 @@ public interface Tenant extends AutoCloseable, TransactionContext { * Runs a transactional function against this {@code Tenant} with retry logic. * {@link Function#apply(Object) apply(Transaction)} will be called on the * supplied {@link Function} until a non-retryable - * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * FDBException (or any {@code Throwable} other than an {@code FDBException}) * is thrown or {@link Transaction#commit() commit()}, * when called after {@code apply()}, returns success. This call is blocking -- this * method will not return until {@code commit()} has been called and returned success.
@@ -200,7 +200,7 @@ public interface Tenant extends AutoCloseable, TransactionContext { * Runs a transactional function against this {@code Tenant} with retry logic. * {@link Function#apply(Object) apply(Transaction)} will be called on the * supplied {@link Function} until a non-retryable - * {@link FDBException} (or any {@code Throwable} other than an {@code FDBException}) + * FDBException (or any {@code Throwable} other than an {@code FDBException}) * is thrown or {@link Transaction#commit() commit()}, * when called after {@code apply()}, returns success. This call is non-blocking -- this * method will return immediately and with a {@link CompletableFuture} that will be From be7315473a5452d00ea93a7dfa2fbeff1be4b069 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 28 Mar 2022 13:54:43 -0700 Subject: [PATCH 76/90] Move tenant creation/deletion into a TenantManagement class --- bindings/java/CMakeLists.txt | 1 + .../main/com/apple/foundationdb/Database.java | 44 ---- .../com/apple/foundationdb/FDBDatabase.java | 57 ----- .../com/apple/foundationdb/FDBTenant.java | 2 - .../apple/foundationdb/TenantManagement.java | 214 ++++++++++++++++++ .../foundationdb/test/AsyncStackTester.java | 5 +- .../apple/foundationdb/test/StackTester.java | 9 +- 7 files changed, 223 insertions(+), 109 deletions(-) create mode 100644 bindings/java/src/main/com/apple/foundationdb/TenantManagement.java diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index f3bb84a552..22564dccc8 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -66,6 +66,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/subspace/package-info.java src/main/com/apple/foundationdb/subspace/Subspace.java src/main/com/apple/foundationdb/Tenant.java + src/main/com/apple/foundationdb/TenantManagement.java src/main/com/apple/foundationdb/Transaction.java src/main/com/apple/foundationdb/TransactionContext.java src/main/com/apple/foundationdb/EventKeeper.java diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index 828ede6b5f..0128234a95 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -41,50 +41,6 @@ import com.apple.foundationdb.tuple.Tuple; * in use in order to free any associated resources. */ public interface Database extends AutoCloseable, TransactionContext { - /** - * Creates a new tenant in the cluster. - * - * @param tenantName The name of the tenant. Can be any byte string that does not begin a 0xFF byte. - * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has - * been created. - */ - CompletableFuture allocateTenant(byte[] tenantName); - - /** - * Creates a new tenant in the cluster. This is a convenience method that generates the tenant name by packing a - * {@code Tuple}. - * - * @param tenantName The name of the tenant, as a Tuple. - * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has - * been created. - */ - CompletableFuture allocateTenant(Tuple tenantName); - - /** - * Deletes a tenant from the cluster.
- *
- * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must - * first use a clear operation to delete all of its keys. - * - * @param tenantName The name of the tenant being deleted. - * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has - * been deleted. - */ - CompletableFuture deleteTenant(byte[] tenantName); - - /** - * Deletes a tenant from the cluster. This is a convenience method that generates the tenant name by packing a - * {@code Tuple}.
- *
- * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty - * tenant, you must first use a clear operation to delete all of its keys. - * - * @param tenantName The name of the tenant being deleted, as a Tuple. - * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has - * been deleted. - */ - CompletableFuture deleteTenant(Tuple tenantName); - /** * Opens an existing tenant to be used for running transactions. * diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index 5214dd62c7..5e0b808242 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -23,7 +23,6 @@ package com.apple.foundationdb; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -119,60 +118,6 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } - @Override - public CompletableFuture allocateTenant(byte[] tenantName) { - final AtomicBoolean checkedExistence = new AtomicBoolean(false); - final byte[] key = ByteArrayUtil.join(FDBTenant.TENANT_MAP_PREFIX, tenantName); - return runAsync(tr -> { - tr.options().setSpecialKeySpaceEnableWrites(); - if(checkedExistence.get()) { - tr.set(key, new byte[0]); - return CompletableFuture.completedFuture(null); - } - else { - return tr.get(key).thenAcceptAsync(result -> { - checkedExistence.set(true); - if(result != null) { - throw new FDBException("A tenant with the given name already exists", 2132); - } - tr.set(key, new byte[0]); - }); - } - }); - } - - @Override - public CompletableFuture allocateTenant(Tuple tenantName) { - return allocateTenant(tenantName.pack()); - } - - @Override - public CompletableFuture deleteTenant(byte[] tenantName) { - final AtomicBoolean checkedExistence = new AtomicBoolean(false); - final byte[] key = ByteArrayUtil.join(FDBTenant.TENANT_MAP_PREFIX, tenantName); - return runAsync(tr -> { - tr.options().setSpecialKeySpaceEnableWrites(); - if(checkedExistence.get()) { - tr.clear(key); - return CompletableFuture.completedFuture(null); - } - else { - return tr.get(key).thenAcceptAsync(result -> { - checkedExistence.set(true); - if (result == null) { - throw new FDBException("Tenant does not exist", 2131); - } - tr.clear(key); - }); - } - }); - } - - @Override - public CompletableFuture deleteTenant(Tuple tenantName) { - return deleteTenant(tenantName.pack()); - } - @Override public Tenant openTenant(byte[] tenantName, Executor e) { return openTenant(tenantName, e, eventKeeper); @@ -265,8 +210,6 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume Database_dispose(cPtr); } - private native long Database_allocateTenant(long cPtr, byte[] tenantName); - private native long Database_deleteTenant(long cPtr, byte[] tenantName); private native long Database_openTenant(long cPtr, byte[] tenantName); private native long Database_createTransaction(long cPtr); private native void Database_dispose(long cPtr); diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java index 2a5315fa4a..36aa8293ba 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java @@ -35,8 +35,6 @@ class FDBTenant extends NativeObjectWrapper implements Tenant { private final Executor executor; private final EventKeeper eventKeeper; - static final byte[] TENANT_MAP_PREFIX = ByteArrayUtil.join(new byte[] { (byte)255, (byte)255 }, "/management/tenant_map/".getBytes()); - protected FDBTenant(long cPtr, Database database, byte[] name, Executor executor) { this(cPtr, database, name, executor, null); } diff --git a/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java b/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java new file mode 100644 index 0000000000..857aeb7f1f --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/TenantManagement.java @@ -0,0 +1,214 @@ +/* + * TenantManagement.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; + +import com.apple.foundationdb.async.AsyncIterable; +import com.apple.foundationdb.async.AsyncIterator; +import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.async.CloseableAsyncIterator; +import com.apple.foundationdb.tuple.ByteArrayUtil; +import com.apple.foundationdb.tuple.Tuple; + +/** + * The FoundationDB API includes function to manage the set of tenants in a cluster. + */ +public class TenantManagement { + static final byte[] TENANT_MAP_PREFIX = ByteArrayUtil.join(new byte[] { (byte)255, (byte)255 }, + "/management/tenant_map/".getBytes()); + + /** + * Creates a new tenant in the cluster. If the tenant already exists, this operation will complete + * successfully without changing anything. The transaction must be committed for the creation to take + * effect or to observe any errors. + * + * @param tr The transaction used to create the tenant. + * @param tenantName The name of the tenant. Can be any byte string that does not begin a 0xFF byte. + */ + public static void createTenant(Transaction tr, byte[] tenantName) { + tr.options().setSpecialKeySpaceEnableWrites(); + tr.set(ByteArrayUtil.join(TENANT_MAP_PREFIX, tenantName), new byte[0]); + } + + /** + * Creates a new tenant in the cluster. If the tenant already exists, this operation will complete + * successfully without changing anything. The transaction must be committed for the creation to take + * effect or to observe any errors.
+ *
+ * This is a convenience method that generates the tenant name by packing a {@code Tuple}. + * + * @param tr The transaction used to create the tenant. + * @param tenantName The name of the tenant, as a Tuple. + */ + public static void createTenant(Transaction tr, Tuple tenantName) { + createTenant(tr, tenantName.pack()); + } + + /** + * Creates a new tenant in the cluster using a transaction created on the specified {@code Database}. + * This operation will first check whether the tenant exists, and if it does it will set the + * {@code CompletableFuture} to a tenant_already_exists error. Otherwise, it will attempt to create + * the tenant in a retry loop. If the tenant is created concurrently by another transaction, this + * function may still return successfully. + * + * @param db The database used to create a transaction for creating the tenant. + * @param tenantName The name of the tenant. Can be any byte string that does not begin a 0xFF byte. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been created. + */ + public static CompletableFuture createTenant(Database db, byte[] tenantName) { + final AtomicBoolean checkedExistence = new AtomicBoolean(false); + final byte[] key = ByteArrayUtil.join(TENANT_MAP_PREFIX, tenantName); + return db.runAsync(tr -> { + tr.options().setSpecialKeySpaceEnableWrites(); + if(checkedExistence.get()) { + tr.set(key, new byte[0]); + return CompletableFuture.completedFuture(null); + } + else { + return tr.get(key).thenAcceptAsync(result -> { + checkedExistence.set(true); + if(result != null) { + throw new FDBException("A tenant with the given name already exists", 2132); + } + tr.set(key, new byte[0]); + }); + } + }); + } + + /** + * Creates a new tenant in the cluster using a transaction created on the specified {@code Database}. + * This operation will first check whether the tenant exists, and if it does it will set the + * {@code CompletableFuture} to a tenant_already_exists error. Otherwise, it will attempt to create + * the tenant in a retry loop. If the tenant is created concurrently by another transaction, this + * function may still return successfully.
+ *
+ * This is a convenience method that generates the tenant name by packing a {@code Tuple}. + * + * @param db The database used to create a transaction for creating the tenant. + * @param tenantName The name of the tenant, as a Tuple. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been created. + */ + public static CompletableFuture createTenant(Database db, Tuple tenantName) { + return createTenant(db, tenantName.pack()); + } + + /** + * Deletes a tenant from the cluster. If the tenant does not exists, this operation will complete + * successfully without changing anything. The transaction must be committed for the deletion to take + * effect or to observe any errors.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must + * first use a clear operation to delete all of its keys. + * + * @param tr The transaction used to delete the tenant. + * @param tenantName The name of the tenant being deleted. + */ + public static void deleteTenant(Transaction tr, byte[] tenantName) { + tr.options().setSpecialKeySpaceEnableWrites(); + tr.clear(ByteArrayUtil.join(TENANT_MAP_PREFIX, tenantName)); + } + + /** + * Deletes a tenant from the cluster. If the tenant does not exists, this operation will complete + * successfully without changing anything. The transaction must be committed for the deletion to take + * effect or to observe any errors.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must + * first use a clear operation to delete all of its keys.
+ *
+ * This is a convenience method that generates the tenant name by packing a {@code Tuple}. + * + * @param tr The transaction used to delete the tenant. + * @param tenantName The name of the tenant being deleted, as a Tuple. + */ + public static void deleteTenant(Transaction tr, Tuple tenantName) { + deleteTenant(tr, tenantName.pack()); + } + + /** + * Deletes a tenant from the cluster using a transaction created on the specified {@code Database}. This + * operation will first check whether the tenant exists, and if it does not it will set the + * {@code CompletableFuture} to a tenant_not_found error. Otherwise, it will attempt to delete the + * tenant in a retry loop. If the tenant is deleted concurrently by another transaction, this function may + * still return successfully.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must + * first use a clear operation to delete all of its keys. + * + * @param db The database used to create a transaction for deleting the tenant. + * @param tenantName The name of the tenant being deleted. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been deleted. + */ + public static CompletableFuture deleteTenant(Database db, byte[] tenantName) { + final AtomicBoolean checkedExistence = new AtomicBoolean(false); + final byte[] key = ByteArrayUtil.join(TENANT_MAP_PREFIX, tenantName); + return db.runAsync(tr -> { + tr.options().setSpecialKeySpaceEnableWrites(); + if(checkedExistence.get()) { + tr.clear(key); + return CompletableFuture.completedFuture(null); + } + else { + return tr.get(key).thenAcceptAsync(result -> { + checkedExistence.set(true); + if(result == null) { + throw new FDBException("Tenant does not exist", 2131); + } + tr.clear(key); + }); + } + }); + } + + /** + * Deletes a tenant from the cluster using a transaction created on the specified {@code Database}. This + * operation will first check whether the tenant exists, and if it does not it will set the + * {@code CompletableFuture} to a tenant_not_found error. Otherwise, it will attempt to delete the + * tenant in a retry loop. If the tenant is deleted concurrently by another transaction, this function may + * still return successfully.
+ *
+ * Note: A tenant cannot be deleted if it has any data in it. To delete a non-empty tenant, you must + * first use a clear operation to delete all of its keys.
+ *
+ * This is a convenience method that generates the tenant name by packing a {@code Tuple}. + * + * @param db The database used to create a transaction for deleting the tenant. + * @param tenantName The name of the tenant being deleted. + * @return a {@code CompletableFuture} that when set without error will indicate that the tenant has + * been deleted. + */ + public static CompletableFuture deleteTenant(Database db, Tuple tenantName) { + return deleteTenant(db, tenantName.pack()); + } + + private TenantManagement() {} +} diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index b303ed3a3a..70263b510a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -43,6 +43,7 @@ import com.apple.foundationdb.KeyArrayResult; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; import com.apple.foundationdb.StreamingMode; +import com.apple.foundationdb.TenantManagement; import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncUtil; import com.apple.foundationdb.tuple.ByteArrayUtil; @@ -473,13 +474,13 @@ public class AsyncStackTester { else if (op == StackOperation.TENANT_CREATE) { return inst.popParam().thenAcceptAsync(param -> { byte[] tenantName = (byte[])param; - inst.push(inst.context.db.allocateTenant(tenantName)); + inst.push(TenantManagement.createTenant(inst.context.db, tenantName)); }, FDB.DEFAULT_EXECUTOR); } else if (op == StackOperation.TENANT_DELETE) { return inst.popParam().thenAcceptAsync(param -> { byte[] tenantName = (byte[])param; - inst.push(inst.context.db.deleteTenant(tenantName)); + inst.push(TenantManagement.deleteTenant(inst.context.db, tenantName)); }, FDB.DEFAULT_EXECUTOR); } else if (op == StackOperation.TENANT_SET_ACTIVE) { diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index ef4c7ec6d6..0fc9141c96 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -45,6 +45,7 @@ import com.apple.foundationdb.LocalityUtil; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; import com.apple.foundationdb.StreamingMode; +import com.apple.foundationdb.TenantManagement; import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncIterable; import com.apple.foundationdb.async.AsyncUtil; @@ -422,11 +423,11 @@ public class StackTester { } else if (op == StackOperation.TENANT_CREATE) { byte[] tenantName = (byte[])inst.popParam().join(); - inst.push(inst.context.db.allocateTenant(tenantName)); + inst.push(TenantManagement.createTenant(inst.context.db, tenantName)); } else if (op == StackOperation.TENANT_DELETE) { byte[] tenantName = (byte[])inst.popParam().join(); - inst.push(inst.context.db.deleteTenant(tenantName)); + inst.push(TenantManagement.deleteTenant(inst.context.db, tenantName)); } else if (op == StackOperation.TENANT_SET_ACTIVE) { byte[] tenantName = (byte[])inst.popParam().join(); @@ -761,7 +762,7 @@ public class StackTester { private static void testTenantTupleNames(Database db) { try { - db.allocateTenant(Tuple.from("tenant")).join(); + TenantManagement.createTenant(db, Tuple.from("tenant")).join(); Tenant tenant = db.openTenant(Tuple.from("tenant")); tenant.run(tr -> { @@ -781,7 +782,7 @@ public class StackTester { return null; }); - db.deleteTenant(Tuple.from("tenant")).join(); + TenantManagement.deleteTenant(db, Tuple.from("tenant")).join(); } catch(Exception e) { e.printStackTrace(); From 84f9e002584eb468140d4656eed78666e9ddbda9 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 16:41:14 -0700 Subject: [PATCH 77/90] Remove duplicative generic actor repeatEvery() since recurring() exists. --- fdbserver/VersionedBTree.actor.cpp | 2 +- flow/genericactors.actor.h | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 724256a353..0f34a5190d 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -10370,7 +10370,7 @@ TEST_CASE(":/redwood/performance/set") { state Future stats = traceMetrics ? Void() - : repeatEvery(1.0, [&]() { printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); }); + : recurring([&]() { printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); }, 1.0); if (scans > 0) { printf("Parallel scans, concurrency=%d, scans=%d, scanWidth=%d, scanPreftchBytes=%d ...\n", diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h index f30ef772e5..cdf21e24fa 100644 --- a/flow/genericactors.actor.h +++ b/flow/genericactors.actor.h @@ -221,6 +221,7 @@ Future delayed(Future what, double time = 0.0, TaskPriority taskID = TaskP } } +// wait then call what() in a loop forever ACTOR template Future recurring(Func what, double interval, TaskPriority taskID = TaskPriority::DefaultDelay) { loop choose { @@ -2048,15 +2049,6 @@ private: Reference data; }; -// Call a lambda every seconds -ACTOR template -Future repeatEvery(double interval, Fn fn) { - loop { - wait(delay(interval)); - fn(); - } -} - #include "flow/unactorcompiler.h" #endif From 6744e9e4f9da50b69bfaa835926902bf0114cc1e Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 30 Mar 2022 17:57:00 -0700 Subject: [PATCH 78/90] Change timestamps used in storage server metadata and perpetual wiggle metrics to epoch seconds, stored as doubles, and stringified as either floating point epoch seconds or timestamp strings of the form "2013-04-28 20:57:01.000 +0000". --- fdbclient/FDBTypes.h | 6 +++--- fdbserver/DataDistribution.actor.cpp | 4 ++-- fdbserver/DataDistribution.actor.h | 18 ++++++++++-------- fdbserver/Status.actor.cpp | 2 +- flow/Platform.actor.cpp | 20 ++++++++++++-------- flow/Platform.h | 6 ++---- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 8975e35bc5..4b638869b8 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -1366,12 +1366,12 @@ struct ReadBlobGranuleContext { // Store metadata associated with each storage server. Now it only contains data be used in perpetual storage wiggle. struct StorageMetadataType { constexpr static FileIdentifier file_identifier = 732123; - // when the SS is initialized - uint64_t createdTime; // comes from currentTime() + // when the SS is initialized, in epoch seconds, comes from currentTime() + double createdTime; StorageMetadataType() : createdTime(0) {} StorageMetadataType(uint64_t t) : createdTime(t) {} - static uint64_t currentTime() { return g_network->timer_int(); } + static double currentTime() { return g_network->timer(); } // To change this serialization, ProtocolVersion::StorageMetadata must be updated, and downgrades need // to be considered diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index de6d815453..8381b073a7 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -296,7 +296,7 @@ Future StorageWiggler::restoreStats() { return map(readFuture, assignFunc); } Future StorageWiggler::startWiggle() { - metrics.last_wiggle_start = g_network->timer_int(); + metrics.last_wiggle_start = StorageMetadataType::currentTime(); if (shouldStartNewRound()) { metrics.last_round_start = metrics.last_wiggle_start; } @@ -304,7 +304,7 @@ Future StorageWiggler::startWiggle() { } Future StorageWiggler::finishWiggle() { - metrics.last_wiggle_finish = g_network->timer_int(); + metrics.last_wiggle_finish = StorageMetadataType::currentTime(); metrics.finished_wiggle += 1; auto duration = metrics.last_wiggle_finish - metrics.last_wiggle_start; metrics.smoothed_wiggle_duration.setTotal((double)duration); diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h index 909d878889..433c798e58 100644 --- a/fdbserver/DataDistribution.actor.h +++ b/fdbserver/DataDistribution.actor.h @@ -299,15 +299,17 @@ struct StorageWiggleMetrics { // round statistics // One StorageServer wiggle round is considered 'complete', when all StorageServers with creationTime < T are // wiggled - uint64_t last_round_start = 0; // wall timer: timer_int() - uint64_t last_round_finish = 0; + // Start and finish are in epoch seconds + double last_round_start = 0; + double last_round_finish = 0; TimerSmoother smoothed_round_duration; int finished_round = 0; // finished round since storage wiggle is open // step statistics // 1 wiggle step as 1 storage server is wiggled in the current round - uint64_t last_wiggle_start = 0; // wall timer: timer_int() - uint64_t last_wiggle_finish = 0; + // Start and finish are in epoch seconds + double last_wiggle_start = 0; + double last_wiggle_finish = 0; TimerSmoother smoothed_wiggle_duration; int finished_wiggle = 0; // finished step since storage wiggle is open @@ -365,15 +367,15 @@ struct StorageWiggleMetrics { StatusObject toJSON() const { StatusObject result; - result["last_round_start_datetime"] = timerIntToGmt(last_round_start); - result["last_round_finish_datetime"] = timerIntToGmt(last_round_finish); + result["last_round_start_datetime"] = epochsToGMTString(last_round_start); + result["last_round_finish_datetime"] = epochsToGMTString(last_round_finish); result["last_round_start_timestamp"] = last_round_start; result["last_round_finish_timestamp"] = last_round_finish; result["smoothed_round_seconds"] = smoothed_round_duration.smoothTotal(); result["finished_round"] = finished_round; - result["last_wiggle_start_datetime"] = timerIntToGmt(last_wiggle_start); - result["last_wiggle_finish_datetime"] = timerIntToGmt(last_wiggle_finish); + result["last_wiggle_start_datetime"] = epochsToGMTString(last_wiggle_start); + result["last_wiggle_finish_datetime"] = epochsToGMTString(last_wiggle_finish); result["last_wiggle_start_timestamp"] = last_wiggle_start; result["last_wiggle_finish_timestamp"] = last_wiggle_finish; result["smoothed_wiggle_seconds"] = smoothed_wiggle_duration.smoothTotal(); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 5549083b8e..78a6711a2b 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1934,7 +1934,7 @@ ACTOR static Future>> ge if (metadata[i].present()) { TraceEventFields metadataField; metadataField.addField("CreatedTimeTimestamp", std::to_string(metadata[i].get().createdTime)); - metadataField.addField("CreatedTimeDatetime", timerIntToGmt(metadata[i].get().createdTime)); + metadataField.addField("CreatedTimeDatetime", epochsToGMTString(metadata[i].get().createdTime)); results[i].second.emplace("Metadata", metadataField); } else if (!servers[i].isTss()) { TraceEventFields metadataField; diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 20a13ac8c7..d1a580b10a 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -1925,16 +1925,20 @@ void getLocalTime(const time_t* timep, struct tm* result) { #endif } -std::string timerIntToGmt(uint64_t timestamp) { - auto time = (time_t)(timestamp / 1e9); // convert to second, see timer_int() implementation - return getGmtTimeStr(&time); -} +// Outputs a GMT time string for the given epoch seconds, which looks like +// 2013-04-28 20:57:01.000 +0000 +std::string epochsToGMTString(double epochs) { + auto time = (time_t)epochs; -std::string getGmtTimeStr(const time_t* time) { char buff[50]; - auto size = strftime(buff, 50, "%c %z", gmtime(time)); - // printf(buff); - return std::string(std::begin(buff), std::begin(buff) + size); + auto size = strftime(buff, 50, "%Y-%m-%d %H:%M:%S", gmtime(&time)); + std::string timeString = std::string(std::begin(buff), std::begin(buff) + size); + + // Add fractional seconds and GMT timezone. + double integerPart; + timeString += format(".%03.3d +0000", (int)(1000 * modf(epochs, &integerPart))); + + return timeString; } void setMemoryQuota(size_t limit) { diff --git a/flow/Platform.h b/flow/Platform.h index 6ccd8618a1..ce336273ab 100644 --- a/flow/Platform.h +++ b/flow/Platform.h @@ -279,10 +279,8 @@ uint64_t timer_int(); // Return timer as uint64_t representing epoch nanoseconds void getLocalTime(const time_t* timep, struct tm* result); -// convert timestamp returned by timer_int() to Gmt format string -std::string timerIntToGmt(uint64_t timestamp); - -std::string getGmtTimeStr(const time_t* time); +// get GMT time string from an epoch seconds double +std::string epochsToGMTString(double epochs); void setMemoryQuota(size_t limit); From 61e374f3012c69b93520c33780a5f213ff22a76c Mon Sep 17 00:00:00 2001 From: Xiaoxi Wang Date: Wed, 30 Mar 2022 21:42:02 -0700 Subject: [PATCH 79/90] fix cmake bug when use gcc --- fdbmonitor/CMakeLists.txt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/fdbmonitor/CMakeLists.txt b/fdbmonitor/CMakeLists.txt index 2c36c7bad3..3787fb22f8 100644 --- a/fdbmonitor/CMakeLists.txt +++ b/fdbmonitor/CMakeLists.txt @@ -6,7 +6,7 @@ assert_no_version_h(fdbmonitor) if(UNIX AND NOT APPLE) target_link_libraries(fdbmonitor PRIVATE rt) endif() -# FIXME: This include directory is an ugly hack. We probably want to fix this +# FIXME: This include directory is an ugly hack. We probably want to fix this. # as soon as we get rid of the old build system target_link_libraries(fdbmonitor PUBLIC Threads::Threads) @@ -14,11 +14,17 @@ target_link_libraries(fdbmonitor PUBLIC Threads::Threads) # appears to change its behavior (it no longer seems to restart killed # processes). fdbmonitor is single-threaded anyway. get_target_property(fdbmonitor_options fdbmonitor COMPILE_OPTIONS) -list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") -set_property(TARGET fdbmonitor PROPERTY COMPILE_OPTIONS ${fdbmonitor_options}) +if (NOT "${fdbmonitor_options}" STREQUAL "fdbmonitor_options-NOTFOUND") + list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") + set_property(TARGET fdbmonitor PROPERTY COMPILE_OPTIONS ${fdbmonitor_options}) +endif () + get_target_property(fdbmonitor_options fdbmonitor LINK_OPTIONS) -list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") -set_property(TARGET fdbmonitor PROPERTY LINK_OPTIONS ${fdbmonitor_options}) + +if (NOT "${fdbmonitor_options}" STREQUAL "fdbmonitor_options-NOTFOUND") + list(REMOVE_ITEM fdbmonitor_options "-fsanitize=thread") + set_property(TARGET fdbmonitor PROPERTY LINK_OPTIONS ${fdbmonitor_options}) +endif () if(GENERATE_DEBUG_PACKAGES) fdb_install(TARGETS fdbmonitor DESTINATION fdbmonitor COMPONENT server) From 860ede0c2e2f30abc1f38782add2479b35b3d7b0 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 21 Mar 2022 16:26:28 -0700 Subject: [PATCH 80/90] Add some basic documentation for tenants --- documentation/sphinx/source/client-design.rst | 3 + .../sphinx/source/developer-guide.rst | 10 ++++ documentation/sphinx/source/tenants.rst | 58 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 documentation/sphinx/source/tenants.rst diff --git a/documentation/sphinx/source/client-design.rst b/documentation/sphinx/source/client-design.rst index ac9345d158..bd213509b8 100644 --- a/documentation/sphinx/source/client-design.rst +++ b/documentation/sphinx/source/client-design.rst @@ -26,6 +26,8 @@ FoundationDB supports language bindings for application development using the or * :doc:`known-limitations` describes both long-term design limitations of FoundationDB and short-term limitations applicable to the current version. +* :doc:`tenants` describes the use of the tenants feature to define named transaction domains. + .. toctree:: :maxdepth: 1 :titlesonly: @@ -42,3 +44,4 @@ FoundationDB supports language bindings for application development using the or known-limitations transaction-profiler-analyzer api-version-upgrade-guide + tenants diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst index 17e75f1f1b..3bf5ec30c0 100644 --- a/documentation/sphinx/source/developer-guide.rst +++ b/documentation/sphinx/source/developer-guide.rst @@ -273,6 +273,16 @@ Directory partitions have the following drawbacks, and in general they should no * Directories in a partition have longer prefixes than their counterparts outside of partitions, which reduces performance. Nesting partitions inside of other partitions results in even longer prefixes. * The root directory of a partition cannot be used to pack/unpack keys and therefore cannot be used to create subspaces. You must create at least one subdirectory of a partition in order to store content in it. +Tenants +------- + +:doc:`tenants` in FoundationDB provide a way to divide the cluster key-space into named transaction domains. Each tenant has a byte-string name that can be used to open transactions on the tenant's data, and tenant transactions are not permitted to access data outside of the tenant. Tenants can be useful for enforcing separation between unrelated use-cases. + +Tenants and directories +~~~~~~~~~~~~~~~~~~~~~~~ + +Because tenants enforce that transactions operate within the tenant boundaries, it is not recommended to use a global directory layer shared between tenants. It is possible, however, to use the directory layer within each tenant. To do so, simply use the directory layer as normal with tenant transactions. + Working with the APIs ===================== diff --git a/documentation/sphinx/source/tenants.rst b/documentation/sphinx/source/tenants.rst new file mode 100644 index 0000000000..f837ec3fbd --- /dev/null +++ b/documentation/sphinx/source/tenants.rst @@ -0,0 +1,58 @@ +####### +Tenants +####### + +.. warning :: Tenants are currently experimental and are not recommended for use in production. + +FoundationDB provides a feature called tenants that allow you to configure one or more named transaction domains in your cluster. A transaction domain is a key-space in which a transaction is allowed to operate, and no tenant operations are allowed to use keys outside the tenant key-space. Tenants can be useful for managing separate, unrelated use-cases and preventing them from interfering with each other. They can also be helpful for defining safe boundaries when moving a subset of data between clusters. + +By default, FoundationDB has a single transaction domain that contains both the normal key-space (``['', '\xff')``) as well as the system keys (``['\xff', '\xff\xff')``) and the :doc:`special-keys` (``['\xff\xff', '\xff\xff\xff')``). + +Overview +======== + +A tenant in a FoundationDB cluster maps a byte-string name to a key-space that can be used to store data associated with that tenant. This key-space is stored in the clusters global key-space under a prefix assigned to that tenant, with each tenant being assigned a separate non-intersecting prefix. + +In addition to being each assigned a separate tenant prefix, tenants can be configured to have a common shared prefix. By default, the shared prefix is empty and tenants are allocated prefixes throughout the normal key-space. To configure an alternate shared prefix, set the ``\xff/tenantDataPrefix`` key to have the desired prefix as the value. + +Tenant operations are implicitly confined to the key-space associated with the tenant. It is not necessary for client applications to use or be aware of the prefix assigned to the tenant. + +Enabling tenants +================ + +In order to use tenants, the cluster must be configured with an appropriate tenant mode using ``fdbcli``:: + + fdb> configure tenant_mode= + +FoundationDB clusters support the following tenant modes: + +* ``disabled`` - Tenants cannot be created or used. Disabled is the default tenant mode. +* ``optional_experimental`` - Tenants can be created. Each transaction can choose whether or not to use a tenant. This mode is primarily intended for migration and testing purposes, and care should be taken to avoid conflicts between tenant and non-tenant data. +* ``required_experimental`` - Tenants can be created. Each normal transaction must use a tenant. To support special access needs, transactions will be permitted to access the raw key-space using the ``RAW_ACCESS`` transaction option. + +Creating and deleting tenants +============================= + +Tenants can be created and deleted using the ``\xff\xff/management/tenant_map/`` :doc:`special key ` range as well as by using APIs provided in some language bindings. + +Tenants can be created with any byte-string name that does not begin with the ``\xff`` character. Once created, a tenant will be assigned an ID and a prefix where its data will reside. + +In order to delete a tenant, it must first be empty. If a tenant contains any keys, they must be cleared prior to deleting the tenant. + +Using tenants +============= + +In order to use the key-space associated with an existing tenant, you must open the tenant using the ``Database`` object provided by your language binding. The resulting ``Tenant`` object can be used to create transactions much like with a ``Database``, and the resulting transactions will be restricted to the tenant's key-space. + +All operations performed within a tenant transaction will occur within the tenant key-space. It is not necessary to use or even be aware of the prefix assigned to a tenant in the global key-space. Operations that could resolve outside of the tenant key-space (e.g. resolving key selectors) will be clamped to the tenant. + +.. note :: Tenant transactions are not permitted to access system keys. + +Raw access +---------- + +When operating in the tenant mode ``required_experimental``, transactions are not ordinarily permitted to run without using a tenant. In order to access the system keys or perform maintenance operations that span multiple tenants, it is required to use the ``RAW_ACCESS`` transaction option to access the global key-space. It is an error to specify ``RAW_ACCESS`` on a transaction that is configured to use a tenant. + +.. note :: Setting the ``READ_SYSTEM_KEYS`` or ``ACCESS_SYSTEM_KEYS`` options implies ``RAW_ACCESS`` for your transaction. + +.. warning :: Care should be taken when using raw access to run transactions spanning multiple tenants if the tenant feature is being utilized to aid in moving data between clusters. In such scenarios, it may not be guaranteed that all of the data you intend to access is on a single cluster. From b6350a2535b84d61552a998505bf7555a7beb3f7 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 30 Mar 2022 16:26:47 -0700 Subject: [PATCH 81/90] Add a note that use of special keys may implicitly enable raw access on a transaction. --- documentation/sphinx/source/tenants.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/sphinx/source/tenants.rst b/documentation/sphinx/source/tenants.rst index f837ec3fbd..531d956c2f 100644 --- a/documentation/sphinx/source/tenants.rst +++ b/documentation/sphinx/source/tenants.rst @@ -55,4 +55,6 @@ When operating in the tenant mode ``required_experimental``, transactions are no .. note :: Setting the ``READ_SYSTEM_KEYS`` or ``ACCESS_SYSTEM_KEYS`` options implies ``RAW_ACCESS`` for your transaction. +.. note :: Many :doc:`special keys ` operations access parts of the system keys and will implictly enable raw access on the transactions in which they are used. + .. warning :: Care should be taken when using raw access to run transactions spanning multiple tenants if the tenant feature is being utilized to aid in moving data between clusters. In such scenarios, it may not be guaranteed that all of the data you intend to access is on a single cluster. From 68f15650a1d8ff90f80cb473753344bdd4181292 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 30 Mar 2022 13:05:49 -0700 Subject: [PATCH 82/90] Make sure closed and tenantUpdater are read/written in the same critical section. --- fdbclient/MultiVersionTransaction.actor.cpp | 11 +++++------ fdbclient/MultiVersionTransaction.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 75e252d4d7..09a6875d65 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -1235,10 +1235,6 @@ MultiVersionTenant::TenantState::TenantState(Reference db, // Creates a new underlying tenant object whenever the database connection changes. This change is signaled // to open transactions via an AsyncVar. void MultiVersionTenant::TenantState::updateTenant() { - if (closed) { - return; - } - Reference tenant; auto currentDb = db->dbState->dbVar->get(); if (currentDb.value) { @@ -1252,6 +1248,10 @@ void MultiVersionTenant::TenantState::updateTenant() { Reference self = Reference::addRef(this); MutexHolder holder(tenantLock); + if (closed) { + return; + } + tenantUpdater = mapThreadFuture(currentDb.onChange, [self](ErrorOr result) { self->updateTenant(); return Void(); @@ -1259,9 +1259,8 @@ void MultiVersionTenant::TenantState::updateTenant() { } void MultiVersionTenant::TenantState::close() { - closed = true; - MutexHolder holder(tenantLock); + closed = true; if (tenantUpdater.isValid()) { tenantUpdater.cancel(); } diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index a8df462d88..c915329681 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -666,7 +666,7 @@ public: Mutex tenantLock; ThreadFuture tenantUpdater; - std::atomic_bool closed; + bool closed; }; Reference tenantState; From 5469b57a2b5e8cc8bf8d4560fd8a320eac456f95 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 31 Mar 2022 11:39:50 -0700 Subject: [PATCH 83/90] Add a note that opening a tenant does not check whether that tenant exists in the cluster --- .../java/src/main/com/apple/foundationdb/Database.java | 10 ++++++++-- documentation/sphinx/source/api-python.rst | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index 0128234a95..8606d7ec39 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -42,7 +42,10 @@ import com.apple.foundationdb.tuple.Tuple; */ public interface Database extends AutoCloseable, TransactionContext { /** - * Opens an existing tenant to be used for running transactions. + * Opens an existing tenant to be used for running transactions.
+ *
+ * Note: opening a tenant does not check its existence in the cluster. If the tenant does not exist, + * attempts to read or write data with it will fail. * * @param tenantName The name of the tenant to open. * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. @@ -53,7 +56,10 @@ public interface Database extends AutoCloseable, TransactionContext { /** * Opens an existing tenant to be used for running transactions. This is a convenience method that generates the - * tenant name by packing a {@code Tuple}. + * tenant name by packing a {@code Tuple}.
+ *
+ * Note: opening a tenant does not check its existence in the cluster. If the tenant does not exist, + * attempts to read or write data with it will fail. * * @param tenantName The name of the tenant to open, as a Tuple. * @return a {@link Tenant} that can be used to create transactions that will operate in the tenant's key-space. diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index eb8326654c..0f8c16d6bd 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -322,6 +322,8 @@ A |database-blurb1| |database-blurb2| The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + .. note :: Opening a tenant does not check its existence in the cluster. If the tenant does not exist, attempts to read or write data with it will fail. + .. |sync-read| replace:: This read is fully synchronous. .. |sync-write| replace:: This change will be committed immediately, and is fully synchronous. From 9e0688167393185f523c0786a3a318938d109d3a Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Thu, 31 Mar 2022 09:22:56 -0500 Subject: [PATCH 84/90] fix destination limiting and cancelling logic in move_to_removed_server case --- fdbserver/DataDistributionQueue.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp index d0d12f0387..60c88b00d4 100644 --- a/fdbserver/DataDistributionQueue.actor.cpp +++ b/fdbserver/DataDistributionQueue.actor.cpp @@ -1378,6 +1378,7 @@ ACTOR Future dataDistributionRelocator(DDQueueData* self, RelocateData rd, } else { TEST(true); // move to removed server healthyDestinations.addDataInFlightToTeam(-metrics.bytes); + rd.completeDests.clear(); wait(delay(SERVER_KNOBS->RETRY_RELOCATESHARD_DELAY, TaskPriority::DataDistributionLaunch)); } } From 001909be082d8c2b8a911578fe7163187d76bb09 Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Thu, 31 Mar 2022 14:06:45 -0700 Subject: [PATCH 85/90] Fixes for when getMappedRange cannot parse as tuple (#6665) --- bindings/c/test/unit/unit_tests.cpp | 37 +++++++++++++++++++++++++---- fdbserver/storageserver.actor.cpp | 25 ++++++++++++++++--- flow/error_definitions.h | 4 ++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index 78cb2ee2e9..1dde194a6e 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -949,12 +949,10 @@ std::map fillInRecords(int n) { return data; } -GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr) { +GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr, std::string mapper) { std::string indexEntryKeyBegin = indexEntryKey(beginId); std::string indexEntryKeyEnd = indexEntryKey(endId); - std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString(); - return get_mapped_range( tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)indexEntryKeyBegin.c_str(), indexEntryKeyBegin.size()), @@ -969,6 +967,11 @@ GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transact /* reverse */ 0); } +GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr) { + std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString(); + return getMappedIndexEntries(beginId, endId, tr, mapper); +} + TEST_CASE("fdb_transaction_get_mapped_range") { const int TOTAL_RECORDS = 20; fillInRecords(TOTAL_RECORDS); @@ -1009,7 +1012,6 @@ TEST_CASE("fdb_transaction_get_mapped_range") { TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") { std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString(); fdb::Transaction tr(db); - fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0)); auto result = get_mapped_range( tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)indexEntryKey(0).c_str(), indexEntryKey(0).size()), @@ -1039,11 +1041,36 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_ryw_enable") { /* target_bytes */ 0, /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0, - /* snapshot */ true, + /* snapshot */ false, /* reverse */ 0); ASSERT(result.err == error_code_unsupported_operation); } +void assertNotTuple(std::string str) { + try { + Tuple::unpack(str); + } catch (Error& e) { + return; + } + UNREACHABLE(); +} + +TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") { + // A string that cannot be parsed as tuple. + // "\x15:\x152\x15E\x15\x09\x15\x02\x02MySimpleRecord$repeater-version\x00\x15\x013\x00\x00\x00\x00\x1aU\x90\xba\x00\x00\x00\x02\x15\x04" + std::string mapper = { + '\x15', ':', '\x15', '2', '\x15', 'E', '\x15', '\t', '\x15', '\x02', '\x02', 'M', + 'y', 'S', 'i', 'm', 'p', 'l', 'e', 'R', 'e', 'c', 'o', 'r', + 'd', '$', 'r', 'e', 'p', 'e', 'a', 't', 'e', 'r', '-', 'v', + 'e', 'r', 's', 'i', 'o', 'n', '\x00', '\x15', '\x01', '3', '\x00', '\x00', + '\x00', '\x00', '\x1a', 'U', '\x90', '\xba', '\x00', '\x00', '\x00', '\x02', '\x15', '\x04' + }; + assertNotTuple(mapper); + fdb::Transaction tr(db); + auto result = getMappedIndexEntries(1, 3, tr, mapper); + ASSERT(result.err == error_code_mapper_not_tuple); +} + TEST_CASE("fdb_transaction_get_range reverse") { std::map data = create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } }); insert_data(db, data); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index a56632c1e6..4b8e483b05 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -100,6 +100,9 @@ bool canReplyWith(Error e) { case error_code_quick_get_value_miss: case error_code_quick_get_key_values_miss: case error_code_get_mapped_key_values_has_more: + case error_code_key_not_tuple: + case error_code_value_not_tuple: + case error_code_mapper_not_tuple: // case error_code_all_alternatives_failed: return true; default: @@ -3437,14 +3440,24 @@ Key constructMappedKey(KeyValueRef* keyValue, Tuple& mappedKeyFormatTuple, bool& // Use keyTuple as reference. if (!keyTuple.present()) { // May throw exception if the key is not parsable as a tuple. - keyTuple = Tuple::unpack(keyValue->key); + try { + keyTuple = Tuple::unpack(keyValue->key); + } catch (Error& e) { + TraceEvent("KeyNotTuple").error(e).detail("Key", keyValue->key.printable()); + throw key_not_tuple(); + } } referenceTuple = &keyTuple.get(); } else if (s[1] == 'V') { // Use valueTuple as reference. if (!valueTuple.present()) { // May throw exception if the value is not parsable as a tuple. - valueTuple = Tuple::unpack(keyValue->value); + try { + valueTuple = Tuple::unpack(keyValue->value); + } catch (Error& e) { + TraceEvent("ValueNotTuple").error(e).detail("Value", keyValue->value.printable()); + throw value_not_tuple(); + } } referenceTuple = &valueTuple.get(); } else { @@ -3578,7 +3591,13 @@ ACTOR Future mapKeyValues(StorageServer* data, result.data.reserve(result.arena, input.data.size()); - state Tuple mappedKeyFormatTuple = Tuple::unpack(mapper); + state Tuple mappedKeyFormatTuple; + try { + mappedKeyFormatTuple = Tuple::unpack(mapper); + } catch (Error& e) { + TraceEvent("MapperNotTuple").error(e).detail("Mapper", mapper.printable()); + throw mapper_not_tuple(); + } state KeyValueRef* it = input.data.begin(); for (; it != input.data.end(); it++) { state MappedKeyValueRef kvm; diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 7710bca9ca..ecd7ab1d28 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -178,6 +178,10 @@ ERROR( blob_granule_not_materialized, 2037, "Blob Granule Read was not materiali ERROR( get_mapped_key_values_has_more, 2038, "getMappedRange does not support continuation for now" ) ERROR( get_mapped_range_reads_your_writes, 2039, "getMappedRange tries to read data that were previously written in the transaction" ) ERROR( checkpoint_not_found, 2040, "Checkpoint not found" ) +ERROR( key_not_tuple, 2041, "The key cannot be parsed as a tuple" ); +ERROR( value_not_tuple, 2042, "The value cannot be parsed as a tuple" ); +ERROR( mapper_not_tuple, 2043, "The mapper cannot be parsed as a tuple" ); + ERROR( incompatible_protocol_version, 2100, "Incompatible protocol version" ) ERROR( transaction_too_large, 2101, "Transaction exceeds byte limit" ) From 7d365bd1bb0fb134f3b94e77af26bfc27a1feab5 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 31 Mar 2022 17:08:59 -0700 Subject: [PATCH 86/90] Remote ikvs debugging (#6465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * initial structure for remote IKVS server * moved struct to .h file, added new files to CMakeList * happy path implementation, connection error when testing * saved minor local change * changed tracing to debug * fixed onClosed and getError being called before init is finished * fix spawn process bug, now use absolute path * added server knob to set ikvs process port number * added server knob for remote/local kv store * implement simulator remote process spawning * fixed bug for simulator timeout * commit all changes * removed print lines in trace * added FlowProcess implementation by Markus * initial debug of FlowProcess, stuck at parent sending OpenKVStoreRequest to child * temporary fix for process factory throwing segfault on create * specify public address in command * change remote kv store knob to false for jenkins build * made port 0 open random unused port * change remote store knob to true for benchmark * set listening port to randomly opened port * added print lines for jenkins run open kv store timeout debug * removed most tracing and print lines * removed tutorial changes * update handleIOErrors error handling to handle remote-ikvs cases * Push all debugging changes * A version where worker bug exists * A version where restarting tests fail * Use both the name and the port to determine the child process * Remove unnecessary update on local address * Disable remote-kvs for DiskFailureCycle test * A version where restarting stuck * A version where most restarting tests green * Reset connection with child process explicitly * Remove change on unnecessary files * Unify flags from _ to - * fix merging unexpected changes * fix trac.error to .errorUnsuppressed * Add license header * Remove unnecessary header in FlowProcess.actor.cpp * Fix Windows build * Fix Windows build, add missing ; * Fix a stupid bug caused by code dropped by code merging * Disable remote kvs by default * Pass the conn_file path to the flow process, though not needed, but the buildNetwork is difficult to tune * serialization change on readrange * Update traces * Refactor the RemoteIKVS interface * Format files * Update sim2 interface to not clog connections between parent and child processes in simulation * Update comments; remove debugging symbols; Add error handling for remote_kvs_cancelled * Add comments, format files * Change method name from isBuggifyDisabled to isStableConnection; Decrease(0.1x) latency for stable connections * Commit the IConnection interface change, forgot in previous commit * Fix the issue that onClosed request is cancelled by ActorCollection * Enable the remote kv store knob * Remove FlowProcess.actor.cpp and move functions to RemoteIKeyValueStore.actor.cpp; Add remote kv store delay to avoid race; Bind the child process to die with parent process * Fix the bug where one process starts storage server more than once * Add a please_reboot_remote_kv_store error to restart the storage server worker if remote kvs died abnormally * Remove unreachable code path and add comments * Clang format the code * Fix a simple wait error * Clang format after merging the main branch * Testing mixed mode in simulation if remote_kvs knob is enabled, setting the default to false * Disable remote kvs for PhysicalShardMove which is for RocksDB * Cleanup #include orders, remove debugging traces * Revert the reorder in fdbserver.actor.cpp, which fails the gcc build Co-authored-by: “Lincoln <“lincoln.xiao@snowflake.com”> --- bindings/python/tests/fdbcli_tests.py | 3 +- fdbclient/FDBTypes.h | 2 + fdbclient/ServerKnobs.cpp | 4 + fdbclient/ServerKnobs.h | 9 + fdbrpc/CMakeLists.txt | 1 + fdbrpc/FlowProcess.actor.h | 94 ++++ fdbrpc/FlowTransport.actor.cpp | 16 +- fdbrpc/sim2.actor.cpp | 87 +++- fdbrpc/simulator.h | 29 +- fdbserver/CMakeLists.txt | 2 + fdbserver/FDBExecHelper.actor.cpp | 164 ++++++- fdbserver/FDBExecHelper.actor.h | 11 +- fdbserver/IKeyValueStore.h | 13 +- fdbserver/RemoteIKeyValueStore.actor.cpp | 246 +++++++++++ fdbserver/RemoteIKeyValueStore.actor.h | 504 ++++++++++++++++++++++ fdbserver/SimulatedCluster.actor.cpp | 16 + fdbserver/fdbserver.actor.cpp | 74 +++- fdbserver/storageserver.actor.cpp | 3 +- fdbserver/tester.actor.cpp | 3 +- fdbserver/worker.actor.cpp | 76 +++- fdbserver/workloads/SaveAndKill.actor.cpp | 7 +- flow/Net2.actor.cpp | 7 + flow/Platform.actor.cpp | 34 ++ flow/Platform.h | 3 + flow/error_definitions.h | 2 + flow/genericactors.actor.h | 2 +- flow/network.h | 4 + tests/fast/PhysicalShardMove.toml | 1 + tests/slow/DiskFailureCycle.toml | 1 + 29 files changed, 1355 insertions(+), 63 deletions(-) create mode 100644 fdbrpc/FlowProcess.actor.h create mode 100644 fdbserver/RemoteIKeyValueStore.actor.cpp create mode 100644 fdbserver/RemoteIKeyValueStore.actor.h diff --git a/bindings/python/tests/fdbcli_tests.py b/bindings/python/tests/fdbcli_tests.py index d24ddb876f..7be7c75f4d 100755 --- a/bindings/python/tests/fdbcli_tests.py +++ b/bindings/python/tests/fdbcli_tests.py @@ -233,7 +233,8 @@ def suspend(logger): port = address.split(':')[1] logger.debug("Port: {}".format(port)) # use the port number to find the exact fdb process we are connecting to - pinfo = list(filter(lambda x: port in x, pinfos)) + # child process like fdbserver -r flowprocess does not provide `datadir` in the command line + pinfo = list(filter(lambda x: port in x and 'datadir' in x, pinfos)) assert len(pinfo) == 1 pid = pinfo[0].split(' ')[0] logger.debug("Pid: {}".format(pid)) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index bbc2cb0adf..acc1d3253c 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -652,6 +652,7 @@ struct GetRangeLimits { }; struct RangeResultRef : VectorRef { + constexpr static FileIdentifier file_identifier = 3985192; bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the // limits requested) False implies that no such values remain Optional readThrough; // Only present when 'more' is true. When present, this value represent the end (or @@ -958,6 +959,7 @@ struct TLogSpillType { // Contains the amount of free and total space for a storage server, in bytes struct StorageBytes { + constexpr static FileIdentifier file_identifier = 3928581; // Free space on the filesystem int64_t free; // Total space on the filesystem diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index f53efac786..32a8738f61 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -250,6 +250,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DEBOUNCE_RECRUITING_DELAY, 5.0 ); init( DD_FAILURE_TIME, 1.0 ); if( randomize && BUGGIFY ) DD_FAILURE_TIME = 10.0; init( DD_ZERO_HEALTHY_TEAM_DELAY, 1.0 ); + init( REMOTE_KV_STORE, false ); if( randomize && BUGGIFY ) REMOTE_KV_STORE = true; + init( REMOTE_KV_STORE_INIT_DELAY, 0.1 ); + init( REMOTE_KV_STORE_MAX_INIT_DURATION, 10.0 ); init( REBALANCE_MAX_RETRIES, 100 ); init( DD_OVERLAP_PENALTY, 10000 ); init( DD_EXCLUDE_MIN_REPLICAS, 1 ); @@ -555,6 +558,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( MIN_REBOOT_TIME, 4.0 ); if( longReboots ) MIN_REBOOT_TIME = 10.0; init( MAX_REBOOT_TIME, 5.0 ); if( longReboots ) MAX_REBOOT_TIME = 20.0; init( LOG_DIRECTORY, "."); // Will be set to the command line flag. + init( CONN_FILE, ""); // Will be set to the command line flag. init( SERVER_MEM_LIMIT, 8LL << 30 ); init( SYSTEM_MONITOR_FREQUENCY, 5.0 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index de69ef43dc..830d462883 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -233,6 +233,14 @@ public: double DD_FAILURE_TIME; double DD_ZERO_HEALTHY_TEAM_DELAY; + // Run storage enginee on a child process on the same machine with storage process + bool REMOTE_KV_STORE; + // A delay to avoid race on file resources if the new kv store process started immediately after the previous kv + // store process died + double REMOTE_KV_STORE_INIT_DELAY; + // max waiting time for the remote kv store to initialize + double REMOTE_KV_STORE_MAX_INIT_DURATION; + // KeyValueStore SQLITE int CLEAR_BUFFER_SIZE; double READ_VALUE_TIME_ESTIMATE; @@ -488,6 +496,7 @@ public: double MIN_REBOOT_TIME; double MAX_REBOOT_TIME; std::string LOG_DIRECTORY; + std::string CONN_FILE; int64_t SERVER_MEM_LIMIT; double SYSTEM_MONITOR_FREQUENCY; diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt index baff60b4f0..59ae21bc9e 100644 --- a/fdbrpc/CMakeLists.txt +++ b/fdbrpc/CMakeLists.txt @@ -10,6 +10,7 @@ set(FDBRPC_SRCS AsyncFileNonDurable.actor.cpp AsyncFileWriteChecker.cpp FailureMonitor.actor.cpp + FlowProcess.actor.h FlowTransport.actor.cpp genericactors.actor.h genericactors.actor.cpp diff --git a/fdbrpc/FlowProcess.actor.h b/fdbrpc/FlowProcess.actor.h new file mode 100644 index 0000000000..bd734198d8 --- /dev/null +++ b/fdbrpc/FlowProcess.actor.h @@ -0,0 +1,94 @@ +/* + * FlowProcess.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBRPC_FLOW_PROCESS_ACTOR_G_H) +#define FDBRPC_FLOW_PROCESS_ACTOR_G_H +#include "fdbrpc/FlowProcess.actor.g.h" +#elif !defined(FDBRPC_FLOW_PROCESS_ACTOR_H) +#define FDBRPC_FLOW_PROCESS_ACTOR_H + +#include "fdbrpc/fdbrpc.h" + +#include +#include + +#include // has to be last include + +struct FlowProcessInterface { + constexpr static FileIdentifier file_identifier = 3491839; + RequestStream registerProcess; + + template + void serialize(Ar& ar) { + serializer(ar, registerProcess); + } +}; + +struct FlowProcessRegistrationRequest { + constexpr static FileIdentifier file_identifier = 3411838; + Standalone flowProcessInterface; + + template + void serialize(Ar& ar) { + serializer(ar, flowProcessInterface); + } +}; + +class FlowProcess { + +public: + virtual ~FlowProcess() {} + virtual StringRef name() const = 0; + virtual StringRef serializedInterface() const = 0; + virtual Future run() = 0; + virtual void registerEndpoint(Endpoint p) = 0; +}; + +struct IProcessFactory { + static FlowProcess* create(std::string const& name) { + auto it = factories().find(name); + if (it == factories().end()) + return nullptr; // or throw? + return it->second->create(); + } + static std::map& factories() { + static std::map theFactories; + return theFactories; + } + + virtual FlowProcess* create() = 0; + + virtual const char* getName() = 0; +}; + +template +struct ProcessFactory : IProcessFactory { + ProcessFactory(const char* name) : name(name) { factories()[name] = this; } + FlowProcess* create() override { return new ProcessType(); } + const char* getName() override { return this->name; } + +private: + const char* name; +}; + +#include +#endif diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp index bac1a9b145..9e5a224d4c 100644 --- a/fdbrpc/FlowTransport.actor.cpp +++ b/fdbrpc/FlowTransport.actor.cpp @@ -991,7 +991,8 @@ static void scanPackets(TransportData* transport, Arena& arena, NetworkAddress const& peerAddress, ProtocolVersion peerProtocolVersion, - Future disconnect) { + Future disconnect, + bool isStableConnection) { // Find each complete packet in the given byte range and queue a ready task to deliver it. // Remove the complete packets from the range by increasing unprocessed_begin. // There won't be more than 64K of data plus one packet, so this shouldn't take a long time. @@ -1030,7 +1031,7 @@ static void scanPackets(TransportData* transport, if (checksumEnabled) { bool isBuggifyEnabled = false; - if (g_network->isSimulated() && + if (g_network->isSimulated() && !isStableConnection && g_network->now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration && BUGGIFY_WITH_PROB(0.0001)) { g_simulator.lastConnectionFailure = g_network->now(); @@ -1057,7 +1058,8 @@ static void scanPackets(TransportData* transport, if (isBuggifyEnabled) { TraceEvent(SevInfo, "ChecksumMismatchExp") .detail("PacketChecksum", packetChecksum) - .detail("CalculatedChecksum", calculatedChecksum); + .detail("CalculatedChecksum", calculatedChecksum) + .detail("PeerAddress", peerAddress.toString()); } else { TraceEvent(SevWarnAlways, "ChecksumMismatchUnexp") .detail("PacketChecksum", packetChecksum) @@ -1305,7 +1307,8 @@ ACTOR static Future connectionReader(TransportData* transport, arena, peerAddress, peerProtocolVersion, - peer->disconnect.getFuture()); + peer->disconnect.getFuture(), + g_network->isSimulated() && conn->isStableConnection()); } else { unprocessed_begin = unprocessed_end; peer->resetPing.trigger(); @@ -1364,6 +1367,11 @@ ACTOR static Future listen(TransportData* self, NetworkAddress listenAddr) state ActorCollectionNoErrors incoming; // Actors monitoring incoming connections that haven't yet been associated with a peer state Reference listener = INetworkConnections::net()->listen(listenAddr); + if (!g_network->isSimulated() && self->localAddresses.address.port == 0) { + TraceEvent(SevInfo, "UpdatingListenAddress") + .detail("AssignedListenAddress", listener->getListenAddress().toString()); + self->localAddresses.address = listener->getListenAddress(); + } state uint64_t connectionCount = 0; try { loop { diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index ba3a247bb6..268005eaff 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "contrib/fmt-8.1.1/include/fmt/format.h" #include "fdbrpc/simulator.h" @@ -121,20 +122,24 @@ void ISimulator::displayWorkers() const { int openCount = 0; struct SimClogging { - double getSendDelay(NetworkAddress from, NetworkAddress to) const { return halfLatency(); } + double getSendDelay(NetworkAddress from, NetworkAddress to, bool stableConnection = false) const { + // stable connection here means it's a local connection between processes on the same machine + // we expect it to have much lower latency + return (stableConnection ? 0.1 : 1.0) * halfLatency(); + } - double getRecvDelay(NetworkAddress from, NetworkAddress to) { + double getRecvDelay(NetworkAddress from, NetworkAddress to, bool stableConnection = false) { auto pair = std::make_pair(from.ip, to.ip); double tnow = now(); - double t = tnow + halfLatency(); - if (!g_simulator.speedUpSimulation) + double t = tnow + (stableConnection ? 0.1 : 1.0) * halfLatency(); + if (!g_simulator.speedUpSimulation && !stableConnection) t += clogPairLatency[pair]; - if (!g_simulator.speedUpSimulation && clogPairUntil.count(pair)) + if (!g_simulator.speedUpSimulation && !stableConnection && clogPairUntil.count(pair)) t = std::max(t, clogPairUntil[pair]); - if (!g_simulator.speedUpSimulation && clogRecvUntil.count(to.ip)) + if (!g_simulator.speedUpSimulation && !stableConnection && clogRecvUntil.count(to.ip)) t = std::max(t, clogRecvUntil[to.ip]); return t - tnow; @@ -182,8 +187,8 @@ SimClogging g_clogging; struct Sim2Conn final : IConnection, ReferenceCounted { Sim2Conn(ISimulator::ProcessInfo* process) - : opened(false), closedByCaller(false), process(process), dbgid(deterministicRandom()->randomUniqueID()), - stopReceive(Never()) { + : opened(false), closedByCaller(false), stableConnection(false), process(process), + dbgid(deterministicRandom()->randomUniqueID()), stopReceive(Never()) { pipes = sender(this) && receiver(this); } @@ -202,7 +207,18 @@ struct Sim2Conn final : IConnection, ReferenceCounted { process->address.ip, FLOW_KNOBS->MAX_CLOGGING_LATENCY * deterministicRandom()->random01()); sendBufSize = std::max(deterministicRandom()->randomInt(0, 5000000), 25e6 * (latency + .002)); - TraceEvent("Sim2Connection").detail("SendBufSize", sendBufSize).detail("Latency", latency); + // options like clogging or bitsflip are disabled for stable connections + stableConnection = std::any_of(process->childs.begin(), + process->childs.end(), + [&](ISimulator::ProcessInfo* child) { return child && child == peerProcess; }) || + std::any_of(peerProcess->childs.begin(), + peerProcess->childs.end(), + [&](ISimulator::ProcessInfo* child) { return child && child == process; }); + + TraceEvent("Sim2Connection") + .detail("SendBufSize", sendBufSize) + .detail("Latency", latency) + .detail("StableConnection", stableConnection); } ~Sim2Conn() { ASSERT_ABORT(!opened || closedByCaller); } @@ -222,6 +238,8 @@ struct Sim2Conn final : IConnection, ReferenceCounted { bool isPeerGone() const { return !peer || peerProcess->failed; } + bool isStableConnection() const override { return stableConnection; } + void peerClosed() { leakedConnectionTracker = trackLeakedConnection(this); stopReceive = delay(1.0); @@ -249,7 +267,7 @@ struct Sim2Conn final : IConnection, ReferenceCounted { ASSERT(limit > 0); int toSend = 0; - if (BUGGIFY) { + if (BUGGIFY && !stableConnection) { toSend = std::min(limit, buffer->bytes_written - buffer->bytes_sent); } else { for (auto p = buffer; p; p = p->next) { @@ -262,7 +280,7 @@ struct Sim2Conn final : IConnection, ReferenceCounted { } } ASSERT(toSend); - if (BUGGIFY) + if (BUGGIFY && !stableConnection) toSend = std::min(toSend, deterministicRandom()->randomInt(0, 1000)); if (!peer) @@ -286,7 +304,7 @@ struct Sim2Conn final : IConnection, ReferenceCounted { NetworkAddress getPeerAddress() const override { return peerEndpoint; } UID getDebugID() const override { return dbgid; } - bool opened, closedByCaller; + bool opened, closedByCaller, stableConnection; private: ISimulator::ProcessInfo *process, *peerProcess; @@ -336,10 +354,12 @@ private: deterministicRandom()->random01() < .5 ? self->sentBytes.get() : deterministicRandom()->randomInt64(self->receivedBytes.get(), self->sentBytes.get() + 1); - wait(delay(g_clogging.getSendDelay(self->process->address, self->peerProcess->address))); + wait(delay(g_clogging.getSendDelay( + self->process->address, self->peerProcess->address, self->isStableConnection()))); wait(g_simulator.onProcess(self->process)); ASSERT(g_simulator.getCurrentProcess() == self->process); - wait(delay(g_clogging.getRecvDelay(self->process->address, self->peerProcess->address))); + wait(delay(g_clogging.getRecvDelay( + self->process->address, self->peerProcess->address, self->isStableConnection()))); ASSERT(g_simulator.getCurrentProcess() == self->process); if (self->stopReceive.isReady()) { wait(Future(Never())); @@ -389,7 +409,9 @@ private: } void rollRandomClose() { - if (now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration && + // make sure connections between parenta and their childs are not closed + if (!stableConnection && + now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration && deterministicRandom()->random01() < .00001) { g_simulator.lastConnectionFailure = now(); double a = deterministicRandom()->random01(), b = deterministicRandom()->random01(); @@ -1101,6 +1123,10 @@ public: if (mustBeDurable || deterministicRandom()->random01() < 0.5) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); + TraceEvent(SevDebug, "Sim2DeleteFileImpl") + .detail("CurrentProcess", currentProcess->toString()) + .detail("Filename", filename) + .detail("Durable", mustBeDurable); wait(g_simulator.onMachine(currentProcess)); try { wait(::delay(0.05 * deterministicRandom()->random01())); @@ -1118,6 +1144,9 @@ public: throw err; } } else { + TraceEvent(SevDebug, "Sim2DeleteFileImplNonDurable") + .detail("Filename", filename) + .detail("Durable", mustBeDurable); TEST(true); // Simulated non-durable delete return Void(); } @@ -1163,6 +1192,9 @@ public: MachineInfo& machine = machines[locality.machineId().get()]; if (!machine.machineId.present()) machine.machineId = locality.machineId(); + if (port == 0 && std::string(name) == "remote flow process") { + port = machine.getRandomPort(); + } for (int i = 0; i < machine.processes.size(); i++) { if (machine.processes[i]->locality.machineId() != locality.machineId()) { // SOMEDAY: compute ip from locality to avoid this check @@ -1220,6 +1252,11 @@ public: .detail("Excluded", m->excluded) .detail("Cleared", m->cleared); + if (std::string(name) == "remote flow process") { + protectedAddresses.insert(m->address); + TraceEvent(SevDebug, "NewFlowProcessProtected").detail("Address", m->address); + } + // FIXME: Sometimes, connections to/from this process will explicitly close return m; @@ -1497,6 +1534,7 @@ public: .detail("MachineId", p->locality.machineId()); currentlyRebootingProcesses.insert(std::pair(p->address, p)); std::vector& processes = machines[p->locality.machineId().get()].processes; + machines[p->locality.machineId().get()].removeRemotePort(p->address.port); if (p != processes.back()) { auto it = std::find(processes.begin(), processes.end(), p); std::swap(*it, processes.back()); @@ -1520,7 +1558,8 @@ public: .detail("Protected", protectedAddresses.count(machine->address)) .backtrace(); // This will remove all the "tracked" messages that came from the machine being killed - latestEventCache.clear(); + if (std::string(machine->name) != "remote flow process") + latestEventCache.clear(); machine->failed = true; } else if (kt == InjectFaults) { TraceEvent(SevWarn, "FaultMachine") @@ -1548,7 +1587,8 @@ public: } else { ASSERT(false); } - ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting); + ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting || + std::string(machine->name) == "remote flow process"); } void rebootProcess(ProcessInfo* process, KillType kt) override { if (kt == RebootProcessAndDelete && protectedAddresses.count(process->address)) { @@ -2390,8 +2430,19 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) { kt == ISimulator::RebootProcessAndDelete); // Simulated process rebooted with data and coordination state deletion - if (p->rebooting || !p->isReliable()) + if (p->rebooting || !p->isReliable()) { + TraceEvent(SevDebug, "DoRebootFailed") + .detail("Rebooting", p->rebooting) + .detail("Reliable", p->isReliable()); return; + } else if (std::string(p->name) == "remote flow process") { + TraceEvent(SevDebug, "DoRebootFailed").detail("Name", p->name).detail("Address", p->address); + return; + } else if (p->getChilds().size()) { + TraceEvent(SevDebug, "DoRebootFailedOnParentProcess").detail("Address", p->address); + return; + } + TraceEvent("RebootingProcess") .detail("KillType", kt) .detail("Address", p->address) diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 80f17b3971..8f23db0400 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -21,6 +21,7 @@ #ifndef FLOW_SIMULATOR_H #define FLOW_SIMULATOR_H #include "flow/ProtocolVersion.h" +#include #include #pragma once @@ -87,6 +88,8 @@ public: ProtocolVersion protocolVersion; + std::vector childs; + ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, @@ -117,6 +120,7 @@ public: << " fault_injection_p2:" << fault_injection_p2; return ss.str(); } + std::vector const& getChilds() const { return childs; } // Return true if the class type is suitable for stateful roles, such as tLog and StorageServer. bool isAvailableClass() const { @@ -202,7 +206,30 @@ public: std::set closingFiles; Optional> machineId; - MachineInfo() : machineProcess(nullptr) {} + const uint16_t remotePortStart; + std::vector usedRemotePorts; + + MachineInfo() : machineProcess(nullptr), remotePortStart(1000) {} + + short getRandomPort() { + for (uint16_t i = remotePortStart; i < 60000; i++) { + if (std::find(usedRemotePorts.begin(), usedRemotePorts.end(), i) == usedRemotePorts.end()) { + TraceEvent(SevDebug, "RandomPortOpened").detail("PortNum", i); + usedRemotePorts.push_back(i); + return i; + } + } + UNREACHABLE(); + } + + void removeRemotePort(uint16_t port) { + if (port < remotePortStart) + return; + auto pos = std::find(usedRemotePorts.begin(), usedRemotePorts.end(), port); + if (pos != usedRemotePorts.end()) { + usedRemotePorts.erase(pos); + } + } }; ProcessInfo* getProcess(Endpoint const& endpoint) { return getProcessByAddress(endpoint.getPrimaryAddress()); } diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 2726c039fe..7970dd18ef 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -98,6 +98,8 @@ set(FDBSERVER_SRCS Ratekeeper.h RatekeeperInterface.h RecoveryState.h + RemoteIKeyValueStore.actor.h + RemoteIKeyValueStore.actor.cpp ResolutionBalancer.actor.cpp ResolutionBalancer.actor.h Resolver.actor.cpp diff --git a/fdbserver/FDBExecHelper.actor.cpp b/fdbserver/FDBExecHelper.actor.cpp index 9b690d4d35..3e34fc8e25 100644 --- a/fdbserver/FDBExecHelper.actor.cpp +++ b/fdbserver/FDBExecHelper.actor.cpp @@ -18,17 +18,30 @@ * limitations under the License. */ +#include "flow/TLSConfig.actor.h" +#include "flow/Trace.h" +#include "flow/Platform.h" +#include "flow/flow.h" +#include "flow/genericactors.actor.h" +#include "flow/network.h" +#include "fdbrpc/FlowProcess.actor.h" +#include "fdbrpc/Net2FileSystem.h" +#include "fdbrpc/simulator.h" +#include "fdbclient/WellKnownEndpoints.h" +#include "fdbclient/versions.h" +#include "fdbserver/CoroFlow.h" +#include "fdbserver/FDBExecHelper.actor.h" +#include "fdbserver/Knobs.h" +#include "fdbserver/RemoteIKeyValueStore.actor.h" + #if !defined(_WIN32) && !defined(__APPLE__) && !defined(__INTEL_COMPILER) #define BOOST_SYSTEM_NO_LIB #define BOOST_DATE_TIME_NO_LIB #define BOOST_REGEX_NO_LIB #include #endif -#include "fdbserver/FDBExecHelper.actor.h" -#include "flow/Trace.h" -#include "flow/flow.h" -#include "fdbclient/versions.h" -#include "fdbserver/Knobs.h" +#include + #include "flow/actorcompiler.h" // This must be the last #include. ExecCmdValueString::ExecCmdValueString(StringRef pCmdValueString) { @@ -90,12 +103,138 @@ void ExecCmdValueString::dbgPrint() const { return; } +ACTOR void destoryChildProcess(Future parentSSClosed, ISimulator::ProcessInfo* childInfo, std::string message) { + // This code path should be bug free + wait(parentSSClosed); + TraceEvent(SevDebug, message.c_str()).log(); + // This one is root cause for most failures, make sure it's okay to destory + g_pSimulator->destroyProcess(childInfo); + // Explicitly reset the connection with the child process in case re-spawn very quickly + FlowTransport::transport().resetConnection(childInfo->address); +} + +ACTOR Future spawnSimulated(std::vector paramList, + double maxWaitTime, + bool isSync, + double maxSimDelayTime, + IClosable* parent) { + state ISimulator::ProcessInfo* self = g_pSimulator->getCurrentProcess(); + state ISimulator::ProcessInfo* child; + + state std::string role; + state std::string addr; + state std::string flowProcessName; + state Endpoint parentProcessEndpoint; + state int i = 0; + // fdbserver -r flowprocess --process-name ikvs --process-endpoint ip:port,token,id + for (; i < paramList.size(); i++) { + if (paramList.size() > i + 1) { + // temporary args parser that only supports the flowprocess role + if (paramList[i] == "-r") { + role = paramList[i + 1]; + } else if (paramList[i] == "-p" || paramList[i] == "--public_address") { + addr = paramList[i + 1]; + } else if (paramList[i] == "--process-name") { + flowProcessName = paramList[i + 1]; + } else if (paramList[i] == "--process-endpoint") { + state std::vector addressArray; + boost::split(addressArray, paramList[i + 1], [](char c) { return c == ','; }); + if (addressArray.size() != 3) { + std::cerr << "Invalid argument, expected 3 elements in --process-endpoint got " + << addressArray.size() << std::endl; + flushAndExit(FDB_EXIT_ERROR); + } + try { + auto addr = NetworkAddress::parse(addressArray[0]); + uint64_t fst = std::stoul(addressArray[1]); + uint64_t snd = std::stoul(addressArray[2]); + UID token(fst, snd); + NetworkAddressList l; + l.address = addr; + parentProcessEndpoint = Endpoint(l, token); + } catch (Error& e) { + std::cerr << "Could not parse network address " << addressArray[0] << std::endl; + flushAndExit(FDB_EXIT_ERROR); + } + } + } + } + state int result = 0; + child = g_pSimulator->newProcess("remote flow process", + self->address.ip, + 0, + self->address.isTLS(), + self->addresses.secondaryAddress.present() ? 2 : 1, + self->locality, + ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource), + self->dataFolder, + self->coordinationFolder, // do we need to customize this coordination folder path? + self->protocolVersion); + wait(g_pSimulator->onProcess(child)); + state Future onShutdown = child->onShutdown(); + state Future parentShutdown = self->onShutdown(); + state Future flowProcessF; + + try { + TraceEvent(SevDebug, "SpawnedChildProcess") + .detail("Child", child->toString()) + .detail("Parent", self->toString()); + std::string role = ""; + std::string addr = ""; + for (int i = 0; i < paramList.size(); i++) { + if (paramList.size() > i + 1 && paramList[i] == "-r") { + role = paramList[i + 1]; + } + } + if (role == "flowprocess" && !parentShutdown.isReady()) { + self->childs.push_back(child); + state Future parentSSClosed = parent->onClosed(); + FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT); + FlowTransport::transport().bind(child->address, child->address); + Sim2FileSystem::newFileSystem(); + ProcessFactory(flowProcessName.c_str()); + flowProcessF = runFlowProcess(flowProcessName, parentProcessEndpoint); + + choose { + when(wait(flowProcessF)) { + TraceEvent(SevDebug, "ChildProcessKilled").log(); + wait(g_pSimulator->onProcess(self)); + TraceEvent(SevDebug, "BackOnParentProcess").detail("Result", std::to_string(result)); + destoryChildProcess(parentSSClosed, child, "StorageServerReceivedClosedMessage"); + } + when(wait(success(onShutdown))) { + ASSERT(false); + // In prod, we use prctl to bind parent and child processes to die together + // In simulation, we simply disable killing parent or child processes as we cannot use the same + // mechanism here + } + when(wait(success(parentShutdown))) { + ASSERT(false); + // Parent process is not killed, see above + } + } + } else { + ASSERT(false); + } + } catch (Error& e) { + TraceEvent(SevError, "RemoteIKVSDied").errorUnsuppressed(e); + result = -1; + } + + return result; +} + #if defined(_WIN32) || defined(__APPLE__) || defined(__INTEL_COMPILER) ACTOR Future spawnProcess(std::string binPath, std::vector paramList, double maxWaitTime, bool isSync, - double maxSimDelayTime) { + double maxSimDelayTime, + IClosable* parent) { + if (g_network->isSimulated() && getExecPath() == binPath) { + int res = wait(spawnSimulated(paramList, maxWaitTime, isSync, maxSimDelayTime, parent)); + return res; + } wait(delay(0.0)); return 0; } @@ -125,6 +264,9 @@ static auto fork_child(const std::string& path, std::vector& paramList) { } static void setupTraceWithOutput(TraceEvent& event, size_t bytesRead, char* outputBuffer) { + // get some errors printed for spawned process + std::cout << "Output bytesRead: " << bytesRead << std::endl; + std::cout << "output buffer: " << std::string(outputBuffer) << std::endl; if (bytesRead == 0) return; ASSERT(bytesRead <= SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT); @@ -139,7 +281,12 @@ ACTOR Future spawnProcess(std::string path, std::vector args, double maxWaitTime, bool isSync, - double maxSimDelayTime) { + double maxSimDelayTime, + IClosable* parent) { + if (g_network->isSimulated() && getExecPath() == path) { + int res = wait(spawnSimulated(args, maxWaitTime, isSync, maxSimDelayTime, parent)); + return res; + } // for async calls in simulator, always delay by a deterministic amount of time and then // do the call synchronously, otherwise the predictability of the simulator breaks if (!isSync && g_network->isSimulated()) { @@ -182,7 +329,7 @@ ACTOR Future spawnProcess(std::string path, int flags = fcntl(readFD.get(), F_GETFL, 0); fcntl(readFD.get(), F_SETFL, flags | O_NONBLOCK); while (true) { - if (runTime > maxWaitTime) { + if (maxWaitTime >= 0 && runTime > maxWaitTime) { // timing out TraceEvent(SevWarnAlways, "SpawnProcessFailure") @@ -203,7 +350,6 @@ ACTOR Future spawnProcess(std::string path, break; bytesRead += bytes; } - if (err < 0) { TraceEvent event(SevWarnAlways, "SpawnProcessFailure"); setupTraceWithOutput(event, bytesRead, outputBuffer); diff --git a/fdbserver/FDBExecHelper.actor.h b/fdbserver/FDBExecHelper.actor.h index f5f07a000d..4a191663d8 100644 --- a/fdbserver/FDBExecHelper.actor.h +++ b/fdbserver/FDBExecHelper.actor.h @@ -63,16 +63,19 @@ private: // data StringRef binaryPath; }; +class IClosable; // Forward declaration + // FIXME: move this function to a common location // spawns a process pointed by `binPath` and the arguments provided at `paramList`, -// if the process spawned takes more than `maxWaitTime` then it will be killed -// if isSync is set to true then the process will be synchronously executed -// if async and in simulator then delay spawning the process to max of maxSimDelayTime +// if the process spawned takes more than `maxWaitTime` then it will be killed, if `maxWaitTime` < 0, then there won't +// be timeout if isSync is set to true then the process will be synchronously executed if async and in simulator then +// delay spawning the process to max of maxSimDelayTime ACTOR Future spawnProcess(std::string binPath, std::vector paramList, double maxWaitTime, bool isSync, - double maxSimDelayTime); + double maxSimDelayTime, + IClosable* parent = nullptr); // helper to run all the work related to running the exec command ACTOR Future execHelper(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role); diff --git a/fdbserver/IKeyValueStore.h b/fdbserver/IKeyValueStore.h index 479a7c544b..2020eebb1a 100644 --- a/fdbserver/IKeyValueStore.h +++ b/fdbserver/IKeyValueStore.h @@ -159,12 +159,23 @@ extern IKeyValueStore* keyValueStoreLogSystem(class IDiskQueue* queue, bool replaceContent, bool exactRecovery); +extern IKeyValueStore* openRemoteKVStore(KeyValueStoreType storeType, + std::string const& filename, + UID logID, + int64_t memoryLimit, + bool checkChecksums = false, + bool checkIntegrity = false); + inline IKeyValueStore* openKVStore(KeyValueStoreType storeType, std::string const& filename, UID logID, int64_t memoryLimit, bool checkChecksums = false, - bool checkIntegrity = false) { + bool checkIntegrity = false, + bool openRemotely = false) { + if (openRemotely) { + return openRemoteKVStore(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity); + } switch (storeType) { case KeyValueStoreType::SSD_BTREE_V1: return keyValueStoreSQLite(filename, logID, KeyValueStoreType::SSD_BTREE_V1, false, checkIntegrity); diff --git a/fdbserver/RemoteIKeyValueStore.actor.cpp b/fdbserver/RemoteIKeyValueStore.actor.cpp new file mode 100644 index 0000000000..bead82267a --- /dev/null +++ b/fdbserver/RemoteIKeyValueStore.actor.cpp @@ -0,0 +1,246 @@ +/* + * RemoteIKeyValueStore.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flow/ActorCollection.h" +#include "flow/Error.h" +#include "flow/Platform.h" +#include "flow/Trace.h" +#include "fdbrpc/FlowProcess.actor.h" +#include "fdbrpc/fdbrpc.h" +#include "fdbclient/FDBTypes.h" +#include "fdbserver/FDBExecHelper.actor.h" +#include "fdbserver/Knobs.h" +#include "fdbserver/RemoteIKeyValueStore.actor.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +StringRef KeyValueStoreProcess::_name = "KeyValueStoreProcess"_sr; + +// A guard for guaranteed killing of machine after runIKVS returns +struct AfterReturn { + IKeyValueStore* kvStore; + UID id; + AfterReturn() : kvStore(nullptr) {} + AfterReturn(IKeyValueStore* store, UID& uid) : kvStore(store), id(uid) {} + ~AfterReturn() { + TraceEvent(SevDebug, "RemoteKVStoreAfterReturn") + .detail("Valid", kvStore != nullptr ? "True" : "False") + .detail("UID", id) + .log(); + if (kvStore != nullptr) { + kvStore->close(); + } + } + // called when we already explicitly closed the kv store + void invalidate() { kvStore = nullptr; } +}; + +ACTOR void sendCommitReply(IKVSCommitRequest commitReq, IKeyValueStore* kvStore, Future onClosed) { + try { + choose { + when(wait(onClosed)) { commitReq.reply.sendError(remote_kvs_cancelled()); } + when(wait(kvStore->commit(commitReq.sequential))) { + StorageBytes storageBytes = kvStore->getStorageBytes(); + commitReq.reply.send(IKVSCommitReply(storageBytes)); + } + } + } catch (Error& e) { + TraceEvent(SevDebug, "RemoteKVSCommitReplyError").errorUnsuppressed(e); + commitReq.reply.sendError(e.code() == error_code_actor_cancelled ? remote_kvs_cancelled() : e); + } +} + +ACTOR template +Future cancellableForwardPromise(ReplyPromise output, Future input) { + try { + T value = wait(input); + output.send(value); + } catch (Error& e) { + TraceEvent(SevDebug, "CancellableForwardPromiseError").errorUnsuppressed(e).backtrace(); + output.sendError(e.code() == error_code_actor_cancelled ? remote_kvs_cancelled() : e); + } + return Void(); +} + +ACTOR Future runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterface) { + state IKeyValueStore* kvStore = openKVStore(openReq.storeType, + openReq.filename, + openReq.logID, + openReq.memoryLimit, + openReq.checkChecksums, + openReq.checkIntegrity); + state UID kvsId(ikvsInterface.id()); + state ActorCollection actors(false); + state AfterReturn guard(kvStore, kvsId); + state Promise onClosed; + TraceEvent(SevDebug, "RemoteKVStoreInitializing").detail("UID", kvsId); + wait(kvStore->init()); + openReq.reply.send(ikvsInterface); + TraceEvent(SevInfo, "RemoteKVStoreInitialized").detail("IKVSInterfaceUID", kvsId); + + loop { + try { + choose { + when(IKVSGetValueRequest getReq = waitNext(ikvsInterface.getValue.getFuture())) { + actors.add(cancellableForwardPromise(getReq.reply, + kvStore->readValue(getReq.key, getReq.type, getReq.debugID))); + } + when(IKVSSetRequest req = waitNext(ikvsInterface.set.getFuture())) { kvStore->set(req.keyValue); } + when(IKVSClearRequest req = waitNext(ikvsInterface.clear.getFuture())) { kvStore->clear(req.range); } + when(IKVSCommitRequest commitReq = waitNext(ikvsInterface.commit.getFuture())) { + sendCommitReply(commitReq, kvStore, onClosed.getFuture()); + } + when(IKVSReadValuePrefixRequest readPrefixReq = waitNext(ikvsInterface.readValuePrefix.getFuture())) { + actors.add(cancellableForwardPromise( + readPrefixReq.reply, + kvStore->readValuePrefix( + readPrefixReq.key, readPrefixReq.maxLength, readPrefixReq.type, readPrefixReq.debugID))); + } + when(IKVSReadRangeRequest readRangeReq = waitNext(ikvsInterface.readRange.getFuture())) { + actors.add(cancellableForwardPromise( + readRangeReq.reply, + fmap( + [](const RangeResult& result) { return IKVSReadRangeReply(result); }, + kvStore->readRange( + readRangeReq.keys, readRangeReq.rowLimit, readRangeReq.byteLimit, readRangeReq.type)))); + } + when(IKVSGetStorageByteRequest req = waitNext(ikvsInterface.getStorageBytes.getFuture())) { + StorageBytes storageBytes = kvStore->getStorageBytes(); + req.reply.send(storageBytes); + } + when(IKVSGetErrorRequest getFutureReq = waitNext(ikvsInterface.getError.getFuture())) { + actors.add(cancellableForwardPromise(getFutureReq.reply, kvStore->getError())); + } + when(IKVSOnClosedRequest onClosedReq = waitNext(ikvsInterface.onClosed.getFuture())) { + // onClosed request is not cancelled even this actor is cancelled + forwardPromise(onClosedReq.reply, kvStore->onClosed()); + } + when(IKVSDisposeRequest disposeReq = waitNext(ikvsInterface.dispose.getFuture())) { + TraceEvent(SevDebug, "RemoteIKVSDisposeReceivedRequest").detail("UID", kvsId); + kvStore->dispose(); + guard.invalidate(); + onClosed.send(Void()); + return Void(); + } + when(IKVSCloseRequest closeReq = waitNext(ikvsInterface.close.getFuture())) { + TraceEvent(SevDebug, "RemoteIKVSCloseReceivedRequest").detail("UID", kvsId); + kvStore->close(); + guard.invalidate(); + onClosed.send(Void()); + return Void(); + } + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + TraceEvent(SevInfo, "RemoteKVStoreCancelled").detail("UID", kvsId).backtrace(); + onClosed.send(Void()); + return Void(); + } else { + TraceEvent(SevError, "RemoteKVStoreError").error(e).detail("UID", kvsId).backtrace(); + throw; + } + } + } +} + +ACTOR static Future flowProcessRunner(RemoteIKeyValueStore* self, Promise ready) { + state FlowProcessInterface processInterface; + state Future process; + + auto path = abspath(getExecPath()); + auto endpoint = processInterface.registerProcess.getEndpoint(); + auto address = endpoint.addresses.address.toString(); + auto token = endpoint.token; + + // port 0 means we will find a random available port number for it + std::string flowProcessAddr = g_network->getLocalAddress().ip.toString().append(":0"); + std::vector args = { "bin/fdbserver", + "-r", + "flowprocess", + "-C", + SERVER_KNOBS->CONN_FILE, + "--logdir", + SERVER_KNOBS->LOG_DIRECTORY, + "-p", + flowProcessAddr, + "--process-name", + KeyValueStoreProcess::_name.toString(), + "--process-endpoint", + format("%s,%lu,%lu", address.c_str(), token.first(), token.second()) }; + // For remote IKV store, we need to make sure the shutdown signal is sent back until we can destroy it in the + // simulation + process = spawnProcess(path, args, -1.0, false, 0.01 /*not used*/, self); + choose { + when(FlowProcessRegistrationRequest req = waitNext(processInterface.registerProcess.getFuture())) { + self->consumeInterface(req.flowProcessInterface); + ready.send(Void()); + } + when(int res = wait(process)) { + // 0 means process normally shut down; non-zero means errors + // process should not shut down normally before not ready + ASSERT(res); + return res; + } + } + int res = wait(process); + return res; +} + +ACTOR static Future initializeRemoteKVStore(RemoteIKeyValueStore* self, OpenKVStoreRequest openKVSReq) { + TraceEvent(SevInfo, "WaitingOnFlowProcess").detail("StoreType", openKVSReq.storeType).log(); + Promise ready; + self->returnCode = flowProcessRunner(self, ready); + wait(ready.getFuture()); + IKVSInterface ikvsInterface = wait(self->kvsProcess.openKVStore.getReply(openKVSReq)); + TraceEvent(SevInfo, "IKVSInterfaceReceived").detail("UID", ikvsInterface.id()); + self->interf = ikvsInterface; + self->interf.storeType = openKVSReq.storeType; + return Void(); +} + +IKeyValueStore* openRemoteKVStore(KeyValueStoreType storeType, + std::string const& filename, + UID logID, + int64_t memoryLimit, + bool checkChecksums, + bool checkIntegrity) { + RemoteIKeyValueStore* self = new RemoteIKeyValueStore(); + self->initialized = initializeRemoteKVStore( + self, OpenKVStoreRequest(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity)); + return self; +} + +ACTOR static Future delayFlowProcessRunAction(FlowProcess* self, double time) { + wait(delay(time)); + wait(self->run()); + return Void(); +} + +Future runFlowProcess(std::string const& name, Endpoint endpoint) { + TraceEvent(SevInfo, "RunFlowProcessStart").log(); + FlowProcess* self = IProcessFactory::create(name.c_str()); + self->registerEndpoint(endpoint); + RequestStream registerProcess(endpoint); + FlowProcessRegistrationRequest req; + req.flowProcessInterface = self->serializedInterface(); + registerProcess.send(req); + TraceEvent(SevDebug, "FlowProcessInitFinished").log(); + return delayFlowProcessRunAction(self, g_network->isSimulated() ? 0 : SERVER_KNOBS->REMOTE_KV_STORE_INIT_DELAY); +} diff --git a/fdbserver/RemoteIKeyValueStore.actor.h b/fdbserver/RemoteIKeyValueStore.actor.h new file mode 100644 index 0000000000..7df95aa2e8 --- /dev/null +++ b/fdbserver/RemoteIKeyValueStore.actor.h @@ -0,0 +1,504 @@ +/* + * RemoteIKeyValueStore.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_G_H) +#define FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_G_H +#include "fdbserver/RemoteIKeyValueStore.actor.g.h" +#elif !defined(FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_H) +#define FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_H + +#include "flow/ActorCollection.h" +#include "flow/IRandom.h" +#include "flow/Knobs.h" +#include "flow/Trace.h" +#include "flow/flow.h" +#include "flow/network.h" +#include "fdbrpc/FlowProcess.actor.h" +#include "fdbrpc/FlowTransport.h" +#include "fdbrpc/fdbrpc.h" +#include "fdbclient/FDBTypes.h" +#include "fdbserver/FDBExecHelper.actor.h" +#include "fdbserver/IKeyValueStore.h" +#include "fdbserver/Knobs.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +struct IKVSCommitReply { + constexpr static FileIdentifier file_identifier = 3958189; + StorageBytes storeBytes; + + IKVSCommitReply() : storeBytes(0, 0, 0, 0) {} + IKVSCommitReply(const StorageBytes& sb) : storeBytes(sb) {} + + template + void serialize(Ar& ar) { + serializer(ar, storeBytes); + } +}; + +struct RemoteKVSProcessInterface { + + constexpr static FileIdentifier file_identifier = 3491838; + RequestStream getProcessInterface; + RequestStream openKVStore; + + UID uniqueID = deterministicRandom()->randomUniqueID(); + + UID id() const { return uniqueID; } + + template + void serialize(Ar& ar) { + serializer(ar, getProcessInterface, openKVStore); + } +}; + +struct IKVSInterface { + constexpr static FileIdentifier file_identifier = 4929113; + RequestStream getValue; + RequestStream set; + RequestStream clear; + RequestStream commit; + RequestStream readValuePrefix; + RequestStream readRange; + RequestStream getStorageBytes; + RequestStream getError; + RequestStream onClosed; + RequestStream dispose; + RequestStream close; + + UID uniqueID; + + UID id() const { return uniqueID; } + + KeyValueStoreType storeType; + + KeyValueStoreType type() const { return storeType; } + + IKVSInterface() {} + + IKVSInterface(KeyValueStoreType type) : uniqueID(deterministicRandom()->randomUniqueID()), storeType(type) {} + + template + void serialize(Ar& ar) { + serializer(ar, + getValue, + set, + clear, + commit, + readValuePrefix, + readRange, + getStorageBytes, + getError, + onClosed, + dispose, + close, + uniqueID); + } +}; + +struct GetRemoteKVSProcessInterfaceRequest { + constexpr static FileIdentifier file_identifier = 8382983; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, reply); + } +}; + +struct OpenKVStoreRequest { + constexpr static FileIdentifier file_identifier = 5918682; + KeyValueStoreType storeType; + std::string filename; + UID logID; + int64_t memoryLimit; + bool checkChecksums; + bool checkIntegrity; + ReplyPromise reply; + + OpenKVStoreRequest(){}; + + OpenKVStoreRequest(KeyValueStoreType storeType, + std::string filename, + UID logID, + int64_t memoryLimit, + bool checkChecksums = false, + bool checkIntegrity = false) + : storeType(storeType), filename(filename), logID(logID), memoryLimit(memoryLimit), + checkChecksums(checkChecksums), checkIntegrity(checkIntegrity) {} + + template + void serialize(Ar& ar) { + serializer(ar, storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity, reply); + } +}; + +struct IKVSGetValueRequest { + constexpr static FileIdentifier file_identifier = 1029439; + KeyRef key; + IKeyValueStore::ReadType type; + Optional debugID = Optional(); + ReplyPromise> reply; + + template + void serialize(Ar& ar) { + serializer(ar, key, type, debugID, reply); + } +}; + +struct IKVSSetRequest { + constexpr static FileIdentifier file_identifier = 7283948; + KeyValueRef keyValue; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, keyValue, reply); + } +}; + +struct IKVSClearRequest { + constexpr static FileIdentifier file_identifier = 2838575; + KeyRangeRef range; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, range, reply); + } +}; + +struct IKVSCommitRequest { + constexpr static FileIdentifier file_identifier = 2985129; + bool sequential; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, sequential, reply); + } +}; + +struct IKVSReadValuePrefixRequest { + constexpr static FileIdentifier file_identifier = 1928374; + KeyRef key; + int maxLength; + IKeyValueStore::ReadType type; + Optional debugID = Optional(); + ReplyPromise> reply; + + template + void serialize(Ar& ar) { + serializer(ar, key, maxLength, type, debugID, reply); + } +}; + +// Use this instead of RangeResult as reply for better serialization performance +struct IKVSReadRangeReply { + constexpr static FileIdentifier file_identifier = 6682449; + Arena arena; + VectorRef data; + bool more; + Optional readThrough; + bool readToBegin; + bool readThroughEnd; + + IKVSReadRangeReply() = default; + + explicit IKVSReadRangeReply(const RangeResult& res) + : arena(res.arena()), data(static_cast&>(res)), more(res.more), + readThrough(res.readThrough), readToBegin(res.readToBegin), readThroughEnd(res.readThroughEnd) {} + + template + void serialize(Ar& ar) { + serializer(ar, data, more, readThrough, readToBegin, readThroughEnd, arena); + } + + RangeResult toRangeResult() const { + RangeResult r(RangeResultRef(data, more, readThrough), arena); + r.readToBegin = readToBegin; + r.readThroughEnd = readThroughEnd; + return r; + } +}; + +struct IKVSReadRangeRequest { + constexpr static FileIdentifier file_identifier = 5918394; + KeyRangeRef keys; + int rowLimit; + int byteLimit; + IKeyValueStore::ReadType type; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, keys, rowLimit, byteLimit, type, reply); + } +}; + +struct IKVSGetStorageByteRequest { + constexpr static FileIdentifier file_identifier = 3512344; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, reply); + } +}; + +struct IKVSGetErrorRequest { + constexpr static FileIdentifier file_identifier = 3942891; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, reply); + } +}; + +struct IKVSOnClosedRequest { + constexpr static FileIdentifier file_identifier = 1923894; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, reply); + } +}; + +struct IKVSDisposeRequest { + constexpr static FileIdentifier file_identifier = 1235952; + + template + void serialize(Ar& ar) { + serializer(ar); + } +}; + +struct IKVSCloseRequest { + constexpr static FileIdentifier file_identifier = 13859172; + + template + void serialize(Ar& ar) { + serializer(ar); + } +}; + +ACTOR Future runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterface); + +struct KeyValueStoreProcess : FlowProcess { + RemoteKVSProcessInterface kvsIf; + Standalone serializedIf; + + Endpoint ssProcess; // endpoint for the storage process + RequestStream ssRequestStream; + + KeyValueStoreProcess() { + TraceEvent(SevDebug, "InitKeyValueStoreProcess").log(); + ObjectWriter writer(IncludeVersion()); + writer.serialize(kvsIf); + serializedIf = writer.toString(); + } + + void registerEndpoint(Endpoint p) override { + ssProcess = p; + ssRequestStream = RequestStream(p); + } + + StringRef name() const override { return _name; } + StringRef serializedInterface() const override { return serializedIf; } + + ACTOR static Future _run(KeyValueStoreProcess* self) { + state ActorCollection actors(true); + TraceEvent("WaitingForOpenKVStoreRequest").log(); + loop { + choose { + when(OpenKVStoreRequest req = waitNext(self->kvsIf.openKVStore.getFuture())) { + TraceEvent("OpenKVStoreRequestReceived").log(); + IKVSInterface interf; + actors.add(runIKVS(req, interf)); + } + when(ErrorOr e = wait(errorOr(actors.getResult()))) { + if (e.isError()) { + TraceEvent("KeyValueStoreProcessRunActorError").errorUnsuppressed(e.getError()); + throw e.getError(); + } else { + TraceEvent("KeyValueStoreProcessFinished").log(); + return e.get(); + } + } + } + } + } + + Future run() override { return _run(this); } + + static StringRef _name; +}; + +struct RemoteIKeyValueStore : public IKeyValueStore { + RemoteKVSProcessInterface kvsProcess; + IKVSInterface interf; + Future initialized; + Future returnCode; + StorageBytes storageBytes; + + RemoteIKeyValueStore() : storageBytes(0, 0, 0, 0) {} + + Future init() override { + TraceEvent(SevInfo, "RemoteIKeyValueStoreInit").log(); + return initialized; + } + + Future getError() const override { return getErrorImpl(this, returnCode); } + Future onClosed() const override { return onCloseImpl(this); } + + void dispose() override { + TraceEvent(SevDebug, "RemoteIKVSDisposeRequest").backtrace(); + interf.dispose.send(IKVSDisposeRequest{}); + // hold the future to not cancel the spawned process + uncancellable(returnCode); + delete this; + } + void close() override { + TraceEvent(SevDebug, "RemoteIKVSCloseRequest").backtrace(); + interf.close.send(IKVSCloseRequest{}); + // hold the future to not cancel the spawned process + uncancellable(returnCode); + delete this; + } + + KeyValueStoreType getType() const override { return interf.type(); } + + void set(KeyValueRef keyValue, const Arena* arena = nullptr) override { + interf.set.send(IKVSSetRequest{ keyValue, ReplyPromise() }); + } + void clear(KeyRangeRef range, const Arena* arena = nullptr) override { + interf.clear.send(IKVSClearRequest{ range, ReplyPromise() }); + } + + Future commit(bool sequential = false) override { + Future commitReply = + interf.commit.getReply(IKVSCommitRequest{ sequential, ReplyPromise() }); + return commitAndGetStorageBytes(this, commitReply); + } + + Future> readValue(KeyRef key, + ReadType type = ReadType::NORMAL, + Optional debugID = Optional()) override { + return readValueImpl(this, IKVSGetValueRequest{ key, type, debugID, ReplyPromise>() }); + } + + Future> readValuePrefix(KeyRef key, + int maxLength, + ReadType type = ReadType::NORMAL, + Optional debugID = Optional()) override { + return interf.readValuePrefix.getReply( + IKVSReadValuePrefixRequest{ key, maxLength, type, debugID, ReplyPromise>() }); + } + + Future readRange(KeyRangeRef keys, + int rowLimit = 1 << 30, + int byteLimit = 1 << 30, + ReadType type = ReadType::NORMAL) override { + IKVSReadRangeRequest req{ keys, rowLimit, byteLimit, type, ReplyPromise() }; + return fmap([](const IKVSReadRangeReply& reply) { return reply.toRangeResult(); }, + interf.readRange.getReply(req)); + } + + StorageBytes getStorageBytes() const override { return storageBytes; } + + void consumeInterface(StringRef intf) { + kvsProcess = ObjectReader::fromStringRef(intf, IncludeVersion()); + } + + ACTOR static Future commitAndGetStorageBytes(RemoteIKeyValueStore* self, + Future commitReplyFuture) { + IKVSCommitReply commitReply = wait(commitReplyFuture); + self->storageBytes = commitReply.storeBytes; + return Void(); + } + + ACTOR static Future> readValueImpl(RemoteIKeyValueStore* self, IKVSGetValueRequest req) { + Optional val = wait(self->interf.getValue.getReply(req)); + return val; + } + + ACTOR static Future getErrorImpl(const RemoteIKeyValueStore* self, Future returnCode) { + choose { + when(wait(self->initialized)) {} + when(wait(delay(SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION))) { + TraceEvent(SevError, "RemoteIKVSInitTooLong") + .detail("TimeLimit", SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION); + throw please_reboot_remote_kv_store(); + } + } + state Future connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY); + state Future> storeError = errorOr(self->interf.getError.getReply(IKVSGetErrorRequest{})); + loop choose { + when(ErrorOr e = wait(storeError)) { + TraceEvent(SevDebug, "RemoteIKVSGetError") + .errorUnsuppressed(e.isError() ? e.getError() : success()) + .backtrace(); + if (e.isError()) + throw e.getError(); + else + return e.get(); + } + when(int res = wait(returnCode)) { + TraceEvent(res != 0 ? SevError : SevInfo, "SpawnedProcessDied").detail("Res", res); + if (res) + throw please_reboot_remote_kv_store(); // this will reboot the worker + else + return Void(); + } + when(wait(connectionCheckingDelay)) { + // for the corner case where the child process stuck and waitpid also does not give update on it + // In this scenario, we need to manually reboot the storage engine process + if (IFailureMonitor::failureMonitor() + .getState(self->interf.getError.getEndpoint().getPrimaryAddress()) + .isFailed()) { + TraceEvent(SevError, "RemoteKVStoreConnectionStuck").log(); + throw please_reboot_remote_kv_store(); // this will reboot the worker + } + connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY); + } + } + } + + ACTOR static Future onCloseImpl(const RemoteIKeyValueStore* self) { + try { + wait(self->initialized); + wait(self->interf.onClosed.getReply(IKVSOnClosedRequest{})); + TraceEvent(SevDebug, "RemoteIKVSOnCloseImplOnClosedFinished"); + } catch (Error& e) { + TraceEvent(SevInfo, "RemoteIKVSOnCloseImplError").errorUnsuppressed(e).backtrace(); + throw; + } + return Void(); + } +}; + +Future runFlowProcess(std::string const& name, Endpoint endpoint); + +#include "flow/unactorcompiler.h" +#endif \ No newline at end of file diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index b369f5d3df..3883dd50d3 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -263,6 +263,9 @@ class TestConfig { if (attrib == "disableHostname") { disableHostname = strcmp(value.c_str(), "true") == 0; } + if (attrib == "disableRemoteKVS") { + disableRemoteKVS = strcmp(value.c_str(), "true") == 0; + } if (attrib == "restartInfoLocation") { isFirstTestInRestart = true; } @@ -298,6 +301,8 @@ public: bool disableTss = false; // 7.1 cannot be downgraded to 7.0 and below after enabling hostname, so disable hostname for 7.0 downgrade tests bool disableHostname = false; + // remote key value store is a child process spawned by the SS process to run the storage engine + bool disableRemoteKVS = false; // Storage Engine Types: Verify match with SimulationConfig::generateNormalConfig // 0 = "ssd" // 1 = "memory" @@ -357,6 +362,7 @@ public: .add("maxTLogVersion", &maxTLogVersion) .add("disableTss", &disableTss) .add("disableHostname", &disableHostname) + .add("disableRemoteKVS", &disableRemoteKVS) .add("simpleConfig", &simpleConfig) .add("generateFearless", &generateFearless) .add("datacenters", &datacenters) @@ -1084,6 +1090,11 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor INetworkConnections::net()->parseMockDNSFromString(mockDNSStr); } } + if (testConfig.disableRemoteKVS) { + IKnobCollection::getMutableGlobalKnobCollection().setKnob("remote_kv_store", + KnobValueRef::create(bool{ false })); + TraceEvent(SevDebug, "DisaableRemoteKVS").log(); + } *pConnString = conn; *pTesterCount = testerCount; bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1; @@ -1836,6 +1847,11 @@ void setupSimulatedSystem(std::vector>* systemActors, if (testConfig.configureLocked) { startingConfigString += " locked"; } + if (testConfig.disableRemoteKVS) { + IKnobCollection::getMutableGlobalKnobCollection().setKnob("remote_kv_store", + KnobValueRef::create(bool{ false })); + TraceEvent(SevDebug, "DisaableRemoteKVS").log(); + } auto configDBType = testConfig.getConfigDBType(); for (auto kv : startingConfigJSON) { if ("tss_storage_engine" == kv.first) { diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 17f1bcf9d2..75c47cf6a8 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -45,16 +45,20 @@ #include "fdbclient/WellKnownEndpoints.h" #include "fdbclient/SimpleIni.h" #include "fdbrpc/AsyncFileCached.actor.h" +#include "fdbrpc/FlowProcess.actor.h" #include "fdbrpc/Net2FileSystem.h" #include "fdbrpc/PerfMetric.h" +#include "fdbrpc/fdbrpc.h" #include "fdbrpc/simulator.h" #include "fdbserver/ConflictSet.h" #include "fdbserver/CoordinationInterface.h" #include "fdbserver/CoroFlow.h" #include "fdbserver/DataDistribution.actor.h" +#include "fdbserver/FDBExecHelper.actor.h" #include "fdbserver/IKeyValueStore.h" #include "fdbserver/MoveKeys.actor.h" #include "fdbserver/NetworkTest.h" +#include "fdbserver/RemoteIKeyValueStore.actor.h" #include "fdbserver/RestoreWorkerInterface.actor.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/SimulatedCluster.h" @@ -74,10 +78,13 @@ #include "flow/WriteOnlySet.h" #include "flow/UnitTest.h" #include "flow/FaultInjection.h" +#include "flow/flow.h" +#include "flow/network.h" #if defined(__linux__) || defined(__FreeBSD__) #include #include +#include #ifdef ALLOC_INSTRUMENTATION #include #endif @@ -100,7 +107,7 @@ enum { OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE, OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE, - OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME, + OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME, OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT }; CSimpleOpt::SOption g_rgOptions[] = { @@ -187,8 +194,10 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_USE_TEST_CONFIG_DB, "--use-test-config-db", SO_NONE }, { OPT_FAULT_INJECTION, "-fi", SO_REQ_SEP }, { OPT_FAULT_INJECTION, "--fault-injection", SO_REQ_SEP }, - { OPT_PROFILER, "--profiler-", SO_REQ_SEP}, + { OPT_PROFILER, "--profiler-", SO_REQ_SEP }, { OPT_PRINT_SIMTIME, "--print-sim-time", SO_NONE }, + { OPT_FLOW_PROCESS_NAME, "--process-name", SO_REQ_SEP }, + { OPT_FLOW_PROCESS_ENDPOINT, "--process-endpoint", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS @@ -959,7 +968,8 @@ enum class ServerRole { SkipListTest, Test, VersionedMapTest, - UnitTests + UnitTests, + FlowProcess }; struct CLIOptions { std::string commandLine; @@ -1015,6 +1025,8 @@ struct CLIOptions { UnitTestParameters testParams; std::map profilerConfig; + std::string flowProcessName; + Endpoint flowProcessEndpoint; bool printSimTime = false; static CLIOptions parseArgs(int argc, char* argv[]) { @@ -1193,6 +1205,8 @@ private: role = ServerRole::ConsistencyCheck; else if (!strcmp(sRole, "unittests")) role = ServerRole::UnitTests; + else if (!strcmp(sRole, "flowprocess")) + role = ServerRole::FlowProcess; else { fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole); printHelpTeaser(argv[0]); @@ -1517,6 +1531,42 @@ private: case OPT_USE_TEST_CONFIG_DB: configDBType = ConfigDBType::SIMPLE; break; + case OPT_FLOW_PROCESS_NAME: + flowProcessName = args.OptionArg(); + std::cout << flowProcessName << std::endl; + break; + case OPT_FLOW_PROCESS_ENDPOINT: { + std::vector strings; + std::cout << args.OptionArg() << std::endl; + boost::split(strings, args.OptionArg(), [](char c) { return c == ','; }); + for (auto& str : strings) { + std::cout << str << " "; + } + std::cout << "\n"; + if (strings.size() != 3) { + std::cerr << "Invalid argument, expected 3 elements in --process-endpoint got " << strings.size() + << std::endl; + flushAndExit(FDB_EXIT_ERROR); + } + try { + auto addr = NetworkAddress::parse(strings[0]); + uint64_t fst = std::stoul(strings[1]); + uint64_t snd = std::stoul(strings[2]); + UID token(fst, snd); + NetworkAddressList l; + l.address = addr; + flowProcessEndpoint = Endpoint(l, token); + std::cout << "flowProcessEndpoint: " << flowProcessEndpoint.getPrimaryAddress().toString() + << ", token: " << flowProcessEndpoint.token.toString() << "\n"; + } catch (Error& e) { + std::cerr << "Could not parse network address " << strings[0] << std::endl; + flushAndExit(FDB_EXIT_ERROR); + } catch (std::exception& e) { + std::cerr << "Could not parse token " << strings[1] << "," << strings[2] << std::endl; + flushAndExit(FDB_EXIT_ERROR); + } + break; + } case OPT_PRINT_SIMTIME: printSimTime = true; break; @@ -1723,6 +1773,7 @@ int main(int argc, char* argv[]) { role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False); IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder)); + IKnobCollection::getMutableGlobalKnobCollection().setKnob("conn_file", KnobValue::create(opts.connFile)); if (role != ServerRole::Simulation) { IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit", KnobValue::create(int64_t{ opts.memLimit })); @@ -1802,8 +1853,8 @@ int main(int argc, char* argv[]) { FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT); opts.buildNetwork(argv[0]); - const bool expectsPublicAddress = - (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::Restore); + const bool expectsPublicAddress = (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || + role == ServerRole::Restore || role == ServerRole::FlowProcess); if (opts.publicAddressStrs.empty()) { if (expectsPublicAddress) { fprintf(stderr, "ERROR: The -p or --public-address option is required\n"); @@ -2139,6 +2190,19 @@ int main(int argc, char* argv[]) { } f = result; + } else if (role == ServerRole::FlowProcess) { + TraceEvent(SevDebug, "StartingFlowProcess").detail("From", "fdbserver"); +#if defined(__linux__) || defined(__FreeBSD__) + prctl(PR_SET_PDEATHSIG, SIGTERM); + if (getppid() == 1) /* parent already died before prctl */ + flushAndExit(FDB_EXIT_SUCCESS); +#endif + + if (opts.flowProcessName == "KeyValueStoreProcess") { + ProcessFactory(opts.flowProcessName.c_str()); + } + f = stopAfter(runFlowProcess(opts.flowProcessName, opts.flowProcessEndpoint)); + g_network->run(); } else if (role == ServerRole::KVFileDump) { f = stopAfter(KVFileDump(opts.kvFile)); g_network->run(); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 4b8e483b05..f627fb82c1 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -8421,7 +8421,8 @@ bool storageServerTerminated(StorageServer& self, IKeyValueStore* persistentData } if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed || - e.code() == error_code_file_not_found || e.code() == error_code_actor_cancelled) { + e.code() == error_code_file_not_found || e.code() == error_code_actor_cancelled || + e.code() == error_code_remote_kvs_cancelled) { TraceEvent("StorageServerTerminated", self.thisServerID).errorUnsuppressed(e); return true; } else diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 1596da1362..710ad420d7 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1092,7 +1092,8 @@ std::map> testSpecGlobalKey [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMaxTLogVersion", ""); } }, { "disableTss", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableTSS", ""); } }, { "disableHostname", - [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } } + [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } }, + { "disableRemoteKVS", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } } }; std::map> testSpecTestKeys = { diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 0cc0faa57d..bd721b437b 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include #include #include @@ -49,6 +50,7 @@ #include "fdbserver/CoordinationInterface.h" #include "fdbserver/ConfigNode.h" #include "fdbserver/LocalConfiguration.h" +#include "fdbserver/RemoteIKeyValueStore.actor.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/ClientWorkerInterface.h" #include "flow/Profiler.h" @@ -208,31 +210,44 @@ ACTOR Future handleIOErrors(Future actor, IClosable* store, UID id, state Future> storeError = actor.isReady() ? Never() : errorOr(store->getError()); choose { when(state ErrorOr e = wait(errorOr(actor))) { + TraceEvent(SevDebug, "HandleIOErrorsActorIsReady") + .detail("Error", e.isError() ? e.getError().code() : -1) + .detail("UID", id); if (e.isError() && e.getError().code() == error_code_please_reboot) { // no need to wait. } else { + TraceEvent(SevDebug, "HandleIOErrorsActorBeforeOnClosed").detail("IsClosed", onClosed.isReady()); wait(onClosed); + TraceEvent(SevDebug, "HandleIOErrorsActorOnClosedFinished") + .detail("StoreError", + storeError.isReady() ? (storeError.get().isError() ? storeError.get().getError().code() : 0) + : -1); } if (e.isError() && e.getError().code() == error_code_broken_promise && !storeError.isReady()) { wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY)); } - if (storeError.isReady()) - throw storeError.get().getError(); - if (e.isError()) + if (storeError.isReady() && + !((storeError.get().isError() && storeError.get().getError().code() == error_code_file_not_found))) { + throw storeError.get().isError() ? storeError.get().getError() : actor_cancelled(); + } + if (e.isError()) { throw e.getError(); - else + } else return e.get(); } when(ErrorOr e = wait(storeError)) { - TraceEvent("WorkerTerminatingByIOError", id).errorUnsuppressed(e.getError()); + // for remote kv store, worker can terminate without an error, so throws actor_cancelled + // (there's probably a better way tho) + TraceEvent("WorkerTerminatingByIOError", id) + .errorUnsuppressed(e.isError() ? e.getError() : actor_cancelled()); actor.cancel(); // file_not_found can occur due to attempting to open a partially deleted DiskQueue, which should not be // reported SevError. - if (e.getError().code() == error_code_file_not_found) { + if (e.isError() && e.getError().code() == error_code_file_not_found) { TEST(true); // Worker terminated with file_not_found error return Void(); } - throw e.getError(); + throw e.isError() ? e.getError() : actor_cancelled(); } } } @@ -243,6 +258,7 @@ ACTOR Future workerHandleErrors(FutureStream errors) { ErrorInfo err = _err; bool ok = err.error.code() == error_code_success || err.error.code() == error_code_please_reboot || err.error.code() == error_code_actor_cancelled || + err.error.code() == error_code_remote_kvs_cancelled || err.error.code() == error_code_coordinators_changed || // The worker server was cancelled err.error.code() == error_code_shutdown_in_progress; @@ -253,6 +269,7 @@ ACTOR Future workerHandleErrors(FutureStream errors) { endRole(err.role, err.id, "Error", ok, err.error); if (err.error.code() == error_code_please_reboot || + err.error.code() == error_code_please_reboot_remote_kv_store || (err.role == Role::SHARED_TRANSACTION_LOG && (err.error.code() == error_code_io_error || err.error.code() == error_code_io_timeout))) throw err.error; @@ -1090,9 +1107,13 @@ struct TrackRunningStorage { KeyValueStoreType storeType, std::set>* runningStorages) : self(self), storeType(storeType), runningStorages(runningStorages) { + TraceEvent(SevDebug, "TrackingRunningStorageConstruction").detail("StorageID", self); runningStorages->emplace(self, storeType); } - ~TrackRunningStorage() { runningStorages->erase(std::make_pair(self, storeType)); }; + ~TrackRunningStorage() { + runningStorages->erase(std::make_pair(self, storeType)); + TraceEvent(SevDebug, "TrackingRunningStorageDesctruction").detail("StorageID", self); + }; }; ACTOR Future storageServerRollbackRebooter(std::set>* runningStorages, @@ -1523,8 +1544,15 @@ ACTOR Future workerServer(Reference connRecord, if (s.storedComponent == DiskStore::Storage) { LocalLineage _; getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::Storage; - IKeyValueStore* kv = - openKVStore(s.storeType, s.filename, s.storeID, memoryLimit, false, validateDataFiles); + IKeyValueStore* kv = openKVStore( + s.storeType, + s.filename, + s.storeID, + memoryLimit, + false, + validateDataFiles, + SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled*/ + (g_network->isSimulated() ? deterministicRandom()->coinflip() : true)); Future kvClosed = kv->onClosed(); filesClosed.add(kvClosed); @@ -1598,6 +1626,7 @@ ACTOR Future workerServer(Reference connRecord, logQueueBasename = fileLogQueuePrefix.toString() + optionsString.toString() + "-"; } ASSERT_WE_THINK(abspath(parentDirectory(s.filename)) == folder); + // TraceEvent(SevDebug, "openRemoteKVStore").detail("storeType", "TlogData"); IKeyValueStore* kv = openKVStore(s.storeType, s.filename, s.storeID, memoryLimit, validateDataFiles); const DiskQueueVersion dqv = s.tLogOptions.getDiskQueueVersion(); const int64_t diskQueueWarnSize = @@ -2002,6 +2031,7 @@ ACTOR Future workerServer(Reference connRecord, req.logVersion > TLogVersion::V2 ? fileVersionedLogDataPrefix : fileLogDataPrefix; std::string filename = filenameFromId(req.storeType, folder, prefix.toString() + tLogOptions.toPrefix(), logId); + // TraceEvent(SevDebug, "openRemoteKVStore").detail("storeType", "3"); IKeyValueStore* data = openKVStore(req.storeType, filename, logId, memoryLimit); const DiskQueueVersion dqv = tLogOptions.getDiskQueueVersion(); IDiskQueue* queue = openDiskQueue( @@ -2086,7 +2116,17 @@ ACTOR Future workerServer(Reference connRecord, folder, isTss ? testingStoragePrefix.toString() : fileStoragePrefix.toString(), recruited.id()); - IKeyValueStore* data = openKVStore(req.storeType, filename, recruited.id(), memoryLimit); + + IKeyValueStore* data = openKVStore( + req.storeType, + filename, + recruited.id(), + memoryLimit, + false, + false, + SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled*/ + (g_network->isSimulated() ? deterministicRandom()->coinflip() : true)); + Future kvClosed = data->onClosed(); filesClosed.add(kvClosed); ReplyPromise storageReady = req.reply; @@ -2333,20 +2373,26 @@ ACTOR Future workerServer(Reference connRecord, when(wait(handleErrors)) {} } } catch (Error& err) { + TraceEvent(SevDebug, "WorkerServer").detail("Error", err.code()).backtrace(); // Make sure actors are cancelled before "recovery" promises are destructed. for (auto f : recoveries) f.cancel(); state Error e = err; bool ok = e.code() == error_code_please_reboot || e.code() == error_code_actor_cancelled || - e.code() == error_code_please_reboot_delete; + e.code() == error_code_please_reboot_delete || e.code() == error_code_please_reboot_remote_kv_store; endRole(Role::WORKER, interf.id(), "WorkerError", ok, e); errorForwarders.clear(false); sharedLogs.clear(); - if (e.code() != - error_code_actor_cancelled) { // We get cancelled e.g. when an entire simulation times out, but in that case - // we won't be restarted and don't need to wait for shutdown + if (e.code() != error_code_actor_cancelled && e.code() != error_code_please_reboot_remote_kv_store) { + // actor_cancelled: + // We get cancelled e.g. when an entire simulation times out, but in that case + // we won't be restarted and don't need to wait for shutdown + // reboot_remote_kv_store: + // The child process running the storage engine died abnormally, + // the current solution is to reboot the worker. + // Some refactoring work in the future can make it only reboot the storage server stopping.send(Void()); wait(filesClosed.getResult()); // Wait for complete shutdown of KV stores wait(delay(0.0)); // Unwind the callstack to make sure that IAsyncFile references are all gone diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.actor.cpp index 316d9b13c9..3f3aa66f4c 100644 --- a/fdbserver/workloads/SaveAndKill.actor.cpp +++ b/fdbserver/workloads/SaveAndKill.actor.cpp @@ -22,6 +22,7 @@ #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbrpc/simulator.h" +#include "boost/algorithm/string/predicate.hpp" #undef state #include "fdbclient/SimpleIni.h" @@ -70,12 +71,14 @@ struct SaveAndKillWorkload : TestWorkload { std::map rebootingProcesses = g_simulator.currentlyRebootingProcesses; std::map allProcessesMap; for (const auto& [_, process] : rebootingProcesses) { - if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) { + if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end() && + std::string(process->name) != "remote flow process") { allProcessesMap[process->dataFolder] = process; } } for (const auto& process : processes) { - if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) { + if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end() && + std::string(process->name) != "remote flow process") { allProcessesMap[process->dataFolder] = process; } } diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 1e65e828c3..47a09eb1a0 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -743,6 +743,13 @@ class Listener final : public IListener, ReferenceCounted { public: Listener(boost::asio::io_context& io_service, NetworkAddress listenAddress) : io_service(io_service), listenAddress(listenAddress), acceptor(io_service, tcpEndpoint(listenAddress)) { + // when port 0 is passed in, a random port will be opened + // set listenAddress as the address with the actual port opened instead of port 0 + if (listenAddress.port == 0) { + this->listenAddress = + NetworkAddress::parse(acceptor.local_endpoint().address().to_string().append(":").append( + std::to_string(acceptor.local_endpoint().port()))); + } platform::setCloseOnExec(acceptor.native_handle()); } diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 20a13ac8c7..466549419f 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -3755,6 +3755,40 @@ void fdb_probe_actor_exit(const char* name, unsigned long id, int index) { } #endif +void throwExecPathError(Error e, char path[]) { + Severity sev = e.code() == error_code_io_error ? SevError : SevWarnAlways; + TraceEvent(sev, "GetPathError").error(e).detail("Path", path); + throw e; +} + +std::string getExecPath() { + char path[1024]; + uint32_t size = sizeof(path); +#if defined(__APPLE__) + if (_NSGetExecutablePath(path, &size) == 0) { + return std::string(path); + } else { + throwExecPathError(platform_error(), path); + } +#elif defined(__linux__) + ssize_t len = ::readlink("/proc/self/exe", path, size); + if (len != -1) { + path[len] = '\0'; + return std::string(path); + } else { + throwExecPathError(platform_error(), path); + } +#elif defined(_WIN32) + auto len = GetModuleFileName(nullptr, path, size); + if (len != 0) { + return std::string(path); + } else { + throwExecPathError(platform_error(), path); + } +#endif + return "unsupported OS"; +} + void setupRunLoopProfiler() { #ifdef __linux__ if (!profileThread && FLOW_KNOBS->RUN_LOOP_PROFILING_INTERVAL > 0) { diff --git a/flow/Platform.h b/flow/Platform.h index dae2a63a08..5ce6cd6640 100644 --- a/flow/Platform.h +++ b/flow/Platform.h @@ -703,6 +703,9 @@ void* loadFunction(void* lib, const char* func_name); std::string exePath(); +// get the absolute path +std::string getExecPath(); + #ifdef _WIN32 inline static int ctzll(uint64_t value) { unsigned long count = 0; diff --git a/flow/error_definitions.h b/flow/error_definitions.h index ecd7ab1d28..318aa1d7d2 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -87,6 +87,7 @@ ERROR( blob_granule_file_load_error, 1063, "Error loading a blob file during gra ERROR( blob_granule_transaction_too_old, 1064, "Read version is older than blob granule history supports" ) ERROR( blob_manager_replaced, 1065, "This blob manager has been replaced." ) ERROR( change_feed_popped, 1066, "Tried to read a version older than what has been popped from the change feed" ) +ERROR( remote_kvs_cancelled, 1067, "The remote key-value store is cancelled" ) ERROR( broken_promise, 1100, "Broken promise" ) ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" ) @@ -113,6 +114,7 @@ ERROR( dd_tracker_cancelled, 1215, "The data distribution tracker has been cance ERROR( failed_to_progress, 1216, "Process has failed to make sufficient progress" ) ERROR( invalid_cluster_id, 1217, "Attempted to join cluster with a different cluster ID" ) ERROR( restart_cluster_controller, 1218, "Restart cluster controller process" ) +ERROR( please_reboot_remote_kv_store, 1219, "Need to reboot the storage engine process as it died abnormally") // 15xx Platform errors ERROR( platform_error, 1500, "Platform error" ) diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h index f30ef772e5..f5f2aedba1 100644 --- a/flow/genericactors.actor.h +++ b/flow/genericactors.actor.h @@ -80,7 +80,7 @@ Future> stopAfter(Future what) { ret = Optional(_); } catch (Error& e) { bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete || - e.code() == error_code_actor_cancelled; + e.code() == error_code_actor_cancelled || e.code() == error_code_please_reboot_remote_kv_store; TraceEvent(ok ? SevInfo : SevError, "StopAfterError").error(e); if (!ok) { fprintf(stderr, "Fatal Error: %s\n", e.what()); diff --git a/flow/network.h b/flow/network.h index 967a145b7e..5617f96501 100644 --- a/flow/network.h +++ b/flow/network.h @@ -507,6 +507,10 @@ public: virtual NetworkAddress getPeerAddress() const = 0; virtual UID getDebugID() const = 0; + + // At present, implemented by Sim2Conn where we want to disable bits flip for connections between parent process and + // child process, also reduce latency for this kind of connection + virtual bool isStableConnection() const { throw unsupported_operation(); } }; class IListener { diff --git a/tests/fast/PhysicalShardMove.toml b/tests/fast/PhysicalShardMove.toml index 72d1f0331c..6377f8d6a2 100644 --- a/tests/fast/PhysicalShardMove.toml +++ b/tests/fast/PhysicalShardMove.toml @@ -4,6 +4,7 @@ storageEngineType = 4 processesPerMachine = 1 coordinators = 3 machineCount = 15 +disableRemoteKVS = true [[test]] testTitle = 'PhysicalShardMove' diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml index b61bdebc61..3f09821bf4 100644 --- a/tests/slow/DiskFailureCycle.toml +++ b/tests/slow/DiskFailureCycle.toml @@ -4,6 +4,7 @@ minimumReplication = 3 minimumRegions = 3 logAntiQuorum = 0 storageEngineExcludeTypes = [4] +disableRemoteKVS = true [[test]] testTitle = 'DiskFailureCycle' From 377e252fcf6eeead8f72dee89db70914cc47f6e9 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Fri, 1 Apr 2022 18:09:46 -0500 Subject: [PATCH 87/90] Better split sizing in blob manager (#6725) --- fdbserver/BlobManager.actor.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index 192475f4dd..e0eadbca4e 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -273,7 +273,8 @@ struct BlobManagerData : NonCopyable, ReferenceCounted { ACTOR Future>> splitRange(Reference bmData, KeyRange range, - bool writeHot) { + bool writeHot, + bool initialSplit) { try { if (BM_DEBUG) { fmt::print("Splitting new range [{0} - {1}): {2}\n", @@ -290,8 +291,24 @@ ACTOR Future>> splitRange(ReferenceBG_SNAPSHOT_FILE_TARGET_BYTES; + if (!initialSplit) { + // If we have X MB target granule size, we want to do the initial split to split up into X MB chunks. + // However, if we already have a granule that we are evaluating for split, if we split it as soon as it is + // larger than X MB, we will end up with 2 X/2 MB granules. + // To ensure an average size of X MB, we split granules at 4/3*X, so that they range between 2/3*X and + // 4/3*X, averaging X + splitThreshold = (splitThreshold * 4) / 3; + } + // if write-hot, we want to be able to split smaller, but not infinitely. Allow write-hot granules to be 3x + // smaller + // TODO knob? + // TODO: re-evaluate after we have granule merging? + if (writeHot) { + splitThreshold /= 3; + } TEST(writeHot); // Change feed write hot split - if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES || writeHot) { + if (estimated.bytes > splitThreshold) { // only split on bytes and write rate state StorageMetrics splitMetrics; splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES; @@ -325,6 +342,7 @@ ACTOR Future>> splitRange(Reference monitorClientRanges(Reference bmData) { // Divide new ranges up into equal chunks by using SS byte sample for (KeyRangeRef range : rangesToAdd) { TraceEvent("ClientBlobRangeAdded", bmData->id).detail("Range", range); - splitFutures.push_back(splitRange(bmData, range, false)); + splitFutures.push_back(splitRange(bmData, range, false, true)); } for (auto f : splitFutures) { @@ -892,7 +910,7 @@ ACTOR Future maybeSplitRange(Reference bmData, state Standalone> newRanges; // first get ranges to split - Standalone> _newRanges = wait(splitRange(bmData, granuleRange, writeHot)); + Standalone> _newRanges = wait(splitRange(bmData, granuleRange, writeHot, false)); newRanges = _newRanges; ASSERT(newRanges.size() >= 2); From b179813989769e065b116f0f3972e2021b040641 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Fri, 1 Apr 2022 17:21:35 -0700 Subject: [PATCH 88/90] Updated status schema and fixed spacing. --- fdbclient/Schemas.cpp | 62 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 18ffac2fa2..fbfd81558f 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -24,37 +24,37 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( { "cluster":{ - "storage_wiggler": { - "wiggle_server_ids":["0ccb4e0feddb55"], - "wiggle_server_addresses": ["127.0.0.1"], + "storage_wiggler": { + "wiggle_server_ids":["0ccb4e0feddb55"], + "wiggle_server_addresses": ["127.0.0.1"], "primary": { - "last_round_start_datetime": "Wed Feb 4 09:36:37 2022 +0000", - "last_round_start_timestamp": 63811229797, - "last_round_finish_datetime": "Thu Jan 1 00:00:00 1970 +0000", - "last_round_finish_timestamp": 0, - "smoothed_round_seconds": 1, - "finished_round": 1, - "last_wiggle_start_datetime": "Wed Feb 4 09:36:37 2022 +0000", - "last_wiggle_start_timestamp": 63811229797, - "last_wiggle_finish_datetime": "Thu Jan 1 00:00:00 1970 +0000", - "last_wiggle_finish_timestamp": 0, - "smoothed_wiggle_seconds": 1, - "finished_wiggle": 1 - }, - "remote": { - "last_round_start_datetime": "Wed Feb 4 09:36:37 2022 +0000", - "last_round_start_timestamp": 63811229797, - "last_round_finish_datetime": "Thu Jan 1 00:00:00 1970 +0000", - "last_round_finish_timestamp": 0, - "smoothed_round_seconds": 1, - "finished_round": 1, - "last_wiggle_start_datetime": "Wed Feb 4 09:36:37 2022 +0000", - "last_wiggle_start_timestamp": 63811229797, - "last_wiggle_finish_datetime": "Thu Jan 1 00:00:00 1970 +0000", - "last_wiggle_finish_timestamp": 0, - "smoothed_wiggle_seconds": 1, - "finished_wiggle": 1 - } + "last_round_start_datetime": "2022-04-02 00:05:05.123 +0000", + "last_round_start_timestamp": 1648857905.123, + "last_round_finish_datetime": "1970-01-01 00:00:00.000 +0000", + "last_round_finish_timestamp": 0, + "smoothed_round_seconds": 1, + "finished_round": 1, + "last_wiggle_start_datetime": "2022-04-02 00:05:05.123 +0000", + "last_wiggle_start_timestamp": 1648857905.123, + "last_wiggle_finish_datetime": "1970-01-01 00:00:00.000 +0000", + "last_wiggle_finish_timestamp": 0, + "smoothed_wiggle_seconds": 1, + "finished_wiggle": 1 + }, + "remote": { + "last_round_start_datetime": "2022-04-02 00:05:05.123 +0000", + "last_round_start_timestamp": 1648857905.123, + "last_round_finish_datetime": "1970-01-01 00:00:00.000 +0000", + "last_round_finish_timestamp": 0, + "smoothed_round_seconds": 1, + "finished_round": 1, + "last_wiggle_start_datetime": "2022-04-02 00:05:05.123 +0000", + "last_wiggle_start_timestamp": 1648857905.123, + "last_wiggle_finish_datetime": "1970-01-01 00:00:00.000 +0000", + "last_wiggle_finish_timestamp": 0, + "smoothed_wiggle_seconds": 1, + "finished_wiggle": 1 + } }, "layers":{ "_valid":true, @@ -136,7 +136,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( ] }, "storage_metadata":{ - "created_time_datetime":"Thu Jan 1 00:00:00 1970 +0000", + "created_time_datetime":"1970-01-01 00:00:00.000 +0000", "created_time_timestamp": 0 }, "data_version":12341234, From 268caa5ac86b2fb77cd58924898e189de509cf67 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Fri, 1 Apr 2022 17:57:30 -0500 Subject: [PATCH 89/90] fixing shard size knobs outside of simulation --- fdbclient/ClientKnobs.cpp | 2 +- fdbclient/ServerKnobs.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index 29aff4f4e1..37e64c166c 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -70,7 +70,7 @@ void ClientKnobs::initialize(Randomize randomize) { init( RESOURCE_CONSTRAINED_MAX_BACKOFF, 30.0 ); init( PROXY_COMMIT_OVERHEAD_BYTES, 23 ); //The size of serializing 7 tags (3 primary, 3 remote, 1 log router) + 2 for the tag length init( SHARD_STAT_SMOOTH_AMOUNT, 5.0 ); - init( INIT_MID_SHARD_BYTES, 50000000 ); if( randomize && BUGGIFY ) INIT_MID_SHARD_BYTES = 40000; else if(randomize && !BUGGIFY) INIT_MID_SHARD_BYTES = 200000; // The same value as SERVER_KNOBS->MIN_SHARD_BYTES + init( INIT_MID_SHARD_BYTES, 50000000 ); if( randomize && BUGGIFY ) INIT_MID_SHARD_BYTES = 40000; else if(randomize && BUGGIFY_WITH_PROB(0.75)) INIT_MID_SHARD_BYTES = 200000; // The same value as SERVER_KNOBS->MIN_SHARD_BYTES init( TRANSACTION_SIZE_LIMIT, 1e7 ); init( KEY_SIZE_LIMIT, 1e4 ); diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index ba31b697ea..07214d396f 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -152,7 +152,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( RETRY_RELOCATESHARD_DELAY, 0.1 ); init( DATA_DISTRIBUTION_FAILURE_REACTION_TIME, 60.0 ); if( randomize && BUGGIFY ) DATA_DISTRIBUTION_FAILURE_REACTION_TIME = 1.0; bool buggifySmallShards = randomize && BUGGIFY; - bool simulationMediumShards = !buggifySmallShards && randomize && !BUGGIFY; // prefer smaller shards in simulation + bool simulationMediumShards = !buggifySmallShards && isSimulated && randomize && !BUGGIFY; // prefer smaller shards in simulation init( MIN_SHARD_BYTES, 50000000 ); if( buggifySmallShards ) MIN_SHARD_BYTES = 40000; if (simulationMediumShards) MIN_SHARD_BYTES = 200000; //FIXME: data distribution tracker (specifically StorageMetrics) relies on this number being larger than the maximum size of a key value pair init( SHARD_BYTES_RATIO, 4 ); init( SHARD_BYTES_PER_SQRT_BYTES, 45 ); if( buggifySmallShards ) SHARD_BYTES_PER_SQRT_BYTES = 0;//Approximately 10000 bytes per shard From cb918b9cef96be4296549ce90e7973b4427fb112 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Thu, 31 Mar 2022 12:36:01 -0500 Subject: [PATCH 90/90] Added basic blob granule consistency check --- fdbclient/ServerKnobs.cpp | 3 + fdbclient/ServerKnobs.h | 2 + fdbserver/BlobGranuleValidation.actor.cpp | 165 +++++++++++++++++ fdbserver/BlobGranuleValidation.actor.h | 53 ++++++ fdbserver/BlobManager.actor.cpp | 111 ++++++++++-- fdbserver/CMakeLists.txt | 2 + .../BlobGranuleCorrectnessWorkload.actor.cpp | 37 +--- .../workloads/BlobGranuleVerifier.actor.cpp | 167 ++---------------- 8 files changed, 338 insertions(+), 202 deletions(-) create mode 100644 fdbserver/BlobGranuleValidation.actor.cpp create mode 100644 fdbserver/BlobGranuleValidation.actor.h diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 07214d396f..dacf89e754 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -843,6 +843,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( BG_MAX_SPLIT_FANOUT, 10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15); init( BG_HOT_SNAPSHOT_VERSIONS, 5000000 ); + init( BG_CONSISTENCY_CHECK_ENABLED, true ); if (randomize && BUGGIFY) BG_CONSISTENCY_CHECK_ENABLED = false; + init( BG_CONSISTENCY_CHECK_TARGET_SPEED_KB, 1000 ); if (randomize && BUGGIFY) BG_CONSISTENCY_CHECK_TARGET_SPEED_KB *= (deterministicRandom()->randomInt(2, 50) / 10); + init( BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM, 8 ); if( randomize && BUGGIFY ) BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM = 1; init( BLOB_WORKER_TIMEOUT, 10.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0; init( BLOB_WORKER_REQUEST_TIMEOUT, 5.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0; diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 5f88a9975c..e15a3100b5 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -798,6 +798,8 @@ public: int BG_DELTA_BYTES_BEFORE_COMPACT; int BG_MAX_SPLIT_FANOUT; int BG_HOT_SNAPSHOT_VERSIONS; + int BG_CONSISTENCY_CHECK_ENABLED; + int BG_CONSISTENCY_CHECK_TARGET_SPEED_KB; int BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM; double BLOB_WORKER_TIMEOUT; // Blob Manager's reaction time to a blob worker failure diff --git a/fdbserver/BlobGranuleValidation.actor.cpp b/fdbserver/BlobGranuleValidation.actor.cpp new file mode 100644 index 0000000000..9f8168ffc8 --- /dev/null +++ b/fdbserver/BlobGranuleValidation.actor.cpp @@ -0,0 +1,165 @@ +/* + * BlobGranuleValidation.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbserver/BlobGranuleValidation.actor.h" +#include "flow/actorcompiler.h" // has to be last include + +ACTOR Future> readFromFDB(Database cx, KeyRange range) { + state bool first = true; + state Version v; + state RangeResult out; + state Transaction tr(cx); + state KeyRange currentRange = range; + loop { + try { + state RangeResult r = wait(tr.getRange(currentRange, CLIENT_KNOBS->TOO_MANY)); + Version grv = wait(tr.getReadVersion()); + // need consistent version snapshot of range + if (first) { + v = grv; + first = false; + } else if (v != grv) { + // reset the range and restart the read at a higher version + first = true; + out = RangeResult(); + currentRange = range; + tr.reset(); + continue; + } + out.arena().dependsOn(r.arena()); + out.append(out.arena(), r.begin(), r.size()); + if (r.more) { + currentRange = KeyRangeRef(keyAfter(r.back().key), currentRange.end); + } else { + break; + } + } catch (Error& e) { + wait(tr.onError(e)); + } + } + return std::pair(out, v); +} + +// FIXME: typedef this pair type and/or chunk list +ACTOR Future>>> readFromBlob( + Database cx, + Reference bstore, + KeyRange range, + Version beginVersion, + Version readVersion) { + state RangeResult out; + state Standalone> chunks; + state Transaction tr(cx); + + loop { + try { + Standalone> chunks_ = + wait(tr.readBlobGranules(range, beginVersion, readVersion)); + chunks = chunks_; + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + for (const BlobGranuleChunkRef& chunk : chunks) { + RangeResult chunkRows = wait(readBlobGranule(chunk, range, beginVersion, readVersion, bstore)); + out.arena().dependsOn(chunkRows.arena()); + out.append(out.arena(), chunkRows.begin(), chunkRows.size()); + } + return std::pair(out, chunks); +} + +bool compareFDBAndBlob(RangeResult fdb, + std::pair>> blob, + KeyRange range, + Version v, + bool debug) { + bool correct = fdb == blob.first; + if (!correct) { + TraceEvent ev(SevError, "GranuleMismatch"); + ev.detail("RangeStart", range.begin) + .detail("RangeEnd", range.end) + .detail("Version", v) + .detail("FDBSize", fdb.size()) + .detail("BlobSize", blob.first.size()); + + if (debug) { + fmt::print("\nMismatch for [{0} - {1}) @ {2} ({3}). F({4}) B({5}):\n", + range.begin.printable(), + range.end.printable(), + v, + fdb.size(), + blob.first.size()); + + Optional lastCorrect; + for (int i = 0; i < std::max(fdb.size(), blob.first.size()); i++) { + if (i >= fdb.size() || i >= blob.first.size() || fdb[i] != blob.first[i]) { + printf(" Found mismatch at %d.\n", i); + if (lastCorrect.present()) { + printf(" last correct: %s=%s\n", + lastCorrect.get().key.printable().c_str(), + lastCorrect.get().value.printable().c_str()); + } + if (i < fdb.size()) { + printf(" FDB: %s=%s\n", fdb[i].key.printable().c_str(), fdb[i].value.printable().c_str()); + } else { + printf(" FDB: \n"); + } + if (i < blob.first.size()) { + printf(" BLB: %s=%s\n", + blob.first[i].key.printable().c_str(), + blob.first[i].value.printable().c_str()); + } else { + printf(" BLB: \n"); + } + printf("\n"); + break; + } + if (i < fdb.size()) { + lastCorrect = fdb[i]; + } else { + lastCorrect = blob.first[i]; + } + } + + printf("Chunks:\n"); + for (auto& chunk : blob.second) { + printf("[%s - %s)\n", chunk.keyRange.begin.printable().c_str(), chunk.keyRange.end.printable().c_str()); + + printf(" SnapshotFile:\n %s\n", + chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : ""); + printf(" DeltaFiles:\n"); + for (auto& df : chunk.deltaFiles) { + printf(" %s\n", df.toString().c_str()); + } + printf(" Deltas: (%d)", chunk.newDeltas.size()); + if (chunk.newDeltas.size() > 0) { + fmt::print(" with version [{0} - {1}]", + chunk.newDeltas[0].version, + chunk.newDeltas[chunk.newDeltas.size() - 1].version); + } + fmt::print(" IncludedVersion: {}\n", chunk.includedVersion); + } + printf("\n"); + } + } + return correct; +} \ No newline at end of file diff --git a/fdbserver/BlobGranuleValidation.actor.h b/fdbserver/BlobGranuleValidation.actor.h new file mode 100644 index 0000000000..749027055a --- /dev/null +++ b/fdbserver/BlobGranuleValidation.actor.h @@ -0,0 +1,53 @@ +/* + * BlobGranuleValidation.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BLOBGRANULEVALIDATION_ACTOR_G_H) +#define FDBSERVER_BLOBGRANULEVALIDATION_ACTOR_G_H +#include "fdbserver/BlobGranuleValidation.actor.g.h" +#elif !defined(FDBSERVER_BLOBGRANULEVALIDATION_ACTOR_H) +#define FDBSERVER_BLOBGRANULEVALIDATION_ACTOR_H + +#pragma once + +#include "flow/flow.h" +#include "fdbclient/BlobGranuleReader.actor.h" +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/BlobGranuleCommon.h" +#include "flow/actorcompiler.h" // has to be last include + +/* Contains utility functions for validating blob granule data */ + +ACTOR Future>>> readFromBlob( + Database cx, + Reference bstore, + KeyRange range, + Version beginVersion, + Version readVersion); + +ACTOR Future> readFromFDB(Database cx, KeyRange range); + +bool compareFDBAndBlob(RangeResult fdb, + std::pair>> blob, + KeyRange range, + Version v, + bool debug); + +#endif \ No newline at end of file diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index e0eadbca4e..912b0842e8 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -34,6 +34,7 @@ #include "fdbclient/SystemData.h" #include "fdbserver/BlobManagerInterface.h" #include "fdbserver/Knobs.h" +#include "fdbserver/BlobGranuleValidation.actor.h" #include "fdbserver/BlobGranuleServerCommon.actor.h" #include "fdbserver/QuietDatabase.h" #include "fdbserver/WaitFailure.h" @@ -195,10 +196,11 @@ struct RangeAssignment { }; // SOMEDAY: track worker's reads/writes eventually -struct BlobWorkerStats { +// FIXME: namespace? +struct BlobWorkerInfo { int numGranulesAssigned; - BlobWorkerStats(int numGranulesAssigned = 0) : numGranulesAssigned(numGranulesAssigned) {} + BlobWorkerInfo(int numGranulesAssigned = 0) : numGranulesAssigned(numGranulesAssigned) {} }; struct SplitEvaluation { @@ -218,12 +220,17 @@ struct BlobManagerStats { Counter granuleSplits; Counter granuleWriteHotSplits; + Counter ccGranulesChecked; + Counter ccRowsChecked; + Counter ccBytesChecked; + Counter ccMismatches; Future logger; // Current stats maintained for a given blob worker process explicit BlobManagerStats(UID id, double interval, std::unordered_map* workers) : cc("BlobManagerStats", id.toString()), granuleSplits("GranuleSplits", cc), - granuleWriteHotSplits("GranuleWriteHotSplits", cc) { + granuleWriteHotSplits("GranuleWriteHotSplits", cc), ccGranulesChecked("CCGranulesChecked", cc), + ccRowsChecked("CCRowsChecked", cc), ccBytesChecked("CCBytesChecked", cc), ccMismatches("CCMismatches", cc) { specialCounter(cc, "WorkerCount", [workers]() { return workers->size(); }); logger = traceCounters("BlobManagerMetrics", id, interval, &cc, "BlobManagerMetrics"); } @@ -241,7 +248,7 @@ struct BlobManagerData : NonCopyable, ReferenceCounted { Reference bstore; std::unordered_map workersById; - std::unordered_map workerStats; // mapping between workerID -> workerStats + std::unordered_map workerStats; // mapping between workerID -> workerStats std::unordered_set workerAddresses; std::unordered_set deadWorkers; KeyRangeMap workerAssignments; @@ -269,6 +276,19 @@ struct BlobManagerData : NonCopyable, ReferenceCounted { : id(id), db(db), dcId(dcId), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &workersById), knownBlobRanges(false, normalKeys.end), restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), recruitingStream(0) {} + + // only initialize blob store if actually needed + void initBStore() { + if (!bstore.isValid()) { + if (BM_DEBUG) { + fmt::print("BM {} constructing backup container from {}\n", epoch, SERVER_KNOBS->BG_URL.c_str()); + } + bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); + if (BM_DEBUG) { + fmt::print("BM {} constructed backup container\n", epoch); + } + } + } }; ACTOR Future>> splitRange(Reference bmData, @@ -1519,7 +1539,7 @@ ACTOR Future checkBlobWorkerList(Reference bmData, Promis worker.locality.dcId() == bmData->dcId) { bmData->workerAddresses.insert(worker.stableAddress()); bmData->workersById[worker.id()] = worker; - bmData->workerStats[worker.id()] = BlobWorkerStats(); + bmData->workerStats[worker.id()] = BlobWorkerInfo(); bmData->addActor.send(monitorBlobWorker(bmData, worker)); foundAnyNew = true; } else if (!bmData->workersById.count(worker.id())) { @@ -2022,7 +2042,7 @@ ACTOR Future initializeBlobWorker(Reference self, Recruit if (!self->workerAddresses.count(bwi.stableAddress()) && bwi.locality.dcId() == self->dcId) { self->workerAddresses.insert(bwi.stableAddress()); self->workersById[bwi.id()] = bwi; - self->workerStats[bwi.id()] = BlobWorkerStats(); + self->workerStats[bwi.id()] = BlobWorkerInfo(); self->addActor.send(monitorBlobWorker(self, bwi)); } else if (!self->workersById.count(bwi.id())) { self->addActor.send(killBlobWorker(self, bwi, false)); @@ -2554,14 +2574,7 @@ ACTOR Future pruneRange(Reference self, KeyRangeRef range * case that the timer is up before any new prune intents arrive). */ ACTOR Future monitorPruneKeys(Reference self) { - // setup bstore - if (BM_DEBUG) { - fmt::print("BM constructing backup container from {}\n", SERVER_KNOBS->BG_URL.c_str()); - } - self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL, {}, {}); - if (BM_DEBUG) { - printf("BM constructed backup container\n"); - } + self->initBStore(); loop { state Reference tr = makeReference(self->db); @@ -2730,6 +2743,73 @@ static void blobManagerExclusionSafetyCheck(Reference self, req.reply.send(reply); } +// FIXME: could eventually make this more thorough by storing some state in the DB or something +// FIXME: simpler solution could be to shuffle ranges +ACTOR Future bgConsistencyCheck(Reference bmData) { + + state Reference rateLimiter = + Reference(new SpeedLimit(SERVER_KNOBS->BG_CONSISTENCY_CHECK_TARGET_SPEED_KB * 1024, 1)); + bmData->initBStore(); + + if (BM_DEBUG) { + fmt::print("BGCC starting\n"); + } + + loop { + if (g_network->isSimulated() && g_simulator.speedUpSimulation) { + if (BM_DEBUG) { + printf("BGCC stopping\n"); + } + return Void(); + } + + if (bmData->workersById.size() >= 1) { + int tries = 10; + state KeyRange range; + while (tries > 0) { + auto randomRange = bmData->workerAssignments.randomRange(); + if (randomRange.value() != UID()) { + range = randomRange.range(); + break; + } + tries--; + } + + if (tries == 0) { + if (BM_DEBUG) { + printf("BGCC couldn't find random range to check, skipping\n"); + } + wait(rateLimiter->getAllowance(SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES)); + } else { + state std::pair fdbResult = wait(readFromFDB(bmData->db, range)); + + std::pair>> blobResult = + wait(readFromBlob(bmData->db, bmData->bstore, range, 0, fdbResult.second)); + + if (!compareFDBAndBlob(fdbResult.first, blobResult, range, fdbResult.second, BM_DEBUG)) { + ++bmData->stats.ccMismatches; + } + + int64_t bytesRead = fdbResult.first.expectedSize(); + + ++bmData->stats.ccGranulesChecked; + bmData->stats.ccRowsChecked += fdbResult.first.size(); + bmData->stats.ccBytesChecked += bytesRead; + + // clear fdb result to release memory since it is a state variable + fdbResult = std::pair(RangeResult(), 0); + + wait(rateLimiter->getAllowance(bytesRead)); + } + } else { + if (BM_DEBUG) { + fmt::print("BGCC found no workers, skipping\n", bmData->workerAssignments.size()); + } + wait(delay(60.0)); + } + } +} + // Simulation validation that multiple blob managers aren't started with the same epoch static std::map managerEpochsSeen; @@ -2776,6 +2856,9 @@ ACTOR Future blobManager(BlobManagerInterface bmInterf, self->addActor.send(doLockChecks(self)); self->addActor.send(monitorClientRanges(self)); self->addActor.send(monitorPruneKeys(self)); + if (SERVER_KNOBS->BG_CONSISTENCY_CHECK_ENABLED) { + self->addActor.send(bgConsistencyCheck(self)); + } if (BUGGIFY) { self->addActor.send(chaosRangeMover(self)); diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 7970dd18ef..3ffe1febbd 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -7,6 +7,8 @@ set(FDBSERVER_SRCS BackupWorker.actor.cpp BlobGranuleServerCommon.actor.cpp BlobGranuleServerCommon.actor.h + BlobGranuleValidation.actor.cpp + BlobGranuleValidation.actor.h BlobManager.actor.cpp BlobManagerInterface.h BlobWorker.actor.cpp diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp index fc6d3035ae..d7ffd4e92c 100644 --- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp +++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp @@ -29,6 +29,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" #include "fdbclient/SystemData.h" +#include "fdbserver/BlobGranuleValidation.actor.h" #include "fdbserver/Knobs.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" @@ -271,36 +272,6 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload { return Void(); } - // FIXME: typedef this pair type and/or chunk list - ACTOR Future>>> readFromBlob( - Database cx, - BlobGranuleCorrectnessWorkload* self, - KeyRange range, - Version beginVersion, - Version readVersion) { - state RangeResult out; - state Standalone> chunks; - state Transaction tr(cx); - - loop { - try { - Standalone> chunks_ = - wait(tr.readBlobGranules(range, beginVersion, readVersion)); - chunks = chunks_; - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - - for (const BlobGranuleChunkRef& chunk : chunks) { - RangeResult chunkRows = wait(readBlobGranule(chunk, range, beginVersion, readVersion, self->bstore)); - out.arena().dependsOn(chunkRows.arena()); - out.append(out.arena(), chunkRows.begin(), chunkRows.size()); - } - return std::pair(out, chunks); - } - // handle retries + errors // It's ok to reset the transaction here because its read version is only used for reading the granule mapping from // the system keyspace @@ -326,7 +297,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload { Version rv = wait(self->doGrv(&tr)); state Version readVersion = rv; std::pair>> blob = - wait(self->readFromBlob(cx, self, threadData->directoryRange, 0, readVersion)); + wait(readFromBlob(cx, self->bstore, threadData->directoryRange, 0, readVersion)); fmt::print("Directory {0} got {1} RV {2}\n", threadData->directoryID, doSetup ? "initial" : "final", @@ -690,7 +661,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload { } std::pair>> blob = - wait(self->readFromBlob(cx, self, range, beginVersion, readVersion)); + wait(readFromBlob(cx, self->bstore, range, beginVersion, readVersion)); self->validateResult(threadData, blob, startKey, endKey, beginVersion, readVersion); int resultBytes = blob.first.expectedSize(); @@ -884,7 +855,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload { fmt::print("Directory {0} doing final data check @ {1}\n", threadData->directoryID, readVersion); } std::pair>> blob = - wait(self->readFromBlob(cx, self, threadData->directoryRange, 0, readVersion)); + wait(readFromBlob(cx, self->bstore, threadData->directoryRange, 0, readVersion)); result = self->validateResult(threadData, blob, 0, std::numeric_limits::max(), 0, readVersion); finalRowsValidated = blob.first.size(); diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index ba49923bf1..a2f0062720 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -28,6 +28,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" #include "fdbclient/SystemData.h" +#include "fdbserver/BlobGranuleValidation.actor.h" #include "fdbserver/Knobs.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" @@ -167,154 +168,6 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } } - // assumes we can read the whole range in one transaction at a single version - ACTOR Future> readFromFDB(Database cx, KeyRange range) { - state bool first = true; - state Version v; - state RangeResult out; - state Transaction tr(cx); - state KeyRange currentRange = range; - loop { - try { - state RangeResult r = wait(tr.getRange(currentRange, CLIENT_KNOBS->TOO_MANY)); - Version grv = wait(tr.getReadVersion()); - // need consistent version snapshot of range - if (first) { - v = grv; - first = false; - } else if (v != grv) { - // reset the range and restart the read at a higher version - TraceEvent(SevDebug, "BGVFDBReadReset").detail("ReadVersion", v); - TEST(true); // BGV transaction reset - fmt::print("Resetting BGV GRV {0} -> {1}\n", v, grv); - first = true; - out = RangeResult(); - currentRange = range; - tr.reset(); - continue; - } - out.arena().dependsOn(r.arena()); - out.append(out.arena(), r.begin(), r.size()); - if (r.more) { - currentRange = KeyRangeRef(keyAfter(r.back().key), currentRange.end); - } else { - break; - } - } catch (Error& e) { - wait(tr.onError(e)); - } - } - return std::pair(out, v); - } - - // FIXME: typedef this pair type and/or chunk list - ACTOR Future>>> - readFromBlob(Database cx, BlobGranuleVerifierWorkload* self, KeyRange range, Version version) { - state RangeResult out; - state Standalone> chunks; - state Transaction tr(cx); - - loop { - try { - Standalone> chunks_ = wait(tr.readBlobGranules(range, 0, version)); - chunks = chunks_; - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - - for (const BlobGranuleChunkRef& chunk : chunks) { - RangeResult chunkRows = wait(readBlobGranule(chunk, range, 0, version, self->bstore)); - out.arena().dependsOn(chunkRows.arena()); - out.append(out.arena(), chunkRows.begin(), chunkRows.size()); - } - return std::pair(out, chunks); - } - - bool compareResult(RangeResult fdb, - std::pair>> blob, - KeyRange range, - Version v, - bool initialRequest) { - bool correct = fdb == blob.first; - if (!correct) { - mismatches++; - TraceEvent ev(SevError, "GranuleMismatch"); - ev.detail("RangeStart", range.begin) - .detail("RangeEnd", range.end) - .detail("Version", v) - .detail("RequestType", initialRequest ? "RealTime" : "TimeTravel") - .detail("FDBSize", fdb.size()) - .detail("BlobSize", blob.first.size()); - - if (BGV_DEBUG) { - fmt::print("\nMismatch for [{0} - {1}) @ {2} ({3}). F({4}) B({5}):\n", - range.begin.printable(), - range.end.printable(), - v, - initialRequest ? "RealTime" : "TimeTravel", - fdb.size(), - blob.first.size()); - - Optional lastCorrect; - for (int i = 0; i < std::max(fdb.size(), blob.first.size()); i++) { - if (i >= fdb.size() || i >= blob.first.size() || fdb[i] != blob.first[i]) { - printf(" Found mismatch at %d.\n", i); - if (lastCorrect.present()) { - printf(" last correct: %s=%s\n", - lastCorrect.get().key.printable().c_str(), - lastCorrect.get().value.printable().c_str()); - } - if (i < fdb.size()) { - printf( - " FDB: %s=%s\n", fdb[i].key.printable().c_str(), fdb[i].value.printable().c_str()); - } else { - printf(" FDB: \n"); - } - if (i < blob.first.size()) { - printf(" BLB: %s=%s\n", - blob.first[i].key.printable().c_str(), - blob.first[i].value.printable().c_str()); - } else { - printf(" BLB: \n"); - } - printf("\n"); - break; - } - if (i < fdb.size()) { - lastCorrect = fdb[i]; - } else { - lastCorrect = blob.first[i]; - } - } - - printf("Chunks:\n"); - for (auto& chunk : blob.second) { - printf("[%s - %s)\n", - chunk.keyRange.begin.printable().c_str(), - chunk.keyRange.end.printable().c_str()); - - printf(" SnapshotFile:\n %s\n", - chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : ""); - printf(" DeltaFiles:\n"); - for (auto& df : chunk.deltaFiles) { - printf(" %s\n", df.toString().c_str()); - } - printf(" Deltas: (%d)", chunk.newDeltas.size()); - if (chunk.newDeltas.size() > 0) { - fmt::print(" with version [{0} - {1}]", - chunk.newDeltas[0].version, - chunk.newDeltas[chunk.newDeltas.size() - 1].version); - } - fmt::print(" IncludedVersion: {}\n", chunk.includedVersion); - } - printf("\n"); - } - } - return correct; - } - struct OldRead { KeyRange range; Version v; @@ -469,21 +322,23 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } } std::pair>> reReadResult = - wait(self->readFromBlob(cx, self, oldRead.range, oldRead.v)); - self->compareResult(oldRead.oldResult, reReadResult, oldRead.range, oldRead.v, false); + wait(readFromBlob(cx, self->bstore, oldRead.range, 0, oldRead.v)); + if (!compareFDBAndBlob(oldRead.oldResult, reReadResult, oldRead.range, oldRead.v, BGV_DEBUG)) { + self->mismatches++; + } self->timeTravelReads++; if (doPruning) { wait(self->killBlobWorkers(cx, self)); std::pair>> versionRead = - wait(self->readFromBlob(cx, self, oldRead.range, prevPruneVersion)); + wait(readFromBlob(cx, self->bstore, oldRead.range, 0, prevPruneVersion)); try { Version minSnapshotVersion = newPruneVersion; for (auto& it : versionRead.second) { minSnapshotVersion = std::min(minSnapshotVersion, it.snapshotVersion); } std::pair>> versionRead = - wait(self->readFromBlob(cx, self, oldRead.range, minSnapshotVersion - 1)); + wait(readFromBlob(cx, self->bstore, oldRead.range, 0, minSnapshotVersion - 1)); ASSERT(false); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) { @@ -504,10 +359,10 @@ struct BlobGranuleVerifierWorkload : TestWorkload { int rIndex = deterministicRandom()->randomInt(0, self->granuleRanges.get().size()); state KeyRange range = self->granuleRanges.get()[rIndex]; - state std::pair fdb = wait(self->readFromFDB(cx, range)); + state std::pair fdb = wait(readFromFDB(cx, range)); std::pair>> blob = - wait(self->readFromBlob(cx, self, range, fdb.second)); - if (self->compareResult(fdb.first, blob, range, fdb.second, true)) { + wait(readFromBlob(cx, self->bstore, range, 0, fdb.second)); + if (compareFDBAndBlob(fdb.first, blob, range, fdb.second, BGV_DEBUG)) { // TODO: bias for immediately re-reading to catch rollback cases double reReadTime = currentTime + deterministicRandom()->random01() * self->timeTravelLimit; int memory = fdb.first.expectedSize(); @@ -516,6 +371,8 @@ struct BlobGranuleVerifierWorkload : TestWorkload { timeTravelChecks[reReadTime] = OldRead(range, fdb.second, fdb.first); timeTravelChecksMemory += memory; } + } else { + self->mismatches++; } self->rowsRead += fdb.first.size(); self->bytesRead += fdb.first.expectedSize();