From df39c5a44ef4d6ee3075f55341871b35a60b4d04 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 30 Jun 2021 17:05:04 -0700 Subject: [PATCH 001/338] Implement Disk Throttling Chaos workload. --- fdbclient/ClientWorkerInterface.h | 23 ++++++++++++++++++- fdbrpc/AsyncFileEIO.actor.h | 26 +++++++++++++++++---- fdbrpc/AsyncFileKAIO.actor.h | 14 +++++++++++- fdbrpc/IAsyncFile.h | 4 ++++ fdbrpc/sim2.actor.cpp | 17 +++++++++++++- fdbrpc/simulator.h | 7 ++++-- fdbserver/CMakeLists.txt | 1 + fdbserver/worker.actor.cpp | 14 ++++++++++++ flow/Knobs.cpp | 4 ++++ flow/Knobs.h | 3 +++ flow/network.h | 38 ++++++++++++++++++++++++++++++- tests/CMakeLists.txt | 1 + 12 files changed, 141 insertions(+), 11 deletions(-) diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index cff4172387..b73c43ebd7 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -31,8 +31,10 @@ // A ClientWorkerInterface is embedded as the first element of a WorkerInterface. struct ClientWorkerInterface { constexpr static FileIdentifier file_identifier = 12418152; + RequestStream reboot; RequestStream profiler; + RequestStream setFailureInjection; bool operator==(ClientWorkerInterface const& r) const { return id() == r.id(); } bool operator!=(ClientWorkerInterface const& r) const { return id() != r.id(); } @@ -43,7 +45,7 @@ struct ClientWorkerInterface { template void serialize(Ar& ar) { - serializer(ar, reboot, profiler); + serializer(ar, reboot, profiler, setFailureInjection); } }; @@ -88,4 +90,23 @@ struct ProfilerRequest { } }; +struct SetFailureInjection { + constexpr static FileIdentifier file_identifier = 15439864; + ReplyPromise reply; + struct ThrottleDiskCommand { + double time; + Optional address; // TODO: NEELAM: how do we identify the machine + + template + void serialize(Ar& ar) { + serializer(ar, time, address); + } + }; + Optional throttleDisk; + + template + void serialize(Ar& ar) { + serializer(ar, reply, throttleDisk); + } +}; #endif diff --git a/fdbrpc/AsyncFileEIO.actor.h b/fdbrpc/AsyncFileEIO.actor.h index 44fe6448db..c962e60098 100644 --- a/fdbrpc/AsyncFileEIO.actor.h +++ b/fdbrpc/AsyncFileEIO.actor.h @@ -162,14 +162,16 @@ public: Future read(void* data, int length, int64_t offset) override { ++countFileLogicalReads; ++countLogicalReads; - return read_impl(fd, data, length, offset); + double throttleFor = diskFailureInjector->getDiskDelay(); + return read_impl(fd, data, length, offset, throttleFor); } Future write(void const* data, int length, int64_t offset) override // Copies data synchronously { ++countFileLogicalWrites; ++countLogicalWrites; + double throttleFor = diskFailureInjector->getDiskDelay(); // Standalone copy = StringRef((const uint8_t*)data, length); - return write_impl(fd, err, StringRef((const uint8_t*)data, length), offset); + return write_impl(fd, err, StringRef((const uint8_t*)data, length), offset, throttleFor); } Future truncate(int64_t size) override { ++countFileLogicalWrites; @@ -270,6 +272,7 @@ private: int fd, flags; Reference err; std::string filename; + //DiskFailureInjector* diskFailureInjector; mutable Int64MetricHandle countFileLogicalWrites; mutable Int64MetricHandle countFileLogicalReads; @@ -277,7 +280,8 @@ private: mutable Int64MetricHandle countLogicalReads; AsyncFileEIO(int fd, int flags, std::string const& filename) - : fd(fd), flags(flags), filename(filename), err(new ErrorInfo) { + : fd(fd), flags(flags), filename(filename), err(new ErrorInfo), + diskFailureInjector(DiskFailureInjector::injector()) { if (!g_network->isSimulated()) { countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename); countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename); @@ -329,13 +333,18 @@ private: TraceEvent("AsyncFileClosed").suppressFor(1.0).detail("Fd", fd); } - ACTOR static Future read_impl(int fd, void* data, int length, int64_t offset) { + ACTOR static Future read_impl(int fd, void* data, int length, int64_t offset, double throttleFor) { state TaskPriority taskID = g_network->getCurrentTask(); state Promise p; // fprintf(stderr, "eio_read (fd=%d length=%d offset=%lld)\n", fd, length, offset); state eio_req* r = eio_read(fd, data, length, offset, 0, eio_callback, &p); try { wait(p.getFuture()); + // throttleDisk if enabled + //double throttleFor = diskFailureInjector->getDiskDelay(); + if (throttleFor > 0.0) { + wait(delay(throttleFor)); + } } catch (...) { g_network->setCurrentTask(taskID); eio_cancel(r); @@ -358,12 +367,17 @@ private: } } - ACTOR static Future write_impl(int fd, Reference err, StringRef data, int64_t offset) { + ACTOR static Future write_impl(int fd, Reference err, StringRef data, int64_t offset, double throttleFor) { state TaskPriority taskID = g_network->getCurrentTask(); state Promise p; state eio_req* r = eio_write(fd, (void*)data.begin(), data.size(), offset, 0, eio_callback, &p); try { wait(p.getFuture()); + // throttleDisk if enabled + //double throttleFor = diskFailureInjector->getDiskDelay(); + if (throttleFor > 0.0) { + wait(delay(throttleFor)); + } } catch (...) { g_network->setCurrentTask(taskID); eio_cancel(r); @@ -553,6 +567,8 @@ private: static void apple_fsync(eio_req* req) { req->result = fcntl(req->int1, F_FULLFSYNC, 0); } static void free_req(eio_req* req) { free(req); } #endif +public: + DiskFailureInjector* diskFailureInjector; }; #ifdef FILESYSTEM_IMPL diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h index 5e6592e6ba..15553a85e2 100644 --- a/fdbrpc/AsyncFileKAIO.actor.h +++ b/fdbrpc/AsyncFileKAIO.actor.h @@ -195,7 +195,10 @@ public: void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } - + ACTOR static void throttleDisk(double throttleFor) { + if (throttleFor > 0.0) + wait(delay(throttleFor)); + } Future read(void* data, int length, int64_t offset) override { ++countFileLogicalReads; ++countLogicalReads; @@ -213,6 +216,9 @@ public: enqueue(io, "read", this); Future result = io->result.getFuture(); + // throttleDisk if enabled + throttleDisk(diskFailureInjector->getDiskDelay()); + #if KAIO_LOGGING // result = map(result, [=](int r) mutable { KAIOLogBlockEvent(io, OpLogEntry::READY, r); return r; }); #endif @@ -238,6 +244,9 @@ public: enqueue(io, "write", this); Future result = io->result.getFuture(); + // throttleDisk if enabled + throttleDisk(diskFailureInjector->getDiskDelay()); + #if KAIO_LOGGING // result = map(result, [=](int r) mutable { KAIOLogBlockEvent(io, OpLogEntry::READY, r); return r; }); #endif @@ -749,6 +758,9 @@ private: } } } + +public: + DiskFailureInjector* diskFailureInjector; }; #if KAIO_LOGGING diff --git a/fdbrpc/IAsyncFile.h b/fdbrpc/IAsyncFile.h index ed703514c6..f21760cb00 100644 --- a/fdbrpc/IAsyncFile.h +++ b/fdbrpc/IAsyncFile.h @@ -34,6 +34,7 @@ // must complete or cancel, but you should probably look at the file implementations you'll be using. class IAsyncFile { public: + //explicit IAsyncFile() : diskFailureInjector(DiskFailureInjector::injector()) {} virtual ~IAsyncFile(); // Pass these to g_network->open to get an IAsyncFile enum { @@ -95,6 +96,9 @@ public: // Used for rate control, at present, only AsyncFileCached supports it virtual Reference const& getRateControl() { throw unsupported_operation(); } virtual void setRateControl(Reference const& rc) { throw unsupported_operation(); } + +//public: + //DiskFailureInjector* diskFailureInjector; }; typedef void (*runCycleFuncPtr)(); diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index ee735b963a..093ef389ac 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -1949,6 +1949,13 @@ public: void clogPair(const IPAddress& from, const IPAddress& to, double seconds) override { g_clogging.clogPairFor(from, to, seconds); } + void throttleDisk(ProcessInfo* machine, double seconds) override { + machine->throttleDiskFor = seconds; + TraceEvent("ThrottleDisk").detail("Delay", seconds). + detail("Roles", getRoles(machine->address)). + detail("Address", machine->address). + detail("StartingClass", machine->startingClass.toString()); + } std::vector getAllProcesses() const override { std::vector processes; for (auto& c : machines) { @@ -2390,11 +2397,19 @@ Future waitUntilDiskReady(Reference diskParameters, int64_ diskParameters->nextOperation += (1.0 / diskParameters->iops) + (size / diskParameters->bandwidth); double randomLatency; - if (sync) { + if (g_simulator.getCurrentProcess()->throttleDiskFor) { + randomLatency = g_simulator.getCurrentProcess()->throttleDiskFor; + TraceEvent("WaitUntilDiskReadyThrottling") + .detail("Delay", randomLatency); + } else if (sync) { randomLatency = .005 + deterministicRandom()->random01() * (BUGGIFY ? 1.0 : .010); } else randomLatency = 10 * deterministicRandom()->random01() / diskParameters->iops; + TraceEvent("WaitUntilDiskReady").detail("Delay", randomLatency). + detail("Roles", g_simulator.getRoles(g_simulator.getCurrentProcess()->address)). + detail("Address", g_simulator.getCurrentProcess()->address). + detail("ThrottleDiskFor", g_simulator.getCurrentProcess()->throttleDiskFor); return delayUntil(diskParameters->nextOperation + randomLatency); } diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 6404eafc17..1da850e48c 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -87,6 +87,7 @@ public: uint64_t fault_injection_r; double fault_injection_p1, fault_injection_p2; bool failedDisk; + double throttleDiskFor; UID uid; @@ -102,7 +103,7 @@ public: : name(name), locality(locality), startingClass(startingClass), addresses(addresses), address(addresses.address), dataFolder(dataFolder), network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), rebooting(false), fault_injection_p1(0), fault_injection_p2(0), - fault_injection_r(0), machine(0), cleared(false), failedDisk(false) { + fault_injection_r(0), machine(0), cleared(false), failedDisk(false), throttleDiskFor(0) { uid = deterministicRandom()->randomUniqueID(); } @@ -374,6 +375,7 @@ public: virtual void clogInterface(const IPAddress& ip, double seconds, ClogMode mode = ClogDefault) = 0; virtual void clogPair(const IPAddress& from, const IPAddress& to, double seconds) = 0; + virtual void throttleDisk(ProcessInfo* machine, double seconds) = 0; virtual std::vector getAllProcesses() const = 0; virtual ProcessInfo* getProcessByAddress(NetworkAddress const& address) = 0; virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0; @@ -462,8 +464,9 @@ struct DiskParameters : ReferenceCounted { double nextOperation; int64_t iops; int64_t bandwidth; + double throttleFor; - DiskParameters(int64_t iops, int64_t bandwidth) : nextOperation(0), iops(iops), bandwidth(bandwidth) {} + DiskParameters(int64_t iops, int64_t bandwidth) : nextOperation(0), iops(iops), bandwidth(bandwidth), throttleFor(0) {} }; // Simulates delays for performing operations on disk diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 0f7d5dc860..efa2c7fbf1 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -158,6 +158,7 @@ set(FDBSERVER_SRCS workloads/ChangeConfig.actor.cpp workloads/ClientTransactionProfileCorrectness.actor.cpp workloads/TriggerRecovery.actor.cpp + workloads/DiskThrottling.actor.cpp workloads/SuspendProcesses.actor.cpp workloads/CommitBugCheck.actor.cpp workloads/ConfigureDatabase.actor.cpp diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index ad91d4dd34..dd6ee5e39d 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1209,6 +1209,10 @@ ACTOR Future workerServer(Reference connFile, state Reference>> issues(new AsyncVar>()); + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { + TraceEvent(SevWarnAlways, "ChaosFeaturesEnabled"); + } + folder = abspath(folder); if (metricsPrefix.size() > 0) { @@ -1509,6 +1513,16 @@ ACTOR Future workerServer(Reference connFile, flushAndExit(0); } } + when(SetFailureInjection req = waitNext(interf.clientInterface.setFailureInjection.getFuture())) { + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { + if (req.throttleDisk.present()) { + DiskFailureInjector::injector()->throttleFor(req.throttleDisk.get().time); + } + req.reply.send(Void()); + } else { + req.reply.sendError(client_invalid_operation()); + } + } when(ProfilerRequest req = waitNext(interf.clientInterface.profiler.getFuture())) { state ProfilerRequest profilerReq = req; // There really isn't a great "filepath sanitizer" or "filepath escape" function available, diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 7ceeb95801..1d91a7e8da 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -64,6 +64,10 @@ void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { init( HUGE_ARENA_LOGGING_BYTES, 100e6 ); init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 ); + // Chaos testing + init( ENABLE_CHAOS_FEATURES, false ); + + init( WRITE_TRACING_ENABLED, true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false; init( TRACING_UDP_LISTENER_PORT, 8889 ); // Only applicable if TracerType is set to a network option. diff --git a/flow/Knobs.h b/flow/Knobs.h index ef4fdcf2af..340848b68f 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -98,6 +98,9 @@ public: double HUGE_ARENA_LOGGING_BYTES; double HUGE_ARENA_LOGGING_INTERVAL; + // Chaos testing + bool ENABLE_CHAOS_FEATURES; + bool WRITE_TRACING_ENABLED; int TRACING_UDP_LISTENER_PORT; diff --git a/flow/network.h b/flow/network.h index 00f430fb86..d174601fec 100644 --- a/flow/network.h +++ b/flow/network.h @@ -486,7 +486,8 @@ public: enNetworkAddressesFunc = 11, enClientFailureMonitor = 12, enSQLiteInjectedError = 13, - enGlobalConfig = 14 + enGlobalConfig = 14, + enFailureInjector = 15 }; virtual void longTaskCheck(const char* name) {} @@ -646,4 +647,39 @@ public: // Returns the interface that should be used to make and accept socket connections }; +struct DiskFailureInjector : FastAllocated { + static DiskFailureInjector* injector() { + auto res = g_network->global(INetwork::enFailureInjector); + if (!res) { + res = new DiskFailureInjector(); + g_network->setGlobal(INetwork::enFailureInjector, res); + } + return static_cast(res); + } + + //double getSendDelay(NetworkAddress const& peer); + //double getReceiveDelay(NetworkAddress const& peer); + + //virtual void throttleFor(double time) = 0; + //virtual double getDiskDelay() = 0; + + void throttleFor(double time) { + throttleUntil = std::max(throttleUntil, timer_monotonic() + time); + } + + double getDiskDelay() { + if (!FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { + return 0.0; + } + return throttleUntil; + } + +private: // members + double throttleUntil = 0.0; + +private: // construction + DiskFailureInjector() = default; + DiskFailureInjector(DiskFailureInjector const&) = delete; +}; + #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 913b39413b..5a5bf2c208 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -124,6 +124,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/ConstrainedRandomSelector.toml) add_fdb_test(TEST_FILES fast/CycleAndLock.toml) add_fdb_test(TEST_FILES fast/CycleTest.toml) + add_fdb_test(TEST_FILES fast/DiskThrottledCycle.toml IGNORE) add_fdb_test(TEST_FILES fast/FuzzApiCorrectness.toml) add_fdb_test(TEST_FILES fast/FuzzApiCorrectnessClean.toml) add_fdb_test(TEST_FILES fast/IncrementalBackup.toml) From 96bde8919f29b0c8720d924b1c1706f5ad5c2af8 Mon Sep 17 00:00:00 2001 From: negoyal Date: Thu, 1 Jul 2021 15:00:19 -0700 Subject: [PATCH 002/338] Adding the Disk throttle workload file that I forgot earlier. --- fdbserver/workloads/DiskThrottling.actor.cpp | 133 +++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 fdbserver/workloads/DiskThrottling.actor.cpp diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskThrottling.actor.cpp new file mode 100644 index 0000000000..1eb4c1639d --- /dev/null +++ b/fdbserver/workloads/DiskThrottling.actor.cpp @@ -0,0 +1,133 @@ +#include "fdbclient/NativeAPI.actor.h" +#include "fdbserver/TesterInterface.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "fdbrpc/simulator.h" +#include "fdbserver/WorkerInterface.actor.h" +#include "fdbserver/ServerDBInfo.h" +#include "fdbserver/QuietDatabase.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +struct DiskThrottlingWorkload : TestWorkload { + bool enabled; + double testDuration; + double throttleFor; + DiskThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + enabled = !clientId; // only do this on the "first" client + testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); + throttleFor = getOption(options, LiteralStringRef("throttleDelay"), 2.0); + TraceEvent("DiskThrottlingWorkload").detail("TestDuration", testDuration).detail("For", throttleFor); + } + + std::string description() const override { + if (&g_simulator == g_network) + return "DiskThrottling"; + else + return "NoSimDiskThrolling"; + } + + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { + if (&g_simulator == g_network && enabled) { + TraceEvent("DiskThrottlingStart").detail("For", throttleFor); + return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), + testDuration, + Void()); + } else if (enabled) { + return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), + testDuration, + Void()); + } else + return Void(); + } + + Future check(Database const& cx) override { return true; } + + void getMetrics(vector& m) override {} + + ACTOR void doThrottle(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) { + wait(::delay(delay)); + TraceEvent("ThrottleDisk").detail("For", t); + g_simulator.throttleDisk(machine, t); + TraceEvent("ThrottleDiskSet").detail("For", t); + } + + static void checkDiskThrottleResult(Future res, WorkerInterface worker) { + if (res.isError()) { + auto err = res.getError(); + if (err.code() == error_code_client_invalid_operation) { + TraceEvent(SevError, "ChaosDisabled") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); + } else { + TraceEvent(SevError, "DiskThrottlingFailed") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) + .error(err); + } + } + } + + ACTOR void doThrottle(WorkerInterface worker, double t, double delay = 0.0) { + state Future res; + wait(::delay(delay)); + SetFailureInjection::ThrottleDiskCommand throttleDisk; + throttleDisk.time = t; + SetFailureInjection req; + req.throttleDisk = throttleDisk; + TraceEvent("ThrottleDisk").detail("For", t); + res = worker.clientInterface.setFailureInjection.getReply(req); + wait(ready(res)); + checkDiskThrottleResult(res, worker); + } + + static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { + result->clear(); + *result = g_simulator.getAllProcesses(); + return Void(); + } + + static Future getAllStorageWorkers(Database cx, DiskThrottlingWorkload* self, std::vector* result) { + vector all = g_simulator.getAllProcesses(); + for (int i = 0; i < all.size(); i++) + if (!all[i]->failed && + all[i]->name == std::string("Server") && + ((all[i]->startingClass == ProcessClass::StorageClass) || + (all[i]->startingClass == ProcessClass::UnsetClass))) + result->emplace_back(all[i]); + return Void(); + } + + ACTOR static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { + result->clear(); + std::vector res = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + for (auto& worker : res) { + result->emplace_back(worker.interf); + } + return Void(); + } + + ACTOR static Future getAllStorageWorkers(Database cx, DiskThrottlingWorkload* self, std::vector* result) { + result->clear(); + state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); + for (auto& worker : res) { + result->emplace_back(worker); + } + return Void(); + } + + ACTOR template + Future throttleDiskClient(Database cx, DiskThrottlingWorkload* self) { + state double lastTime = now(); + state double workloadEnd = now() + self->testDuration; + state std::vector machines; + loop { + wait(poisson(&lastTime, 1)); + wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); + //wait(DiskThrottlingWorkload::getAllWorkers(self, &machines)); + auto machine = deterministicRandom()->randomChoice(machines); + TraceEvent("DoThrottleDisk").detail("For", self->throttleFor); + self->doThrottle(machine, self->throttleFor); + } + } +}; +WorkloadFactory DiskThrottlingWorkloadFactory("DiskThrottling"); From 957eceb14cf52c7030277c1026fac0bae9d82b85 Mon Sep 17 00:00:00 2001 From: negoyal Date: Thu, 1 Jul 2021 15:01:13 -0700 Subject: [PATCH 003/338] And the test file. --- tests/fast/DiskThrottledCycle.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/fast/DiskThrottledCycle.toml diff --git a/tests/fast/DiskThrottledCycle.toml b/tests/fast/DiskThrottledCycle.toml new file mode 100644 index 0000000000..c0f35293aa --- /dev/null +++ b/tests/fast/DiskThrottledCycle.toml @@ -0,0 +1,13 @@ +[[test]] +testTitle = 'DiskThrottledCycle' + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = 'DiskThrottling' + testDuration = 30.0 + From 2b5a96f745e23d557ce725a068ba5ace0145a2f6 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 7 Jul 2021 23:58:14 -0700 Subject: [PATCH 004/338] Single code path for sim and non-sim modes. --- fdbrpc/AsyncFileNonDurable.actor.h | 43 ++++++++++++++++---- fdbserver/worker.actor.cpp | 1 + fdbserver/workloads/DiskThrottling.actor.cpp | 20 ++++----- flow/Knobs.cpp | 2 +- flow/network.h | 8 ++-- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index f813c1a354..0c63846169 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -31,6 +31,7 @@ #include "flow/flow.h" #include "fdbrpc/IAsyncFile.h" #include "flow/ActorCollection.h" +#include "flow/network.h" #include "fdbrpc/simulator.h" #include "fdbrpc/TraceFileIO.h" #include "fdbrpc/RangeMap.h" @@ -61,7 +62,7 @@ private: Future shutdown; public: - explicit AsyncFileDetachable(Reference file) : file(file) { shutdown = doShutdown(this); } + explicit AsyncFileDetachable(Reference file) : file(file), diskFailureInjector(DiskFailureInjector::injector()) { shutdown = doShutdown(this); } ACTOR Future doShutdown(AsyncFileDetachable* self) { wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture())); @@ -84,12 +85,20 @@ public: Future read(void* data, int length, int64_t offset) override { if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady()) return io_error().asInjectedFault(); + // throttleDisk if enabled + auto throttleFor = diskFailureInjector->getDiskDelay(); + if (throttleFor > 0.0) { + TraceEvent("AsyncFileDetachable_Read").detail("ThrottleDelay", throttleFor); + //wait(delay(throttleFor)); + } return sendErrorOnShutdown(file->read(data, length, offset)); } Future write(void const* data, int length, int64_t offset) override { if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady()) return io_error().asInjectedFault(); + if (diskFailureInjector->getDiskDelay() > 0.0) + TraceEvent("AsyncFileDetachable_Write").detail("ThrottleDelay", diskFailureInjector->getDiskDelay()); return sendErrorOnShutdown(file->write(data, length, offset)); } @@ -121,6 +130,8 @@ public: throw io_error().asInjectedFault(); return file->getFilename(); } +public: + DiskFailureInjector* diskFailureInjector; }; // An async file implementation which wraps another async file and will randomly destroy sectors that it is writing when @@ -190,11 +201,12 @@ private: Reference diskParameters, NetworkAddress openedAddress, bool aio) - : filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters), - openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false), - aio(aio) { + : filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters), + openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false), + aio(aio), diskFailureInjector(DiskFailureInjector::injector()) + { - // This is only designed to work in simulation + // This is only designed to work in simulation ASSERT(g_network->isSimulated()); this->id = deterministicRandom()->randomUniqueID(); @@ -309,7 +321,7 @@ public: // Passes along reads straight to the underlying file, waiting for any outstanding changes that could affect the // results - Future read(void* data, int length, int64_t offset) override { return read(this, data, length, offset); } + Future read(void* data, int length, int64_t offset) override { return read(this, data, length, offset, diskFailureInjector->getDiskDelay()); } // Writes data to the file. Writes are delayed a random amount of time before being // passed to the underlying file @@ -324,7 +336,7 @@ public: Promise writeStarted; Promise> writeEnded; - writeEnded.send(write(this, writeStarted, writeEnded.getFuture(), data, length, offset)); + writeEnded.send(write(this, writeStarted, writeEnded.getFuture(), data, length, offset, diskFailureInjector->getDiskDelay())); return writeStarted.getFuture(); } @@ -432,7 +444,7 @@ private: return readFuture.get(); } - ACTOR Future read(AsyncFileNonDurable* self, void* data, int length, int64_t offset) { + ACTOR Future read(AsyncFileNonDurable* self, void* data, int length, int64_t offset, double throttleFor = 0.0) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); @@ -441,6 +453,11 @@ private: state int rep = wait(self->onRead(self, data, length, offset)); wait(g_simulator.onProcess(currentProcess, currentTaskID)); + // throttleDisk if enabled + if (throttleFor > 0.0) { + TraceEvent("AsyncFileNonDurable_ReadDone", self->id).detail("ThrottleDelay", throttleFor).detail("Filename", self->filename).detail("ReadLength", length).detail("Offset", offset); + wait(delay(throttleFor)); + } return rep; } catch (Error& e) { state Error err = e; @@ -457,7 +474,8 @@ private: Future> ownFuture, void const* data, int length, - int64_t offset) { + int64_t offset, + double throttleFor = 0.0) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); @@ -621,6 +639,11 @@ private: } wait(waitForAll(writeFutures)); + // throttleDisk if enabled + if (throttleFor > 0.0) { + TraceEvent("AsyncFileNonDurable_WriteDone", self->id).detail("ThrottleDelay", throttleFor).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset); + wait(delay(throttleFor)); + } //TraceEvent("AsyncFileNonDurable_WriteDone", self->id).detail("Delay", delayDuration).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset); return Void(); } @@ -866,6 +889,8 @@ private: throw err; } } +public: + DiskFailureInjector* diskFailureInjector; }; #include "flow/unactorcompiler.h" diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index dd6ee5e39d..4a99e02265 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1516,6 +1516,7 @@ ACTOR Future workerServer(Reference connFile, when(SetFailureInjection req = waitNext(interf.clientInterface.setFailureInjection.getFuture())) { if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { if (req.throttleDisk.present()) { + TraceEvent("DiskThrottleRequest").detail("Delay", req.throttleDisk.get().time); DiskFailureInjector::injector()->throttleFor(req.throttleDisk.get().time); } req.reply.send(Void()); diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskThrottling.actor.cpp index 1eb4c1639d..61c465ae7f 100644 --- a/fdbserver/workloads/DiskThrottling.actor.cpp +++ b/fdbserver/workloads/DiskThrottling.actor.cpp @@ -28,12 +28,13 @@ struct DiskThrottlingWorkload : TestWorkload { Future setup(Database const& cx) override { return Void(); } Future start(Database const& cx) override { - if (&g_simulator == g_network && enabled) { - TraceEvent("DiskThrottlingStart").detail("For", throttleFor); - return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), - testDuration, - Void()); - } else if (enabled) { + //if (&g_simulator == g_network && enabled) { + // TraceEvent("DiskThrottlingStart").detail("For", throttleFor); + // return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), + // testDuration, + // Void()); + //} else + if (enabled) { return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), testDuration, Void()); @@ -45,7 +46,7 @@ struct DiskThrottlingWorkload : TestWorkload { void getMetrics(vector& m) override {} - ACTOR void doThrottle(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) { + ACTOR void doThrottle_unused(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) { wait(::delay(delay)); TraceEvent("ThrottleDisk").detail("For", t); g_simulator.throttleDisk(machine, t); @@ -79,13 +80,13 @@ struct DiskThrottlingWorkload : TestWorkload { checkDiskThrottleResult(res, worker); } - static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { + static Future getAllWorkers_unused(DiskThrottlingWorkload* self, std::vector* result) { result->clear(); *result = g_simulator.getAllProcesses(); return Void(); } - static Future getAllStorageWorkers(Database cx, DiskThrottlingWorkload* self, std::vector* result) { + static Future getAllStorageWorkers_unused(Database cx, DiskThrottlingWorkload* self, std::vector* result) { vector all = g_simulator.getAllProcesses(); for (int i = 0; i < all.size(); i++) if (!all[i]->failed && @@ -123,7 +124,6 @@ struct DiskThrottlingWorkload : TestWorkload { loop { wait(poisson(&lastTime, 1)); wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); - //wait(DiskThrottlingWorkload::getAllWorkers(self, &machines)); auto machine = deterministicRandom()->randomChoice(machines); TraceEvent("DoThrottleDisk").detail("For", self->throttleFor); self->doThrottle(machine, self->throttleFor); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 1d91a7e8da..12bc0d70c9 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -65,7 +65,7 @@ void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 ); // Chaos testing - init( ENABLE_CHAOS_FEATURES, false ); + init( ENABLE_CHAOS_FEATURES, true ); init( WRITE_TRACING_ENABLED, true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false; diff --git a/flow/network.h b/flow/network.h index d174601fec..651882d23e 100644 --- a/flow/network.h +++ b/flow/network.h @@ -657,25 +657,25 @@ struct DiskFailureInjector : FastAllocated { return static_cast(res); } - //double getSendDelay(NetworkAddress const& peer); - //double getReceiveDelay(NetworkAddress const& peer); - //virtual void throttleFor(double time) = 0; //virtual double getDiskDelay() = 0; void throttleFor(double time) { + TraceEvent("DiskFailureInjectorBefore").detail("ThrottleUntil", throttleUntil); throttleUntil = std::max(throttleUntil, timer_monotonic() + time); + TraceEvent("DiskFailureInjectorAfter").detail("ThrottleUntil", throttleUntil); } double getDiskDelay() { if (!FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { return 0.0; } - return throttleUntil; + return std::max(0.0, throttleUntil - timer_monotonic()); } private: // members double throttleUntil = 0.0; + std::unordered_map throttleDisk; private: // construction DiskFailureInjector() = default; From 1b8b22deccfb940b669bbd65a8884089b2d3bd91 Mon Sep 17 00:00:00 2001 From: negoyal Date: Mon, 12 Jul 2021 17:51:01 -0700 Subject: [PATCH 005/338] Wrapper class to avoid adding overhead to all async disk calls --- fdbclient/ClientWorkerInterface.h | 10 ++- fdbrpc/AsyncFileDelayed.actor.h | 89 ++++++++++++++++++++ fdbrpc/AsyncFileEIO.actor.h | 26 ++---- fdbrpc/AsyncFileKAIO.actor.h | 13 --- fdbrpc/AsyncFileNonDurable.actor.h | 36 ++------ fdbrpc/Net2FileSystem.cpp | 3 + fdbrpc/sim2.actor.cpp | 20 +---- fdbrpc/simulator.h | 7 +- fdbserver/worker.actor.cpp | 10 ++- fdbserver/workloads/DiskThrottling.actor.cpp | 54 ++++-------- flow/network.h | 52 +++++++++--- tests/fast/DiskThrottledCycle.toml | 1 + 12 files changed, 182 insertions(+), 139 deletions(-) create mode 100644 fdbrpc/AsyncFileDelayed.actor.h diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index b73c43ebd7..181017cfcf 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -94,12 +94,16 @@ struct SetFailureInjection { constexpr static FileIdentifier file_identifier = 15439864; ReplyPromise reply; struct ThrottleDiskCommand { - double time; - Optional address; // TODO: NEELAM: how do we identify the machine + // how often should the delay be inserted (0 meaning once, 10 meaning every 10 secs) + double delayFrequency; + // min delay to be inserted + double delayMin; + //max delay to be inserted + double delayMax; template void serialize(Ar& ar) { - serializer(ar, time, address); + serializer(ar, delayFrequency, delayMin, delayMax); } }; Optional throttleDisk; diff --git a/fdbrpc/AsyncFileDelayed.actor.h b/fdbrpc/AsyncFileDelayed.actor.h new file mode 100644 index 0000000000..5dfb9c655a --- /dev/null +++ b/fdbrpc/AsyncFileDelayed.actor.h @@ -0,0 +1,89 @@ +/* + * VersionedBTree.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flow/flow.h" +#include "flow/serialize.h" +#include "flow/genericactors.actor.h" +#include "fdbrpc/IAsyncFile.h" +#include "flow/network.h" +#include "flow/ActorCollection.h" +#include "flow/actorcompiler.h" + + +//template +class AsyncFileDelayed final : public IAsyncFile, public ReferenceCounted { +private: + Reference file; +public: + explicit AsyncFileDelayed(Reference file) : file(file) {} + + void addref() override { ReferenceCounted::addref(); } + void delref() override { ReferenceCounted::delref(); } + + Future read(void* data, int length, int64_t offset) override { + double delay = 0.0; + auto res = g_network->global(INetwork::enFailureInjector); + if (res) + delay = static_cast(res)->getDiskDelay(); + TraceEvent("AsyncFileDelayedRead").detail("ThrottleDelay", delay); + return delayed(file->read(data, length, offset), delay); + } + + Future write(void const* data, int length, int64_t offset) override { + double delay = 0.0; + auto res = g_network->global(INetwork::enFailureInjector); + if (res) + delay = static_cast(res)->getDiskDelay(); + TraceEvent("AsyncFileDelayedWrite").detail("ThrottleDelay", delay); + return delayed(file->write(data, length, offset), delay); + } + + Future truncate(int64_t size) override { + double delay = 0.0; + auto res = g_network->global(INetwork::enFailureInjector); + if (res) + delay = static_cast(res)->getDiskDelay(); + return delayed(file->truncate(size), delay); + } + + Future sync() override { + double delay = 0.0; + auto res = g_network->global(INetwork::enFailureInjector); + if (res) + delay = static_cast(res)->getDiskDelay(); + return delayed(file->sync(), delay); + } + + Future size() const override { + double delay = 0.0; + auto res = g_network->global(INetwork::enFailureInjector); + if (res) + delay = static_cast(res)->getDiskDelay(); + return delayed(file->size(), delay); + } + + int64_t debugFD() const override { + return file->debugFD(); + } + + std::string getFilename() const override { + return file->getFilename(); + } +}; diff --git a/fdbrpc/AsyncFileEIO.actor.h b/fdbrpc/AsyncFileEIO.actor.h index c962e60098..1d3ab4791e 100644 --- a/fdbrpc/AsyncFileEIO.actor.h +++ b/fdbrpc/AsyncFileEIO.actor.h @@ -162,16 +162,14 @@ public: Future read(void* data, int length, int64_t offset) override { ++countFileLogicalReads; ++countLogicalReads; - double throttleFor = diskFailureInjector->getDiskDelay(); - return read_impl(fd, data, length, offset, throttleFor); + return read_impl(fd, data, length, offset); } Future write(void const* data, int length, int64_t offset) override // Copies data synchronously { ++countFileLogicalWrites; ++countLogicalWrites; - double throttleFor = diskFailureInjector->getDiskDelay(); // Standalone copy = StringRef((const uint8_t*)data, length); - return write_impl(fd, err, StringRef((const uint8_t*)data, length), offset, throttleFor); + return write_impl(fd, err, StringRef((const uint8_t*)data, length), offset); } Future truncate(int64_t size) override { ++countFileLogicalWrites; @@ -272,7 +270,6 @@ private: int fd, flags; Reference err; std::string filename; - //DiskFailureInjector* diskFailureInjector; mutable Int64MetricHandle countFileLogicalWrites; mutable Int64MetricHandle countFileLogicalReads; @@ -280,8 +277,7 @@ private: mutable Int64MetricHandle countLogicalReads; AsyncFileEIO(int fd, int flags, std::string const& filename) - : fd(fd), flags(flags), filename(filename), err(new ErrorInfo), - diskFailureInjector(DiskFailureInjector::injector()) { + : fd(fd), flags(flags), filename(filename), err(new ErrorInfo) { if (!g_network->isSimulated()) { countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename); countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename); @@ -333,18 +329,13 @@ private: TraceEvent("AsyncFileClosed").suppressFor(1.0).detail("Fd", fd); } - ACTOR static Future read_impl(int fd, void* data, int length, int64_t offset, double throttleFor) { + ACTOR static Future read_impl(int fd, void* data, int length, int64_t offset) { state TaskPriority taskID = g_network->getCurrentTask(); state Promise p; // fprintf(stderr, "eio_read (fd=%d length=%d offset=%lld)\n", fd, length, offset); state eio_req* r = eio_read(fd, data, length, offset, 0, eio_callback, &p); try { wait(p.getFuture()); - // throttleDisk if enabled - //double throttleFor = diskFailureInjector->getDiskDelay(); - if (throttleFor > 0.0) { - wait(delay(throttleFor)); - } } catch (...) { g_network->setCurrentTask(taskID); eio_cancel(r); @@ -367,17 +358,12 @@ private: } } - ACTOR static Future write_impl(int fd, Reference err, StringRef data, int64_t offset, double throttleFor) { + ACTOR static Future write_impl(int fd, Reference err, StringRef data, int64_t offset) { state TaskPriority taskID = g_network->getCurrentTask(); state Promise p; state eio_req* r = eio_write(fd, (void*)data.begin(), data.size(), offset, 0, eio_callback, &p); try { wait(p.getFuture()); - // throttleDisk if enabled - //double throttleFor = diskFailureInjector->getDiskDelay(); - if (throttleFor > 0.0) { - wait(delay(throttleFor)); - } } catch (...) { g_network->setCurrentTask(taskID); eio_cancel(r); @@ -567,8 +553,6 @@ private: static void apple_fsync(eio_req* req) { req->result = fcntl(req->int1, F_FULLFSYNC, 0); } static void free_req(eio_req* req) { free(req); } #endif -public: - DiskFailureInjector* diskFailureInjector; }; #ifdef FILESYSTEM_IMPL diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h index 15553a85e2..c82b57161b 100644 --- a/fdbrpc/AsyncFileKAIO.actor.h +++ b/fdbrpc/AsyncFileKAIO.actor.h @@ -195,10 +195,6 @@ public: void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } - ACTOR static void throttleDisk(double throttleFor) { - if (throttleFor > 0.0) - wait(delay(throttleFor)); - } Future read(void* data, int length, int64_t offset) override { ++countFileLogicalReads; ++countLogicalReads; @@ -216,9 +212,6 @@ public: enqueue(io, "read", this); Future result = io->result.getFuture(); - // throttleDisk if enabled - throttleDisk(diskFailureInjector->getDiskDelay()); - #if KAIO_LOGGING // result = map(result, [=](int r) mutable { KAIOLogBlockEvent(io, OpLogEntry::READY, r); return r; }); #endif @@ -244,9 +237,6 @@ public: enqueue(io, "write", this); Future result = io->result.getFuture(); - // throttleDisk if enabled - throttleDisk(diskFailureInjector->getDiskDelay()); - #if KAIO_LOGGING // result = map(result, [=](int r) mutable { KAIOLogBlockEvent(io, OpLogEntry::READY, r); return r; }); #endif @@ -758,9 +748,6 @@ private: } } } - -public: - DiskFailureInjector* diskFailureInjector; }; #if KAIO_LOGGING diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index 0c63846169..98bbe0c4e8 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -62,7 +62,7 @@ private: Future shutdown; public: - explicit AsyncFileDetachable(Reference file) : file(file), diskFailureInjector(DiskFailureInjector::injector()) { shutdown = doShutdown(this); } + explicit AsyncFileDetachable(Reference file) : file(file) { shutdown = doShutdown(this); } ACTOR Future doShutdown(AsyncFileDetachable* self) { wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture())); @@ -85,20 +85,12 @@ public: Future read(void* data, int length, int64_t offset) override { if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady()) return io_error().asInjectedFault(); - // throttleDisk if enabled - auto throttleFor = diskFailureInjector->getDiskDelay(); - if (throttleFor > 0.0) { - TraceEvent("AsyncFileDetachable_Read").detail("ThrottleDelay", throttleFor); - //wait(delay(throttleFor)); - } return sendErrorOnShutdown(file->read(data, length, offset)); } Future write(void const* data, int length, int64_t offset) override { if (!file.getPtr() || g_simulator.getCurrentProcess()->shutdownSignal.getFuture().isReady()) return io_error().asInjectedFault(); - if (diskFailureInjector->getDiskDelay() > 0.0) - TraceEvent("AsyncFileDetachable_Write").detail("ThrottleDelay", diskFailureInjector->getDiskDelay()); return sendErrorOnShutdown(file->write(data, length, offset)); } @@ -130,8 +122,6 @@ public: throw io_error().asInjectedFault(); return file->getFilename(); } -public: - DiskFailureInjector* diskFailureInjector; }; // An async file implementation which wraps another async file and will randomly destroy sectors that it is writing when @@ -203,7 +193,7 @@ private: bool aio) : filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters), openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false), - aio(aio), diskFailureInjector(DiskFailureInjector::injector()) + aio(aio) { // This is only designed to work in simulation @@ -321,7 +311,7 @@ public: // Passes along reads straight to the underlying file, waiting for any outstanding changes that could affect the // results - Future read(void* data, int length, int64_t offset) override { return read(this, data, length, offset, diskFailureInjector->getDiskDelay()); } + Future read(void* data, int length, int64_t offset) override { return read(this, data, length, offset); } // Writes data to the file. Writes are delayed a random amount of time before being // passed to the underlying file @@ -336,7 +326,7 @@ public: Promise writeStarted; Promise> writeEnded; - writeEnded.send(write(this, writeStarted, writeEnded.getFuture(), data, length, offset, diskFailureInjector->getDiskDelay())); + writeEnded.send(write(this, writeStarted, writeEnded.getFuture(), data, length, offset)); return writeStarted.getFuture(); } @@ -444,7 +434,7 @@ private: return readFuture.get(); } - ACTOR Future read(AsyncFileNonDurable* self, void* data, int length, int64_t offset, double throttleFor = 0.0) { + ACTOR Future read(AsyncFileNonDurable* self, void* data, int length, int64_t offset) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); @@ -452,12 +442,6 @@ private: try { state int rep = wait(self->onRead(self, data, length, offset)); wait(g_simulator.onProcess(currentProcess, currentTaskID)); - - // throttleDisk if enabled - if (throttleFor > 0.0) { - TraceEvent("AsyncFileNonDurable_ReadDone", self->id).detail("ThrottleDelay", throttleFor).detail("Filename", self->filename).detail("ReadLength", length).detail("Offset", offset); - wait(delay(throttleFor)); - } return rep; } catch (Error& e) { state Error err = e; @@ -474,8 +458,7 @@ private: Future> ownFuture, void const* data, int length, - int64_t offset, - double throttleFor = 0.0) { + int64_t offset) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); @@ -639,11 +622,6 @@ private: } wait(waitForAll(writeFutures)); - // throttleDisk if enabled - if (throttleFor > 0.0) { - TraceEvent("AsyncFileNonDurable_WriteDone", self->id).detail("ThrottleDelay", throttleFor).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset); - wait(delay(throttleFor)); - } //TraceEvent("AsyncFileNonDurable_WriteDone", self->id).detail("Delay", delayDuration).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset); return Void(); } @@ -889,8 +867,6 @@ private: throw err; } } -public: - DiskFailureInjector* diskFailureInjector; }; #include "flow/unactorcompiler.h" diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index 71a7d784a1..a71115a859 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -31,6 +31,7 @@ #define FILESYSTEM_IMPL 1 #include "fdbrpc/AsyncFileCached.actor.h" +#include "fdbrpc/AsyncFileDelayed.actor.h" #include "fdbrpc/AsyncFileEIO.actor.h" #include "fdbrpc/AsyncFileWinASIO.actor.h" #include "fdbrpc/AsyncFileKAIO.actor.h" @@ -76,6 +77,8 @@ Future> Net2FileSystem::open(const std::string& file static_cast((void*)g_network->global(INetwork::enASIOService))); if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) + f = map(f, [=](Reference r) { return Reference(new AsyncFileDelayed(r)); }); return f; } diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 093ef389ac..1e30618279 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -34,6 +34,7 @@ #include "fdbrpc/IAsyncFile.h" #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbrpc/AsyncFileNonDurable.actor.h" +#include "fdbrpc/AsyncFileDelayed.actor.h" #include "flow/crc32c.h" #include "fdbrpc/TraceFileIO.h" #include "flow/FaultInjection.h" @@ -1949,13 +1950,6 @@ public: void clogPair(const IPAddress& from, const IPAddress& to, double seconds) override { g_clogging.clogPairFor(from, to, seconds); } - void throttleDisk(ProcessInfo* machine, double seconds) override { - machine->throttleDiskFor = seconds; - TraceEvent("ThrottleDisk").detail("Delay", seconds). - detail("Roles", getRoles(machine->address)). - detail("Address", machine->address). - detail("StartingClass", machine->startingClass.toString()); - } std::vector getAllProcesses() const override { std::vector processes; for (auto& c : machines) { @@ -2397,19 +2391,11 @@ Future waitUntilDiskReady(Reference diskParameters, int64_ diskParameters->nextOperation += (1.0 / diskParameters->iops) + (size / diskParameters->bandwidth); double randomLatency; - if (g_simulator.getCurrentProcess()->throttleDiskFor) { - randomLatency = g_simulator.getCurrentProcess()->throttleDiskFor; - TraceEvent("WaitUntilDiskReadyThrottling") - .detail("Delay", randomLatency); - } else if (sync) { + if (sync) { randomLatency = .005 + deterministicRandom()->random01() * (BUGGIFY ? 1.0 : .010); } else randomLatency = 10 * deterministicRandom()->random01() / diskParameters->iops; - TraceEvent("WaitUntilDiskReady").detail("Delay", randomLatency). - detail("Roles", g_simulator.getRoles(g_simulator.getCurrentProcess()->address)). - detail("Address", g_simulator.getCurrentProcess()->address). - detail("ThrottleDiskFor", g_simulator.getCurrentProcess()->throttleDiskFor); return delayUntil(diskParameters->nextOperation + randomLatency); } @@ -2488,6 +2474,8 @@ Future> Sim2FileSystem::open(const std::string& file f = AsyncFileDetachable::open(f); if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) + f = map(f, [=](Reference r) { return Reference(new AsyncFileDelayed(r)); }); return f; } else return AsyncFileCached::open(filename, flags, mode); diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 1da850e48c..6404eafc17 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -87,7 +87,6 @@ public: uint64_t fault_injection_r; double fault_injection_p1, fault_injection_p2; bool failedDisk; - double throttleDiskFor; UID uid; @@ -103,7 +102,7 @@ public: : name(name), locality(locality), startingClass(startingClass), addresses(addresses), address(addresses.address), dataFolder(dataFolder), network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), rebooting(false), fault_injection_p1(0), fault_injection_p2(0), - fault_injection_r(0), machine(0), cleared(false), failedDisk(false), throttleDiskFor(0) { + fault_injection_r(0), machine(0), cleared(false), failedDisk(false) { uid = deterministicRandom()->randomUniqueID(); } @@ -375,7 +374,6 @@ public: virtual void clogInterface(const IPAddress& ip, double seconds, ClogMode mode = ClogDefault) = 0; virtual void clogPair(const IPAddress& from, const IPAddress& to, double seconds) = 0; - virtual void throttleDisk(ProcessInfo* machine, double seconds) = 0; virtual std::vector getAllProcesses() const = 0; virtual ProcessInfo* getProcessByAddress(NetworkAddress const& address) = 0; virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0; @@ -464,9 +462,8 @@ struct DiskParameters : ReferenceCounted { double nextOperation; int64_t iops; int64_t bandwidth; - double throttleFor; - DiskParameters(int64_t iops, int64_t bandwidth) : nextOperation(0), iops(iops), bandwidth(bandwidth), throttleFor(0) {} + DiskParameters(int64_t iops, int64_t bandwidth) : nextOperation(0), iops(iops), bandwidth(bandwidth) {} }; // Simulates delays for performing operations on disk diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 4a99e02265..2b20f695ae 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1516,8 +1516,14 @@ ACTOR Future workerServer(Reference connFile, when(SetFailureInjection req = waitNext(interf.clientInterface.setFailureInjection.getFuture())) { if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { if (req.throttleDisk.present()) { - TraceEvent("DiskThrottleRequest").detail("Delay", req.throttleDisk.get().time); - DiskFailureInjector::injector()->throttleFor(req.throttleDisk.get().time); + TraceEvent("DiskThrottleRequest").detail("DelayFrequency",req.throttleDisk.get().delayFrequency). + detail("DelayMin", req.throttleDisk.get().delayMin). + detail("DelayMax", req.throttleDisk.get().delayMax); + auto diskFailureInjector = DiskFailureInjector::injector(); + //DiskFailureInjector::injector()->throttleFor(req.throttleDisk.get()); + diskFailureInjector->throttleFor(req.throttleDisk.get().delayFrequency, + req.throttleDisk.get().delayMin, + req.throttleDisk.get().delayMax); } req.reply.send(Void()); } else { diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskThrottling.actor.cpp index 61c465ae7f..8529be5fc3 100644 --- a/fdbserver/workloads/DiskThrottling.actor.cpp +++ b/fdbserver/workloads/DiskThrottling.actor.cpp @@ -10,12 +10,18 @@ struct DiskThrottlingWorkload : TestWorkload { bool enabled; double testDuration; - double throttleFor; + double throttleFrequency; + double throttleMin; + double throttleMax; DiskThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { enabled = !clientId; // only do this on the "first" client testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); - throttleFor = getOption(options, LiteralStringRef("throttleDelay"), 2.0); - TraceEvent("DiskThrottlingWorkload").detail("TestDuration", testDuration).detail("For", throttleFor); + throttleFrequency = getOption(options, LiteralStringRef("throttleFrequency"), 0.0); + throttleMin = getOption(options, LiteralStringRef("throttleMin"), 2.0); + throttleMax = getOption(options, LiteralStringRef("throttleMax"), 2.0); + TraceEvent("DiskThrottlingWorkload") + .detail("TestDuration", testDuration).detail("Frequency", throttleFrequency) + .detail("Min", throttleMin).detail("Max", throttleMax); } std::string description() const override { @@ -28,12 +34,6 @@ struct DiskThrottlingWorkload : TestWorkload { Future setup(Database const& cx) override { return Void(); } Future start(Database const& cx) override { - //if (&g_simulator == g_network && enabled) { - // TraceEvent("DiskThrottlingStart").detail("For", throttleFor); - // return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), - // testDuration, - // Void()); - //} else if (enabled) { return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), testDuration, @@ -46,13 +46,6 @@ struct DiskThrottlingWorkload : TestWorkload { void getMetrics(vector& m) override {} - ACTOR void doThrottle_unused(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) { - wait(::delay(delay)); - TraceEvent("ThrottleDisk").detail("For", t); - g_simulator.throttleDisk(machine, t); - TraceEvent("ThrottleDiskSet").detail("For", t); - } - static void checkDiskThrottleResult(Future res, WorkerInterface worker) { if (res.isError()) { auto err = res.getError(); @@ -67,36 +60,20 @@ struct DiskThrottlingWorkload : TestWorkload { } } - ACTOR void doThrottle(WorkerInterface worker, double t, double delay = 0.0) { + ACTOR void doThrottle(WorkerInterface worker, double frequency, double minDelay, double maxDelay, double startDelay = 0.0) { state Future res; - wait(::delay(delay)); + wait(::delay(startDelay)); SetFailureInjection::ThrottleDiskCommand throttleDisk; - throttleDisk.time = t; + throttleDisk.delayFrequency = frequency; + throttleDisk.delayMin = minDelay; + throttleDisk.delayMax = maxDelay; SetFailureInjection req; req.throttleDisk = throttleDisk; - TraceEvent("ThrottleDisk").detail("For", t); res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); checkDiskThrottleResult(res, worker); } - static Future getAllWorkers_unused(DiskThrottlingWorkload* self, std::vector* result) { - result->clear(); - *result = g_simulator.getAllProcesses(); - return Void(); - } - - static Future getAllStorageWorkers_unused(Database cx, DiskThrottlingWorkload* self, std::vector* result) { - vector all = g_simulator.getAllProcesses(); - for (int i = 0; i < all.size(); i++) - if (!all[i]->failed && - all[i]->name == std::string("Server") && - ((all[i]->startingClass == ProcessClass::StorageClass) || - (all[i]->startingClass == ProcessClass::UnsetClass))) - result->emplace_back(all[i]); - return Void(); - } - ACTOR static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { result->clear(); std::vector res = @@ -125,8 +102,7 @@ struct DiskThrottlingWorkload : TestWorkload { wait(poisson(&lastTime, 1)); wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); auto machine = deterministicRandom()->randomChoice(machines); - TraceEvent("DoThrottleDisk").detail("For", self->throttleFor); - self->doThrottle(machine, self->throttleFor); + self->doThrottle(machine, self->throttleFrequency, self->throttleMin, self->throttleMax); } } }; diff --git a/flow/network.h b/flow/network.h index 651882d23e..4b34a648cc 100644 --- a/flow/network.h +++ b/flow/network.h @@ -647,6 +647,44 @@ public: // Returns the interface that should be used to make and accept socket connections }; +struct DelayGenerator : FastAllocated { + + void setDelay(double frequency, double min, double max) { + delayFrequency = frequency; + delayMin = min; + delayMax = max; + delayFor = (delayMin == delayMax) ? delayMin : deterministicRandom()->randomInt(delayMin, delayMax); + delayUntil = std::max(delayUntil, timer_monotonic() + delayFor); + TraceEvent("DelayGeneratorSetDelay").detail("DelayFrequency", frequency).detail("DelayMin", min). + detail("DelayMax", max).detail("DelayFor", delayFor).detail("DelayUntil", delayUntil); + } + + double getDelay() { + // If a delayFrequency was specified, this logic determins the delay to be inserted at any point in time + if (delayFrequency) { + auto timeElapsed = fmod(timer_monotonic(), delayFrequency); + TraceEvent("DelayGeneratorGetDelay").detail("DelayFrequency", delayFrequency). + detail("TimeElapsed", timeElapsed).detail("DelayFor", delayFor); + return std::max(0.0, delayFor - timeElapsed); + } + TraceEvent("DelayGeneratorGetDelay").detail("DelayFrequency", delayFrequency). + detail("CurTime", timer_monotonic()).detail("DelayUntil", delayUntil); + return std::max(0.0, delayUntil - timer_monotonic()); + } + +private: //members + double delayFrequency = 0.0; // how often should the delay be inserted (0 meaning once, 10 meaning every 10 secs) + double delayMin; // min delay to be inserted + double delayMax; // max delay to be inserted + double delayFor = 0.0; // randomly chosen delay between min and max + double delayUntil = 0.0; // used when the delayFrequency is 0 + +public: // construction + DelayGenerator() = default; + DelayGenerator(DelayGenerator const&) = delete; + +}; + struct DiskFailureInjector : FastAllocated { static DiskFailureInjector* injector() { auto res = g_network->global(INetwork::enFailureInjector); @@ -657,25 +695,19 @@ struct DiskFailureInjector : FastAllocated { return static_cast(res); } - //virtual void throttleFor(double time) = 0; - //virtual double getDiskDelay() = 0; - - void throttleFor(double time) { - TraceEvent("DiskFailureInjectorBefore").detail("ThrottleUntil", throttleUntil); - throttleUntil = std::max(throttleUntil, timer_monotonic() + time); - TraceEvent("DiskFailureInjectorAfter").detail("ThrottleUntil", throttleUntil); + void throttleFor(double frequency, double delayMin, double delayMax) { + delayGenerator.setDelay(frequency, delayMin, delayMax); } double getDiskDelay() { if (!FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { return 0.0; } - return std::max(0.0, throttleUntil - timer_monotonic()); + return delayGenerator.getDelay(); } private: // members - double throttleUntil = 0.0; - std::unordered_map throttleDisk; + DelayGenerator delayGenerator; private: // construction DiskFailureInjector() = default; diff --git a/tests/fast/DiskThrottledCycle.toml b/tests/fast/DiskThrottledCycle.toml index c0f35293aa..60429350b6 100644 --- a/tests/fast/DiskThrottledCycle.toml +++ b/tests/fast/DiskThrottledCycle.toml @@ -10,4 +10,5 @@ testTitle = 'DiskThrottledCycle' [[test.workload]] testName = 'DiskThrottling' testDuration = 30.0 + throttleFrequency = 10.0 From f950fe9f9d0fcb18c36927198ded3adafaea1712 Mon Sep 17 00:00:00 2001 From: negoyal Date: Sun, 18 Jul 2021 17:35:05 -0700 Subject: [PATCH 006/338] Chaos workload to randomly flip bits during SS writes. --- fdbclient/ClientWorkerInterface.h | 16 ++++++++- fdbrpc/AsyncFileDelayed.actor.h | 36 ++++++++++++++++++-- fdbserver/CMakeLists.txt | 3 +- fdbserver/worker.actor.cpp | 8 ++++- flow/network.h | 55 ++++++++++++++++++++++++++++++- tests/CMakeLists.txt | 1 + 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index 181017cfcf..9c80312708 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -106,11 +106,25 @@ struct SetFailureInjection { serializer(ar, delayFrequency, delayMin, delayMax); } }; + + struct FlipBitsCommand { + // File that the bit flips are requested for + //Reference filename; + // percent of bits to flip in the given file + double percentBitFlips; + + template + void serialize(Ar& ar) { + serializer(ar, percentBitFlips); + } + }; + Optional throttleDisk; + Optional flipBits; template void serialize(Ar& ar) { - serializer(ar, reply, throttleDisk); + serializer(ar, reply, throttleDisk, flipBits); } }; #endif diff --git a/fdbrpc/AsyncFileDelayed.actor.h b/fdbrpc/AsyncFileDelayed.actor.h index 5dfb9c655a..6e990f276c 100644 --- a/fdbrpc/AsyncFileDelayed.actor.h +++ b/fdbrpc/AsyncFileDelayed.actor.h @@ -37,22 +37,52 @@ public: void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } + uint8_t toggleNthBit(uint8_t b, uint8_t n) { + auto singleBitMask = uint8_t(1) << (n); + return b ^ singleBitMask; + } + + void flipBits(void* data, int length, double percentBitFlips) { + auto toFlip = int(float(length*8) * percentBitFlips / 100); + TraceEvent("AsyncFileFlipBits").detail("ToFlip", toFlip); + for (auto i = 0; i < toFlip; i++) { + auto byteOffset = deterministicRandom()->randomInt64(0, length); + auto bitOffset = uint8_t(deterministicRandom()->randomInt(0, 8)); + ((uint8_t *)data)[byteOffset] = toggleNthBit(((uint8_t *)data)[byteOffset], bitOffset); + } + } + Future read(void* data, int length, int64_t offset) override { double delay = 0.0; auto res = g_network->global(INetwork::enFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); TraceEvent("AsyncFileDelayedRead").detail("ThrottleDelay", delay); - return delayed(file->read(data, length, offset), delay); + return delayed(file->read(data, length, offset), delay); } Future write(void const* data, int length, int64_t offset) override { double delay = 0.0; - auto res = g_network->global(INetwork::enFailureInjector); + char* pdata = nullptr; + auto res = g_network->global(INetwork::enBitFlipper); + if (res) { + auto percentBitFlips = static_cast(res)->getPercentBitFlips(); + if (percentBitFlips > 0.0) { + TraceEvent("AsyncFileCorruptWrite").detail("PercentBitFlips", percentBitFlips); + pdata = new char[length]; + memcpy(pdata, data, length); + flipBits(pdata, length, percentBitFlips); + auto diff = memcmp(pdata, data, length); + if (diff) + TraceEvent("AsyncFileCorruptWriteDiff").detail("Diff", diff); + } + } + + res = g_network->global(INetwork::enFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); TraceEvent("AsyncFileDelayedWrite").detail("ThrottleDelay", delay); - return delayed(file->write(data, length, offset), delay); + return delayed(file->write((pdata != nullptr) ? pdata : data, length, offset), delay); } Future truncate(int64_t size) override { diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index efa2c7fbf1..d1686bf478 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -151,6 +151,7 @@ set(FDBSERVER_SRCS workloads/BackupToDBAbort.actor.cpp workloads/BackupToDBCorrectness.actor.cpp workloads/BackupToDBUpgrade.actor.cpp + workloads/BitFlipping.actor.cpp workloads/BlobStoreWorkload.h workloads/BulkLoad.actor.cpp workloads/BulkSetup.actor.h @@ -158,7 +159,6 @@ set(FDBSERVER_SRCS workloads/ChangeConfig.actor.cpp workloads/ClientTransactionProfileCorrectness.actor.cpp workloads/TriggerRecovery.actor.cpp - workloads/DiskThrottling.actor.cpp workloads/SuspendProcesses.actor.cpp workloads/CommitBugCheck.actor.cpp workloads/ConfigureDatabase.actor.cpp @@ -172,6 +172,7 @@ set(FDBSERVER_SRCS workloads/DDMetricsExclude.actor.cpp workloads/DiskDurability.actor.cpp workloads/DiskDurabilityTest.actor.cpp + workloads/DiskThrottling.actor.cpp workloads/Downgrade.actor.cpp workloads/DummyWorkload.actor.cpp workloads/ExternalWorkload.actor.cpp diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 2b20f695ae..8d70c1ccda 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1520,10 +1520,16 @@ ACTOR Future workerServer(Reference connFile, detail("DelayMin", req.throttleDisk.get().delayMin). detail("DelayMax", req.throttleDisk.get().delayMax); auto diskFailureInjector = DiskFailureInjector::injector(); - //DiskFailureInjector::injector()->throttleFor(req.throttleDisk.get()); diskFailureInjector->throttleFor(req.throttleDisk.get().delayFrequency, req.throttleDisk.get().delayMin, req.throttleDisk.get().delayMax); + } else if (req.flipBits.present()) { + TraceEvent("FlipBitsRequest"). + detail("Percent", req.flipBits.get().percentBitFlips); + //detail("File",req.flipBits.get().file). + auto bitFlipper = BitFlipper::flipper(); + bitFlipper->setPercentBitFlips(req.flipBits.get().percentBitFlips); + //flipBits(req.flipBits.get().file, req.flipBits.get().percent); } req.reply.send(Void()); } else { diff --git a/flow/network.h b/flow/network.h index 4b34a648cc..6f96e993cf 100644 --- a/flow/network.h +++ b/flow/network.h @@ -487,7 +487,8 @@ public: enClientFailureMonitor = 12, enSQLiteInjectedError = 13, enGlobalConfig = 14, - enFailureInjector = 15 + enFailureInjector = 15, + enBitFlipper = 16 }; virtual void longTaskCheck(const char* name) {} @@ -714,4 +715,56 @@ private: // construction DiskFailureInjector(DiskFailureInjector const&) = delete; }; +struct BitFlipper : FastAllocated { + static BitFlipper* flipper() { + auto res = g_network->global(INetwork::enBitFlipper); + if (!res) { + res = new BitFlipper(); + g_network->setGlobal(INetwork::enBitFlipper, res); + } + return static_cast(res); + } + + //uint8_t toggleNthBit(uint8_t b, uint8_t n) { + // auto singleBitMask = uint8(1) << (n); + // return b ^ singleBitMask; + //} + + //void flipBitAtOffset(int64_t byteOffset, uint8_t bitOffset) { + //auto oneByte = make([]byte, 1); + // uint8_t oneByte[1]; + // int readBytes = wait(file->Read(oneByte, 1, byteOffset)); + + // oneByte[0] = toggleNthBit(oneByte[0], bitOffset); + // file->write(oneByte, 1, byteOffset); + //} + + //void flipBits(Reference fileName, double percent) { + // file = fileName; + // auto toFlip = int(float64(file->size()*8) * percent / 100); + // for (auto i = 0; i < toFlip; i++) { + // auto byteOffset = deterministicRandom()->randomInt64(0, file->size()); + // auto bitOffset = uint8_t(deterministicRandom()->randomInt(0, 8)); + // flipBitAtOffset(byteOffset, bitOffset); + // } + //} + + double getPercentBitFlips() { + TraceEvent("BitFlipperGetPercentBitFlips").detail("PercentBitFlips", percentBitFlips); + return percentBitFlips; + } + + void setPercentBitFlips(double percentFlips) { + percentBitFlips = percentFlips; + TraceEvent("BitFlipperSetPercentBitFlips").detail("PercentBitFlips", percentBitFlips); + } + +private: // members + double percentBitFlips = 0.0; + //Reference file; + +private: // construction + BitFlipper() = default; + BitFlipper(BitFlipper const&) = delete; +}; #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5a5bf2c208..758b16949b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,6 +118,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/BackupCorrectnessClean.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectness.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectnessClean.toml) + add_fdb_test(TEST_FILES fast/BitFlippedCycle.toml IGNORE) add_fdb_test(TEST_FILES fast/CacheTest.toml) add_fdb_test(TEST_FILES fast/CloggedSideband.toml) add_fdb_test(TEST_FILES fast/ConfigureLocked.toml) From 596ca92e2fad977a08738d96ede113734f1062e4 Mon Sep 17 00:00:00 2001 From: negoyal Date: Mon, 19 Jul 2021 11:13:57 -0700 Subject: [PATCH 007/338] Add missing files and rename some. --- ...Delayed.actor.h => AsyncFileChaos.actor.h} | 22 ++--- fdbrpc/Net2FileSystem.cpp | 4 +- fdbrpc/sim2.actor.cpp | 4 +- fdbserver/worker.actor.cpp | 2 - fdbserver/workloads/BitFlipping.actor.cpp | 92 +++++++++++++++++++ flow/network.h | 6 +- tests/fast/BitFlippedCycle.toml | 13 +++ 7 files changed, 123 insertions(+), 20 deletions(-) rename fdbrpc/{AsyncFileDelayed.actor.h => AsyncFileChaos.actor.h} (81%) create mode 100644 fdbserver/workloads/BitFlipping.actor.cpp create mode 100644 tests/fast/BitFlippedCycle.toml diff --git a/fdbrpc/AsyncFileDelayed.actor.h b/fdbrpc/AsyncFileChaos.actor.h similarity index 81% rename from fdbrpc/AsyncFileDelayed.actor.h rename to fdbrpc/AsyncFileChaos.actor.h index 6e990f276c..7890ff0b51 100644 --- a/fdbrpc/AsyncFileDelayed.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -28,14 +28,14 @@ //template -class AsyncFileDelayed final : public IAsyncFile, public ReferenceCounted { +class AsyncFileChaos final : public IAsyncFile, public ReferenceCounted { private: Reference file; public: - explicit AsyncFileDelayed(Reference file) : file(file) {} + explicit AsyncFileChaos(Reference file) : file(file) {} - void addref() override { ReferenceCounted::addref(); } - void delref() override { ReferenceCounted::delref(); } + void addref() override { ReferenceCounted::addref(); } + void delref() override { ReferenceCounted::delref(); } uint8_t toggleNthBit(uint8_t b, uint8_t n) { auto singleBitMask = uint8_t(1) << (n); @@ -54,10 +54,10 @@ public: Future read(void* data, int length, int64_t offset) override { double delay = 0.0; - auto res = g_network->global(INetwork::enFailureInjector); + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); - TraceEvent("AsyncFileDelayedRead").detail("ThrottleDelay", delay); + TraceEvent("AsyncFileChaosRead").detail("ThrottleDelay", delay); return delayed(file->read(data, length, offset), delay); } @@ -78,16 +78,16 @@ public: } } - res = g_network->global(INetwork::enFailureInjector); + res = g_network->global(INetwork::enDiskFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); - TraceEvent("AsyncFileDelayedWrite").detail("ThrottleDelay", delay); + TraceEvent("AsyncFileChaosWrite").detail("ThrottleDelay", delay); return delayed(file->write((pdata != nullptr) ? pdata : data, length, offset), delay); } Future truncate(int64_t size) override { double delay = 0.0; - auto res = g_network->global(INetwork::enFailureInjector); + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); return delayed(file->truncate(size), delay); @@ -95,7 +95,7 @@ public: Future sync() override { double delay = 0.0; - auto res = g_network->global(INetwork::enFailureInjector); + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); return delayed(file->sync(), delay); @@ -103,7 +103,7 @@ public: Future size() const override { double delay = 0.0; - auto res = g_network->global(INetwork::enFailureInjector); + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) delay = static_cast(res)->getDiskDelay(); return delayed(file->size(), delay); diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index a71115a859..76128ffd86 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -31,7 +31,7 @@ #define FILESYSTEM_IMPL 1 #include "fdbrpc/AsyncFileCached.actor.h" -#include "fdbrpc/AsyncFileDelayed.actor.h" +#include "fdbrpc/AsyncFileChaos.actor.h" #include "fdbrpc/AsyncFileEIO.actor.h" #include "fdbrpc/AsyncFileWinASIO.actor.h" #include "fdbrpc/AsyncFileKAIO.actor.h" @@ -78,7 +78,7 @@ Future> Net2FileSystem::open(const std::string& file if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) - f = map(f, [=](Reference r) { return Reference(new AsyncFileDelayed(r)); }); + f = map(f, [=](Reference r) { return Reference(new AsyncFileChaos(r)); }); return f; } diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 1e30618279..4051f935a0 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -34,7 +34,7 @@ #include "fdbrpc/IAsyncFile.h" #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbrpc/AsyncFileNonDurable.actor.h" -#include "fdbrpc/AsyncFileDelayed.actor.h" +#include "fdbrpc/AsyncFileChaos.actor.h" #include "flow/crc32c.h" #include "fdbrpc/TraceFileIO.h" #include "flow/FaultInjection.h" @@ -2475,7 +2475,7 @@ Future> Sim2FileSystem::open(const std::string& file if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) - f = map(f, [=](Reference r) { return Reference(new AsyncFileDelayed(r)); }); + f = map(f, [=](Reference r) { return Reference(new AsyncFileChaos(r)); }); return f; } else return AsyncFileCached::open(filename, flags, mode); diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 8d70c1ccda..1c5bf22408 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1526,10 +1526,8 @@ ACTOR Future workerServer(Reference connFile, } else if (req.flipBits.present()) { TraceEvent("FlipBitsRequest"). detail("Percent", req.flipBits.get().percentBitFlips); - //detail("File",req.flipBits.get().file). auto bitFlipper = BitFlipper::flipper(); bitFlipper->setPercentBitFlips(req.flipBits.get().percentBitFlips); - //flipBits(req.flipBits.get().file, req.flipBits.get().percent); } req.reply.send(Void()); } else { diff --git a/fdbserver/workloads/BitFlipping.actor.cpp b/fdbserver/workloads/BitFlipping.actor.cpp new file mode 100644 index 0000000000..ff1941fbe3 --- /dev/null +++ b/fdbserver/workloads/BitFlipping.actor.cpp @@ -0,0 +1,92 @@ +#include "fdbclient/NativeAPI.actor.h" +#include "fdbserver/TesterInterface.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "fdbrpc/simulator.h" +#include "fdbserver/WorkerInterface.actor.h" +#include "fdbserver/ServerDBInfo.h" +#include "fdbserver/QuietDatabase.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +struct BitFlippingWorkload : TestWorkload { + bool enabled; + double testDuration; + double percentBitFlips; + BitFlippingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + enabled = !clientId; // only do this on the "first" client + testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); + percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 1.0); + TraceEvent("BitFlippingWorkload") + .detail("TestDuration", testDuration).detail("Percentage", percentBitFlips); + } + + std::string description() const override { + if (&g_simulator == g_network) + return "BitFlipping"; + else + return "NoSimBitFlipping"; + } + + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { + if (enabled) { + return timeout(reportErrors(flipBitsClient(cx, this), "BitFlippingError"), + testDuration, + Void()); + } else + return Void(); + } + + Future check(Database const& cx) override { return true; } + + void getMetrics(vector& m) override {} + + static void checkBitFlipResult(Future res, WorkerInterface worker) { + if (res.isError()) { + auto err = res.getError(); + if (err.code() == error_code_client_invalid_operation) { + TraceEvent(SevError, "ChaosDisabled") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); + } else { + TraceEvent(SevError, "BitFlippingFailed") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) + .error(err); + } + } + } + + ACTOR void doBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { + state Future res; + wait(::delay(startDelay)); + SetFailureInjection::FlipBitsCommand flipBits; + flipBits.percentBitFlips = percentage; + SetFailureInjection req; + req.flipBits = flipBits; + res = worker.clientInterface.setFailureInjection.getReply(req); + wait(ready(res)); + checkBitFlipResult(res, worker); + } + + ACTOR static Future getAllStorageWorkers(Database cx, BitFlippingWorkload* self, std::vector* result) { + result->clear(); + state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); + for (auto& worker : res) { + result->emplace_back(worker); + } + return Void(); + } + + ACTOR template + Future flipBitsClient(Database cx, BitFlippingWorkload* self) { + state double lastTime = now(); + state double workloadEnd = now() + self->testDuration; + state std::vector machines; + loop { + wait(poisson(&lastTime, 1)); + wait(BitFlippingWorkload::getAllStorageWorkers(cx, self, &machines)); + auto machine = deterministicRandom()->randomChoice(machines); + self->doBitFlips(machine, self->percentBitFlips); + } + } +}; +WorkloadFactory BitFlippingWorkloadFactory("BitFlipping"); diff --git a/flow/network.h b/flow/network.h index 6f96e993cf..0ce243e9f2 100644 --- a/flow/network.h +++ b/flow/network.h @@ -487,7 +487,7 @@ public: enClientFailureMonitor = 12, enSQLiteInjectedError = 13, enGlobalConfig = 14, - enFailureInjector = 15, + enDiskFailureInjector = 15, enBitFlipper = 16 }; @@ -688,10 +688,10 @@ public: // construction struct DiskFailureInjector : FastAllocated { static DiskFailureInjector* injector() { - auto res = g_network->global(INetwork::enFailureInjector); + auto res = g_network->global(INetwork::enDiskFailureInjector); if (!res) { res = new DiskFailureInjector(); - g_network->setGlobal(INetwork::enFailureInjector, res); + g_network->setGlobal(INetwork::enDiskFailureInjector, res); } return static_cast(res); } diff --git a/tests/fast/BitFlippedCycle.toml b/tests/fast/BitFlippedCycle.toml new file mode 100644 index 0000000000..3cab1f74fe --- /dev/null +++ b/tests/fast/BitFlippedCycle.toml @@ -0,0 +1,13 @@ +[[test]] +testTitle = 'BitFlippedCycle' + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 2500.0 + testDuration = 60.0 + expectedRate = 0 + + [[test.workload]] + testName = 'BitFlipping' + testDuration = 60.0 + percentBitFlips = 20.0 From fa3ce6d98712732b9649446a4ab0f0ff5d9e98b0 Mon Sep 17 00:00:00 2001 From: negoyal Date: Tue, 20 Jul 2021 15:28:46 -0700 Subject: [PATCH 008/338] Adding the clear range workload. --- fdbserver/CMakeLists.txt | 1 + fdbserver/workloads/BitFlipping.actor.cpp | 20 +++++++ .../workloads/ClearSingleRange.actor.cpp | 59 +++++++++++++++++++ fdbserver/workloads/DiskThrottling.actor.cpp | 20 +++++++ 4 files changed, 100 insertions(+) create mode 100644 fdbserver/workloads/ClearSingleRange.actor.cpp diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index d1686bf478..9aedb11e9a 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -157,6 +157,7 @@ set(FDBSERVER_SRCS workloads/BulkSetup.actor.h workloads/Cache.actor.cpp workloads/ChangeConfig.actor.cpp + workloads/ClearSingleRange.actor.cpp workloads/ClientTransactionProfileCorrectness.actor.cpp workloads/TriggerRecovery.actor.cpp workloads/SuspendProcesses.actor.cpp diff --git a/fdbserver/workloads/BitFlipping.actor.cpp b/fdbserver/workloads/BitFlipping.actor.cpp index ff1941fbe3..b3e2f3f235 100644 --- a/fdbserver/workloads/BitFlipping.actor.cpp +++ b/fdbserver/workloads/BitFlipping.actor.cpp @@ -1,3 +1,23 @@ +/* + * BitFlipping.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "fdbclient/NativeAPI.actor.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.actor.cpp new file mode 100644 index 0000000000..3419da80c6 --- /dev/null +++ b/fdbserver/workloads/ClearSingleRange.actor.cpp @@ -0,0 +1,59 @@ +/* + * ClearSingleRange.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/NativeAPI.actor.h" +#include "fdbserver/TesterInterface.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "fdbserver/workloads/BulkSetup.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +struct ClearSingleRange : TestWorkload { + Key begin; + Key end; + double startDelay; + + ClearSingleRange(WorkloadContext const& wcx) : TestWorkload(wcx) { + begin = getOption(options, LiteralStringRef("begin"), normalKeys.begin); + end = getOption(options, LiteralStringRef("end"), normalKeys.end); + startDelay = getOption(options, LiteralStringRef("beginClearRange"), 10.0); + } + + std::string description() const override { return "ClearSingleRangeWorkload"; } + + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { + return clientId != 0 ? Void() : fdbClientClearRange(cx, this); + } + + Future check(Database const& cx) override { return true; } + + void getMetrics(vector& m) override {} + + ACTOR static Future fdbClientClearRange(Database db, ClearSingleRange* self) { + state Transaction tr(db); + TraceEvent("ClearSingleRangeWaiting").detail("StartDelay", self->startDelay); + wait(delay(self->startDelay)); + tr.clear(KeyRangeRef(self->begin, self->end)); + return Void(); + } +}; + +WorkloadFactory ClearSingleRangeWorkloadFactory("ClearSingleRange"); diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskThrottling.actor.cpp index 8529be5fc3..5eead212f4 100644 --- a/fdbserver/workloads/DiskThrottling.actor.cpp +++ b/fdbserver/workloads/DiskThrottling.actor.cpp @@ -1,3 +1,23 @@ +/* + * DiskThrottling.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "fdbclient/NativeAPI.actor.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" From 050c218502c671bb6b32c335517a4781dd702eb8 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 28 Jul 2021 16:03:37 -0700 Subject: [PATCH 009/338] New Disk Delay Logic and ChaosMetrics. --- fdbclient/ClientWorkerInterface.h | 22 +- fdbrpc/AsyncFileChaos.actor.h | 119 +++---- fdbrpc/AsyncFileNonDurable.actor.h | 11 +- fdbrpc/IAsyncFile.h | 4 - fdbrpc/sim2.actor.cpp | 3 + fdbrpc/simulator.h | 1 + fdbserver/worker.actor.cpp | 50 +-- fdbserver/workloads/BitFlipping.actor.cpp | 259 +++++++++++---- .../workloads/ClearSingleRange.actor.cpp | 4 +- fdbserver/workloads/DiskThrottling.actor.cpp | 302 +++++++++++++----- flow/Knobs.cpp | 1 + flow/Knobs.h | 1 + flow/Net2.actor.cpp | 6 +- flow/network.h | 152 +++++---- tests/fast/DiskThrottledCycle.toml | 4 +- 15 files changed, 602 insertions(+), 337 deletions(-) diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index 9c80312708..28caa48739 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -93,23 +93,21 @@ struct ProfilerRequest { struct SetFailureInjection { constexpr static FileIdentifier file_identifier = 15439864; ReplyPromise reply; - struct ThrottleDiskCommand { - // how often should the delay be inserted (0 meaning once, 10 meaning every 10 secs) - double delayFrequency; - // min delay to be inserted - double delayMin; - //max delay to be inserted - double delayMax; + struct DiskFailureCommand { + // how often should the disk be stalled (0 meaning once, 10 meaning every 10 secs) + double stallInterval; + // Period of time disk stalls will be injected for + double stallPeriod; + // Period of time the disk will be slowed down for + double throttlePeriod; template void serialize(Ar& ar) { - serializer(ar, delayFrequency, delayMin, delayMax); + serializer(ar, stallInterval, stallPeriod, throttlePeriod); } }; struct FlipBitsCommand { - // File that the bit flips are requested for - //Reference filename; // percent of bits to flip in the given file double percentBitFlips; @@ -119,12 +117,12 @@ struct SetFailureInjection { } }; - Optional throttleDisk; + Optional diskFailure; Optional flipBits; template void serialize(Ar& ar) { - serializer(ar, reply, throttleDisk, flipBits); + serializer(ar, reply, diskFailure, flipBits); } }; #endif diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index 7890ff0b51..c7eaaefe15 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -1,5 +1,5 @@ /* - * VersionedBTree.actor.cpp + * AsyncFileChaos.actor.h * * This source file is part of the FoundationDB open source project * @@ -26,94 +26,103 @@ #include "flow/ActorCollection.h" #include "flow/actorcompiler.h" - -//template +// template class AsyncFileChaos final : public IAsyncFile, public ReferenceCounted { private: Reference file; + Arena arena; + public: explicit AsyncFileChaos(Reference file) : file(file) {} void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } - uint8_t toggleNthBit(uint8_t b, uint8_t n) { - auto singleBitMask = uint8_t(1) << (n); - return b ^ singleBitMask; - } + static double getDelay() { + double delayFor = 0.0; + auto res = g_network->global(INetwork::enDiskFailureInjector); + if (res) { + DiskFailureInjector* delayInjector = static_cast(res); + delayFor = delayInjector->getDiskDelay(); - void flipBits(void* data, int length, double percentBitFlips) { - auto toFlip = int(float(length*8) * percentBitFlips / 100); - TraceEvent("AsyncFileFlipBits").detail("ToFlip", toFlip); - for (auto i = 0; i < toFlip; i++) { - auto byteOffset = deterministicRandom()->randomInt64(0, length); - auto bitOffset = uint8_t(deterministicRandom()->randomInt(0, 8)); - ((uint8_t *)data)[byteOffset] = toggleNthBit(((uint8_t *)data)[byteOffset], bitOffset); + // increment the metric for disk delays + if (delayFor > 0.0) { + auto res = g_network->global(INetwork::enChaosMetrics); + if (res) { + ChaosMetrics* chaosMetrics = static_cast(res); + chaosMetrics->diskDelays++; + } + } } + return delayFor; } Future read(void* data, int length, int64_t offset) override { - double delay = 0.0; - auto res = g_network->global(INetwork::enDiskFailureInjector); - if (res) - delay = static_cast(res)->getDiskDelay(); - TraceEvent("AsyncFileChaosRead").detail("ThrottleDelay", delay); - return delayed(file->read(data, length, offset), delay); + double diskDelay = getDelay(); + + // Wait for diskDelay before submitting the I/O + // Template types are being provided explicitly because they can't be automatically deduced for some reason. + return mapAsync(Void)>, int>( + delay(diskDelay), [=](Void _) -> Future { return file->read(data, length, offset); }); } Future write(void const* data, int length, int64_t offset) override { - double delay = 0.0; char* pdata = nullptr; + + // Check if a bit flip event was injected, if so, copy the buffer contents + // with a random bit flipped in a new buffer and use that for the write auto res = g_network->global(INetwork::enBitFlipper); if (res) { - auto percentBitFlips = static_cast(res)->getPercentBitFlips(); - if (percentBitFlips > 0.0) { - TraceEvent("AsyncFileCorruptWrite").detail("PercentBitFlips", percentBitFlips); - pdata = new char[length]; + auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); + if (bitFlipPercentage > 0.0) { + pdata = (char*)arena.allocate4kAlignedBuffer(length); memcpy(pdata, data, length); - flipBits(pdata, length, percentBitFlips); - auto diff = memcmp(pdata, data, length); - if (diff) - TraceEvent("AsyncFileCorruptWriteDiff").detail("Diff", diff); + if (deterministicRandom()->random01() < bitFlipPercentage) { + // copy buffer with a flipped bit + pdata[deterministicRandom()->randomInt(0, length)] ^= (1 << deterministicRandom()->randomInt(0, 8)); + + // increment the metric for bit flips + auto res = g_network->global(INetwork::enChaosMetrics); + if (res) { + ChaosMetrics* chaosMetrics = static_cast(res); + chaosMetrics->bitFlips++; + } + } } } - res = g_network->global(INetwork::enDiskFailureInjector); - if (res) - delay = static_cast(res)->getDiskDelay(); - TraceEvent("AsyncFileChaosWrite").detail("ThrottleDelay", delay); - return delayed(file->write((pdata != nullptr) ? pdata : data, length, offset), delay); + double diskDelay = getDelay(); + // Wait for diskDelay before submitting the I/O + return mapAsync(Void)>, Void>(delay(diskDelay), [=](Void _) -> Future { + if (pdata) + return holdWhile(pdata, file->write(pdata, length, offset)); + + return file->write(data, length, offset); + }); } Future truncate(int64_t size) override { - double delay = 0.0; - auto res = g_network->global(INetwork::enDiskFailureInjector); - if (res) - delay = static_cast(res)->getDiskDelay(); - return delayed(file->truncate(size), delay); + double diskDelay = getDelay(); + // Wait for diskDelay before submitting the I/O + return mapAsync(Void)>, Void>( + delay(diskDelay), [=](Void _) -> Future { return file->truncate(size); }); } Future sync() override { - double delay = 0.0; - auto res = g_network->global(INetwork::enDiskFailureInjector); - if (res) - delay = static_cast(res)->getDiskDelay(); - return delayed(file->sync(), delay); + double diskDelay = getDelay(); + // Wait for diskDelay before submitting the I/O + return mapAsync(Void)>, Void>( + delay(diskDelay), [=](Void _) -> Future { return file->sync(); }); } Future size() const override { - double delay = 0.0; - auto res = g_network->global(INetwork::enDiskFailureInjector); - if (res) - delay = static_cast(res)->getDiskDelay(); - return delayed(file->size(), delay); + double diskDelay = getDelay(); + // Wait for diskDelay before submitting the I/O + return mapAsync(Void)>, int64_t>( + delay(diskDelay), [=](Void _) -> Future { return file->size(); }); } - int64_t debugFD() const override { - return file->debugFD(); - } + int64_t debugFD() const override { return file->debugFD(); } - std::string getFilename() const override { - return file->getFilename(); - } + std::string getFilename() const override { return file->getFilename(); } }; diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index 98bbe0c4e8..8489a3842d 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -191,12 +191,11 @@ private: Reference diskParameters, NetworkAddress openedAddress, bool aio) - : filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters), - openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false), - aio(aio) - { + : filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters), + openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false), + aio(aio) { - // This is only designed to work in simulation + // This is only designed to work in simulation ASSERT(g_network->isSimulated()); this->id = deterministicRandom()->randomUniqueID(); @@ -458,7 +457,7 @@ private: Future> ownFuture, void const* data, int length, - int64_t offset) { + int64_t offset) { state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess(); state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); diff --git a/fdbrpc/IAsyncFile.h b/fdbrpc/IAsyncFile.h index f21760cb00..ed703514c6 100644 --- a/fdbrpc/IAsyncFile.h +++ b/fdbrpc/IAsyncFile.h @@ -34,7 +34,6 @@ // must complete or cancel, but you should probably look at the file implementations you'll be using. class IAsyncFile { public: - //explicit IAsyncFile() : diskFailureInjector(DiskFailureInjector::injector()) {} virtual ~IAsyncFile(); // Pass these to g_network->open to get an IAsyncFile enum { @@ -96,9 +95,6 @@ public: // Used for rate control, at present, only AsyncFileCached supports it virtual Reference const& getRateControl() { throw unsupported_operation(); } virtual void setRateControl(Reference const& rc) { throw unsupported_operation(); } - -//public: - //DiskFailureInjector* diskFailureInjector; }; typedef void (*runCycleFuncPtr)(); diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 4051f935a0..94b78f1ede 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -1186,6 +1186,9 @@ public: m->protocolVersion = protocol; m->setGlobal(enTDMetrics, (flowGlobalType)&m->tdmetrics); + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { + m->setGlobal(enChaosMetrics, (flowGlobalType)&m->chaosMetrics); + } m->setGlobal(enNetworkConnections, (flowGlobalType)m->network); m->setGlobal(enASIOTimedOut, (flowGlobalType) false); diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 6404eafc17..764b8b125b 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -73,6 +73,7 @@ public: LocalityData locality; ProcessClass startingClass; TDMetricCollection tdmetrics; + ChaosMetrics chaosMetrics; HistogramRegistry histograms; std::map> listenerMap; std::map> boundUDPSockets; diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 1c5bf22408..ba57c9505c 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1165,6 +1165,28 @@ struct SharedLogsValue { : actor(actor), uid(uid), requests(requests) {} }; +ACTOR Future chaosMetricsLogger() { + + auto res = g_network->global(INetwork::enChaosMetrics); + if (!res) + return Void(); + + state ChaosMetrics* chaosMetrics = static_cast(res); + chaosMetrics->clear(); + + loop { + wait(delay(FLOW_KNOBS->CHAOS_LOGGING_INTERVAL)); + + TraceEvent e("ChaosMetrics"); + // double elapsed = now() - chaosMetrics->startTime; + double elapsed = timer_monotonic() - chaosMetrics->startTime; + e.detail("Elapsed", elapsed); + chaosMetrics->getFields(&e); + e.trackLatest("ChaosMetrics"); + chaosMetrics->clear(); + } +} + ACTOR Future workerServer(Reference connFile, Reference>> ccInterface, LocalityData locality, @@ -1191,6 +1213,7 @@ ACTOR Future workerServer(Reference connFile, state Promise stopping; state WorkerCache storageCache; state Future metricsLogger; + state Future chaosMetricsActor; state Reference> degraded = FlowTransport::transport().getDegraded(); // tLogFnForOptions() can return a function that doesn't correspond with the FDB version that the // TLogVersion represents. This can be done if the newer TLog doesn't support a requested option. @@ -1211,6 +1234,7 @@ ACTOR Future workerServer(Reference connFile, if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { TraceEvent(SevWarnAlways, "ChaosFeaturesEnabled"); + chaosMetricsActor = chaosMetricsLogger(); } folder = abspath(folder); @@ -1436,15 +1460,8 @@ ACTOR Future workerServer(Reference connFile, wait(waitForAll(recoveries)); recoveredDiskFiles.send(Void()); - errorForwarders.add(registrationClient(ccInterface, - interf, - asyncPriorityInfo, - initialClass, - ddInterf, - rkInterf, - degraded, - connFile, - issues)); + errorForwarders.add(registrationClient( + ccInterface, interf, asyncPriorityInfo, initialClass, ddInterf, rkInterf, degraded, connFile, issues)); if (SERVER_KNOBS->ENABLE_WORKER_HEALTH_MONITOR) { errorForwarders.add(healthMonitor(ccInterface, interf, locality, dbInfo)); @@ -1515,19 +1532,14 @@ ACTOR Future workerServer(Reference connFile, } when(SetFailureInjection req = waitNext(interf.clientInterface.setFailureInjection.getFuture())) { if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { - if (req.throttleDisk.present()) { - TraceEvent("DiskThrottleRequest").detail("DelayFrequency",req.throttleDisk.get().delayFrequency). - detail("DelayMin", req.throttleDisk.get().delayMin). - detail("DelayMax", req.throttleDisk.get().delayMax); + if (req.diskFailure.present()) { auto diskFailureInjector = DiskFailureInjector::injector(); - diskFailureInjector->throttleFor(req.throttleDisk.get().delayFrequency, - req.throttleDisk.get().delayMin, - req.throttleDisk.get().delayMax); + diskFailureInjector->setDiskFailure(req.diskFailure.get().stallInterval, + req.diskFailure.get().stallPeriod, + req.diskFailure.get().throttlePeriod); } else if (req.flipBits.present()) { - TraceEvent("FlipBitsRequest"). - detail("Percent", req.flipBits.get().percentBitFlips); auto bitFlipper = BitFlipper::flipper(); - bitFlipper->setPercentBitFlips(req.flipBits.get().percentBitFlips); + bitFlipper->setBitFlipPercentage(req.flipBits.get().percentBitFlips); } req.reply.send(Void()); } else { diff --git a/fdbserver/workloads/BitFlipping.actor.cpp b/fdbserver/workloads/BitFlipping.actor.cpp index b3e2f3f235..8dd8781b6f 100644 --- a/fdbserver/workloads/BitFlipping.actor.cpp +++ b/fdbserver/workloads/BitFlipping.actor.cpp @@ -28,85 +28,202 @@ #include "flow/actorcompiler.h" // This must be the last #include. struct BitFlippingWorkload : TestWorkload { - bool enabled; - double testDuration; - double percentBitFlips; - BitFlippingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - enabled = !clientId; // only do this on the "first" client - testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); - percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 1.0); - TraceEvent("BitFlippingWorkload") - .detail("TestDuration", testDuration).detail("Percentage", percentBitFlips); - } + bool enabled; + double testDuration; + double percentBitFlips; + double periodicCheckInterval; + std::vector chosenWorkers; + std::vector> clients; - std::string description() const override { - if (&g_simulator == g_network) - return "BitFlipping"; - else - return "NoSimBitFlipping"; - } + BitFlippingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + enabled = !clientId; // only do this on the "first" client + testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); + percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 10.0); + periodicCheckInterval = getOption(options, LiteralStringRef("periodicCheckInterval"), 10.0); + } - Future setup(Database const& cx) override { return Void(); } + std::string description() const override { + if (&g_simulator == g_network) + return "BitFlipping"; + else + return "NoSimBitFlipping"; + } - Future start(Database const& cx) override { - if (enabled) { - return timeout(reportErrors(flipBitsClient(cx, this), "BitFlippingError"), - testDuration, - Void()); - } else - return Void(); - } + Future setup(Database const& cx) override { return Void(); } - Future check(Database const& cx) override { return true; } + // Starts the workload by - + // 1. Starting the actor to periodically check chaosMetrics, and + // 2. Starting the actor that injects failures on chosen storage servers + Future start(Database const& cx) override { + if (enabled) { + clients.push_back(periodicMetricCheck(this)); + clients.push_back(flipBitsClient(cx, this)); + return timeout(waitForAll(clients), testDuration, Void()); + } else + return Void(); + } - void getMetrics(vector& m) override {} + Future check(Database const& cx) override { return true; } - static void checkBitFlipResult(Future res, WorkerInterface worker) { - if (res.isError()) { - auto err = res.getError(); - if (err.code() == error_code_client_invalid_operation) { - TraceEvent(SevError, "ChaosDisabled") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); - } else { - TraceEvent(SevError, "BitFlippingFailed") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) - .error(err); - } - } - } + void getMetrics(vector& m) override {} - ACTOR void doBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { - state Future res; - wait(::delay(startDelay)); - SetFailureInjection::FlipBitsCommand flipBits; - flipBits.percentBitFlips = percentage; - SetFailureInjection req; - req.flipBits = flipBits; - res = worker.clientInterface.setFailureInjection.getReply(req); - wait(ready(res)); - checkBitFlipResult(res, worker); - } + static void checkBitFlipResult(Future res, WorkerInterface worker) { + if (res.isError()) { + auto err = res.getError(); + if (err.code() == error_code_client_invalid_operation) { + TraceEvent(SevError, "ChaosDisabled") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); + } else { + TraceEvent(SevError, "BitFlippingFailed") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) + .error(err); + } + } + } - ACTOR static Future getAllStorageWorkers(Database cx, BitFlippingWorkload* self, std::vector* result) { - result->clear(); - state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); - for (auto& worker : res) { - result->emplace_back(worker); - } - return Void(); - } + ACTOR void doBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { + state Future res; + wait(::delay(startDelay)); + SetFailureInjection::FlipBitsCommand flipBits; + flipBits.percentBitFlips = percentage; + SetFailureInjection req; + req.flipBits = flipBits; + res = worker.clientInterface.setFailureInjection.getReply(req); + wait(ready(res)); + checkBitFlipResult(res, worker); + } - ACTOR template - Future flipBitsClient(Database cx, BitFlippingWorkload* self) { - state double lastTime = now(); - state double workloadEnd = now() + self->testDuration; - state std::vector machines; - loop { - wait(poisson(&lastTime, 1)); - wait(BitFlippingWorkload::getAllStorageWorkers(cx, self, &machines)); - auto machine = deterministicRandom()->randomChoice(machines); - self->doBitFlips(machine, self->percentBitFlips); - } - } + ACTOR static Future getAllStorageWorkers(Database cx, + BitFlippingWorkload* self, + std::vector* result) { + result->clear(); + state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); + for (auto& worker : res) { + result->emplace_back(worker); + } + return Void(); + } + + ACTOR template + Future flipBitsClient(Database cx, BitFlippingWorkload* self) { + state double lastTime = now(); + state double workloadEnd = now() + self->testDuration; + state std::vector machines; + loop { + wait(poisson(&lastTime, 1)); + wait(BitFlippingWorkload::getAllStorageWorkers(cx, self, &machines)); + auto machine = deterministicRandom()->randomChoice(machines); + + // If we have already chosen this worker, then just continue + if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) != + self->chosenWorkers.end()) + continue; + + // Keep track of chosen workers for verification purpose + self->chosenWorkers.emplace_back(machine.address()); + self->doBitFlips(machine, self->percentBitFlips); + } + } + + // Resend the chaos event to previosuly chosen workers, in case some workers got restarted and lost their chaos + // config + ACTOR static Future reSendChaos(BitFlippingWorkload* self) { + std::vector workers = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + std::map workersMap; + for (auto worker : workers) { + workersMap[worker.interf.address()] = worker.interf; + } + for (auto& workerAddress : self->chosenWorkers) { + auto itr = workersMap.find(workerAddress); + if (itr != workersMap.end()) + self->doBitFlips(itr->second, self->percentBitFlips); + } + return Void(); + } + // For fetching chaosMetrics to ensure chaos events are happening + // This is borrowed code from Status.actor.cpp + struct WorkerEvents : std::map {}; + + ACTOR static Future>>> latestEventOnWorkers( + std::vector workers, + std::string eventName) { + try { + state vector>> eventTraces; + for (int c = 0; c < workers.size(); c++) { + EventLogRequest req = + eventName.size() > 0 ? EventLogRequest(Standalone(eventName)) : EventLogRequest(); + eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0))); + } + + wait(waitForAll(eventTraces)); + + std::set failed; + WorkerEvents results; + + for (int i = 0; i < eventTraces.size(); i++) { + const ErrorOr& v = eventTraces[i].get(); + if (v.isError()) { + failed.insert(workers[i].interf.address().toString()); + results[workers[i].interf.address()] = TraceEventFields(); + } else { + results[workers[i].interf.address()] = v.get(); + } + } + + std::pair> val; + val.first = results; + val.second = failed; + + return val; + } catch (Error& e) { + ASSERT(e.code() == + error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above + throw; + } + } + + // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers + ACTOR static Future chaosGetStatus(BitFlippingWorkload* self) { + std::vector workers = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + + Future>>> latestEventsFuture; + latestEventsFuture = latestEventOnWorkers(workers, "ChaosMetrics"); + state Optional>> workerEvents = wait(latestEventsFuture); + + state WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents(); + + // Now verify that all chosen workers for chaos events have non-zero chaosMetrics + for (auto& workerAddress : self->chosenWorkers) { + auto chaosMetrics = cMetrics.find(workerAddress); + if (chaosMetrics != cMetrics.end()) { + int bitFlips = chaosMetrics->second.getInt("BitFlips"); + + // we expect bitFlips to be non-zero for chosenWorkers + if (bitFlips == 0) { + TraceEvent(SevError, "ChaosGetStatus") + .detail("OnEndpoint", workerAddress.toString()) + .detail("BitFlips", bitFlips); + } + } + } + + return Void(); + } + + // Periodically fetches chaosMetrics to ensure that chaas events are taking place + ACTOR static Future periodicMetricCheck(BitFlippingWorkload* self) { + state double start = now(); + state double elapsed = 0.0; + + loop { + // re-send the chaos event in case of a process restart + wait(reSendChaos(self)); + elapsed += self->periodicCheckInterval; + wait(delayUntil(start + elapsed)); + wait(chaosGetStatus(self)); + } + } }; WorkloadFactory BitFlippingWorkloadFactory("BitFlipping"); diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.actor.cpp index 3419da80c6..f8f48be929 100644 --- a/fdbserver/workloads/ClearSingleRange.actor.cpp +++ b/fdbserver/workloads/ClearSingleRange.actor.cpp @@ -39,9 +39,7 @@ struct ClearSingleRange : TestWorkload { Future setup(Database const& cx) override { return Void(); } - Future start(Database const& cx) override { - return clientId != 0 ? Void() : fdbClientClearRange(cx, this); - } + Future start(Database const& cx) override { return clientId != 0 ? Void() : fdbClientClearRange(cx, this); } Future check(Database const& cx) override { return true; } diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskThrottling.actor.cpp index 5eead212f4..264ef477a2 100644 --- a/fdbserver/workloads/DiskThrottling.actor.cpp +++ b/fdbserver/workloads/DiskThrottling.actor.cpp @@ -28,102 +28,230 @@ #include "flow/actorcompiler.h" // This must be the last #include. struct DiskThrottlingWorkload : TestWorkload { - bool enabled; - double testDuration; - double throttleFrequency; - double throttleMin; - double throttleMax; - DiskThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - enabled = !clientId; // only do this on the "first" client - testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); - throttleFrequency = getOption(options, LiteralStringRef("throttleFrequency"), 0.0); - throttleMin = getOption(options, LiteralStringRef("throttleMin"), 2.0); - throttleMax = getOption(options, LiteralStringRef("throttleMax"), 2.0); - TraceEvent("DiskThrottlingWorkload") - .detail("TestDuration", testDuration).detail("Frequency", throttleFrequency) - .detail("Min", throttleMin).detail("Max", throttleMax); - } + bool enabled; + double testDuration; + double startDelay; + double stallInterval; + double stallPeriod; + double throttlePeriod; + double periodicCheckInterval; + std::vector chosenWorkers; + std::vector> clients; - std::string description() const override { - if (&g_simulator == g_network) - return "DiskThrottling"; - else - return "NoSimDiskThrolling"; - } + DiskThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + enabled = !clientId; // only do this on the "first" client + startDelay = getOption(options, LiteralStringRef("startDelay"), 0.0); + testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0); + stallInterval = getOption(options, LiteralStringRef("stallInterval"), 0.0); + stallPeriod = getOption(options, LiteralStringRef("stallPeriod"), 60.0); + throttlePeriod = getOption(options, LiteralStringRef("throttlePeriod"), 60.0); + periodicCheckInterval = getOption(options, LiteralStringRef("periodicCheckInterval"), 10.0); + } - Future setup(Database const& cx) override { return Void(); } + std::string description() const override { + if (&g_simulator == g_network) + return "DiskThrottling"; + else + return "NoSimDiskThrolling"; + } - Future start(Database const& cx) override { - if (enabled) { - return timeout(reportErrors(throttleDiskClient(cx, this), "DiskThrottlingError"), - testDuration, - Void()); - } else - return Void(); - } + Future setup(Database const& cx) override { return Void(); } - Future check(Database const& cx) override { return true; } + // Starts the workload by - + // 1. Starting the actor to periodically check chaosMetrics, and + // 2. Starting the actor that injects failures on chosen storage servers + Future start(Database const& cx) override { + if (enabled) { + clients.push_back(periodicMetricCheck(this)); + clients.push_back(throttleDiskClient(cx, this)); + return timeout(waitForAll(clients), testDuration, Void()); + } else + return Void(); + } - void getMetrics(vector& m) override {} + Future check(Database const& cx) override { return true; } - static void checkDiskThrottleResult(Future res, WorkerInterface worker) { - if (res.isError()) { - auto err = res.getError(); - if (err.code() == error_code_client_invalid_operation) { - TraceEvent(SevError, "ChaosDisabled") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); - } else { - TraceEvent(SevError, "DiskThrottlingFailed") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) - .error(err); - } - } - } + void getMetrics(vector& m) override {} - ACTOR void doThrottle(WorkerInterface worker, double frequency, double minDelay, double maxDelay, double startDelay = 0.0) { - state Future res; - wait(::delay(startDelay)); - SetFailureInjection::ThrottleDiskCommand throttleDisk; - throttleDisk.delayFrequency = frequency; - throttleDisk.delayMin = minDelay; - throttleDisk.delayMax = maxDelay; - SetFailureInjection req; - req.throttleDisk = throttleDisk; - res = worker.clientInterface.setFailureInjection.getReply(req); - wait(ready(res)); - checkDiskThrottleResult(res, worker); - } + static void checkDiskThrottleResult(Future res, WorkerInterface worker) { + if (res.isError()) { + auto err = res.getError(); + if (err.code() == error_code_client_invalid_operation) { + TraceEvent(SevError, "ChaosDisabled") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); + } else { + TraceEvent(SevError, "DiskThrottlingFailed") + .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) + .error(err); + } + } + } - ACTOR static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { - result->clear(); - std::vector res = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); - for (auto& worker : res) { - result->emplace_back(worker.interf); - } - return Void(); - } + // Sets the disk failure request + ACTOR void doThrottle(WorkerInterface worker, + double stallInterval, + double stallPeriod, + double throttlePeriod, + double startDelay) { + state Future res; + wait(::delay(startDelay)); + SetFailureInjection::DiskFailureCommand diskFailure; + diskFailure.stallInterval = stallInterval; + diskFailure.stallPeriod = stallPeriod; + diskFailure.throttlePeriod = throttlePeriod; + SetFailureInjection req; + req.diskFailure = diskFailure; + res = worker.clientInterface.setFailureInjection.getReply(req); + wait(ready(res)); + checkDiskThrottleResult(res, worker); + } - ACTOR static Future getAllStorageWorkers(Database cx, DiskThrottlingWorkload* self, std::vector* result) { - result->clear(); - state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); - for (auto& worker : res) { - result->emplace_back(worker); - } - return Void(); - } + // Currently unused, because we only inject disk failures on storage servers + ACTOR static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { + result->clear(); + std::vector res = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + for (auto& worker : res) { + result->emplace_back(worker.interf); + } + return Void(); + } - ACTOR template - Future throttleDiskClient(Database cx, DiskThrottlingWorkload* self) { - state double lastTime = now(); - state double workloadEnd = now() + self->testDuration; - state std::vector machines; - loop { - wait(poisson(&lastTime, 1)); - wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); - auto machine = deterministicRandom()->randomChoice(machines); - self->doThrottle(machine, self->throttleFrequency, self->throttleMin, self->throttleMax); - } - } + ACTOR static Future getAllStorageWorkers(Database cx, + DiskThrottlingWorkload* self, + std::vector* result) { + result->clear(); + state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); + for (auto& worker : res) { + result->emplace_back(worker); + } + return Void(); + } + + // Choose random storage servers to inject disk failures + ACTOR template + Future throttleDiskClient(Database cx, DiskThrottlingWorkload* self) { + state double lastTime = now(); + state std::vector machines; + loop { + wait(poisson(&lastTime, 1)); + wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); + auto machine = deterministicRandom()->randomChoice(machines); + + // If we have already chosen this worker, then just continue + if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) != + self->chosenWorkers.end()) + continue; + + // Keep track of chosen workers for verification purpose + self->chosenWorkers.emplace_back(machine.address()); + self->doThrottle(machine, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + } + } + + // Resend the chaos event to previosuly chosen workers, in case some workers got restarted and lost their chaos + // config + ACTOR static Future reSendChaos(DiskThrottlingWorkload* self) { + std::vector workers = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + std::map workersMap; + for (auto worker : workers) { + workersMap[worker.interf.address()] = worker.interf; + } + for (auto& workerAddress : self->chosenWorkers) { + auto itr = workersMap.find(workerAddress); + if (itr != workersMap.end()) + self->doThrottle( + itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + } + return Void(); + } + + // For fetching chaosMetrics to ensure chaos events are happening + // This is borrowed code from Status.actor.cpp + struct WorkerEvents : std::map {}; + + ACTOR static Future>>> latestEventOnWorkers( + std::vector workers, + std::string eventName) { + try { + state vector>> eventTraces; + for (int c = 0; c < workers.size(); c++) { + EventLogRequest req = + eventName.size() > 0 ? EventLogRequest(Standalone(eventName)) : EventLogRequest(); + eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0))); + } + + wait(waitForAll(eventTraces)); + + std::set failed; + WorkerEvents results; + + for (int i = 0; i < eventTraces.size(); i++) { + const ErrorOr& v = eventTraces[i].get(); + if (v.isError()) { + failed.insert(workers[i].interf.address().toString()); + results[workers[i].interf.address()] = TraceEventFields(); + } else { + results[workers[i].interf.address()] = v.get(); + } + } + + std::pair> val; + val.first = results; + val.second = failed; + + return val; + } catch (Error& e) { + ASSERT(e.code() == + error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above + throw; + } + } + + // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers + ACTOR static Future chaosGetStatus(DiskThrottlingWorkload* self) { + std::vector workers = + wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + + Future>>> latestEventsFuture; + latestEventsFuture = latestEventOnWorkers(workers, "ChaosMetrics"); + state Optional>> workerEvents = wait(latestEventsFuture); + + state WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents(); + + // Now verify that all chosen workers for chaos events have non-zero chaosMetrics + std::vector>>>> futures; + + for (auto& workerAddress : self->chosenWorkers) { + auto chaosMetrics = cMetrics.find(workerAddress); + if (chaosMetrics != cMetrics.end()) { + int diskDelays = chaosMetrics->second.getInt("DiskDelays"); + + // we expect diskDelays to be non-zero for chosenWorkers + if (diskDelays == 0) { + TraceEvent(SevError, "ChaosGetStatus") + .detail("OnEndpoint", workerAddress.toString()) + .detail("DiskDelays", diskDelays); + } + } + } + + return Void(); + } + + // Periodically fetches chaosMetrics to ensure that chaas events are taking place + ACTOR static Future periodicMetricCheck(DiskThrottlingWorkload* self) { + state double start = now(); + state double elapsed = 0.0; + + loop { + // re-send the chaos event in case of a process restart + wait(reSendChaos(self)); + elapsed += self->periodicCheckInterval; + wait(delayUntil(start + elapsed)); + wait(chaosGetStatus(self)); + } + } }; WorkloadFactory DiskThrottlingWorkloadFactory("DiskThrottling"); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 12bc0d70c9..322eb5d52d 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -66,6 +66,7 @@ void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { // Chaos testing init( ENABLE_CHAOS_FEATURES, true ); + init( CHAOS_LOGGING_INTERVAL, 5.0 ); init( WRITE_TRACING_ENABLED, true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false; diff --git a/flow/Knobs.h b/flow/Knobs.h index 340848b68f..a6f9006c12 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -100,6 +100,7 @@ public: // Chaos testing bool ENABLE_CHAOS_FEATURES; + double CHAOS_LOGGING_INTERVAL; bool WRITE_TRACING_ENABLED; int TRACING_UDP_LISTENER_PORT; diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index c3b35f1203..85aea2c5f9 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -226,6 +226,7 @@ public: TaskPriority currentTaskID; uint64_t tasksIssued; TDMetricCollection tdmetrics; + ChaosMetrics chaosMetrics; double currentTime; // May be accessed off the network thread, e.g. by onMainThread std::atomic stopped; @@ -1188,6 +1189,9 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) if (useMetrics) { setGlobal(INetwork::enTDMetrics, (flowGlobalType)&tdmetrics); } + if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { + setGlobal(INetwork::enChaosMetrics, (flowGlobalType)&chaosMetrics); + } setGlobal(INetwork::enNetworkConnections, (flowGlobalType)network); setGlobal(INetwork::enASIOService, (flowGlobalType)&reactor.ios); setGlobal(INetwork::enBlobCredentialFiles, &blobCredentialFiles); @@ -1513,7 +1517,7 @@ void Net2::run() { double newTaskBegin = timer_monotonic(); if (check_yield(TaskPriority::Max, tscNow)) { checkForSlowTask(tscBegin, tscNow, newTaskBegin - taskBegin, currentTaskID); - taskBegin = newTaskBegin; + taskBegin = newTaskBegin; FDB_TRACE_PROBE(run_loop_yield); ++countYields; break; diff --git a/flow/network.h b/flow/network.h index 0ce243e9f2..ec9923052b 100644 --- a/flow/network.h +++ b/flow/network.h @@ -347,7 +347,8 @@ struct NetworkMetrics { std::unordered_map activeTrackers; double lastRunLoopBusyness; // network thread busyness (measured every 5s by default) - std::atomic networkBusyness; // network thread busyness which is returned to the the client (measured every 1s by default) + std::atomic + networkBusyness; // network thread busyness which is returned to the the client (measured every 1s by default) // starvation trackers which keeps track of different task priorities std::vector starvationTrackers; @@ -487,8 +488,9 @@ public: enClientFailureMonitor = 12, enSQLiteInjectedError = 13, enGlobalConfig = 14, - enDiskFailureInjector = 15, - enBitFlipper = 16 + enChaosMetrics = 15, + enDiskFailureInjector = 16, + enBitFlipper = 17 }; virtual void longTaskCheck(const char* name) {} @@ -648,45 +650,40 @@ public: // Returns the interface that should be used to make and accept socket connections }; -struct DelayGenerator : FastAllocated { +// Chaos Metrics - We periodically log chaosMetrics to make sure that chaos events are happening +// Only includes DiskDelays which encapsulates all type delays and BitFlips for now +// Expand as per need +struct ChaosMetrics { - void setDelay(double frequency, double min, double max) { - delayFrequency = frequency; - delayMin = min; - delayMax = max; - delayFor = (delayMin == delayMax) ? delayMin : deterministicRandom()->randomInt(delayMin, delayMax); - delayUntil = std::max(delayUntil, timer_monotonic() + delayFor); - TraceEvent("DelayGeneratorSetDelay").detail("DelayFrequency", frequency).detail("DelayMin", min). - detail("DelayMax", max).detail("DelayFor", delayFor).detail("DelayUntil", delayUntil); + ChaosMetrics() { clear(); } + + void clear() { + memset(this, 0, sizeof(ChaosMetrics)); + startTime = timer_monotonic(); } - - double getDelay() { - // If a delayFrequency was specified, this logic determins the delay to be inserted at any point in time - if (delayFrequency) { - auto timeElapsed = fmod(timer_monotonic(), delayFrequency); - TraceEvent("DelayGeneratorGetDelay").detail("DelayFrequency", delayFrequency). - detail("TimeElapsed", timeElapsed).detail("DelayFor", delayFor); - return std::max(0.0, delayFor - timeElapsed); + + unsigned int diskDelays; + unsigned int bitFlips; + double startTime; + + void getFields(TraceEvent* e) { + std::pair metrics[] = { { "DiskDelays", diskDelays }, { "BitFlips", bitFlips } }; + if (e != nullptr) { + for (auto& m : metrics) { + char c = m.first[0]; + if (c != 0) { + e->detail(m.first, m.second); + } + } } - TraceEvent("DelayGeneratorGetDelay").detail("DelayFrequency", delayFrequency). - detail("CurTime", timer_monotonic()).detail("DelayUntil", delayUntil); - return std::max(0.0, delayUntil - timer_monotonic()); } - -private: //members - double delayFrequency = 0.0; // how often should the delay be inserted (0 meaning once, 10 meaning every 10 secs) - double delayMin; // min delay to be inserted - double delayMax; // max delay to be inserted - double delayFor = 0.0; // randomly chosen delay between min and max - double delayUntil = 0.0; // used when the delayFrequency is 0 - -public: // construction - DelayGenerator() = default; - DelayGenerator(DelayGenerator const&) = delete; - }; -struct DiskFailureInjector : FastAllocated { +// This class supports injecting two type of disk failures +// 1. Stalls: Every interval seconds, the disk will stall and no IO will complete for x seconds, where x is a randomly +// chosen interval +// 2. Slowdown: Random slowdown is injected to each disk operation for specified period of time +struct DiskFailureInjector { static DiskFailureInjector* injector() { auto res = g_network->global(INetwork::enDiskFailureInjector); if (!res) { @@ -696,26 +693,56 @@ struct DiskFailureInjector : FastAllocated { return static_cast(res); } - void throttleFor(double frequency, double delayMin, double delayMax) { - delayGenerator.setDelay(frequency, delayMin, delayMax); + void setDiskFailure(double interval, double stallFor, double throttleFor) { + stallInterval = interval; + stallPeriod = stallFor; + stallUntil = std::max(stallUntil, timer_monotonic() + stallFor); + // random stall duration in ms (chosen once) + stallDuration = 0.001 * deterministicRandom()->randomInt(1, 5); + throttlePeriod = throttleFor; + throttleUntil = std::max(throttleUntil, timer_monotonic() + throttleFor); + TraceEvent("SetDiskFailure") + .detail("StallInterval", interval) + .detail("StallPeriod", stallFor) + .detail("StallUntil", stallUntil) + .detail("ThrottlePeriod", throttleFor) + .detail("ThrottleUntil", throttleUntil); } - double getDiskDelay() { - if (!FLOW_KNOBS->ENABLE_CHAOS_FEATURES) { - return 0.0; + double getStallDelay() { + // If we are in a stall period and a stallInterval was specified, determine the + // delay to be inserted + if (((stallUntil - timer_monotonic()) > 0.0) && stallInterval) { + auto timeElapsed = fmod(timer_monotonic(), stallInterval); + return std::max(0.0, stallDuration - timeElapsed); } - return delayGenerator.getDelay(); + return 0.0; } + double getThrottleDelay() { + // If we are in the throttle period, insert a random delay (in ms) + if ((throttleUntil - timer_monotonic()) > 0.0) + return (0.001 * deterministicRandom()->randomInt(1, 3)); + + return 0.0; + } + + double getDiskDelay() { return getStallDelay() + getThrottleDelay(); } + private: // members - DelayGenerator delayGenerator; + double stallInterval = 0.0; // how often should the disk be stalled (0 meaning once, 10 meaning every 10 secs) + double stallPeriod; // Period of time disk stalls will be injected for + double stallUntil; // End of disk stall period + double stallDuration; // Duration of each stall + double throttlePeriod; // Period of time the disk will be slowed down for + double throttleUntil; // End of disk slowdown period private: // construction DiskFailureInjector() = default; DiskFailureInjector(DiskFailureInjector const&) = delete; }; -struct BitFlipper : FastAllocated { +struct BitFlipper { static BitFlipper* flipper() { auto res = g_network->global(INetwork::enBitFlipper); if (!res) { @@ -725,43 +752,12 @@ struct BitFlipper : FastAllocated { return static_cast(res); } - //uint8_t toggleNthBit(uint8_t b, uint8_t n) { - // auto singleBitMask = uint8(1) << (n); - // return b ^ singleBitMask; - //} + double getBitFlipPercentage() { return bitFlipPercentage; } - //void flipBitAtOffset(int64_t byteOffset, uint8_t bitOffset) { - //auto oneByte = make([]byte, 1); - // uint8_t oneByte[1]; - // int readBytes = wait(file->Read(oneByte, 1, byteOffset)); - - // oneByte[0] = toggleNthBit(oneByte[0], bitOffset); - // file->write(oneByte, 1, byteOffset); - //} - - //void flipBits(Reference fileName, double percent) { - // file = fileName; - // auto toFlip = int(float64(file->size()*8) * percent / 100); - // for (auto i = 0; i < toFlip; i++) { - // auto byteOffset = deterministicRandom()->randomInt64(0, file->size()); - // auto bitOffset = uint8_t(deterministicRandom()->randomInt(0, 8)); - // flipBitAtOffset(byteOffset, bitOffset); - // } - //} - - double getPercentBitFlips() { - TraceEvent("BitFlipperGetPercentBitFlips").detail("PercentBitFlips", percentBitFlips); - return percentBitFlips; - } - - void setPercentBitFlips(double percentFlips) { - percentBitFlips = percentFlips; - TraceEvent("BitFlipperSetPercentBitFlips").detail("PercentBitFlips", percentBitFlips); - } + void setBitFlipPercentage(double percentage) { bitFlipPercentage = percentage; } private: // members - double percentBitFlips = 0.0; - //Reference file; + double bitFlipPercentage = 0.0; private: // construction BitFlipper() = default; diff --git a/tests/fast/DiskThrottledCycle.toml b/tests/fast/DiskThrottledCycle.toml index 60429350b6..83df7fdb1d 100644 --- a/tests/fast/DiskThrottledCycle.toml +++ b/tests/fast/DiskThrottledCycle.toml @@ -10,5 +10,7 @@ testTitle = 'DiskThrottledCycle' [[test.workload]] testName = 'DiskThrottling' testDuration = 30.0 - throttleFrequency = 10.0 + stallInterval = 10.0 + stallPeriod = 30.0 + throttlePeriod = 30.0 From 4b8771647555bf3f0715d3d4cd39e70c8f8011e2 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 28 Jul 2021 18:19:55 -0700 Subject: [PATCH 010/338] Turn the chaos knob off by default. --- fdbrpc/AsyncFileNonDurable.actor.h | 1 - flow/Knobs.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index 8489a3842d..2a74908517 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -31,7 +31,6 @@ #include "flow/flow.h" #include "fdbrpc/IAsyncFile.h" #include "flow/ActorCollection.h" -#include "flow/network.h" #include "fdbrpc/simulator.h" #include "fdbrpc/TraceFileIO.h" #include "fdbrpc/RangeMap.h" diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index ec1f041c18..5233f65757 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -67,7 +67,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 ); // Chaos testing - init( ENABLE_CHAOS_FEATURES, true ); + init( ENABLE_CHAOS_FEATURES, false ); init( CHAOS_LOGGING_INTERVAL, 5.0 ); From 9e7197faba0378bf9120dbbeac3d1c85a9b2478c Mon Sep 17 00:00:00 2001 From: negoyal Date: Fri, 30 Jul 2021 01:32:43 -0700 Subject: [PATCH 011/338] Bunch of changes based on review comments and discussions. --- fdbrpc/AsyncFileChaos.actor.h | 40 ++- fdbrpc/simulator.h | 8 + fdbserver/CMakeLists.txt | 3 +- fdbserver/Status.actor.cpp | 3 +- fdbserver/Status.h | 4 + fdbserver/VFSAsync.h | 3 +- fdbserver/VersionedBTree.actor.cpp | 3 + fdbserver/worker.actor.cpp | 10 +- fdbserver/workloads/BitFlipping.actor.cpp | 229 ------------------ ...tor.cpp => DiskFailureInjection.actor.cpp} | 208 ++++++++-------- flow/Knobs.cpp | 6 +- flow/network.h | 12 +- tests/CMakeLists.txt | 3 +- tests/fast/BitFlippedCycle.toml | 13 - tests/fast/DiskThrottledCycle.toml | 16 -- tests/slow/DiskFailureCycle.toml | 30 +++ 16 files changed, 199 insertions(+), 392 deletions(-) delete mode 100644 fdbserver/workloads/BitFlipping.actor.cpp rename fdbserver/workloads/{DiskThrottling.actor.cpp => DiskFailureInjection.actor.cpp} (50%) delete mode 100644 tests/fast/BitFlippedCycle.toml delete mode 100644 tests/fast/DiskThrottledCycle.toml create mode 100644 tests/slow/DiskFailureCycle.toml diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index c7eaaefe15..11b60f6692 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -30,16 +30,22 @@ class AsyncFileChaos final : public IAsyncFile, public ReferenceCounted { private: Reference file; - Arena arena; + bool enabled; public: - explicit AsyncFileChaos(Reference file) : file(file) {} + explicit AsyncFileChaos(Reference file) : file(file) { + // We onlyl allow chaod events on storage files + enabled = StringRef(file->getFilename()).startsWith(LiteralStringRef("storage-")); + } void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } - static double getDelay() { + double getDelay() const { double delayFor = 0.0; + if (!enabled) + return delayFor; + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) { DiskFailureInjector* delayInjector = static_cast(res); @@ -60,6 +66,9 @@ public: Future read(void* data, int length, int64_t offset) override { double diskDelay = getDelay(); + if (diskDelay == 0.0) + return file->read(data, length, offset); + // Wait for diskDelay before submitting the I/O // Template types are being provided explicitly because they can't be automatically deduced for some reason. return mapAsync(Void)>, int>( @@ -67,18 +76,19 @@ public: } Future write(void const* data, int length, int64_t offset) override { + Arena arena; char* pdata = nullptr; // Check if a bit flip event was injected, if so, copy the buffer contents // with a random bit flipped in a new buffer and use that for the write auto res = g_network->global(INetwork::enBitFlipper); - if (res) { + if (enabled && res) { auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); if (bitFlipPercentage > 0.0) { - pdata = (char*)arena.allocate4kAlignedBuffer(length); - memcpy(pdata, data, length); if (deterministicRandom()->random01() < bitFlipPercentage) { - // copy buffer with a flipped bit + pdata = (char*)arena.allocate4kAlignedBuffer(length); + memcpy(pdata, data, length); + // flip a random bit in the copied buffer pdata[deterministicRandom()->randomInt(0, length)] ^= (1 << deterministicRandom()->randomInt(0, 8)); // increment the metric for bit flips @@ -92,6 +102,13 @@ public: } double diskDelay = getDelay(); + if (diskDelay == 0.0) { + if (pdata) + return holdWhile(arena, file->write(pdata, length, offset)); + + return file->write(data, length, offset); + } + // Wait for diskDelay before submitting the I/O return mapAsync(Void)>, Void>(delay(diskDelay), [=](Void _) -> Future { if (pdata) @@ -103,6 +120,9 @@ public: Future truncate(int64_t size) override { double diskDelay = getDelay(); + if (diskDelay == 0.0) + return file->truncate(size); + // Wait for diskDelay before submitting the I/O return mapAsync(Void)>, Void>( delay(diskDelay), [=](Void _) -> Future { return file->truncate(size); }); @@ -110,6 +130,9 @@ public: Future sync() override { double diskDelay = getDelay(); + if (diskDelay == 0.0) + return file->sync(); + // Wait for diskDelay before submitting the I/O return mapAsync(Void)>, Void>( delay(diskDelay), [=](Void _) -> Future { return file->sync(); }); @@ -117,6 +140,9 @@ public: Future size() const override { double diskDelay = getDelay(); + if (diskDelay == 0.0) + return file->size(); + // Wait for diskDelay before submitting the I/O return mapAsync(Void)>, int64_t>( delay(diskDelay), [=](Void _) -> Future { return file->size(); }); diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 764b8b125b..13c493d434 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -410,6 +410,7 @@ public: std::vector>> primarySatelliteDcIds; std::vector>> remoteSatelliteDcIds; TSSMode tssMode; + std::map corruptWorkerMap; // Used by workloads that perform reconfigurations int testerCount; @@ -440,6 +441,13 @@ public: static thread_local ProcessInfo* currentProcess; + bool checkInjectedCorruption() { + auto iter = corruptWorkerMap.find(currentProcess->address); + if (iter != corruptWorkerMap.end()) + return iter->second; + return false; + } + protected: Mutex mutex; diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index cabf457bdd..f18d247c4e 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -151,7 +151,6 @@ set(FDBSERVER_SRCS workloads/BackupToDBAbort.actor.cpp workloads/BackupToDBCorrectness.actor.cpp workloads/BackupToDBUpgrade.actor.cpp - workloads/BitFlipping.actor.cpp workloads/BlobStoreWorkload.h workloads/BulkLoad.actor.cpp workloads/BulkSetup.actor.h @@ -173,7 +172,7 @@ set(FDBSERVER_SRCS workloads/DDMetricsExclude.actor.cpp workloads/DiskDurability.actor.cpp workloads/DiskDurabilityTest.actor.cpp - workloads/DiskThrottling.actor.cpp + workloads/DiskFailureInjection.actor.cpp workloads/Downgrade.actor.cpp workloads/DummyWorkload.actor.cpp workloads/ExternalWorkload.actor.cpp diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index ac15df2d04..828cb094be 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -95,7 +95,6 @@ extern int limitReasonEnd; extern const char* limitReasonName[]; extern const char* limitReasonDesc[]; -struct WorkerEvents : std::map {}; typedef std::map EventMap; ACTOR static Future> latestEventOnWorker(WorkerInterface worker, std::string eventName) { @@ -115,7 +114,7 @@ ACTOR static Future> latestEventOnWorker(WorkerInterf } } -ACTOR static Future>>> latestEventOnWorkers( +ACTOR Future>>> latestEventOnWorkers( std::vector workers, std::string eventName) { try { diff --git a/fdbserver/Status.h b/fdbserver/Status.h index 3cfb019a8e..f56780e1d1 100644 --- a/fdbserver/Status.h +++ b/fdbserver/Status.h @@ -46,4 +46,8 @@ Future clusterGetStatus( Version const& datacenterVersionDifference, ConfigBroadcaster const* const& conifgBroadcaster); +struct WorkerEvents : std::map {}; +Future>>> latestEventOnWorkers( + std::vector const& workers, + std::string const& eventName); #endif diff --git a/fdbserver/VFSAsync.h b/fdbserver/VFSAsync.h index e2dbd18d28..77aea71348 100644 --- a/fdbserver/VFSAsync.h +++ b/fdbserver/VFSAsync.h @@ -22,6 +22,7 @@ #include #include #include "fdbrpc/IAsyncFile.h" +#include "fdbrpc/simulator.h" /* ** When using this VFS, the sqlite3_file* handles that SQLite uses are @@ -71,7 +72,7 @@ struct VFSAsyncFile { .detail("Found", e) .detail("ErrorCode", (int64_t)g_network->global(INetwork::enSQLiteInjectedError)) .backtrace(); - return e; + return e || (g_network->isSimulated() && g_simulator.checkInjectedCorruption()); } uint32_t* const pLockCount; // +1 for each SHARED_LOCK, or 1+X_COUNT for lock level X diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 15a1ae2a35..2dd22544db 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -26,6 +26,7 @@ #include #include #include "fdbrpc/ContinuousSample.h" +#include "fdbrpc/simulator.h" #include "fdbserver/IPager.h" #include "fdbclient/Tuple.h" #include "flow/serialize.h" @@ -2727,6 +2728,8 @@ public: debug_printf( "DWALPager(%s) checksum failed for %s\n", self->filename.c_str(), toString(pageID).c_str()); Error e = checksum_failed(); + if (g_network->isSimulated() && g_simulator.checkInjectedCorruption()) + e = e.asInjectedFault(); TraceEvent(SevError, "RedwoodChecksumFailed") .detail("Filename", self->filename.c_str()) .detail("PageID", pageID) diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index b0339fcf05..bf5d7d5d0c 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -613,7 +613,6 @@ bool addressInDbAndPrimaryDc(const NetworkAddress& address, Reference(Endpoint({ grvProxyAddress }, UID(1, 2))); + grvProxyInterf.getConsistentReadVersion = + RequestStream(Endpoint({ grvProxyAddress }, UID(1, 2))); testDbInfo.client.grvProxies.push_back(grvProxyInterf); ASSERT(addressInDbAndPrimaryDc(grvProxyAddress, makeReference>(testDbInfo))); NetworkAddress commitProxyAddress(IPAddress(0x37373737), 1); CommitProxyInterface commitProxyInterf; - commitProxyInterf.commit = RequestStream(Endpoint({ commitProxyAddress }, UID(1, 2))); + commitProxyInterf.commit = + RequestStream(Endpoint({ commitProxyAddress }, UID(1, 2))); testDbInfo.client.commitProxies.push_back(commitProxyInterf); ASSERT(addressInDbAndPrimaryDc(commitProxyAddress, makeReference>(testDbInfo))); @@ -1204,8 +1205,7 @@ ACTOR Future chaosMetricsLogger() { wait(delay(FLOW_KNOBS->CHAOS_LOGGING_INTERVAL)); TraceEvent e("ChaosMetrics"); - // double elapsed = now() - chaosMetrics->startTime; - double elapsed = timer_monotonic() - chaosMetrics->startTime; + double elapsed = now() - chaosMetrics->startTime; e.detail("Elapsed", elapsed); chaosMetrics->getFields(&e); e.trackLatest("ChaosMetrics"); diff --git a/fdbserver/workloads/BitFlipping.actor.cpp b/fdbserver/workloads/BitFlipping.actor.cpp deleted file mode 100644 index 8dd8781b6f..0000000000 --- a/fdbserver/workloads/BitFlipping.actor.cpp +++ /dev/null @@ -1,229 +0,0 @@ -/* - * BitFlipping.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbclient/NativeAPI.actor.h" -#include "fdbserver/TesterInterface.actor.h" -#include "fdbserver/workloads/workloads.actor.h" -#include "fdbrpc/simulator.h" -#include "fdbserver/WorkerInterface.actor.h" -#include "fdbserver/ServerDBInfo.h" -#include "fdbserver/QuietDatabase.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -struct BitFlippingWorkload : TestWorkload { - bool enabled; - double testDuration; - double percentBitFlips; - double periodicCheckInterval; - std::vector chosenWorkers; - std::vector> clients; - - BitFlippingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - enabled = !clientId; // only do this on the "first" client - testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0); - percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 10.0); - periodicCheckInterval = getOption(options, LiteralStringRef("periodicCheckInterval"), 10.0); - } - - std::string description() const override { - if (&g_simulator == g_network) - return "BitFlipping"; - else - return "NoSimBitFlipping"; - } - - Future setup(Database const& cx) override { return Void(); } - - // Starts the workload by - - // 1. Starting the actor to periodically check chaosMetrics, and - // 2. Starting the actor that injects failures on chosen storage servers - Future start(Database const& cx) override { - if (enabled) { - clients.push_back(periodicMetricCheck(this)); - clients.push_back(flipBitsClient(cx, this)); - return timeout(waitForAll(clients), testDuration, Void()); - } else - return Void(); - } - - Future check(Database const& cx) override { return true; } - - void getMetrics(vector& m) override {} - - static void checkBitFlipResult(Future res, WorkerInterface worker) { - if (res.isError()) { - auto err = res.getError(); - if (err.code() == error_code_client_invalid_operation) { - TraceEvent(SevError, "ChaosDisabled") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); - } else { - TraceEvent(SevError, "BitFlippingFailed") - .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) - .error(err); - } - } - } - - ACTOR void doBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { - state Future res; - wait(::delay(startDelay)); - SetFailureInjection::FlipBitsCommand flipBits; - flipBits.percentBitFlips = percentage; - SetFailureInjection req; - req.flipBits = flipBits; - res = worker.clientInterface.setFailureInjection.getReply(req); - wait(ready(res)); - checkBitFlipResult(res, worker); - } - - ACTOR static Future getAllStorageWorkers(Database cx, - BitFlippingWorkload* self, - std::vector* result) { - result->clear(); - state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); - for (auto& worker : res) { - result->emplace_back(worker); - } - return Void(); - } - - ACTOR template - Future flipBitsClient(Database cx, BitFlippingWorkload* self) { - state double lastTime = now(); - state double workloadEnd = now() + self->testDuration; - state std::vector machines; - loop { - wait(poisson(&lastTime, 1)); - wait(BitFlippingWorkload::getAllStorageWorkers(cx, self, &machines)); - auto machine = deterministicRandom()->randomChoice(machines); - - // If we have already chosen this worker, then just continue - if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) != - self->chosenWorkers.end()) - continue; - - // Keep track of chosen workers for verification purpose - self->chosenWorkers.emplace_back(machine.address()); - self->doBitFlips(machine, self->percentBitFlips); - } - } - - // Resend the chaos event to previosuly chosen workers, in case some workers got restarted and lost their chaos - // config - ACTOR static Future reSendChaos(BitFlippingWorkload* self) { - std::vector workers = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); - std::map workersMap; - for (auto worker : workers) { - workersMap[worker.interf.address()] = worker.interf; - } - for (auto& workerAddress : self->chosenWorkers) { - auto itr = workersMap.find(workerAddress); - if (itr != workersMap.end()) - self->doBitFlips(itr->second, self->percentBitFlips); - } - return Void(); - } - // For fetching chaosMetrics to ensure chaos events are happening - // This is borrowed code from Status.actor.cpp - struct WorkerEvents : std::map {}; - - ACTOR static Future>>> latestEventOnWorkers( - std::vector workers, - std::string eventName) { - try { - state vector>> eventTraces; - for (int c = 0; c < workers.size(); c++) { - EventLogRequest req = - eventName.size() > 0 ? EventLogRequest(Standalone(eventName)) : EventLogRequest(); - eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0))); - } - - wait(waitForAll(eventTraces)); - - std::set failed; - WorkerEvents results; - - for (int i = 0; i < eventTraces.size(); i++) { - const ErrorOr& v = eventTraces[i].get(); - if (v.isError()) { - failed.insert(workers[i].interf.address().toString()); - results[workers[i].interf.address()] = TraceEventFields(); - } else { - results[workers[i].interf.address()] = v.get(); - } - } - - std::pair> val; - val.first = results; - val.second = failed; - - return val; - } catch (Error& e) { - ASSERT(e.code() == - error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above - throw; - } - } - - // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers - ACTOR static Future chaosGetStatus(BitFlippingWorkload* self) { - std::vector workers = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); - - Future>>> latestEventsFuture; - latestEventsFuture = latestEventOnWorkers(workers, "ChaosMetrics"); - state Optional>> workerEvents = wait(latestEventsFuture); - - state WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents(); - - // Now verify that all chosen workers for chaos events have non-zero chaosMetrics - for (auto& workerAddress : self->chosenWorkers) { - auto chaosMetrics = cMetrics.find(workerAddress); - if (chaosMetrics != cMetrics.end()) { - int bitFlips = chaosMetrics->second.getInt("BitFlips"); - - // we expect bitFlips to be non-zero for chosenWorkers - if (bitFlips == 0) { - TraceEvent(SevError, "ChaosGetStatus") - .detail("OnEndpoint", workerAddress.toString()) - .detail("BitFlips", bitFlips); - } - } - } - - return Void(); - } - - // Periodically fetches chaosMetrics to ensure that chaas events are taking place - ACTOR static Future periodicMetricCheck(BitFlippingWorkload* self) { - state double start = now(); - state double elapsed = 0.0; - - loop { - // re-send the chaos event in case of a process restart - wait(reSendChaos(self)); - elapsed += self->periodicCheckInterval; - wait(delayUntil(start + elapsed)); - wait(chaosGetStatus(self)); - } - } -}; -WorkloadFactory BitFlippingWorkloadFactory("BitFlipping"); diff --git a/fdbserver/workloads/DiskThrottling.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp similarity index 50% rename from fdbserver/workloads/DiskThrottling.actor.cpp rename to fdbserver/workloads/DiskFailureInjection.actor.cpp index 264ef477a2..bb1cf9b124 100644 --- a/fdbserver/workloads/DiskThrottling.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -1,5 +1,5 @@ /* - * DiskThrottling.actor.cpp + * DiskFailureInjection.actor.cpp * * This source file is part of the FoundationDB open source project * @@ -25,34 +25,45 @@ #include "fdbserver/WorkerInterface.actor.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/QuietDatabase.h" +#include "fdbserver/Status.h" #include "flow/actorcompiler.h" // This must be the last #include. -struct DiskThrottlingWorkload : TestWorkload { +struct DiskFailureInjectionWorkload : TestWorkload { bool enabled; double testDuration; double startDelay; + bool throttleDisk; + int workersToThrottle; double stallInterval; double stallPeriod; double throttlePeriod; - double periodicCheckInterval; + bool corruptFile; + int workersToCorrupt; + double percentBitFlips; + double periodicBroadcastInterval; std::vector chosenWorkers; std::vector> clients; - DiskThrottlingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + DiskFailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { enabled = !clientId; // only do this on the "first" client startDelay = getOption(options, LiteralStringRef("startDelay"), 0.0); testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0); + throttleDisk = getOption(options, LiteralStringRef("throttleDisk"), false); + workersToThrottle = getOption(options, LiteralStringRef("workersToThrottle"), 3); stallInterval = getOption(options, LiteralStringRef("stallInterval"), 0.0); stallPeriod = getOption(options, LiteralStringRef("stallPeriod"), 60.0); throttlePeriod = getOption(options, LiteralStringRef("throttlePeriod"), 60.0); - periodicCheckInterval = getOption(options, LiteralStringRef("periodicCheckInterval"), 10.0); + corruptFile = getOption(options, LiteralStringRef("corruptFile"), false); + workersToCorrupt = getOption(options, LiteralStringRef("workersToCorrupt"), 1); + percentBitFlips = getOption(options, LiteralStringRef("percentBitFlips"), 10.0); + periodicBroadcastInterval = getOption(options, LiteralStringRef("periodicBroadcastInterval"), 5.0); } std::string description() const override { if (&g_simulator == g_network) - return "DiskThrottling"; + return "DiskFailureInjection"; else - return "NoSimDiskThrolling"; + return "NoSimDiskFailureInjection"; } Future setup(Database const& cx) override { return Void(); } @@ -62,8 +73,8 @@ struct DiskThrottlingWorkload : TestWorkload { // 2. Starting the actor that injects failures on chosen storage servers Future start(Database const& cx) override { if (enabled) { - clients.push_back(periodicMetricCheck(this)); - clients.push_back(throttleDiskClient(cx, this)); + clients.push_back(diskFailureInjectionClient(cx, this)); + clients.push_back(periodicEventBroadcast(this)); return timeout(waitForAll(clients), testDuration, Void()); } else return Void(); @@ -73,26 +84,26 @@ struct DiskThrottlingWorkload : TestWorkload { void getMetrics(vector& m) override {} - static void checkDiskThrottleResult(Future res, WorkerInterface worker) { + static void checkDiskFailureInjectionResult(Future res, WorkerInterface worker) { if (res.isError()) { auto err = res.getError(); if (err.code() == error_code_client_invalid_operation) { TraceEvent(SevError, "ChaosDisabled") .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()); } else { - TraceEvent(SevError, "DiskThrottlingFailed") + TraceEvent(SevError, "DiskFailureInjectionFailed") .detail("OnEndpoint", worker.waitFailure.getEndpoint().addresses.address.toString()) .error(err); } } } - // Sets the disk failure request - ACTOR void doThrottle(WorkerInterface worker, - double stallInterval, - double stallPeriod, - double throttlePeriod, - double startDelay) { + // Sets the disk delay request + ACTOR void injectDiskDelays(WorkerInterface worker, + double stallInterval, + double stallPeriod, + double throttlePeriod, + double startDelay) { state Future res; wait(::delay(startDelay)); SetFailureInjection::DiskFailureCommand diskFailure; @@ -103,39 +114,34 @@ struct DiskThrottlingWorkload : TestWorkload { req.diskFailure = diskFailure; res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); - checkDiskThrottleResult(res, worker); + checkDiskFailureInjectionResult(res, worker); } - // Currently unused, because we only inject disk failures on storage servers - ACTOR static Future getAllWorkers(DiskThrottlingWorkload* self, std::vector* result) { - result->clear(); - std::vector res = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); - for (auto& worker : res) { - result->emplace_back(worker.interf); - } - return Void(); + // Sets the disk corruption request + ACTOR void injectBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { + state Future res; + wait(::delay(startDelay)); + SetFailureInjection::FlipBitsCommand flipBits; + flipBits.percentBitFlips = percentage; + SetFailureInjection req; + req.flipBits = flipBits; + res = worker.clientInterface.setFailureInjection.getReply(req); + wait(ready(res)); + checkDiskFailureInjectionResult(res, worker); } - ACTOR static Future getAllStorageWorkers(Database cx, - DiskThrottlingWorkload* self, - std::vector* result) { - result->clear(); - state std::vector res = wait(getStorageWorkers(cx, self->dbInfo, false)); - for (auto& worker : res) { - result->emplace_back(worker); - } - return Void(); - } - - // Choose random storage servers to inject disk failures + // Choose random storage servers to inject disk failures. + // We currently only inject disk failure on storage servers. Can be expanded to include + // other worker types in future ACTOR template - Future throttleDiskClient(Database cx, DiskThrottlingWorkload* self) { + Future diskFailureInjectionClient(Database cx, DiskFailureInjectionWorkload* self) { state double lastTime = now(); state std::vector machines; + state int throttledWorkers = 0; + state int corruptedWorkers = 0; loop { wait(poisson(&lastTime, 1)); - wait(DiskThrottlingWorkload::getAllStorageWorkers(cx, self, &machines)); + wait(store(machines, getStorageWorkers(cx, self->dbInfo, false))); auto machine = deterministicRandom()->randomChoice(machines); // If we have already chosen this worker, then just continue @@ -145,13 +151,22 @@ struct DiskThrottlingWorkload : TestWorkload { // Keep track of chosen workers for verification purpose self->chosenWorkers.emplace_back(machine.address()); - self->doThrottle(machine, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) + self->injectDiskDelays( + machine, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { + if (&g_simulator == g_network) + g_simulator.corruptWorkerMap[machine.address()] = true; + self->injectBitFlips(machine, self->percentBitFlips); + } } } // Resend the chaos event to previosuly chosen workers, in case some workers got restarted and lost their chaos // config - ACTOR static Future reSendChaos(DiskThrottlingWorkload* self) { + ACTOR static Future reSendChaos(DiskFailureInjectionWorkload* self) { + state int throttledWorkers = 0; + state int corruptedWorkers = 0; std::vector workers = wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); std::map workersMap; @@ -160,57 +175,22 @@ struct DiskThrottlingWorkload : TestWorkload { } for (auto& workerAddress : self->chosenWorkers) { auto itr = workersMap.find(workerAddress); - if (itr != workersMap.end()) - self->doThrottle( - itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + if (itr != workersMap.end()) { + if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) + self->injectDiskDelays( + itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { + if (&g_simulator == g_network) + g_simulator.corruptWorkerMap[workerAddress] = true; + self->injectBitFlips(itr->second, self->percentBitFlips); + } + } } return Void(); } - // For fetching chaosMetrics to ensure chaos events are happening - // This is borrowed code from Status.actor.cpp - struct WorkerEvents : std::map {}; - - ACTOR static Future>>> latestEventOnWorkers( - std::vector workers, - std::string eventName) { - try { - state vector>> eventTraces; - for (int c = 0; c < workers.size(); c++) { - EventLogRequest req = - eventName.size() > 0 ? EventLogRequest(Standalone(eventName)) : EventLogRequest(); - eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0))); - } - - wait(waitForAll(eventTraces)); - - std::set failed; - WorkerEvents results; - - for (int i = 0; i < eventTraces.size(); i++) { - const ErrorOr& v = eventTraces[i].get(); - if (v.isError()) { - failed.insert(workers[i].interf.address().toString()); - results[workers[i].interf.address()] = TraceEventFields(); - } else { - results[workers[i].interf.address()] = v.get(); - } - } - - std::pair> val; - val.first = results; - val.second = failed; - - return val; - } catch (Error& e) { - ASSERT(e.code() == - error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above - throw; - } - } - // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers - ACTOR static Future chaosGetStatus(DiskThrottlingWorkload* self) { + ACTOR static Future chaosGetStatus(DiskFailureInjectionWorkload* self) { std::vector workers = wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); @@ -220,38 +200,54 @@ struct DiskThrottlingWorkload : TestWorkload { state WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents(); - // Now verify that all chosen workers for chaos events have non-zero chaosMetrics - std::vector>>>> futures; + // Check if any of the chosen workers for chaos events have non-zero chaosMetrics + try { + int foundChaosMetrics = 0; + for (auto& workerAddress : self->chosenWorkers) { + auto chaosMetrics = cMetrics.find(workerAddress); + if (chaosMetrics != cMetrics.end()) { + // we expect diskDelays to be non-zero for chosenWorkers for throttleDisk event + if (self->throttleDisk) { + int diskDelays = chaosMetrics->second.getInt("DiskDelays"); + if (diskDelays > 0) { + foundChaosMetrics++; + } + } - for (auto& workerAddress : self->chosenWorkers) { - auto chaosMetrics = cMetrics.find(workerAddress); - if (chaosMetrics != cMetrics.end()) { - int diskDelays = chaosMetrics->second.getInt("DiskDelays"); - - // we expect diskDelays to be non-zero for chosenWorkers - if (diskDelays == 0) { - TraceEvent(SevError, "ChaosGetStatus") - .detail("OnEndpoint", workerAddress.toString()) - .detail("DiskDelays", diskDelays); + // we expect bitFlips to be non-zero for chosenWorkers for corruptFile event + if (self->corruptFile) { + int bitFlips = chaosMetrics->second.getInt("BitFlips"); + if (bitFlips > 0) { + foundChaosMetrics++; + } + } } } + if (foundChaosMetrics == 0) + TraceEvent("DiskFailureInjectionFailed").detail("ChaosMetricCount", foundChaosMetrics); + else + TraceEvent("ChaosGetStatus").detail("ChaosMetricCount", foundChaosMetrics); + } catch (Error& e) { + // it's possible to get an empty event, it's okay to ignore + if (e.code() != error_code_attribute_not_found) { + throw e; + } } return Void(); } - // Periodically fetches chaosMetrics to ensure that chaas events are taking place - ACTOR static Future periodicMetricCheck(DiskThrottlingWorkload* self) { + // Periodically re-send the chaos event in case of a process restart + ACTOR static Future periodicEventBroadcast(DiskFailureInjectionWorkload* self) { state double start = now(); state double elapsed = 0.0; loop { - // re-send the chaos event in case of a process restart wait(reSendChaos(self)); - elapsed += self->periodicCheckInterval; + elapsed += self->periodicBroadcastInterval; wait(delayUntil(start + elapsed)); wait(chaosGetStatus(self)); } } }; -WorkloadFactory DiskThrottlingWorkloadFactory("DiskThrottling"); +WorkloadFactory DiskFailureInjectionWorkloadFactory("DiskFailureInjection"); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 5233f65757..b506dd46e1 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -66,9 +66,9 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { init( HUGE_ARENA_LOGGING_BYTES, 100e6 ); init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 ); - // Chaos testing - init( ENABLE_CHAOS_FEATURES, false ); - init( CHAOS_LOGGING_INTERVAL, 5.0 ); + // Chaos testing - enabled for simulation by default + init( ENABLE_CHAOS_FEATURES, isSimulated ); + init( CHAOS_LOGGING_INTERVAL, 5.0 ); init( WRITE_TRACING_ENABLED, true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false; diff --git a/flow/network.h b/flow/network.h index c4c8f46ffc..59a1ec9a0a 100644 --- a/flow/network.h +++ b/flow/network.h @@ -666,7 +666,7 @@ struct ChaosMetrics { void clear() { memset(this, 0, sizeof(ChaosMetrics)); - startTime = timer_monotonic(); + startTime = g_network ? g_network->now() : 0; } unsigned int diskDelays; @@ -703,11 +703,11 @@ struct DiskFailureInjector { void setDiskFailure(double interval, double stallFor, double throttleFor) { stallInterval = interval; stallPeriod = stallFor; - stallUntil = std::max(stallUntil, timer_monotonic() + stallFor); + stallUntil = std::max(stallUntil, g_network->now() + stallFor); // random stall duration in ms (chosen once) stallDuration = 0.001 * deterministicRandom()->randomInt(1, 5); throttlePeriod = throttleFor; - throttleUntil = std::max(throttleUntil, timer_monotonic() + throttleFor); + throttleUntil = std::max(throttleUntil, g_network->now() + throttleFor); TraceEvent("SetDiskFailure") .detail("StallInterval", interval) .detail("StallPeriod", stallFor) @@ -719,8 +719,8 @@ struct DiskFailureInjector { double getStallDelay() { // If we are in a stall period and a stallInterval was specified, determine the // delay to be inserted - if (((stallUntil - timer_monotonic()) > 0.0) && stallInterval) { - auto timeElapsed = fmod(timer_monotonic(), stallInterval); + if (((stallUntil - g_network->now()) > 0.0) && stallInterval) { + auto timeElapsed = fmod(g_network->now(), stallInterval); return std::max(0.0, stallDuration - timeElapsed); } return 0.0; @@ -728,7 +728,7 @@ struct DiskFailureInjector { double getThrottleDelay() { // If we are in the throttle period, insert a random delay (in ms) - if ((throttleUntil - timer_monotonic()) > 0.0) + if ((throttleUntil - g_network->now()) > 0.0) return (0.001 * deterministicRandom()->randomInt(1, 3)); return 0.0; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c906f8f93e..f2c1681fee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,14 +119,12 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/BackupCorrectnessClean.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectness.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectnessClean.toml) - add_fdb_test(TEST_FILES fast/BitFlippedCycle.toml IGNORE) add_fdb_test(TEST_FILES fast/CacheTest.toml) add_fdb_test(TEST_FILES fast/CloggedSideband.toml) add_fdb_test(TEST_FILES fast/ConfigureLocked.toml) add_fdb_test(TEST_FILES fast/ConstrainedRandomSelector.toml) add_fdb_test(TEST_FILES fast/CycleAndLock.toml) add_fdb_test(TEST_FILES fast/CycleTest.toml) - add_fdb_test(TEST_FILES fast/DiskThrottledCycle.toml IGNORE) add_fdb_test(TEST_FILES fast/FuzzApiCorrectness.toml) add_fdb_test(TEST_FILES fast/FuzzApiCorrectnessClean.toml) add_fdb_test(TEST_FILES fast/IncrementalBackup.toml) @@ -231,6 +229,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES slow/DDBalanceAndRemove.toml) add_fdb_test(TEST_FILES slow/DDBalanceAndRemoveStatus.toml) add_fdb_test(TEST_FILES slow/DifferentClustersSameRV.toml) + add_fdb_test(TEST_FILES slow/DiskFailureCycle.toml) add_fdb_test(TEST_FILES slow/FastTriggeredWatches.toml) add_fdb_test(TEST_FILES slow/LowLatencyWithFailures.toml) add_fdb_test(TEST_FILES slow/MoveKeysClean.toml) diff --git a/tests/fast/BitFlippedCycle.toml b/tests/fast/BitFlippedCycle.toml deleted file mode 100644 index 3cab1f74fe..0000000000 --- a/tests/fast/BitFlippedCycle.toml +++ /dev/null @@ -1,13 +0,0 @@ -[[test]] -testTitle = 'BitFlippedCycle' - - [[test.workload]] - testName = 'Cycle' - transactionsPerSecond = 2500.0 - testDuration = 60.0 - expectedRate = 0 - - [[test.workload]] - testName = 'BitFlipping' - testDuration = 60.0 - percentBitFlips = 20.0 diff --git a/tests/fast/DiskThrottledCycle.toml b/tests/fast/DiskThrottledCycle.toml deleted file mode 100644 index 83df7fdb1d..0000000000 --- a/tests/fast/DiskThrottledCycle.toml +++ /dev/null @@ -1,16 +0,0 @@ -[[test]] -testTitle = 'DiskThrottledCycle' - - [[test.workload]] - testName = 'Cycle' - transactionsPerSecond = 2500.0 - testDuration = 30.0 - expectedRate = 0 - - [[test.workload]] - testName = 'DiskThrottling' - testDuration = 30.0 - stallInterval = 10.0 - stallPeriod = 30.0 - throttlePeriod = 30.0 - diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml new file mode 100644 index 0000000000..0f4d1365f7 --- /dev/null +++ b/tests/slow/DiskFailureCycle.toml @@ -0,0 +1,30 @@ +[configuration] +buggify = false +minimumReplication = 3 +minimumRegions = 3 +logAntiQuorum = 0 + +[[test]] +testTitle = 'DiskFailureCycle' + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 2500.0 + testDuration = 60.0 + expectedRate = 0 + + [[test.workload]] + testName = 'DiskFailureInjection' + testDuration = 20.0 + startDelay = 20.0 + throttleDisk = true + stallInterval = 10.0 + stallPeriod = 20.0 + throttlePeriod = 20.0 + + [[test.workload]] + testName = 'DiskFailureInjection' + testDuration = 20.0 + startDelay = 40.0 + corruptFile = true + percentBitFlips = 10 From a2d8ab71523755aa260574276a7a052a424c68e2 Mon Sep 17 00:00:00 2001 From: negoyal Date: Fri, 30 Jul 2021 13:21:45 -0700 Subject: [PATCH 012/338] Ignore the errors from getStorageServers. --- fdbserver/workloads/DiskFailureInjection.actor.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index bb1cf9b124..27e3cd74b4 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -141,7 +141,14 @@ struct DiskFailureInjectionWorkload : TestWorkload { state int corruptedWorkers = 0; loop { wait(poisson(&lastTime, 1)); - wait(store(machines, getStorageWorkers(cx, self->dbInfo, false))); + try { + wait(store(machines, getStorageWorkers(cx, self->dbInfo, false))); + } catch (Error& e) { + // If we failed to get a list of storage servers, we can't inject failure events + // But don't throw the error in that case + TraceEvent("DiskFailureInjectionFailed"); + return Void(); + } auto machine = deterministicRandom()->randomChoice(machines); // If we have already chosen this worker, then just continue From a8baeb75d00e113e28f2a0c0d9c2b790cf712496 Mon Sep 17 00:00:00 2001 From: negoyal Date: Fri, 3 Sep 2021 15:03:12 -0700 Subject: [PATCH 013/338] Misc fixes. --- fdbrpc/AsyncFileChaos.actor.h | 7 +++-- .../workloads/DiskFailureInjection.actor.cpp | 29 +++++++++++++------ fdbserver/workloads/TargetedKill.actor.cpp | 12 +++++++- flow/network.h | 1 + tests/slow/DiskFailureCycle.toml | 4 ++- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index 11b60f6692..2aa7ceedcf 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -34,8 +34,11 @@ private: public: explicit AsyncFileChaos(Reference file) : file(file) { - // We onlyl allow chaod events on storage files - enabled = StringRef(file->getFilename()).startsWith(LiteralStringRef("storage-")); + // We only allow chaos events on storage files + enabled = (file->getFilename().find("storage-") != std::string::npos); + //enabled = StringRef(file->getFilename()).startsWith(LiteralStringRef("storage-")); + + TraceEvent("AsyncFileChaos").detail("Enabled", enabled).detail("FileName", file->getFilename()); } void addref() override { ReferenceCounted::addref(); } diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 27e3cd74b4..15bcad5c82 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -43,11 +43,16 @@ struct DiskFailureInjectionWorkload : TestWorkload { double periodicBroadcastInterval; std::vector chosenWorkers; std::vector> clients; + // Verification Mode: We run the workload indefinitely in this mode. + // The idea is to keep going until we get a non-zero chaosMetric to ensure + // that we haven't lost the chaos event. testDuration is ignored in this mode + bool verificationMode; DiskFailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { enabled = !clientId; // only do this on the "first" client startDelay = getOption(options, LiteralStringRef("startDelay"), 0.0); testDuration = getOption(options, LiteralStringRef("testDuration"), 60.0); + verificationMode = getOption(options, LiteralStringRef("verificationMode"), false); throttleDisk = getOption(options, LiteralStringRef("throttleDisk"), false); workersToThrottle = getOption(options, LiteralStringRef("workersToThrottle"), 3); stallInterval = getOption(options, LiteralStringRef("stallInterval"), 0.0); @@ -69,12 +74,18 @@ struct DiskFailureInjectionWorkload : TestWorkload { Future setup(Database const& cx) override { return Void(); } // Starts the workload by - - // 1. Starting the actor to periodically check chaosMetrics, and + // 1. Starting the actor to periodically check chaosMetrics and re-broadcast chaos events, and // 2. Starting the actor that injects failures on chosen storage servers Future start(Database const& cx) override { if (enabled) { clients.push_back(diskFailureInjectionClient(cx, this)); clients.push_back(periodicEventBroadcast(this)); + // In verification mode, we want to wait until the first actor returns which indicates that + // a non-zero chaosMetric was found + if (verificationMode) { + return waitForAny(clients); + } + // Else we honor testDuration return timeout(waitForAll(clients), testDuration, Void()); } else return Void(); @@ -197,7 +208,8 @@ struct DiskFailureInjectionWorkload : TestWorkload { } // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers - ACTOR static Future chaosGetStatus(DiskFailureInjectionWorkload* self) { + ACTOR static Future chaosGetStatus(DiskFailureInjectionWorkload* self) { + state int foundChaosMetrics = 0; std::vector workers = wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); @@ -209,7 +221,6 @@ struct DiskFailureInjectionWorkload : TestWorkload { // Check if any of the chosen workers for chaos events have non-zero chaosMetrics try { - int foundChaosMetrics = 0; for (auto& workerAddress : self->chosenWorkers) { auto chaosMetrics = cMetrics.find(workerAddress); if (chaosMetrics != cMetrics.end()) { @@ -230,10 +241,6 @@ struct DiskFailureInjectionWorkload : TestWorkload { } } } - if (foundChaosMetrics == 0) - TraceEvent("DiskFailureInjectionFailed").detail("ChaosMetricCount", foundChaosMetrics); - else - TraceEvent("ChaosGetStatus").detail("ChaosMetricCount", foundChaosMetrics); } catch (Error& e) { // it's possible to get an empty event, it's okay to ignore if (e.code() != error_code_attribute_not_found) { @@ -241,7 +248,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { } } - return Void(); + return foundChaosMetrics; } // Periodically re-send the chaos event in case of a process restart @@ -253,7 +260,11 @@ struct DiskFailureInjectionWorkload : TestWorkload { wait(reSendChaos(self)); elapsed += self->periodicBroadcastInterval; wait(delayUntil(start + elapsed)); - wait(chaosGetStatus(self)); + int foundChaosMetrics = wait(chaosGetStatus(self)); + if (foundChaosMetrics > 0) { + TraceEvent("FoundChaos").detail("ChaosMetricCount", foundChaosMetrics); + return Void(); + } } } }; diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.actor.cpp index 48d5da4629..1c40cd47b6 100644 --- a/fdbserver/workloads/TargetedKill.actor.cpp +++ b/fdbserver/workloads/TargetedKill.actor.cpp @@ -33,10 +33,14 @@ struct TargetedKillWorkload : TestWorkload { std::string machineToKill; bool enabled, killAllMachineProcesses; double killAt; + bool reboot; + double suspendDuration; TargetedKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { enabled = !clientId; // only do this on the "first" client killAt = getOption(options, LiteralStringRef("killAt"), 5.0); + reboot = getOption(options, LiteralStringRef("reboot"), false); + suspendDuration = getOption(options, LiteralStringRef("suspendDuration"), 1.0); machineToKill = getOption(options, LiteralStringRef("machineToKill"), LiteralStringRef("master")).toString(); killAllMachineProcesses = getOption(options, LiteralStringRef("killWholeMachine"), false); } @@ -61,13 +65,19 @@ struct TargetedKillWorkload : TestWorkload { state vector workers = wait(getWorkers(self->dbInfo)); int killed = 0; + state RebootRequest rbReq; + if (self->reboot) { + rbReq.waitForDuration = self->suspendDuration; + } else { + rbReq.waitForDuration = std::numeric_limits::max(); + } for (int i = 0; i < workers.size(); i++) { if (workers[i].interf.master.getEndpoint().getPrimaryAddress() == address || (self->killAllMachineProcesses && workers[i].interf.master.getEndpoint().getPrimaryAddress().ip == address.ip && workers[i].processClass != ProcessClass::TesterClass)) { TraceEvent("WorkerKill").detail("TargetedMachine", address).detail("Worker", workers[i].interf.id()); - workers[i].interf.clientInterface.reboot.send(RebootRequest()); + workers[i].interf.clientInterface.reboot.send(rbReq); } } diff --git a/flow/network.h b/flow/network.h index b70070a9fa..5a692a58d6 100644 --- a/flow/network.h +++ b/flow/network.h @@ -715,6 +715,7 @@ struct DiskFailureInjector { throttlePeriod = throttleFor; throttleUntil = std::max(throttleUntil, g_network->now() + throttleFor); TraceEvent("SetDiskFailure") + .detail("Now", g_network->now()) .detail("StallInterval", interval) .detail("StallPeriod", stallFor) .detail("StallUntil", stallUntil) diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml index 0f4d1365f7..da1eee421c 100644 --- a/tests/slow/DiskFailureCycle.toml +++ b/tests/slow/DiskFailureCycle.toml @@ -10,12 +10,13 @@ testTitle = 'DiskFailureCycle' [[test.workload]] testName = 'Cycle' transactionsPerSecond = 2500.0 - testDuration = 60.0 + testDuration = 600.0 expectedRate = 0 [[test.workload]] testName = 'DiskFailureInjection' testDuration = 20.0 + verificationMode = true startDelay = 20.0 throttleDisk = true stallInterval = 10.0 @@ -25,6 +26,7 @@ testTitle = 'DiskFailureCycle' [[test.workload]] testName = 'DiskFailureInjection' testDuration = 20.0 + verificationMode = true startDelay = 40.0 corruptFile = true percentBitFlips = 10 From 337d0df13ce536739205f5a8cb4625e4683a311e Mon Sep 17 00:00:00 2001 From: negoyal Date: Tue, 7 Sep 2021 10:07:01 -0700 Subject: [PATCH 014/338] Add verification mode to chaos workload. --- .../workloads/DiskFailureInjection.actor.cpp | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 15bcad5c82..47fdc9344b 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -78,20 +78,23 @@ struct DiskFailureInjectionWorkload : TestWorkload { // 2. Starting the actor that injects failures on chosen storage servers Future start(Database const& cx) override { if (enabled) { - clients.push_back(diskFailureInjectionClient(cx, this)); - clients.push_back(periodicEventBroadcast(this)); - // In verification mode, we want to wait until the first actor returns which indicates that - // a non-zero chaosMetric was found + clients.push_back(timeout(diskFailureInjectionClient(cx, this), testDuration, Void())); + // In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that + // a non-zero chaosMetric was found. if (verificationMode) { - return waitForAny(clients); - } - // Else we honor testDuration - return timeout(waitForAll(clients), testDuration, Void()); + clients.push_back(periodicEventBroadcast(this)); + } else + //Else we honor the testDuration + clients.push_back(timeout(periodicEventBroadcast(this), testDuration, Void())); + return waitForAll(clients); } else return Void(); } - Future check(Database const& cx) override { return true; } + Future check(Database const& cx) override { + clients.clear(); + return true; + } void getMetrics(vector& m) override {} @@ -242,6 +245,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { } } } catch (Error& e) { + TraceEvent(SevDebug, "ChaosGetStatus").error(e); // it's possible to get an empty event, it's okay to ignore if (e.code() != error_code_attribute_not_found) { throw e; From 7729a282ceb73a521408d0002200cc1a29fd8456 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 8 Sep 2021 14:31:09 -0700 Subject: [PATCH 015/338] Misc fixes and updated test toml file. --- fdbrpc/AsyncFileChaos.actor.h | 1 + .../workloads/DiskFailureInjection.actor.cpp | 26 +++++++++---------- flow/network.h | 3 +++ tests/slow/DiskFailureCycle.toml | 14 +++++----- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index 2aa7ceedcf..9b9211cbff 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -87,6 +87,7 @@ public: auto res = g_network->global(INetwork::enBitFlipper); if (enabled && res) { auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); + //TraceEvent("AsyncFileChaosCorrupt").detail("Percentage", bitFlipPercentage); if (bitFlipPercentage > 0.0) { if (deterministicRandom()->random01() < bitFlipPercentage) { pdata = (char*)arena.allocate4kAlignedBuffer(length); diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 47fdc9344b..27b2033a60 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -116,10 +116,8 @@ struct DiskFailureInjectionWorkload : TestWorkload { ACTOR void injectDiskDelays(WorkerInterface worker, double stallInterval, double stallPeriod, - double throttlePeriod, - double startDelay) { + double throttlePeriod) { state Future res; - wait(::delay(startDelay)); SetFailureInjection::DiskFailureCommand diskFailure; diskFailure.stallInterval = stallInterval; diskFailure.stallPeriod = stallPeriod; @@ -132,9 +130,8 @@ struct DiskFailureInjectionWorkload : TestWorkload { } // Sets the disk corruption request - ACTOR void injectBitFlips(WorkerInterface worker, double percentage, double startDelay = 0.0) { + ACTOR void injectBitFlips(WorkerInterface worker, double percentage) { state Future res; - wait(::delay(startDelay)); SetFailureInjection::FlipBitsCommand flipBits; flipBits.percentBitFlips = percentage; SetFailureInjection req; @@ -149,6 +146,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { // other worker types in future ACTOR template Future diskFailureInjectionClient(Database cx, DiskFailureInjectionWorkload* self) { + wait(::delay(self->startDelay)); state double lastTime = now(); state std::vector machines; state int throttledWorkers = 0; @@ -174,7 +172,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { self->chosenWorkers.emplace_back(machine.address()); if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) self->injectDiskDelays( - machine, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + machine, self->stallInterval, self->stallPeriod, self->throttlePeriod); if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { if (&g_simulator == g_network) g_simulator.corruptWorkerMap[machine.address()] = true; @@ -188,9 +186,8 @@ struct DiskFailureInjectionWorkload : TestWorkload { ACTOR static Future reSendChaos(DiskFailureInjectionWorkload* self) { state int throttledWorkers = 0; state int corruptedWorkers = 0; - std::vector workers = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); - std::map workersMap; + state std::map workersMap; + state std::vector workers = wait(getWorkers(self->dbInfo)); for (auto worker : workers) { workersMap[worker.interf.address()] = worker.interf; } @@ -199,7 +196,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { if (itr != workersMap.end()) { if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) self->injectDiskDelays( - itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod, self->startDelay); + itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod); if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { if (&g_simulator == g_network) g_simulator.corruptWorkerMap[workerAddress] = true; @@ -213,8 +210,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { // Fetches chaosMetrics and verifies that chaos events are happening for enabled workers ACTOR static Future chaosGetStatus(DiskFailureInjectionWorkload* self) { state int foundChaosMetrics = 0; - std::vector workers = - wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest{})); + state std::vector workers = wait(getWorkers(self->dbInfo)); Future>>> latestEventsFuture; latestEventsFuture = latestEventOnWorkers(workers, "ChaosMetrics"); @@ -245,9 +241,9 @@ struct DiskFailureInjectionWorkload : TestWorkload { } } } catch (Error& e) { - TraceEvent(SevDebug, "ChaosGetStatus").error(e); // it's possible to get an empty event, it's okay to ignore if (e.code() != error_code_attribute_not_found) { + TraceEvent(SevError, "ChaosGetStatus").error(e); throw e; } } @@ -257,16 +253,18 @@ struct DiskFailureInjectionWorkload : TestWorkload { // Periodically re-send the chaos event in case of a process restart ACTOR static Future periodicEventBroadcast(DiskFailureInjectionWorkload* self) { + wait(::delay(self->startDelay)); state double start = now(); state double elapsed = 0.0; loop { + wait(delayUntil(start + elapsed)); wait(reSendChaos(self)); elapsed += self->periodicBroadcastInterval; wait(delayUntil(start + elapsed)); int foundChaosMetrics = wait(chaosGetStatus(self)); if (foundChaosMetrics > 0) { - TraceEvent("FoundChaos").detail("ChaosMetricCount", foundChaosMetrics); + TraceEvent("FoundChaos").detail("ChaosMetricCount", foundChaosMetrics).detail("ClientID", self->clientId); return Void(); } } diff --git a/flow/network.h b/flow/network.h index 5a692a58d6..7c89ed689c 100644 --- a/flow/network.h +++ b/flow/network.h @@ -735,6 +735,9 @@ struct DiskFailureInjector { double getThrottleDelay() { // If we are in the throttle period, insert a random delay (in ms) + TraceEvent("GetThrottleDelay") + .detail("Now", g_network->now()) + .detail("ThrottleUntil", throttleUntil); if ((throttleUntil - g_network->now()) > 0.0) return (0.001 * deterministicRandom()->randomInt(1, 3)); diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml index da1eee421c..29d3f35f36 100644 --- a/tests/slow/DiskFailureCycle.toml +++ b/tests/slow/DiskFailureCycle.toml @@ -10,23 +10,23 @@ testTitle = 'DiskFailureCycle' [[test.workload]] testName = 'Cycle' transactionsPerSecond = 2500.0 - testDuration = 600.0 + testDuration = 300.0 expectedRate = 0 [[test.workload]] testName = 'DiskFailureInjection' - testDuration = 20.0 + testDuration = 120.0 verificationMode = true - startDelay = 20.0 + startDelay = 30.0 throttleDisk = true stallInterval = 10.0 - stallPeriod = 20.0 - throttlePeriod = 20.0 + stallPeriod = 60.0 + throttlePeriod = 60.0 [[test.workload]] testName = 'DiskFailureInjection' - testDuration = 20.0 + testDuration = 240.0 verificationMode = true - startDelay = 40.0 + startDelay = 120.0 corruptFile = true percentBitFlips = 10 From c8e6bb13c4f00f9938cd4cc99d17438d67efffb5 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 8 Sep 2021 15:18:08 -0700 Subject: [PATCH 016/338] Clang format. --- fdbserver/workloads/DiskFailureInjection.actor.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 27b2033a60..8a08dc1c2a 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -84,7 +84,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { if (verificationMode) { clients.push_back(periodicEventBroadcast(this)); } else - //Else we honor the testDuration + // Else we honor the testDuration clients.push_back(timeout(periodicEventBroadcast(this), testDuration, Void())); return waitForAll(clients); } else @@ -171,8 +171,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { // Keep track of chosen workers for verification purpose self->chosenWorkers.emplace_back(machine.address()); if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) - self->injectDiskDelays( - machine, self->stallInterval, self->stallPeriod, self->throttlePeriod); + self->injectDiskDelays(machine, self->stallInterval, self->stallPeriod, self->throttlePeriod); if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { if (&g_simulator == g_network) g_simulator.corruptWorkerMap[machine.address()] = true; @@ -195,8 +194,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { auto itr = workersMap.find(workerAddress); if (itr != workersMap.end()) { if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) - self->injectDiskDelays( - itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod); + self->injectDiskDelays(itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod); if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { if (&g_simulator == g_network) g_simulator.corruptWorkerMap[workerAddress] = true; @@ -264,7 +262,9 @@ struct DiskFailureInjectionWorkload : TestWorkload { wait(delayUntil(start + elapsed)); int foundChaosMetrics = wait(chaosGetStatus(self)); if (foundChaosMetrics > 0) { - TraceEvent("FoundChaos").detail("ChaosMetricCount", foundChaosMetrics).detail("ClientID", self->clientId); + TraceEvent("FoundChaos") + .detail("ChaosMetricCount", foundChaosMetrics) + .detail("ClientID", self->clientId); return Void(); } } From a48148fdb2f23cb5c07fb110ab994abae6c37b62 Mon Sep 17 00:00:00 2001 From: negoyal Date: Wed, 8 Sep 2021 22:53:52 -0700 Subject: [PATCH 017/338] Tweak the chaos toml file. --- tests/slow/DiskFailureCycle.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml index 29d3f35f36..ca4d5740ca 100644 --- a/tests/slow/DiskFailureCycle.toml +++ b/tests/slow/DiskFailureCycle.toml @@ -22,11 +22,5 @@ testTitle = 'DiskFailureCycle' stallInterval = 10.0 stallPeriod = 60.0 throttlePeriod = 60.0 - - [[test.workload]] - testName = 'DiskFailureInjection' - testDuration = 240.0 - verificationMode = true - startDelay = 120.0 corruptFile = true percentBitFlips = 10 From a63c19c347889c5be26e32c61330967b97926f86 Mon Sep 17 00:00:00 2001 From: negoyal Date: Thu, 9 Sep 2021 11:29:54 -0700 Subject: [PATCH 018/338] Trying clang-format again. --- fdbserver/workloads/DiskFailureInjection.actor.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 8a08dc1c2a..db16901268 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -124,6 +124,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { diskFailure.throttlePeriod = throttlePeriod; SetFailureInjection req; req.diskFailure = diskFailure; + TraceEvent("DiskFailureInjectDiskDelays"); res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); checkDiskFailureInjectionResult(res, worker); @@ -136,6 +137,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { flipBits.percentBitFlips = percentage; SetFailureInjection req; req.flipBits = flipBits; + TraceEvent("DiskFailureInjectBitFlips"); res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); checkDiskFailureInjectionResult(res, worker); @@ -159,14 +161,17 @@ struct DiskFailureInjectionWorkload : TestWorkload { // If we failed to get a list of storage servers, we can't inject failure events // But don't throw the error in that case TraceEvent("DiskFailureInjectionFailed"); - return Void(); + continue; + // return Void(); } auto machine = deterministicRandom()->randomChoice(machines); // If we have already chosen this worker, then just continue if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) != - self->chosenWorkers.end()) + self->chosenWorkers.end()) { + TraceEvent("DiskFailureInjectionSkipped"); continue; + } // Keep track of chosen workers for verification purpose self->chosenWorkers.emplace_back(machine.address()); From a7721d9786b954415e21f999621268b6a7c801fb Mon Sep 17 00:00:00 2001 From: negoyal Date: Fri, 10 Sep 2021 15:41:22 -0700 Subject: [PATCH 019/338] Remove debug trace events and clang-format. --- fdbrpc/AsyncFileChaos.actor.h | 6 +----- fdbserver/workloads/DiskFailureInjection.actor.cpp | 5 ----- flow/network.h | 5 +---- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index 9b9211cbff..3678af63b6 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -36,9 +36,6 @@ public: explicit AsyncFileChaos(Reference file) : file(file) { // We only allow chaos events on storage files enabled = (file->getFilename().find("storage-") != std::string::npos); - //enabled = StringRef(file->getFilename()).startsWith(LiteralStringRef("storage-")); - - TraceEvent("AsyncFileChaos").detail("Enabled", enabled).detail("FileName", file->getFilename()); } void addref() override { ReferenceCounted::addref(); } @@ -48,7 +45,7 @@ public: double delayFor = 0.0; if (!enabled) return delayFor; - + auto res = g_network->global(INetwork::enDiskFailureInjector); if (res) { DiskFailureInjector* delayInjector = static_cast(res); @@ -87,7 +84,6 @@ public: auto res = g_network->global(INetwork::enBitFlipper); if (enabled && res) { auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); - //TraceEvent("AsyncFileChaosCorrupt").detail("Percentage", bitFlipPercentage); if (bitFlipPercentage > 0.0) { if (deterministicRandom()->random01() < bitFlipPercentage) { pdata = (char*)arena.allocate4kAlignedBuffer(length); diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index db16901268..b9079b39eb 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -124,7 +124,6 @@ struct DiskFailureInjectionWorkload : TestWorkload { diskFailure.throttlePeriod = throttlePeriod; SetFailureInjection req; req.diskFailure = diskFailure; - TraceEvent("DiskFailureInjectDiskDelays"); res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); checkDiskFailureInjectionResult(res, worker); @@ -137,7 +136,6 @@ struct DiskFailureInjectionWorkload : TestWorkload { flipBits.percentBitFlips = percentage; SetFailureInjection req; req.flipBits = flipBits; - TraceEvent("DiskFailureInjectBitFlips"); res = worker.clientInterface.setFailureInjection.getReply(req); wait(ready(res)); checkDiskFailureInjectionResult(res, worker); @@ -160,16 +158,13 @@ struct DiskFailureInjectionWorkload : TestWorkload { } catch (Error& e) { // If we failed to get a list of storage servers, we can't inject failure events // But don't throw the error in that case - TraceEvent("DiskFailureInjectionFailed"); continue; - // return Void(); } auto machine = deterministicRandom()->randomChoice(machines); // If we have already chosen this worker, then just continue if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) != self->chosenWorkers.end()) { - TraceEvent("DiskFailureInjectionSkipped"); continue; } diff --git a/flow/network.h b/flow/network.h index 7c89ed689c..51fe05d96d 100644 --- a/flow/network.h +++ b/flow/network.h @@ -715,7 +715,7 @@ struct DiskFailureInjector { throttlePeriod = throttleFor; throttleUntil = std::max(throttleUntil, g_network->now() + throttleFor); TraceEvent("SetDiskFailure") - .detail("Now", g_network->now()) + .detail("Now", g_network->now()) .detail("StallInterval", interval) .detail("StallPeriod", stallFor) .detail("StallUntil", stallUntil) @@ -735,9 +735,6 @@ struct DiskFailureInjector { double getThrottleDelay() { // If we are in the throttle period, insert a random delay (in ms) - TraceEvent("GetThrottleDelay") - .detail("Now", g_network->now()) - .detail("ThrottleUntil", throttleUntil); if ((throttleUntil - g_network->now()) > 0.0) return (0.001 * deterministicRandom()->randomInt(1, 3)); From 8d1e97b329cda898e38953aa24bf97e9318915b2 Mon Sep 17 00:00:00 2001 From: negoyal Date: Mon, 4 Oct 2021 22:43:48 -0700 Subject: [PATCH 020/338] Minor changes. --- fdbrpc/AsyncFileChaos.actor.h | 3 ++- fdbserver/workloads/ClearSingleRange.actor.cpp | 15 ++++++++++++--- .../workloads/DiskFailureInjection.actor.cpp | 4 ++-- fdbserver/workloads/Mako.actor.cpp | 2 +- fdbserver/workloads/TargetedKill.actor.cpp | 1 + 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index 3678af63b6..affd48da06 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -85,7 +85,8 @@ public: if (enabled && res) { auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); if (bitFlipPercentage > 0.0) { - if (deterministicRandom()->random01() < bitFlipPercentage) { + auto bitFlipProb = bitFlipPercentage/100; + if (deterministicRandom()->random01() < bitFlipProb) { pdata = (char*)arena.allocate4kAlignedBuffer(length); memcpy(pdata, data, length); // flip a random bit in the copied buffer diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.actor.cpp index f8f48be929..5e21e35254 100644 --- a/fdbserver/workloads/ClearSingleRange.actor.cpp +++ b/fdbserver/workloads/ClearSingleRange.actor.cpp @@ -47,9 +47,18 @@ struct ClearSingleRange : TestWorkload { ACTOR static Future fdbClientClearRange(Database db, ClearSingleRange* self) { state Transaction tr(db); - TraceEvent("ClearSingleRangeWaiting").detail("StartDelay", self->startDelay); - wait(delay(self->startDelay)); - tr.clear(KeyRangeRef(self->begin, self->end)); + try { + TraceEvent("ClearSingleRange"). + detail("Begin", printable(self->begin)). + detail("End", printable(self->end)).detail("StartDelay", self->startDelay); + tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + wait(delay(self->startDelay)); + tr.clear(KeyRangeRef(self->begin, self->end)); + wait(tr.commit()); + } catch (Error& e) { + TraceEvent("ClearRangeError").error(e); + wait(tr.onError(e)); + } return Void(); } }; diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index b9079b39eb..359e0e19c3 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -225,7 +225,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { if (self->throttleDisk) { int diskDelays = chaosMetrics->second.getInt("DiskDelays"); if (diskDelays > 0) { - foundChaosMetrics++; + foundChaosMetrics += diskDelays; } } @@ -233,7 +233,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { if (self->corruptFile) { int bitFlips = chaosMetrics->second.getInt("BitFlips"); if (bitFlips > 0) { - foundChaosMetrics++; + foundChaosMetrics += bitFlips; } } } diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp index 9611f057bb..67720c2249 100644 --- a/fdbserver/workloads/Mako.actor.cpp +++ b/fdbserver/workloads/Mako.actor.cpp @@ -56,7 +56,7 @@ struct MakoWorkload : TestWorkload { commits("Commits"), totalOps("Operations") { // init parameters from test file // Number of rows populated - rowCount = getOption(options, LiteralStringRef("rows"), 10000); + rowCount = getOption(options, LiteralStringRef("rows"), (uint64_t )10000); // Test duration in seconds testDuration = getOption(options, LiteralStringRef("testDuration"), 30.0); warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0); diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.actor.cpp index 1c40cd47b6..b77b97a18c 100644 --- a/fdbserver/workloads/TargetedKill.actor.cpp +++ b/fdbserver/workloads/TargetedKill.actor.cpp @@ -78,6 +78,7 @@ struct TargetedKillWorkload : TestWorkload { workers[i].processClass != ProcessClass::TesterClass)) { TraceEvent("WorkerKill").detail("TargetedMachine", address).detail("Worker", workers[i].interf.id()); workers[i].interf.clientInterface.reboot.send(rbReq); + killed++; } } From 518065c3edc9f7ef2cd1333248e89162e04469cb Mon Sep 17 00:00:00 2001 From: negoyal Date: Tue, 19 Oct 2021 17:22:27 -0700 Subject: [PATCH 021/338] TargetedKill fixes. --- .../workloads/ClearSingleRange.actor.cpp | 9 ++-- .../workloads/DiskFailureInjection.actor.cpp | 2 +- fdbserver/workloads/TargetedKill.actor.cpp | 41 +++++++++++++------ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.actor.cpp index 5e21e35254..62e7671a7c 100644 --- a/fdbserver/workloads/ClearSingleRange.actor.cpp +++ b/fdbserver/workloads/ClearSingleRange.actor.cpp @@ -43,14 +43,15 @@ struct ClearSingleRange : TestWorkload { Future check(Database const& cx) override { return true; } - void getMetrics(vector& m) override {} + void getMetrics(std::vector& m) override {} ACTOR static Future fdbClientClearRange(Database db, ClearSingleRange* self) { state Transaction tr(db); try { - TraceEvent("ClearSingleRange"). - detail("Begin", printable(self->begin)). - detail("End", printable(self->end)).detail("StartDelay", self->startDelay); + TraceEvent("ClearSingleRange") + .detail("Begin", printable(self->begin)) + .detail("End", printable(self->end)) + .detail("StartDelay", self->startDelay); tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); wait(delay(self->startDelay)); tr.clear(KeyRangeRef(self->begin, self->end)); diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 359e0e19c3..0973a2e61c 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -96,7 +96,7 @@ struct DiskFailureInjectionWorkload : TestWorkload { return true; } - void getMetrics(vector& m) override {} + void getMetrics(std::vector& m) override {} static void checkDiskFailureInjectionResult(Future res, WorkerInterface worker) { if (res.isError()) { diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.actor.cpp index 2270cecab3..cb3d0145c5 100644 --- a/fdbserver/workloads/TargetedKill.actor.cpp +++ b/fdbserver/workloads/TargetedKill.actor.cpp @@ -32,6 +32,7 @@ struct TargetedKillWorkload : TestWorkload { std::string machineToKill; bool enabled, killAllMachineProcesses; + int numKillStorages; double killAt; bool reboot; double suspendDuration; @@ -43,6 +44,7 @@ struct TargetedKillWorkload : TestWorkload { suspendDuration = getOption(options, LiteralStringRef("suspendDuration"), 1.0); machineToKill = getOption(options, LiteralStringRef("machineToKill"), LiteralStringRef("master")).toString(); killAllMachineProcesses = getOption(options, LiteralStringRef("killWholeMachine"), false); + numKillStorages = getOption(options, LiteralStringRef("numKillStorages"), 1); } std::string description() const override { return "TargetedKillWorkload"; } @@ -56,16 +58,17 @@ struct TargetedKillWorkload : TestWorkload { Future check(Database const& cx) override { return true; } void getMetrics(std::vector& m) override {} - ACTOR Future killEndpoint(NetworkAddress address, Database cx, TargetedKillWorkload* self) { + Future killEndpoint(std::vector workers, + NetworkAddress address, + Database cx, + TargetedKillWorkload* self) { if (&g_simulator == g_network) { g_simulator.killInterface(address, ISimulator::KillInstantly); return Void(); } - state std::vector workers = wait(getWorkers(self->dbInfo)); - int killed = 0; - state RebootRequest rbReq; + RebootRequest rbReq; if (self->reboot) { rbReq.waitForDuration = self->suspendDuration; } else { @@ -93,8 +96,13 @@ struct TargetedKillWorkload : TestWorkload { ACTOR Future assassin(Database cx, TargetedKillWorkload* self) { wait(delay(self->killAt)); state std::vector storageServers = wait(getStorageServers(cx)); + state std::vector workers = wait(getWorkers(self->dbInfo)); - NetworkAddress machine; + state NetworkAddress machine; + state NetworkAddress ccAddr; + state int killed = 0; + state int s = 0; + state int j = 0; if (self->machineToKill == "master") { machine = self->dbInfo->get().master.address(); } else if (self->machineToKill == "commitproxy") { @@ -129,13 +137,22 @@ struct TargetedKillWorkload : TestWorkload { } } else if (self->machineToKill == "storage" || self->machineToKill == "ss" || self->machineToKill == "storageserver") { - int o = deterministicRandom()->randomInt(0, storageServers.size()); - for (int i = 0; i < storageServers.size(); i++) { - StorageServerInterface ssi = storageServers[o]; + s = deterministicRandom()->randomInt(0, storageServers.size()); + ccAddr = self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress(); + for (j = 0; j < storageServers.size(); j++) { + StorageServerInterface ssi = storageServers[s]; machine = ssi.address(); - if (machine != self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress()) - break; - o = ++o % storageServers.size(); + if (machine != self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress()) { + TraceEvent("IsolatedMark").detail("TargetedMachine", machine).detail("Role", self->machineToKill); + wait(self->killEndpoint(workers, machine, cx, self)); + killed++; + TraceEvent("SentKillEndpoint") + .detail("Killed", killed) + .detail("NumKillStorages", self->numKillStorages); + if (killed == self->numKillStorages) + return Void(); + } + s = ++s % storageServers.size(); } } else if (self->machineToKill == "clustercontroller" || self->machineToKill == "cc") { machine = self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress(); @@ -143,7 +160,7 @@ struct TargetedKillWorkload : TestWorkload { TraceEvent("IsolatedMark").detail("TargetedMachine", machine).detail("Role", self->machineToKill); - wait(self->killEndpoint(machine, cx, self)); + wait(self->killEndpoint(workers, machine, cx, self)); return Void(); } From 4f0991eb675dc80393784c01f5312ba29597e71e Mon Sep 17 00:00:00 2001 From: Vaidas Gasiunas Date: Tue, 12 Oct 2021 17:29:09 +0200 Subject: [PATCH 022/338] MVC2.0: Introducing client library status values for instructing clients to download and activate a library; Operations to read and change client library status --- fdbclient/ClientLibManagement.actor.cpp | 70 ++++++++++++++++++- fdbclient/ClientLibManagement.actor.h | 8 +++ .../ClientLibManagementWorkload.actor.cpp | 37 ++++++++++ 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/fdbclient/ClientLibManagement.actor.cpp b/fdbclient/ClientLibManagement.actor.cpp index 8b24956ee3..cce3ab783e 100644 --- a/fdbclient/ClientLibManagement.actor.cpp +++ b/fdbclient/ClientLibManagement.actor.cpp @@ -46,7 +46,7 @@ struct ClientLibBinaryInfo { #define ASSERT_INDEX_IN_RANGE(idx, arr) ASSERT(idx >= 0 && idx < sizeof(arr) / sizeof(arr[0])) const std::string& getStatusName(ClientLibStatus status) { - static const std::string statusNames[] = { "disabled", "available", "uploading" }; + static const std::string statusNames[] = { "disabled", "available", "uploading", "download", "active" }; int idx = static_cast(status); ASSERT_INDEX_IN_RANGE(idx, statusNames); return statusNames[idx]; @@ -123,7 +123,13 @@ ClientLibChecksumAlg getChecksumAlgByName(std::string_view checksumAlgName) { namespace { bool isValidTargetStatus(ClientLibStatus status) { - return status == ClientLibStatus::AVAILABLE || status == ClientLibStatus::DISABLED; + return status == ClientLibStatus::AVAILABLE || status == ClientLibStatus::DISABLED || + status == ClientLibStatus::DOWNLOAD || status == ClientLibStatus::ACTIVE; +} + +bool isAvailableForDownload(ClientLibStatus status) { + return status == ClientLibStatus::AVAILABLE || status == ClientLibStatus::DOWNLOAD || + status == ClientLibStatus::ACTIVE; } json_spirit::mObject parseMetadataJson(StringRef metadataString) { @@ -489,7 +495,7 @@ ACTOR Future downloadClientLibrary(Database db, } // Allow downloading only libraries in the available state - if (getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)) != ClientLibStatus::AVAILABLE) { + if (!isAvailableForDownload(getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)))) { throw client_lib_not_available(); } @@ -707,4 +713,62 @@ ACTOR Future>> listClientLibraries(Database db, return result; } +ACTOR Future getClientLibraryStatus(Database db, StringRef clientLibId) { + state Key clientLibMetaKey = metadataKeyFromId(clientLibId.toString()); + state Transaction tr(db); + loop { + try { + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + Optional metadataOpt = wait(tr.get(clientLibMetaKey)); + if (!metadataOpt.present()) { + TraceEvent(SevWarnAlways, "ClientLibraryNotFound").detail("Key", clientLibMetaKey); + throw client_lib_not_found(); + } + json_spirit::mObject metadataJson = parseMetadataJson(metadataOpt.get()); + return getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)); + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + +ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, ClientLibStatus newStatus) { + state Key clientLibMetaKey = metadataKeyFromId(clientLibId.toString()); + state json_spirit::mObject metadataJson; + state std::string jsStr; + + if (!isValidTargetStatus(newStatus)) { + TraceEvent(SevWarnAlways, "ClientLibraryInvalidMetadata") + .detail("Reason", "InvalidTargetStatus") + .detail("Status", getStatusName(newStatus)); + throw client_lib_invalid_metadata(); + } + + loop { + state Transaction tr(db); + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + Optional metadataOpt = wait(tr.get(clientLibMetaKey)); + if (!metadataOpt.present()) { + TraceEvent(SevWarnAlways, "ClientLibraryNotFound").detail("Key", clientLibMetaKey); + throw client_lib_not_found(); + } + metadataJson = parseMetadataJson(metadataOpt.get()); + metadataJson[CLIENTLIB_ATTR_STATUS] = getStatusName(newStatus); + jsStr = json_spirit::write_string(json_spirit::mValue(metadataJson)); + tr.set(clientLibMetaKey, ValueRef(jsStr)); + wait(tr.commit()); + break; + } catch (Error& e) { + if (e.code() == error_code_client_lib_not_found) { + throw; + } + wait(tr.onError(e)); + } + } + + TraceEvent("ClientLibraryStatusChanged").detail("Key", clientLibMetaKey).detail("Status", getStatusName(newStatus)); + return Void(); +} + } // namespace ClientLibManagement \ No newline at end of file diff --git a/fdbclient/ClientLibManagement.actor.h b/fdbclient/ClientLibManagement.actor.h index eaea5ccbf8..fed7be7f79 100644 --- a/fdbclient/ClientLibManagement.actor.h +++ b/fdbclient/ClientLibManagement.actor.h @@ -37,6 +37,8 @@ enum class ClientLibStatus { DISABLED = 0, AVAILABLE, // 1 UPLOADING, // 2 + DOWNLOAD, // 3 + ACTIVE, // 4 COUNT // must be the last one }; @@ -133,6 +135,12 @@ ACTOR Future deleteClientLibrary(Database db, Standalone client // Returns metadata JSON of each library ACTOR Future>> listClientLibraries(Database db, ClientLibFilter filter); +// Get the current status of an uploaded client library +ACTOR Future getClientLibraryStatus(Database db, StringRef clientLibId); + +// Change client library metadata status +ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, ClientLibStatus newStatus); + } // namespace ClientLibManagement #include "flow/unactorcompiler.h" diff --git a/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp b/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp index f259f4c756..7e8a8a2d67 100644 --- a/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp +++ b/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp @@ -107,6 +107,8 @@ struct ClientLibManagementWorkload : public TestWorkload { wait(testClientLibListAfterUpload(self, cx)); wait(testDownloadClientLib(self, cx)); wait(testClientLibDownloadNotExisting(self, cx)); + wait(testChangeClientLibStatusErrors(self, cx)); + wait(testDisableClientLib(self, cx)); wait(testDeleteClientLib(self, cx)); wait(testUploadedClientLibInList(self, cx, ClientLibFilter(), false, "No filter, after delete")); return Void(); @@ -321,6 +323,41 @@ struct ClientLibManagementWorkload : public TestWorkload { return Void(); } + ACTOR static Future testChangeClientLibStatusErrors(ClientLibManagementWorkload* self, Database cx) { + wait(testExpectedError(changeClientLibraryStatus(cx, self->uploadedClientLibId, ClientLibStatus::UPLOADING), + "Setting invalid client library status", + client_lib_invalid_metadata(), + &self->success)); + + wait(testExpectedError(changeClientLibraryStatus(cx, "notExistingClientLib"_sr, ClientLibStatus::DOWNLOAD), + "Changing not existing client library status", + client_lib_not_found(), + &self->success)); + return Void(); + } + + ACTOR static Future testDisableClientLib(ClientLibManagementWorkload* self, Database cx) { + state std::string destFileName = format("clientLibDownload%d", self->clientId); + + // Set disabled status on the uploaded library + wait(changeClientLibraryStatus(cx, self->uploadedClientLibId, ClientLibStatus::DISABLED)); + state ClientLibStatus newStatus = wait(getClientLibraryStatus(cx, self->uploadedClientLibId)); + if (newStatus != ClientLibStatus::DISABLED) { + TraceEvent(SevError, "ClientLibDisableClientLibFailed") + .detail("Reason", "Unexpected status") + .detail("Expected", ClientLibStatus::DISABLED) + .detail("Actual", newStatus); + self->success = false; + } + + // It should not be possible to download a disabled client library + wait(testExpectedError(downloadClientLibrary(cx, self->uploadedClientLibId, StringRef(destFileName)), + "Downloading disabled client library", + client_lib_not_available(), + &self->success)); + return Void(); + } + /* ---------------------------------------------------------------- * Utility methods */ From 875824b1862ea9efae667fbbfc4d4bdc15d94e10 Mon Sep 17 00:00:00 2001 From: Vaidas Gasiunas Date: Fri, 22 Oct 2021 18:45:12 +0200 Subject: [PATCH 023/338] MVC2.0: Notify clients about relevant changes of client libraries --- fdbclient/ClientLibManagement.actor.cpp | 35 ++++++++++++--- fdbclient/ClientLibManagement.actor.h | 4 +- fdbclient/CommitProxyInterface.h | 5 ++- fdbclient/DatabaseContext.h | 4 +- fdbclient/NativeAPI.actor.cpp | 22 +++++++--- fdbclient/SystemData.cpp | 2 + fdbclient/SystemData.h | 2 + fdbserver/ClusterController.actor.cpp | 43 +++++++++++++++++++ .../ClientLibManagementWorkload.actor.cpp | 35 +++++++++++++++ 9 files changed, 135 insertions(+), 17 deletions(-) diff --git a/fdbclient/ClientLibManagement.actor.cpp b/fdbclient/ClientLibManagement.actor.cpp index cce3ab783e..180cdff604 100644 --- a/fdbclient/ClientLibManagement.actor.cpp +++ b/fdbclient/ClientLibManagement.actor.cpp @@ -23,6 +23,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/ClientKnobs.h" +#include "fdbclient/SystemData.h" #include "fdbclient/versions.h" #include "fdbrpc/IAsyncFile.h" #include "flow/Platform.h" @@ -132,6 +133,15 @@ bool isAvailableForDownload(ClientLibStatus status) { status == ClientLibStatus::ACTIVE; } +void updateClientLibChangeCounter(Transaction& tr, ClientLibStatus status) { + static const int64_t counterIncVal = 1; + if (status == ClientLibStatus::DOWNLOAD || status == ClientLibStatus::ACTIVE) { + tr.atomicOp(clientLibChangeCounterKey, + StringRef(reinterpret_cast(&counterIncVal), sizeof(counterIncVal)), + MutationRef::AddValue); + } +} + json_spirit::mObject parseMetadataJson(StringRef metadataString) { json_spirit::mValue parsedMetadata; if (!json_spirit::read_string(metadataString.toString(), parsedMetadata) || @@ -438,6 +448,7 @@ ACTOR Future uploadClientLibrary(Database db, tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.set(clientLibMetaKey, ValueRef(jsStr)); + updateClientLibChangeCounter(tr, targetStatus); wait(tr.commit()); break; } catch (Error& e) { @@ -647,8 +658,8 @@ void applyClientLibFilter(const ClientLibFilter& filter, for (const auto& [k, v] : scanResults) { try { json_spirit::mObject metadataJson = parseMetadataJson(v); - if (filter.matchAvailableOnly && getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)) != - ClientLibStatus::AVAILABLE) { + if (filter.matchAvailableOnly && + !isAvailableForDownload(getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)))) { continue; } if (filter.matchCompatibleAPI && @@ -713,8 +724,8 @@ ACTOR Future>> listClientLibraries(Database db, return result; } -ACTOR Future getClientLibraryStatus(Database db, StringRef clientLibId) { - state Key clientLibMetaKey = metadataKeyFromId(clientLibId.toString()); +ACTOR Future getClientLibraryStatus(Database db, Standalone clientLibId) { + state Key clientLibMetaKey = metadataKeyFromId(clientLibId); state Transaction tr(db); loop { try { @@ -732,10 +743,13 @@ ACTOR Future getClientLibraryStatus(Database db, StringRef clie } } -ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, ClientLibStatus newStatus) { - state Key clientLibMetaKey = metadataKeyFromId(clientLibId.toString()); +ACTOR Future changeClientLibraryStatus(Database db, + Standalone clientLibId, + ClientLibStatus newStatus) { + state Key clientLibMetaKey = metadataKeyFromId(clientLibId); state json_spirit::mObject metadataJson; state std::string jsStr; + state Transaction tr; if (!isValidTargetStatus(newStatus)) { TraceEvent(SevWarnAlways, "ClientLibraryInvalidMetadata") @@ -745,7 +759,7 @@ ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, } loop { - state Transaction tr(db); + tr = Transaction(db); try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); Optional metadataOpt = wait(tr.get(clientLibMetaKey)); @@ -754,9 +768,16 @@ ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, throw client_lib_not_found(); } metadataJson = parseMetadataJson(metadataOpt.get()); + ClientLibStatus prevStatus = getStatusByName(getMetadataStrAttr(metadataJson, CLIENTLIB_ATTR_STATUS)); + if (prevStatus == newStatus) { + return Void(); + } metadataJson[CLIENTLIB_ATTR_STATUS] = getStatusName(newStatus); jsStr = json_spirit::write_string(json_spirit::mValue(metadataJson)); tr.set(clientLibMetaKey, ValueRef(jsStr)); + + updateClientLibChangeCounter(tr, newStatus); + wait(tr.commit()); break; } catch (Error& e) { diff --git a/fdbclient/ClientLibManagement.actor.h b/fdbclient/ClientLibManagement.actor.h index fed7be7f79..95e2e01e8f 100644 --- a/fdbclient/ClientLibManagement.actor.h +++ b/fdbclient/ClientLibManagement.actor.h @@ -136,10 +136,10 @@ ACTOR Future deleteClientLibrary(Database db, Standalone client ACTOR Future>> listClientLibraries(Database db, ClientLibFilter filter); // Get the current status of an uploaded client library -ACTOR Future getClientLibraryStatus(Database db, StringRef clientLibId); +ACTOR Future getClientLibraryStatus(Database db, Standalone clientLibId); // Change client library metadata status -ACTOR Future changeClientLibraryStatus(Database db, StringRef clientLibId, ClientLibStatus newStatus); +ACTOR Future changeClientLibraryStatus(Database db, Standalone clientLibId, ClientLibStatus newStatus); } // namespace ClientLibManagement diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index ecb745d318..c46bdcb709 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -115,6 +115,9 @@ struct ClientDBInfo { firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk Optional forward; std::vector history; + // a counter increased every time a change of uploaded client libraries + // happens, the clients need to be aware of + uint64_t clientLibChangeCounter = 0; ClientDBInfo() {} @@ -126,7 +129,7 @@ struct ClientDBInfo { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, grvProxies, commitProxies, id, forward, history); + serializer(ar, grvProxies, commitProxies, id, forward, history, clientLibChangeCounter); } }; diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 837d4ec793..8392b8b624 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -197,6 +197,7 @@ public: Future> getCommitProxiesFuture(bool useProvisionalProxies); Reference getGrvProxies(bool useProvisionalProxies); Future onProxiesChanged() const; + Future onClientLibStatusChanged() const; Future getHealthMetrics(bool detailed); // Returns the protocol version reported by the coordinator this client is connected to @@ -287,7 +288,8 @@ public: // Key DB-specific information Reference>> connectionRecord; AsyncTrigger proxiesChangeTrigger; - Future monitorProxiesInfoChange; + AsyncTrigger clientLibChangeTrigger; + Future clientDBInfoMonitor; Future monitorTssInfoChange; Future tssMismatchHandler; PromiseStream>> tssMismatchStream; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 117524a43a..b761056eec 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -717,14 +717,17 @@ Future attemptGRVFromOldProxies(std::vector oldProxies, return waitForAll(replies); } -ACTOR static Future monitorProxiesChange(DatabaseContext* cx, - Reference const> clientDBInfo, - AsyncTrigger* triggerVar) { +ACTOR static Future monitorClientDBInfoChange(DatabaseContext* cx, + Reference const> clientDBInfo, + AsyncTrigger* proxyChangeTrigger, + AsyncTrigger* clientLibChangeTrigger) { state std::vector curCommitProxies; state std::vector curGrvProxies; state ActorCollection actors(false); + state uint64_t curClientLibChangeCounter; curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; + curClientLibChangeCounter = clientDBInfo->get().clientLibChangeCounter; loop { choose { @@ -745,7 +748,10 @@ ACTOR static Future monitorProxiesChange(DatabaseContext* cx, } curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; - triggerVar->trigger(); + proxyChangeTrigger->trigger(); + } + if (curClientLibChangeCounter != clientDBInfo->get().clientLibChangeCounter) { + clientLibChangeTrigger->trigger(); } } when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); } @@ -1234,7 +1240,7 @@ DatabaseContext::DatabaseContext(Reference> clientInfo, DatabaseContext::~DatabaseContext() { cacheListMonitor.cancel(); - monitorProxiesInfoChange.cancel(); + clientDBInfoMonitor.cancel(); monitorTssInfoChange.cancel(); tssMismatchHandler.cancel(); for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it)) @@ -1583,6 +1589,10 @@ Future DatabaseContext::onProxiesChanged() const { return this->proxiesChangeTrigger.onTrigger(); } +Future DatabaseContext::onClientLibStatusChanged() const { + return this->clientLibChangeTrigger.onTrigger(); +} + bool DatabaseContext::sampleReadTags() const { double sampleRate = GlobalConfig::globalConfig().get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE); return sampleRate > 0 && deterministicRandom()->random01() <= sampleRate; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 4de1b4aeb6..94627b84df 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1033,6 +1033,8 @@ const KeyRangeRef clientLibBinaryKeys(LiteralStringRef("\xff\x02/clientlib/bin/" LiteralStringRef("\xff\x02/clientlib/bin0")); const KeyRef clientLibBinaryPrefix = clientLibBinaryKeys.begin; +const KeyRef clientLibChangeCounterKey = "\xff\x02/clientlib/changeCounter"_sr; + const KeyRangeRef testOnlyTxnStateStorePrefixRange(LiteralStringRef("\xff/TESTONLYtxnStateStore/"), LiteralStringRef("\xff/TESTONLYtxnStateStore0")); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 4b9c7a22f5..c3d22d162d 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -488,6 +488,8 @@ extern const KeyRef clientLibMetadataPrefix; extern const KeyRangeRef clientLibBinaryKeys; extern const KeyRef clientLibBinaryPrefix; +extern const KeyRef clientLibChangeCounterKey; + // All mutations done to this range are blindly copied into txnStateStore. // Used to create artifically large txnStateStore instances in testing. extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 298b7867f5..f91f7d6d01 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -4719,6 +4719,48 @@ ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { } } +ACTOR Future monitorClientLibChangeCounter(ClusterControllerData::DBInfo* db) { + state ClientDBInfo clientInfo; + state ReadYourWritesTransaction tr; + state Future clientLibChangeFuture; + + loop { + tr = ReadYourWritesTransaction(db->db); + loop { + try { + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE); + + Optional counterVal = wait(tr.get(clientLibChangeCounterKey)); + if (counterVal.present() && counterVal.get().size() == sizeof(uint64_t)) { + uint64_t changeCounter = *reinterpret_cast(counterVal.get().begin()); + + clientInfo = db->serverInfo->get().client; + if (changeCounter != clientInfo.clientLibChangeCounter) { + TraceEvent("ClientLibChangeCounterChanged").detail("Value", changeCounter); + clientInfo.id = deterministicRandom()->randomUniqueID(); + clientInfo.clientLibChangeCounter = changeCounter; + db->clientInfo->set(clientInfo); + + ServerDBInfo serverInfo = db->serverInfo->get(); + serverInfo.id = deterministicRandom()->randomUniqueID(); + serverInfo.infoGeneration = ++db->dbInfoCount; + serverInfo.client = clientInfo; + db->serverInfo->set(serverInfo); + } + } + + clientLibChangeFuture = tr.watch(clientLibChangeCounterKey); + wait(tr.commit()); + wait(clientLibChangeFuture); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } +} + ACTOR Future updatedChangingDatacenters(ClusterControllerData* self) { // do not change the cluster controller until all the processes have had a chance to register wait(delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY)); @@ -5416,6 +5458,7 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, self.addActor.send(monitorProcessClasses(&self)); self.addActor.send(monitorServerInfoConfig(&self.db)); self.addActor.send(monitorGlobalConfig(&self.db)); + self.addActor.send(monitorClientLibChangeCounter(&self.db)); self.addActor.send(updatedChangingDatacenters(&self)); self.addActor.send(updatedChangedDatacenters(&self)); self.addActor.send(updateDatacenterVersionDifference(&self)); diff --git a/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp b/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp index 7e8a8a2d67..6aca7b3adc 100644 --- a/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp +++ b/fdbserver/workloads/ClientLibManagementWorkload.actor.cpp @@ -109,6 +109,7 @@ struct ClientLibManagementWorkload : public TestWorkload { wait(testClientLibDownloadNotExisting(self, cx)); wait(testChangeClientLibStatusErrors(self, cx)); wait(testDisableClientLib(self, cx)); + wait(testChangeStateToDownload(self, cx)); wait(testDeleteClientLib(self, cx)); wait(testUploadedClientLibInList(self, cx, ClientLibFilter(), false, "No filter, after delete")); return Void(); @@ -178,10 +179,13 @@ struct ClientLibManagementWorkload : public TestWorkload { ACTOR static Future testUploadClientLib(ClientLibManagementWorkload* self, Database cx) { state Standalone metadataStr; state std::vector>> concurrentUploads; + state Future clientLibChanged = cx->onClientLibStatusChanged(); + validClientLibMetadataSample(self->uploadedMetadataJson); self->uploadedMetadataJson[CLIENTLIB_ATTR_CHECKSUM] = self->generatedChecksum.toString(); // avoid clientLibId clashes, when multiple clients try to upload the same file self->uploadedMetadataJson[CLIENTLIB_ATTR_TYPE] = format("devbuild%d", self->clientId); + self->uploadedMetadataJson[CLIENTLIB_ATTR_STATUS] = getStatusName(ClientLibStatus::ACTIVE); metadataStr = StringRef(json_spirit::write_string(json_spirit::mValue(self->uploadedMetadataJson))); self->uploadedClientLibId = getClientLibIdFromMetadataJson(metadataStr); @@ -211,6 +215,13 @@ struct ClientLibManagementWorkload : public TestWorkload { TraceEvent(SevError, "ClientLibConflictingUpload").log(); self->success = false; } + + Optional notificationWait = wait(timeout(clientLibChanged, 100.0)); + if (!notificationWait.present()) { + TraceEvent(SevError, "ClientLibChangeNotificationFailed").log(); + self->success = false; + } + return Void(); } @@ -358,6 +369,30 @@ struct ClientLibManagementWorkload : public TestWorkload { return Void(); } + ACTOR static Future testChangeStateToDownload(ClientLibManagementWorkload* self, Database cx) { + state std::string destFileName = format("clientLibDownload%d", self->clientId); + state Future clientLibChanged = cx->onClientLibStatusChanged(); + + // Set disabled status on the uploaded library + wait(changeClientLibraryStatus(cx, self->uploadedClientLibId, ClientLibStatus::DOWNLOAD)); + state ClientLibStatus newStatus = wait(getClientLibraryStatus(cx, self->uploadedClientLibId)); + if (newStatus != ClientLibStatus::DOWNLOAD) { + TraceEvent(SevError, "ClientLibChangeStatusFailed") + .detail("Reason", "Unexpected status") + .detail("Expected", ClientLibStatus::DOWNLOAD) + .detail("Actual", newStatus); + self->success = false; + } + + Optional notificationWait = wait(timeout(clientLibChanged, 100.0)); + if (!notificationWait.present()) { + TraceEvent(SevError, "ClientLibChangeNotificationFailed").log(); + self->success = false; + } + + return Void(); + } + /* ---------------------------------------------------------------- * Utility methods */ From 628317b3b5381d4817c8037d891620e38d5ee91c Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 27 Oct 2021 15:51:21 -0700 Subject: [PATCH 024/338] Separate out memory benchmarks in flowbench --- flowbench/BenchHash.cpp | 27 ---------------------- flowbench/BenchMem.cpp | 48 ++++++++++++++++++++++++++++++++++++++++ flowbench/CMakeLists.txt | 3 ++- 3 files changed, 50 insertions(+), 28 deletions(-) create mode 100644 flowbench/BenchMem.cpp diff --git a/flowbench/BenchHash.cpp b/flowbench/BenchHash.cpp index 2cb0bf428b..e5a2fd8401 100644 --- a/flowbench/BenchHash.cpp +++ b/flowbench/BenchHash.cpp @@ -67,30 +67,3 @@ static void bench_hash(benchmark::State& state) { BENCHMARK_TEMPLATE(bench_hash, HashType::CRC32C)->DenseRange(2, 18)->ReportAggregatesOnly(true); BENCHMARK_TEMPLATE(bench_hash, HashType::HashLittle2)->DenseRange(2, 18)->ReportAggregatesOnly(true); BENCHMARK_TEMPLATE(bench_hash, HashType::XXHash3)->DenseRange(2, 18)->ReportAggregatesOnly(true); - -static void bench_memcmp(benchmark::State& state) { - constexpr int kLength = 10000; - std::unique_ptr b1{ new char[kLength] }; - std::unique_ptr b2{ new char[kLength] }; - memset(b1.get(), 0, kLength); - memset(b2.get(), 0, kLength); - b2.get()[kLength - 1] = 1; - - while (state.KeepRunning()) { - benchmark::DoNotOptimize(memcmp(b1.get(), b2.get(), kLength)); - } -} - -static void bench_memcpy(benchmark::State& state) { - constexpr int kLength = 10000; - std::unique_ptr b1{ new char[kLength] }; - std::unique_ptr b2{ new char[kLength] }; - memset(b1.get(), 0, kLength); - - while (state.KeepRunning()) { - benchmark::DoNotOptimize(memcpy(b2.get(), b1.get(), kLength)); - } -} - -BENCHMARK(bench_memcmp); -BENCHMARK(bench_memcpy); \ No newline at end of file diff --git a/flowbench/BenchMem.cpp b/flowbench/BenchMem.cpp new file mode 100644 index 0000000000..52b8e0c493 --- /dev/null +++ b/flowbench/BenchMem.cpp @@ -0,0 +1,48 @@ +/* + * BenchMem.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark/benchmark.h" + +static void bench_memcmp(benchmark::State& state) { + constexpr int kLength = 10000; + std::unique_ptr b1{ new char[kLength] }; + std::unique_ptr b2{ new char[kLength] }; + memset(b1.get(), 0, kLength); + memset(b2.get(), 0, kLength); + b2.get()[kLength - 1] = 1; + + while (state.KeepRunning()) { + benchmark::DoNotOptimize(memcmp(b1.get(), b2.get(), kLength)); + } +} + +static void bench_memcpy(benchmark::State& state) { + constexpr int kLength = 10000; + std::unique_ptr b1{ new char[kLength] }; + std::unique_ptr b2{ new char[kLength] }; + memset(b1.get(), 0, kLength); + + while (state.KeepRunning()) { + benchmark::DoNotOptimize(memcpy(b2.get(), b1.get(), kLength)); + } +} + +BENCHMARK(bench_memcmp); +BENCHMARK(bench_memcpy); diff --git a/flowbench/CMakeLists.txt b/flowbench/CMakeLists.txt index 8caad0ce02..0a8582afc1 100644 --- a/flowbench/CMakeLists.txt +++ b/flowbench/CMakeLists.txt @@ -1,8 +1,9 @@ set(FLOWBENCH_SRCS flowbench.actor.cpp - BenchMetadataCheck.cpp BenchHash.cpp BenchIterate.cpp + BenchMem.cpp + BenchMetadataCheck.cpp BenchPopulate.cpp BenchRandom.cpp BenchRef.cpp From dbf7f9b04721cc90bd2851fb86b18d57bff94760 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 10:22:50 -0700 Subject: [PATCH 025/338] Add some includes to BenchMem.cpp --- flowbench/BenchMem.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flowbench/BenchMem.cpp b/flowbench/BenchMem.cpp index 52b8e0c493..5373353fb0 100644 --- a/flowbench/BenchMem.cpp +++ b/flowbench/BenchMem.cpp @@ -18,6 +18,9 @@ * limitations under the License. */ +#include +#include + #include "benchmark/benchmark.h" static void bench_memcmp(benchmark::State& state) { From 88e66533ad59e90c6befb9b1c8f948ca4de6b874 Mon Sep 17 00:00:00 2001 From: negoyal Date: Thu, 28 Oct 2021 11:13:12 -0700 Subject: [PATCH 026/338] devFormat --- fdbrpc/AsyncFileChaos.actor.h | 2 +- fdbserver/workloads/Mako.actor.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileChaos.actor.h b/fdbrpc/AsyncFileChaos.actor.h index affd48da06..7f61ab36bb 100644 --- a/fdbrpc/AsyncFileChaos.actor.h +++ b/fdbrpc/AsyncFileChaos.actor.h @@ -85,7 +85,7 @@ public: if (enabled && res) { auto bitFlipPercentage = static_cast(res)->getBitFlipPercentage(); if (bitFlipPercentage > 0.0) { - auto bitFlipProb = bitFlipPercentage/100; + auto bitFlipProb = bitFlipPercentage / 100; if (deterministicRandom()->random01() < bitFlipProb) { pdata = (char*)arena.allocate4kAlignedBuffer(length); memcpy(pdata, data, length); diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp index 67720c2249..9e32b7aa37 100644 --- a/fdbserver/workloads/Mako.actor.cpp +++ b/fdbserver/workloads/Mako.actor.cpp @@ -56,7 +56,7 @@ struct MakoWorkload : TestWorkload { commits("Commits"), totalOps("Operations") { // init parameters from test file // Number of rows populated - rowCount = getOption(options, LiteralStringRef("rows"), (uint64_t )10000); + rowCount = getOption(options, LiteralStringRef("rows"), (uint64_t)10000); // Test duration in seconds testDuration = getOption(options, LiteralStringRef("testDuration"), 30.0); warmingDelay = getOption(options, LiteralStringRef("warmingDelay"), 0.0); From fd0aeaf48ebdc3ab4917149a82f626b95b5712da Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Sat, 21 Aug 2021 20:40:20 -0700 Subject: [PATCH 027/338] Add a new process launcher for FDB on Kube. --- .../.testdata/default_config.json | 36 +++ fdbkubernetesmonitor/.testdata/fdb.cluster | 1 + fdbkubernetesmonitor/.testdata/test_env.sh | 5 + fdbkubernetesmonitor/config.go | 137 ++++++++++ fdbkubernetesmonitor/config_test.go | 102 ++++++++ fdbkubernetesmonitor/go.mod | 27 ++ fdbkubernetesmonitor/go.sum | 6 + fdbkubernetesmonitor/main.go | 41 +++ fdbkubernetesmonitor/monitor.go | 238 ++++++++++++++++++ 9 files changed, 593 insertions(+) create mode 100644 fdbkubernetesmonitor/.testdata/default_config.json create mode 100644 fdbkubernetesmonitor/.testdata/fdb.cluster create mode 100644 fdbkubernetesmonitor/.testdata/test_env.sh create mode 100644 fdbkubernetesmonitor/config.go create mode 100644 fdbkubernetesmonitor/config_test.go create mode 100644 fdbkubernetesmonitor/go.mod create mode 100644 fdbkubernetesmonitor/go.sum create mode 100644 fdbkubernetesmonitor/main.go create mode 100644 fdbkubernetesmonitor/monitor.go diff --git a/fdbkubernetesmonitor/.testdata/default_config.json b/fdbkubernetesmonitor/.testdata/default_config.json new file mode 100644 index 0000000000..ecb09eca28 --- /dev/null +++ b/fdbkubernetesmonitor/.testdata/default_config.json @@ -0,0 +1,36 @@ +{ + "version": "6.3.0", + "arguments": [ + {"value": "--cluster_file"}, + {"value": ".testdata/fdb.cluster"}, + {"value": "--public_address"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_PUBLIC_IP"}, + {"value": ":"}, + {"type": "ProcessNumber", "offset": 4499, "multiplier": 2} + ]}, + {"value": "--listen_address"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_POD_IP"}, + {"value": ":"}, + {"type": "ProcessNumber", "offset": 4499, "multiplier": 2} + ]}, + {"value": "--datadir"}, + {"type": "Concatenate", "values": [ + {"value": ".testdata/data/"}, + {"type": "ProcessNumber"} + ]}, + {"value": "--class"}, + {"value": "storage"}, + {"value": "--locality_zoneid"}, + {"type": "Environment", "source": "FDB_ZONE_ID"}, + {"value": "--locality_instance-id"}, + {"type": "Environment", "source": "FDB_INSTANCE_ID"}, + {"value": "--locality_process-id"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_INSTANCE_ID"}, + {"value": "-"}, + {"type": "ProcessNumber"} + ]} + ] +} diff --git a/fdbkubernetesmonitor/.testdata/fdb.cluster b/fdbkubernetesmonitor/.testdata/fdb.cluster new file mode 100644 index 0000000000..4b36477173 --- /dev/null +++ b/fdbkubernetesmonitor/.testdata/fdb.cluster @@ -0,0 +1 @@ +test:test@127.0.0.1:4501 diff --git a/fdbkubernetesmonitor/.testdata/test_env.sh b/fdbkubernetesmonitor/.testdata/test_env.sh new file mode 100644 index 0000000000..ced881d347 --- /dev/null +++ b/fdbkubernetesmonitor/.testdata/test_env.sh @@ -0,0 +1,5 @@ +export FDB_PUBLIC_IP=127.0.0.1 +export FDB_POD_IP=127.0.0.1 +export FDB_ZONE_ID=localhost +export FDB_MACHINE_ID=localhost +export FDB_INSTANCE_ID=storage-1 diff --git a/fdbkubernetesmonitor/config.go b/fdbkubernetesmonitor/config.go new file mode 100644 index 0000000000..2c89826cb2 --- /dev/null +++ b/fdbkubernetesmonitor/config.go @@ -0,0 +1,137 @@ +// config.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "fmt" + "os" +) + +// ProcessConfiguration models the configuration for starting a FoundationDB +// process. +type ProcessConfiguration struct { + // Version provides the version of FoundationDB the process should run. + Version string `json:"version"` + + // ServerCount defines the number of processes to start. + ServerCount int `json:"serverCount,omitempty"` + + // Arguments provides the arugments to the process. + Arguments []Argument `json:"arguments,omitempty"` +} + +// Argument defines an argument to the process. +type Argument struct { + // ArgumentType determines how the value is generated. + ArgumentType ArgumentType `json:"type,omitempty"` + + // Value provides the value for a Literal type argument. + Value string `json:"value,omitempty"` + + // Values provides the sub-values for a Concatenate type argument. + Values []Argument `json:"values,omitempty"` + + // Source provides the name of the environment variable to use for an + // Environment type argument. + Source string `json:"source,omitempty"` + + // Multiplier provides a multiplier for the process number for ProcessNumber + // type arguments. + Multiplier int `json:"multiplier,omitempty"` + + // Offset provides an offset to add to the process number for ProcessNumber + // type argujments. + Offset int `json:"offset,omitempty"` +} + +// ArgumentType defines the types for arguments. +type ArgumentType string + +const ( + // LiteralArgumentType defines an argument with a literal string value. + LiteralArgumentType ArgumentType = "Literal" + + // ConcatenateArgumentType defines an argument composed of other arguments. + ConcatenateArgumentType = "Concatenate" + + // EnvironmentArgumentType defines an argument that is pulled from an + // environment variable. + EnvironmentArgumentType = "Environment" + + // ProcessNumberArgumentType defines an argument that is calculated using + // the number of the process in the process list. + ProcessNumberArgumentType = "ProcessNumber" +) + +// GenerateArgument processes an argument and generates its string +// representation. +func (argument Argument) GenerateArgument(processNumber int, env map[string]string) (string, error) { + switch argument.ArgumentType { + case "": + fallthrough + case LiteralArgumentType: + return argument.Value, nil + case ConcatenateArgumentType: + concatenated := "" + for _, childArgument := range argument.Values { + childValue, err := childArgument.GenerateArgument(processNumber, env) + if err != nil { + return "", err + } + concatenated += childValue + } + return concatenated, nil + case ProcessNumberArgumentType: + number := processNumber + if argument.Multiplier != 0 { + number = number * argument.Multiplier + } + number = number + argument.Offset + return fmt.Sprintf("%d", number), nil + case EnvironmentArgumentType: + var value string + var present bool + if env != nil { + value, present = env[argument.Source] + } else { + value, present = os.LookupEnv(argument.Source) + } + if !present { + return "", fmt.Errorf("Missing environment variable %s", argument.Source) + } + return value, nil + default: + return "", fmt.Errorf("Unsupported argument type %s", argument.ArgumentType) + } +} + +// GenerateArguments intreprets the arguments in the process configuration and +// generates a command invocation. +func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) { + results := make([]string, len(configuration.Arguments)) + for indexOfArgument, argument := range configuration.Arguments { + result, err := argument.GenerateArgument(processNumber, env) + if err != nil { + return nil, err + } + results[indexOfArgument] = result + } + return results, nil +} diff --git a/fdbkubernetesmonitor/config_test.go b/fdbkubernetesmonitor/config_test.go new file mode 100644 index 0000000000..9aca4043bf --- /dev/null +++ b/fdbkubernetesmonitor/config_test.go @@ -0,0 +1,102 @@ +// config_test.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "encoding/json" + "os" + "reflect" + "testing" +) + +func loadConfigFromFile(path string) (*ProcessConfiguration, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + decoder := json.NewDecoder(file) + config := &ProcessConfiguration{} + err = decoder.Decode(config) + if err != nil { + return nil, err + } + return config, nil +} + +func TestGeneratingArgumentsForDefaultConfig(t *testing.T) { + config, err := loadConfigFromFile(".testdata/default_config.json") + if err != nil { + t.Error(err) + return + } + + arguments, err := config.GenerateArguments(1, map[string]string{ + "FDB_PUBLIC_IP": "10.0.0.1", + "FDB_POD_IP": "192.168.0.1", + "FDB_ZONE_ID": "zone1", + "FDB_INSTANCE_ID": "storage-1", + }) + if err != nil { + t.Error(err) + return + } + + expectedArguments := []string{ + "--cluster_file", ".testdata/fdb.cluster", + "--public_address", "10.0.0.1:4501", "--listen_address", "192.168.0.1:4501", + "--datadir", ".testdata/data/1", "--class", "storage", + "--locality_zoneid", "zone1", "--locality_instance-id", "storage-1", + "--locality_process-id", "storage-1-1", + } + + if !reflect.DeepEqual(arguments, expectedArguments) { + t.Logf("Expected arguments %v, but got arguments %v", expectedArguments, arguments) + t.Fail() + } +} + +func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) { + argument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"} + + result, err := argument.GenerateArgument(1, map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"}) + if err != nil { + t.Error(err) + return + } + if result != "zone1" { + t.Logf("Expected result zone1, but got result %v", result) + t.Fail() + return + } + + _, err = argument.GenerateArgument(1, map[string]string{"FDB_MACHINE_ID": "machine1"}) + if err == nil { + t.Logf("Expected error result, but did not get an error") + t.Fail() + return + } + expectedError := "Missing environment variable FDB_ZONE_ID" + if err.Error() != expectedError { + t.Logf("Expected error %s, but got error %s", expectedError, err) + t.Fail() + return + } +} diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod new file mode 100644 index 0000000000..1172226168 --- /dev/null +++ b/fdbkubernetesmonitor/go.mod @@ -0,0 +1,27 @@ +// go.mod +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +module github.com/apple/foundationdb/fdbkubernetesmonitor + +go 1.16 + +require ( + github.com/spf13/pflag v1.0.5 + github.com/fsnotify/fsnotify v1.5.0 +) diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum new file mode 100644 index 0000000000..e77e248c3e --- /dev/null +++ b/fdbkubernetesmonitor/go.sum @@ -0,0 +1,6 @@ +github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k= +github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go new file mode 100644 index 0000000000..7f82af04f2 --- /dev/null +++ b/fdbkubernetesmonitor/main.go @@ -0,0 +1,41 @@ +// main.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "fmt" + + "github.com/spf13/pflag" +) + +var ( + inputDir string + fdbserverPath string + monitorConfFile string +) + +func main() { + pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary") + pflag.StringVar(&inputDir, "input-dir", ".", "Directory containing input files") + pflag.StringVar(&monitorConfFile, "input-monitor-conf", "config.json", "Name of the file in the input directory that contains the monitor configuration") + pflag.Parse() + + StartMonitor(fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) +} diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go new file mode 100644 index 0000000000..08d8a9cf35 --- /dev/null +++ b/fdbkubernetesmonitor/monitor.go @@ -0,0 +1,238 @@ +// monitor.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "encoding/json" + "io" + "log" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/fsnotify/fsnotify" +) + +// errorBackoffSeconds is the time to wait after a process fails before starting +// another process. +const errorBackoffSeconds = 5 + +// Monitor provides the main monitor loop +type Monitor struct { + // ConfigFile defines the path to the config file to load. + ConfigFile string + + // FDBServerPath defines the path to the fdbserver binary. + FDBServerPath string + + // ActiveConfiguration defines the active process configuration. + ActiveConfiguration *ProcessConfiguration + + // ActiveConfigurationBytes defines the source data for the active process + // configuration. + ActiveConfigurationBytes []byte + + // ProcessIDs stores the PIDs of the processes that are running. A PID of + // zero will indicate that a process does not have a run loop. A PID of -1 + // will indicate that a process has a run loop but is not currently running + // the subprocess. + ProcessesIDs []int + + // Mutex defines a mutex around working with configuration. + Mutex sync.Mutex +} + +// StartMonitor starts the monitor loop. +func StartMonitor(configFile string, fdbserverPath string) { + monitor := &Monitor{ConfigFile: configFile, FDBServerPath: fdbserverPath} + monitor.Run() +} + +// LoadConfiguration loads the latest configuration from the config file. +func (monitor *Monitor) LoadConfiguration() { + file, err := os.Open(monitor.ConfigFile) + if err != nil { + log.Print(err.Error()) + return + } + defer file.Close() + configuration := &ProcessConfiguration{} + configurationBytes, err := io.ReadAll(file) + if err != nil { + log.Print(err.Error()) + } + err = json.Unmarshal(configurationBytes, configuration) + if err != nil { + log.Print(err) + return + } + + _, err = configuration.GenerateArguments(1, nil) + if err != nil { + log.Print(err) + return + } + + log.Printf("Received new configuration file") + monitor.Mutex.Lock() + defer monitor.Mutex.Unlock() + + if configuration.ServerCount == 0 { + configuration.ServerCount = 1 + } + + if monitor.ProcessesIDs == nil { + monitor.ProcessesIDs = make([]int, configuration.ServerCount+1) + } else { + for len(monitor.ProcessesIDs) <= configuration.ServerCount { + monitor.ProcessesIDs = append(monitor.ProcessesIDs, 0) + } + } + + monitor.ActiveConfiguration = configuration + monitor.ActiveConfigurationBytes = configurationBytes + + for processNumber := 1; processNumber <= configuration.ServerCount; processNumber++ { + if monitor.ProcessesIDs[processNumber] == 0 { + monitor.ProcessesIDs[processNumber] = -1 + tempNumber := processNumber + go func() { monitor.RunProcess(tempNumber) }() + } + } +} + +// RunProcess runs a loop to continually start and watch a process. +func (monitor *Monitor) RunProcess(processNumber int) { + log.Printf("Starting run loop for subprocess %d", processNumber) + for { + arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, nil) + arguments = append([]string{monitor.FDBServerPath}, arguments...) + if err != nil { + log.Print(err) + time.Sleep(errorBackoffSeconds * time.Second) + } + cmd := exec.Cmd{ + Path: arguments[0], + Args: arguments, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + + log.Printf("Starting subprocess #%d: %v", processNumber, arguments) + err = cmd.Start() + if err != nil { + log.Printf("Error from subprocess %d: %s", processNumber, err.Error()) + log.Printf("Subprocess #%d will restart in %d seconds", processNumber, errorBackoffSeconds) + time.Sleep(errorBackoffSeconds * time.Second) + continue + } + + monitor.Mutex.Lock() + monitor.ProcessesIDs[processNumber] = cmd.Process.Pid + monitor.Mutex.Unlock() + + err = cmd.Wait() + log.Printf("Subprocess #%d terminated", processNumber) + + if err != nil { + log.Printf("Error from subprocess #%d: %s", processNumber, err.Error()) + } + + monitor.Mutex.Lock() + monitor.ProcessesIDs[processNumber] = -1 + if monitor.ActiveConfiguration.ServerCount < processNumber { + log.Printf("Terminating run loop for subprocess %d", processNumber) + monitor.ProcessesIDs[processNumber] = 0 + monitor.Mutex.Unlock() + return + } + monitor.Mutex.Unlock() + + log.Printf("Subprocess #%d will restart in %d seconds", processNumber, errorBackoffSeconds) + time.Sleep(errorBackoffSeconds * time.Second) + } +} + +// WatchConfiguration detects changes to the monitor configuration file. +func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Printf("Detected event on monitor conf file: %v", event) + if event.Op&fsnotify.Write == fsnotify.Write { + monitor.LoadConfiguration() + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Print(err) + } + } +} + +// Run runs the monitor loop. +func (monitor *Monitor) Run() { + done := make(chan bool, 1) + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + go func() { + latestSignal := <-signals + log.Printf("Received signal %v", latestSignal) + for processNumber, processID := range monitor.ProcessesIDs { + if processID > 0 { + process, err := os.FindProcess(processID) + if err != nil { + log.Printf("Error finding subprocess #%d (PID %d): %s", processNumber, processID, err.Error()) + continue + } + log.Printf("Sending signal %v to subprocess #%d (PID %d)", latestSignal, processNumber, processID) + err = process.Signal(latestSignal) + if err != nil { + log.Printf("Error signaling subprocess #%d (PID %d): %s", processNumber, processID, err.Error()) + continue + } + } + } + done <- true + }() + + monitor.LoadConfiguration() + watcher, err := fsnotify.NewWatcher() + if err != nil { + panic(err) + } + err = watcher.Add(monitor.ConfigFile) + if err != nil { + panic(err) + } + + defer watcher.Close() + go func() { monitor.WatchConfiguration(watcher) }() + + <-done +} From 0f173edb47bfc633b1056d90e9c849f97829b41e Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Sat, 21 Aug 2021 22:11:53 -0700 Subject: [PATCH 028/338] Add a dockerfile for fdb-kubernetes-monitor. --- fdbkubernetesmonitor/monitor.go | 27 ++-- packaging/docker/kubernetes/Dockerfile | 76 ++++++++++ packaging/docker/kubernetes/statefulset.yaml | 149 +++++++++++++++++++ 3 files changed, 241 insertions(+), 11 deletions(-) create mode 100644 packaging/docker/kubernetes/Dockerfile create mode 100644 packaging/docker/kubernetes/statefulset.yaml diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index 08d8a9cf35..3949a40681 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -97,10 +97,6 @@ func (monitor *Monitor) LoadConfiguration() { monitor.Mutex.Lock() defer monitor.Mutex.Unlock() - if configuration.ServerCount == 0 { - configuration.ServerCount = 1 - } - if monitor.ProcessesIDs == nil { monitor.ProcessesIDs = make([]int, configuration.ServerCount+1) } else { @@ -125,6 +121,15 @@ func (monitor *Monitor) LoadConfiguration() { func (monitor *Monitor) RunProcess(processNumber int) { log.Printf("Starting run loop for subprocess %d", processNumber) for { + monitor.Mutex.Lock() + if monitor.ActiveConfiguration.ServerCount < processNumber { + log.Printf("Terminating run loop for subprocess %d", processNumber) + monitor.ProcessesIDs[processNumber] = 0 + monitor.Mutex.Unlock() + return + } + monitor.Mutex.Unlock() + arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, nil) arguments = append([]string{monitor.FDBServerPath}, arguments...) if err != nil { @@ -160,12 +165,6 @@ func (monitor *Monitor) RunProcess(processNumber int) { monitor.Mutex.Lock() monitor.ProcessesIDs[processNumber] = -1 - if monitor.ActiveConfiguration.ServerCount < processNumber { - log.Printf("Terminating run loop for subprocess %d", processNumber) - monitor.ProcessesIDs[processNumber] = 0 - monitor.Mutex.Unlock() - return - } monitor.Mutex.Unlock() log.Printf("Subprocess #%d will restart in %d seconds", processNumber, errorBackoffSeconds) @@ -182,7 +181,13 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) { return } log.Printf("Detected event on monitor conf file: %v", event) - if event.Op&fsnotify.Write == fsnotify.Write { + if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create { + monitor.LoadConfiguration() + } else if event.Op&fsnotify.Remove == fsnotify.Remove { + err := watcher.Add(monitor.ConfigFile) + if err != nil { + panic(err) + } monitor.LoadConfiguration() } case err, ok := <-watcher.Errors: diff --git a/packaging/docker/kubernetes/Dockerfile b/packaging/docker/kubernetes/Dockerfile new file mode 100644 index 0000000000..6d8feab19c --- /dev/null +++ b/packaging/docker/kubernetes/Dockerfile @@ -0,0 +1,76 @@ +# Dockerfile +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This docker image assumes that the context for the docker build is pointed +# at the root of the foundationdb repository. + +# Build the Kubernetes monitor + +FROM golang:1.16.7-bullseye AS go-build + +COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor +WORKDIR /fdbkubernetesmonitor +RUN go build -o /fdb-kubernetes-monitor ./... + +# Build the main image + +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y curl>=7.58.0-2ubuntu3.6 \ + dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \ + lsof>=4.89+dfsg-0.1 \ + tcptraceroute>=1.5beta7+debian-4build1 \ + telnet>=0.17-41 \ + netcat>=1.10-41.1 \ + strace>=4.21-1ubuntu1 \ + tcpdump>=4.9.3-0ubuntu0.18.04.1 \ + less>=487-0.1 \ + vim>=2:8.0.1453-1ubuntu1.4 \ + net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \ + jq>=1.5+dfsg-2 && \ + rm -rf /var/lib/apt/lists/* + +ARG FDB_VERSION +ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" +ARG FDB_WEBSITE=https://www.foundationdb.org + +COPY packaging/docker/website /mnt/website/ + +# Install FoundationDB Binaries +RUN mkdir -p /var/fdb/logs && mkdir -p /var/fdb/tmp && \ + curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ + mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin + +# Install additional FoundationDB Client Libraries +ADD packaging/docker/release/download_multiversion_libraries.bash /var/fdb/tmp +RUN bash /var/fdb/tmp/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_LIBRARY_VERSIONS + +# Clean up temporary directories +RUN rm -rf /mnt/website && rm -r /var/fdb/tmp + +# Install the kubernetes monitor binary +COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ + +VOLUME /var/fdb/data + +# Runtime Configuration Options + +ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] diff --git a/packaging/docker/kubernetes/statefulset.yaml b/packaging/docker/kubernetes/statefulset.yaml new file mode 100644 index 0000000000..b8590dbc9e --- /dev/null +++ b/packaging/docker/kubernetes/statefulset.yaml @@ -0,0 +1,149 @@ +# statefulset.yaml +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file provides an example of using a statefulset to launch FDB processes +# using the foundationdb-kubernetes image. +# +# This is not a recommended way to run production clusters, but it can be useful +# to test the image in development. +# +# To start a cluster with this example, run the following steps: +# 1. Apply this file. +# 2. Wait for all pods to start. +# 3. Generate a connection string, using the following bash command: +# ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")'); echo test:test@$ips +# echo test:test@$ips +# 4. Update the ConfigMap below to have the results of that echo statement as +# the `fdb.cluster` entry, and change the `serverCount` field to `1`. +# 5. Apply the file again. +# 6. Watch the logs for the fdb-kubernetes-example pods to confirm that they +# have launched the fdbserver processes. +# 7. Exec into one of the pods, and run `fdbcli --exec "configure new double ssd"`. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: fdb-kubernetes-example + labels: + app: fdb-kubernetes-example +spec: + selector: + matchLabels: + app: fdb-kubernetes-example + replicas: 5 + serviceName: fdb-kubernetes-example + template: + metadata: + labels: + app: fdb-kubernetes-example + spec: + containers: + - name: foundationdb + image: foundationdb/foundationdb-kubernetes:6.3.15 + env: + - name: FDB_CLUSTER_FILE + value: /var/fdb/data/fdb.cluster + - name: FDB_PUBLIC_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FDB_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FDB_MACHINE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: FDB_ZONE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: FDB_INSTANCE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - --input-dir + - /var/fdb/dynamic-conf + volumeMounts: + - name: dynamic-conf + mountPath: /var/fdb/dynamic-conf + - name: data + mountPath: /var/fdb/data + volumes: + - name: dynamic-conf + configMap: + name: fdb-kubernetes-example-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fdb-kubernetes-example-config +data: + fdb.cluster: "" + config.json: | + { + "serverCount": 0 + "version": "6.3.15", + "arguments": [ + {"value": "--cluster_file"}, + {"value": "/var/fdb/data/fdb.cluster"}, + {"value": "--seed_cluster_file"}, + {"value": "/var/fdb/dynamic-conf/fdb.cluster"}, + {"value": "--public_address"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_PUBLIC_IP"}, + {"value": ":"}, + {"type": "ProcessNumber", "offset": 4499, "multiplier": 2} + ]}, + {"value": "--listen_address"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_POD_IP"}, + {"value": ":"}, + {"type": "ProcessNumber", "offset": 4499, "multiplier": 2} + ]}, + {"value": "--datadir"}, + {"type": "Concatenate", "values": [ + {"value": "/var/fdb/data/"}, + {"type": "ProcessNumber"} + ]}, + {"value": "--class"}, + {"value": "storage"}, + {"value": "--locality_zoneid"}, + {"type": "Environment", "source": "FDB_ZONE_ID"}, + {"value": "--locality_instance-id"}, + {"type": "Environment", "source": "FDB_INSTANCE_ID"}, + {"value": "--locality_process-id"}, + {"type": "Concatenate", "values": [ + {"type": "Environment", "source": "FDB_INSTANCE_ID"}, + {"value": "-"}, + {"type": "ProcessNumber"} + ]} + ] + } From 95ad5854beb1348e3d049325b44881d9d7ece085 Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Sun, 22 Aug 2021 01:28:09 -0700 Subject: [PATCH 029/338] Add a mechanism to post updates from fdb-kubernetes-monitor to pod annotations. Remove some of the local test data for fdb-kubernetes-monitor in favor of testing through a Kuberentes statefulset. --- .../.testdata/default_config.json | 1 + fdbkubernetesmonitor/.testdata/fdb.cluster | 1 - fdbkubernetesmonitor/.testdata/test_env.sh | 5 - fdbkubernetesmonitor/README.md | 25 ++ fdbkubernetesmonitor/go.mod | 5 +- fdbkubernetesmonitor/go.sum | 413 ++++++++++++++++++ fdbkubernetesmonitor/kubernetes.go | 185 ++++++++ fdbkubernetesmonitor/monitor.go | 35 +- .../{statefulset.yaml => config.yaml} | 61 ++- 9 files changed, 708 insertions(+), 23 deletions(-) delete mode 100644 fdbkubernetesmonitor/.testdata/fdb.cluster delete mode 100644 fdbkubernetesmonitor/.testdata/test_env.sh create mode 100644 fdbkubernetesmonitor/README.md create mode 100644 fdbkubernetesmonitor/kubernetes.go rename packaging/docker/kubernetes/{statefulset.yaml => config.yaml} (79%) diff --git a/fdbkubernetesmonitor/.testdata/default_config.json b/fdbkubernetesmonitor/.testdata/default_config.json index ecb09eca28..86cb836164 100644 --- a/fdbkubernetesmonitor/.testdata/default_config.json +++ b/fdbkubernetesmonitor/.testdata/default_config.json @@ -1,5 +1,6 @@ { "version": "6.3.0", + "serverCount": 1, "arguments": [ {"value": "--cluster_file"}, {"value": ".testdata/fdb.cluster"}, diff --git a/fdbkubernetesmonitor/.testdata/fdb.cluster b/fdbkubernetesmonitor/.testdata/fdb.cluster deleted file mode 100644 index 4b36477173..0000000000 --- a/fdbkubernetesmonitor/.testdata/fdb.cluster +++ /dev/null @@ -1 +0,0 @@ -test:test@127.0.0.1:4501 diff --git a/fdbkubernetesmonitor/.testdata/test_env.sh b/fdbkubernetesmonitor/.testdata/test_env.sh deleted file mode 100644 index ced881d347..0000000000 --- a/fdbkubernetesmonitor/.testdata/test_env.sh +++ /dev/null @@ -1,5 +0,0 @@ -export FDB_PUBLIC_IP=127.0.0.1 -export FDB_POD_IP=127.0.0.1 -export FDB_ZONE_ID=localhost -export FDB_MACHINE_ID=localhost -export FDB_INSTANCE_ID=storage-1 diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md new file mode 100644 index 0000000000..67e20401d8 --- /dev/null +++ b/fdbkubernetesmonitor/README.md @@ -0,0 +1,25 @@ + +This package provides a launcher program for running FoundationDB in Kubernetes. + +To test this, run the following commands from the root of the FoundationDB +repository: + + docker build -t foundationdb/foundationdb-kubernetes:latest --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . + kubectl apply -f packaging/docker/kubernetes/config.yaml + # Wait for the pods to become ready + ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') + cat packaging/docker/kubernetes/config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - + kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite + # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. + kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" + +You can then make changes to the data in the config map and update the fdbserver processes: + + kubectl apply -f packaging/docker/kubernetes/config.yaml + kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite + # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. + kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status" + +Once you are done, you can tear down the example with the following command: + + kubectl delete -f packaging/docker/kubernetes/config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod index 1172226168..d44f296e55 100644 --- a/fdbkubernetesmonitor/go.mod +++ b/fdbkubernetesmonitor/go.mod @@ -22,6 +22,9 @@ module github.com/apple/foundationdb/fdbkubernetesmonitor go 1.16 require ( - github.com/spf13/pflag v1.0.5 github.com/fsnotify/fsnotify v1.5.0 + github.com/spf13/pflag v1.0.5 + k8s.io/api v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/client-go v0.20.2 ) diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum index e77e248c3e..c0378c9592 100644 --- a/fdbkubernetesmonitor/go.sum +++ b/fdbkubernetesmonitor/go.sum @@ -1,6 +1,419 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k= github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go new file mode 100644 index 0000000000..63e9af76b0 --- /dev/null +++ b/fdbkubernetesmonitor/kubernetes.go @@ -0,0 +1,185 @@ +// kubernetes.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "strconv" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +const ( + // CurrentConfigurationAnnotation is the annotation we use to store the + // latest configuration. + CurrentConfigurationAnnotation = "foundationdb.org/launcher-current-configuration" + + // EnvironmentAnnotation is the annotation we use to store the environment + // variables. + EnvironmentAnnotation = "foundationdb.org/launcher-environment" + + // OutdatedConfigMapAnnotation is the annotation we read to get notified of + // outdated configuration. + OutdatedConfigMapAnnotation = "foundationdb.org/outdated-config-map-seen" +) + +// PodClient is a wrapper around the pod API. +type PodClient struct { + // podApi is the raw API + podApi typedv1.PodInterface + + // pod is the latest pod configuration + pod *corev1.Pod + + // TimestampFeed is a channel where the pod client will send updates with + // the values from OutdatedConfigMapAnnotation. + TimestampFeed chan int64 +} + +// CreatePodClient creates a new client for working with the pod object. +func CreatePodClient() (*PodClient, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + podApi := client.CoreV1().Pods(os.Getenv("FDB_POD_NAMESPACE")) + pod, err := podApi.Get(context.Background(), os.Getenv("FDB_POD_NAME"), metav1.GetOptions{ResourceVersion: "0"}) + if err != nil { + return nil, err + } + + podClient := &PodClient{podApi: podApi, pod: pod, TimestampFeed: make(chan int64, 10)} + err = podClient.watchPod() + if err != nil { + return nil, err + } + + return podClient, nil +} + +// retrieveEnvironmentVariables extracts the environment variables we have for +// an argument into a map. +func retrieveEnvironmentVariables(argument Argument, target map[string]string) { + if argument.Source != "" { + target[argument.Source] = os.Getenv(argument.Source) + } + if argument.Values != nil { + for _, childArgument := range argument.Values { + retrieveEnvironmentVariables(childArgument, target) + } + } +} + +// UpdateAnnotations updates annotations on the pod after loading new +// configuration. +func (client *PodClient) UpdateAnnotations(monitor *Monitor) error { + environment := make(map[string]string) + for _, argument := range monitor.ActiveConfiguration.Arguments { + retrieveEnvironmentVariables(argument, environment) + } + jsonEnvironment, err := json.Marshal(environment) + if err != nil { + return err + } + + patch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]string{ + CurrentConfigurationAnnotation: string(monitor.ActiveConfigurationBytes), + EnvironmentAnnotation: string(jsonEnvironment), + }, + }, + } + + patchJson, err := json.Marshal(patch) + if err != nil { + return err + } + + pod, err := client.podApi.Patch(context.Background(), client.pod.Name, types.MergePatchType, patchJson, metav1.PatchOptions{}) + if err != nil { + return err + } + client.pod = pod + return nil +} + +// watchPod starts a watch on the pod. +func (client *PodClient) watchPod() error { + podWatch, err := client.podApi.Watch( + context.Background(), + metav1.ListOptions{ + Watch: true, + ResourceVersion: "0", + FieldSelector: fmt.Sprintf("metadata.name=%s", os.Getenv("FDB_POD_NAME")), + }, + ) + if err != nil { + return err + } + results := podWatch.ResultChan() + go func() { + for event := range results { + if event.Type == watch.Modified { + pod, valid := event.Object.(*corev1.Pod) + if !valid { + log.Printf("Error getting pod information from watch: %v", event) + } + client.processPodUpdate(pod) + } + } + }() + + return nil +} + +// processPodUpdate handles an update for a pod. +func (client *PodClient) processPodUpdate(pod *corev1.Pod) { + client.pod = pod + if pod.Annotations == nil { + return + } + annotation := client.pod.Annotations[OutdatedConfigMapAnnotation] + if annotation == "" { + return + } + timestamp, err := strconv.ParseInt(annotation, 10, 64) + if err != nil { + log.Printf("Error parsing annotation %s: %s", annotation, err) + return + } + + client.TimestampFeed <- timestamp +} diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index 3949a40681..a9a78e5096 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -52,6 +52,10 @@ type Monitor struct { // configuration. ActiveConfigurationBytes []byte + // LastConfigurationTime is the last time we successfully reloaded the + // configuration file. + LastConfigurationTime time.Time + // ProcessIDs stores the PIDs of the processes that are running. A PID of // zero will indicate that a process does not have a run loop. A PID of -1 // will indicate that a process has a run loop but is not currently running @@ -60,11 +64,26 @@ type Monitor struct { // Mutex defines a mutex around working with configuration. Mutex sync.Mutex + + // PodClient is a client for posting updates about this pod to + // Kubernetes. + PodClient *PodClient } // StartMonitor starts the monitor loop. func StartMonitor(configFile string, fdbserverPath string) { - monitor := &Monitor{ConfigFile: configFile, FDBServerPath: fdbserverPath} + podClient, err := CreatePodClient() + if err != nil { + panic(err) + } + + monitor := &Monitor{ + ConfigFile: configFile, + FDBServerPath: fdbserverPath, + PodClient: podClient, + } + + go func() { monitor.WatchPodTimestamps() }() monitor.Run() } @@ -107,6 +126,7 @@ func (monitor *Monitor) LoadConfiguration() { monitor.ActiveConfiguration = configuration monitor.ActiveConfigurationBytes = configurationBytes + monitor.LastConfigurationTime = time.Now() for processNumber := 1; processNumber <= configuration.ServerCount; processNumber++ { if monitor.ProcessesIDs[processNumber] == 0 { @@ -115,6 +135,11 @@ func (monitor *Monitor) LoadConfiguration() { go func() { monitor.RunProcess(tempNumber) }() } } + + err = monitor.PodClient.UpdateAnnotations(monitor) + if err != nil { + log.Printf("Error updating pod annotations: %s", err) + } } // RunProcess runs a loop to continually start and watch a process. @@ -241,3 +266,11 @@ func (monitor *Monitor) Run() { <-done } + +func (monitor *Monitor) WatchPodTimestamps() { + for timestamp := range monitor.PodClient.TimestampFeed { + if timestamp > monitor.LastConfigurationTime.Unix() { + monitor.LoadConfiguration() + } + } +} diff --git a/packaging/docker/kubernetes/statefulset.yaml b/packaging/docker/kubernetes/config.yaml similarity index 79% rename from packaging/docker/kubernetes/statefulset.yaml rename to packaging/docker/kubernetes/config.yaml index b8590dbc9e..a47a7fe01e 100644 --- a/packaging/docker/kubernetes/statefulset.yaml +++ b/packaging/docker/kubernetes/config.yaml @@ -23,19 +23,7 @@ # This is not a recommended way to run production clusters, but it can be useful # to test the image in development. # -# To start a cluster with this example, run the following steps: -# 1. Apply this file. -# 2. Wait for all pods to start. -# 3. Generate a connection string, using the following bash command: -# ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")'); echo test:test@$ips -# echo test:test@$ips -# 4. Update the ConfigMap below to have the results of that echo statement as -# the `fdb.cluster` entry, and change the `serverCount` field to `1`. -# 5. Apply the file again. -# 6. Watch the logs for the fdb-kubernetes-example pods to confirm that they -# have launched the fdbserver processes. -# 7. Exec into one of the pods, and run `fdbcli --exec "configure new double ssd"`. - +# For more information on using this file, see fdbkubernetesmonitor/doc.go apiVersion: apps/v1 kind: StatefulSet metadata: @@ -55,8 +43,17 @@ spec: spec: containers: - name: foundationdb - image: foundationdb/foundationdb-kubernetes:6.3.15 + image: foundationdb/foundationdb-kubernetes:latest + imagePullPolicy: IfNotPresent env: + - name: FDB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: FDB_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: FDB_CLUSTER_FILE value: /var/fdb/data/fdb.cluster - name: FDB_PUBLIC_IP @@ -87,6 +84,7 @@ spec: mountPath: /var/fdb/dynamic-conf - name: data mountPath: /var/fdb/data + serviceAccountName: fdb-kubernetes-example volumes: - name: dynamic-conf configMap: @@ -109,7 +107,7 @@ data: fdb.cluster: "" config.json: | { - "serverCount": 0 + "serverCount": 0, "version": "6.3.15", "arguments": [ {"value": "--cluster_file"}, @@ -147,3 +145,36 @@ data: ]} ] } +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fdb-kubernetes-example +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fdb-kubernetes-example +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "get" + - "watch" + - "update" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fdb-kubernetes-example +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fdb-kubernetes-example +subjects: +- kind: ServiceAccount + name: fdb-kubernetes-example + + From 7c36123cf81cd909ab4bc4e59b59d1d6319302ac Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Sun, 22 Aug 2021 21:19:10 -0700 Subject: [PATCH 030/338] Adds structured logging in fdb-kubernetes-monitor. Adds a backoff window when restarting processes in fdb-kubernetes-monitor. --- fdbkubernetesmonitor/.testdata/fdb.cluster | 1 + fdbkubernetesmonitor/.testdata/test_env.sh | 7 + fdbkubernetesmonitor/README.md | 13 +- fdbkubernetesmonitor/config_test.go | 12 ++ fdbkubernetesmonitor/go.mod | 3 + fdbkubernetesmonitor/go.sum | 26 +++- fdbkubernetesmonitor/kubernetes.go | 9 +- fdbkubernetesmonitor/main.go | 16 +- fdbkubernetesmonitor/monitor.go | 145 ++++++++++++------ packaging/docker/kubernetes/Dockerfile | 9 +- .../{config.yaml => test_config.yaml} | 22 ++- 11 files changed, 197 insertions(+), 66 deletions(-) create mode 100644 fdbkubernetesmonitor/.testdata/fdb.cluster create mode 100644 fdbkubernetesmonitor/.testdata/test_env.sh rename packaging/docker/kubernetes/{config.yaml => test_config.yaml} (94%) diff --git a/fdbkubernetesmonitor/.testdata/fdb.cluster b/fdbkubernetesmonitor/.testdata/fdb.cluster new file mode 100644 index 0000000000..4b36477173 --- /dev/null +++ b/fdbkubernetesmonitor/.testdata/fdb.cluster @@ -0,0 +1 @@ +test:test@127.0.0.1:4501 diff --git a/fdbkubernetesmonitor/.testdata/test_env.sh b/fdbkubernetesmonitor/.testdata/test_env.sh new file mode 100644 index 0000000000..13bfc76ec9 --- /dev/null +++ b/fdbkubernetesmonitor/.testdata/test_env.sh @@ -0,0 +1,7 @@ +export FDB_PUBLIC_IP=127.0.0.1 +export FDB_POD_IP=127.0.0.1 +export FDB_ZONE_ID=localhost +export FDB_MACHINE_ID=localhost +export FDB_INSTANCE_ID=storage-1 +export KUBERNETES_SERVICE_HOST=kubernetes.docker.internal +export KUBERNETES_SERVICE_PORT=6443 diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md index 67e20401d8..103108c218 100644 --- a/fdbkubernetesmonitor/README.md +++ b/fdbkubernetesmonitor/README.md @@ -5,21 +5,24 @@ To test this, run the following commands from the root of the FoundationDB repository: docker build -t foundationdb/foundationdb-kubernetes:latest --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . - kubectl apply -f packaging/docker/kubernetes/config.yaml + kubectl apply -f packaging/docker/kubernetes/test_config.yaml # Wait for the pods to become ready ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') - cat packaging/docker/kubernetes/config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - + cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" You can then make changes to the data in the config map and update the fdbserver processes: - kubectl apply -f packaging/docker/kubernetes/config.yaml + cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - + + # You can apply an annotation to speed up the propagation of config kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite - # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. + + # Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce. kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status" Once you are done, you can tear down the example with the following command: - kubectl delete -f packaging/docker/kubernetes/config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example + kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example diff --git a/fdbkubernetesmonitor/config_test.go b/fdbkubernetesmonitor/config_test.go index 9aca4043bf..0820fe29ff 100644 --- a/fdbkubernetesmonitor/config_test.go +++ b/fdbkubernetesmonitor/config_test.go @@ -24,6 +24,9 @@ import ( "os" "reflect" "testing" + + "github.com/go-logr/zapr" + "go.uber.org/zap" ) func loadConfigFromFile(path string) (*ProcessConfiguration, error) { @@ -99,4 +102,13 @@ func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) { t.Fail() return } + + zapLogger, err := zap.NewDevelopment() + if err != nil { + panic(err) + } + + log := zapr.NewLogger(zapLogger) + log.Info("JPB test", "key", "value") + t.Fail() } diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod index d44f296e55..fec774f327 100644 --- a/fdbkubernetesmonitor/go.mod +++ b/fdbkubernetesmonitor/go.mod @@ -23,7 +23,10 @@ go 1.16 require ( github.com/fsnotify/fsnotify v1.5.0 + github.com/go-logr/logr v0.4.0 + github.com/go-logr/zapr v0.4.0 github.com/spf13/pflag v1.0.5 + go.uber.org/zap v1.19.0 k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 k8s.io/client-go v0.20.2 diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum index c0378c9592..1c021ce528 100644 --- a/fdbkubernetesmonitor/go.sum +++ b/fdbkubernetesmonitor/go.sum @@ -36,6 +36,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -62,8 +64,11 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= @@ -157,6 +162,8 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -170,12 +177,21 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -205,6 +221,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -303,6 +320,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -317,6 +335,7 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -388,8 +407,9 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go index 63e9af76b0..dab1f23911 100644 --- a/fdbkubernetesmonitor/kubernetes.go +++ b/fdbkubernetesmonitor/kubernetes.go @@ -23,10 +23,10 @@ import ( "context" "encoding/json" "fmt" - "log" "os" "strconv" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -61,6 +61,9 @@ type PodClient struct { // TimestampFeed is a channel where the pod client will send updates with // the values from OutdatedConfigMapAnnotation. TimestampFeed chan int64 + + // Logger is the logger we use for this client. + Logger logr.Logger } // CreatePodClient creates a new client for working with the pod object. @@ -155,7 +158,7 @@ func (client *PodClient) watchPod() error { if event.Type == watch.Modified { pod, valid := event.Object.(*corev1.Pod) if !valid { - log.Printf("Error getting pod information from watch: %v", event) + client.Logger.Error(nil, "Error getting pod information from watch", "event", event) } client.processPodUpdate(pod) } @@ -177,7 +180,7 @@ func (client *PodClient) processPodUpdate(pod *corev1.Pod) { } timestamp, err := strconv.ParseInt(annotation, 10, 64) if err != nil { - log.Printf("Error parsing annotation %s: %s", annotation, err) + client.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation, err) return } diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go index 7f82af04f2..1cb922d2e2 100644 --- a/fdbkubernetesmonitor/main.go +++ b/fdbkubernetesmonitor/main.go @@ -22,20 +22,34 @@ package main import ( "fmt" + "github.com/go-logr/zapr" "github.com/spf13/pflag" + "go.uber.org/zap" ) var ( inputDir string fdbserverPath string monitorConfFile string + logPath string ) func main() { pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary") pflag.StringVar(&inputDir, "input-dir", ".", "Directory containing input files") pflag.StringVar(&monitorConfFile, "input-monitor-conf", "config.json", "Name of the file in the input directory that contains the monitor configuration") + pflag.StringVar(&logPath, "log-path", "", "Name of a file to send logs to. Logs will be sent to stdout in addition the file you pass in this argument. If this is blank, logs will only by sent to stdout") pflag.Parse() - StartMonitor(fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) + zapConfig := zap.NewProductionConfig() + if logPath != "" { + zapConfig.OutputPaths = append(zapConfig.OutputPaths, logPath) + } + zapLogger, err := zapConfig.Build() + if err != nil { + panic(err) + } + + logger := zapr.NewLogger(zapLogger) + StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) } diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index a9a78e5096..99d8e9c698 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -20,9 +20,9 @@ package main import ( + "bufio" "encoding/json" "io" - "log" "os" "os/exec" "os/signal" @@ -31,11 +31,14 @@ import ( "time" "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" ) // errorBackoffSeconds is the time to wait after a process fails before starting // another process. -const errorBackoffSeconds = 5 +// This delay will only be applied when there has been more than one failure +// within this time window. +const errorBackoffSeconds = 60 // Monitor provides the main monitor loop type Monitor struct { @@ -60,7 +63,7 @@ type Monitor struct { // zero will indicate that a process does not have a run loop. A PID of -1 // will indicate that a process has a run loop but is not currently running // the subprocess. - ProcessesIDs []int + ProcessIDs []int // Mutex defines a mutex around working with configuration. Mutex sync.Mutex @@ -68,10 +71,13 @@ type Monitor struct { // PodClient is a client for posting updates about this pod to // Kubernetes. PodClient *PodClient + + // Logger is the logger instance for this monitor. + Logger logr.Logger } // StartMonitor starts the monitor loop. -func StartMonitor(configFile string, fdbserverPath string) { +func StartMonitor(logger logr.Logger, configFile string, fdbserverPath string) { podClient, err := CreatePodClient() if err != nil { panic(err) @@ -81,6 +87,7 @@ func StartMonitor(configFile string, fdbserverPath string) { ConfigFile: configFile, FDBServerPath: fdbserverPath, PodClient: podClient, + Logger: logger, } go func() { monitor.WatchPodTimestamps() }() @@ -91,36 +98,36 @@ func StartMonitor(configFile string, fdbserverPath string) { func (monitor *Monitor) LoadConfiguration() { file, err := os.Open(monitor.ConfigFile) if err != nil { - log.Print(err.Error()) + monitor.Logger.Error(err, "Error reading monitor config file", "monitorConfigPath", monitor.ConfigFile) return } defer file.Close() configuration := &ProcessConfiguration{} configurationBytes, err := io.ReadAll(file) if err != nil { - log.Print(err.Error()) + monitor.Logger.Error(err, "Error reading monitor configuration", "monitorConfigPath", monitor.ConfigFile) } err = json.Unmarshal(configurationBytes, configuration) if err != nil { - log.Print(err) + monitor.Logger.Error(err, "Error parsing monitor configuration", "rawConfiguration", string(configurationBytes)) return } _, err = configuration.GenerateArguments(1, nil) if err != nil { - log.Print(err) + monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration) return } - log.Printf("Received new configuration file") + monitor.Logger.Info("Received new configuration file", "configuration", configuration) monitor.Mutex.Lock() defer monitor.Mutex.Unlock() - if monitor.ProcessesIDs == nil { - monitor.ProcessesIDs = make([]int, configuration.ServerCount+1) + if monitor.ProcessIDs == nil { + monitor.ProcessIDs = make([]int, configuration.ServerCount+1) } else { - for len(monitor.ProcessesIDs) <= configuration.ServerCount { - monitor.ProcessesIDs = append(monitor.ProcessesIDs, 0) + for len(monitor.ProcessIDs) <= configuration.ServerCount { + monitor.ProcessIDs = append(monitor.ProcessIDs, 0) } } @@ -129,8 +136,8 @@ func (monitor *Monitor) LoadConfiguration() { monitor.LastConfigurationTime = time.Now() for processNumber := 1; processNumber <= configuration.ServerCount; processNumber++ { - if monitor.ProcessesIDs[processNumber] == 0 { - monitor.ProcessesIDs[processNumber] = -1 + if monitor.ProcessIDs[processNumber] == 0 { + monitor.ProcessIDs[processNumber] = -1 tempNumber := processNumber go func() { monitor.RunProcess(tempNumber) }() } @@ -138,18 +145,20 @@ func (monitor *Monitor) LoadConfiguration() { err = monitor.PodClient.UpdateAnnotations(monitor) if err != nil { - log.Printf("Error updating pod annotations: %s", err) + monitor.Logger.Error(err, "Error updating pod annotations") } } // RunProcess runs a loop to continually start and watch a process. func (monitor *Monitor) RunProcess(processNumber int) { - log.Printf("Starting run loop for subprocess %d", processNumber) + pid := 0 + logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "RunProcess") + logger.Info("Starting run loop") for { monitor.Mutex.Lock() if monitor.ActiveConfiguration.ServerCount < processNumber { - log.Printf("Terminating run loop for subprocess %d", processNumber) - monitor.ProcessesIDs[processNumber] = 0 + logger.Info("Terminating run loop") + monitor.ProcessIDs[processNumber] = 0 monitor.Mutex.Unlock() return } @@ -158,42 +167,85 @@ func (monitor *Monitor) RunProcess(processNumber int) { arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, nil) arguments = append([]string{monitor.FDBServerPath}, arguments...) if err != nil { - log.Print(err) + logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration) time.Sleep(errorBackoffSeconds * time.Second) } cmd := exec.Cmd{ - Path: arguments[0], - Args: arguments, - Stdout: os.Stdout, - Stderr: os.Stderr, + Path: arguments[0], + Args: arguments, + } + + logger.Info("Starting subprocess", "arguments", arguments) + + stdout, err := cmd.StdoutPipe() + if err != nil { + logger.Error(err, "Error getting stdout from subprocess") + } + + stderr, err := cmd.StderrPipe() + if err != nil { + logger.Error(err, "Error getting stderr from subprocess") } - log.Printf("Starting subprocess #%d: %v", processNumber, arguments) err = cmd.Start() if err != nil { - log.Printf("Error from subprocess %d: %s", processNumber, err.Error()) - log.Printf("Subprocess #%d will restart in %d seconds", processNumber, errorBackoffSeconds) + logger.Error(err, "Error starting subprocess") time.Sleep(errorBackoffSeconds * time.Second) continue } - monitor.Mutex.Lock() - monitor.ProcessesIDs[processNumber] = cmd.Process.Pid - monitor.Mutex.Unlock() - - err = cmd.Wait() - log.Printf("Subprocess #%d terminated", processNumber) - - if err != nil { - log.Printf("Error from subprocess #%d: %s", processNumber, err.Error()) + if cmd.Process != nil { + pid = cmd.Process.Pid + } else { + logger.Error(nil, "No Process information availale for subprocess") } + startTime := time.Now() + logger.Info("Subprocess started", "PID", pid) + monitor.Mutex.Lock() - monitor.ProcessesIDs[processNumber] = -1 + monitor.ProcessIDs[processNumber] = pid monitor.Mutex.Unlock() - log.Printf("Subprocess #%d will restart in %d seconds", processNumber, errorBackoffSeconds) - time.Sleep(errorBackoffSeconds * time.Second) + if stdout != nil { + stdoutScanner := bufio.NewScanner(stdout) + go func() { + for stdoutScanner.Scan() { + logger.Info("Subprocess output", "msg", stdoutScanner.Text(), "PID", pid) + } + }() + } + + if stderr != nil { + stderrScanner := bufio.NewScanner(stderr) + go func() { + for stderrScanner.Scan() { + logger.Error(nil, "Subprocess error log", "msg", stderrScanner.Text(), "PID", pid) + } + }() + } + + err = cmd.Wait() + if err != nil { + logger.Error(err, "Error from subprocess", "PID", pid) + } + exitCode := -1 + if cmd.ProcessState != nil { + exitCode = cmd.ProcessState.ExitCode() + } + + logger.Info("Subprocess terminated", "exitCode", exitCode, "PID", pid) + + endTime := time.Now() + monitor.Mutex.Lock() + monitor.ProcessIDs[processNumber] = -1 + monitor.Mutex.Unlock() + + processDuration := endTime.Sub(startTime) + if processDuration.Seconds() < errorBackoffSeconds { + logger.Info("Backing off from restarting subprocess", "backOffTimeSeconds", errorBackoffSeconds, "lastExecutionDurationSeconds", processDuration) + time.Sleep(errorBackoffSeconds * time.Second) + } } } @@ -205,7 +257,7 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) { if !ok { return } - log.Printf("Detected event on monitor conf file: %v", event) + monitor.Logger.Info("Detected event on monitor conf file", "event", event) if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create { monitor.LoadConfiguration() } else if event.Op&fsnotify.Remove == fsnotify.Remove { @@ -219,7 +271,7 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) { if !ok { return } - log.Print(err) + monitor.Logger.Error(err, "Error watching for file system events") } } } @@ -232,18 +284,19 @@ func (monitor *Monitor) Run() { go func() { latestSignal := <-signals - log.Printf("Received signal %v", latestSignal) - for processNumber, processID := range monitor.ProcessesIDs { + monitor.Logger.Info("Received system signal", "signal", latestSignal) + for processNumber, processID := range monitor.ProcessIDs { if processID > 0 { + subprocessLogger := monitor.Logger.WithValues("processNumber", processNumber, "PID", processID) process, err := os.FindProcess(processID) if err != nil { - log.Printf("Error finding subprocess #%d (PID %d): %s", processNumber, processID, err.Error()) + subprocessLogger.Error(err, "Error finding subprocess") continue } - log.Printf("Sending signal %v to subprocess #%d (PID %d)", latestSignal, processNumber, processID) + subprocessLogger.Info("Sending signal to subprocess", "signal", latestSignal) err = process.Signal(latestSignal) if err != nil { - log.Printf("Error signaling subprocess #%d (PID %d): %s", processNumber, processID, err.Error()) + subprocessLogger.Error(err, "Error signaling subprocess") continue } } diff --git a/packaging/docker/kubernetes/Dockerfile b/packaging/docker/kubernetes/Dockerfile index 6d8feab19c..bee0369a29 100644 --- a/packaging/docker/kubernetes/Dockerfile +++ b/packaging/docker/kubernetes/Dockerfile @@ -69,8 +69,15 @@ RUN rm -rf /mnt/website && rm -r /var/fdb/tmp # Install the kubernetes monitor binary COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ -VOLUME /var/fdb/data +# Set up a non-root user + +RUN groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ + chown -R fdb:fdb /var/fdb # Runtime Configuration Options +USER fdb +WORKDIR /var/fdb ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] +VOLUME /var/fdb/data diff --git a/packaging/docker/kubernetes/config.yaml b/packaging/docker/kubernetes/test_config.yaml similarity index 94% rename from packaging/docker/kubernetes/config.yaml rename to packaging/docker/kubernetes/test_config.yaml index a47a7fe01e..2034f6282d 100644 --- a/packaging/docker/kubernetes/config.yaml +++ b/packaging/docker/kubernetes/test_config.yaml @@ -23,7 +23,7 @@ # This is not a recommended way to run production clusters, but it can be useful # to test the image in development. # -# For more information on using this file, see fdbkubernetesmonitor/doc.go +# For more information on using this file, see fdbkubernetesmonitor/README.md apiVersion: apps/v1 kind: StatefulSet metadata: @@ -45,6 +45,11 @@ spec: - name: foundationdb image: foundationdb/foundationdb-kubernetes:latest imagePullPolicy: IfNotPresent + args: + - --input-dir + - /var/fdb/dynamic-conf + - --log-path + - /var/fdb/logs/monitor.log env: - name: FDB_POD_NAME valueFrom: @@ -76,19 +81,20 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - args: - - --input-dir - - /var/fdb/dynamic-conf volumeMounts: - name: dynamic-conf mountPath: /var/fdb/dynamic-conf - name: data mountPath: /var/fdb/data + - name: logs + mountPath: /var/fdb/logs serviceAccountName: fdb-kubernetes-example volumes: - name: dynamic-conf configMap: name: fdb-kubernetes-example-config + - name: logs + emptyDir: {} volumeClaimTemplates: - metadata: name: data @@ -142,7 +148,11 @@ data: {"type": "Environment", "source": "FDB_INSTANCE_ID"}, {"value": "-"}, {"type": "ProcessNumber"} - ]} + ]}, + {"value": "--logdir"}, + {"value": "/var/fdb/logs"}, + {"value": "--trace_format"}, + {"value": "json"} ] } --- @@ -176,5 +186,3 @@ roleRef: subjects: - kind: ServiceAccount name: fdb-kubernetes-example - - From c7858d24410073a8b0bd00add04d7889b7da85ed Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Mon, 23 Aug 2021 00:31:18 -0700 Subject: [PATCH 031/338] Add the init and sidecar modes for fdb-kubernetes-monitor. Add support for using a special binary path during upgrades in fdb-kubernetes-monitor. --- fdbkubernetesmonitor/README.md | 3 +- fdbkubernetesmonitor/config.go | 12 +- fdbkubernetesmonitor/config_test.go | 39 +++++-- fdbkubernetesmonitor/copy.go | 100 +++++++++++++++++ fdbkubernetesmonitor/main.go | 110 ++++++++++++++++++- fdbkubernetesmonitor/monitor.go | 20 +++- packaging/docker/kubernetes/Dockerfile | 7 +- packaging/docker/kubernetes/test_config.yaml | 99 ++++++++++++++++- 8 files changed, 363 insertions(+), 27 deletions(-) create mode 100644 fdbkubernetesmonitor/copy.go diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md index 103108c218..95a54866b6 100644 --- a/fdbkubernetesmonitor/README.md +++ b/fdbkubernetesmonitor/README.md @@ -4,7 +4,8 @@ This package provides a launcher program for running FoundationDB in Kubernetes. To test this, run the following commands from the root of the FoundationDB repository: - docker build -t foundationdb/foundationdb-kubernetes:latest --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . + docker build -t foundationdb/foundationdb-kubernetes:latest --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . + docker build -t foundationdb/foundationdb-kubernetes:latest-sidecar --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . kubectl apply -f packaging/docker/kubernetes/test_config.yaml # Wait for the pods to become ready ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') diff --git a/fdbkubernetesmonitor/config.go b/fdbkubernetesmonitor/config.go index 2c89826cb2..0899a145d9 100644 --- a/fdbkubernetesmonitor/config.go +++ b/fdbkubernetesmonitor/config.go @@ -33,6 +33,9 @@ type ProcessConfiguration struct { // ServerCount defines the number of processes to start. ServerCount int `json:"serverCount,omitempty"` + // BinaryPath provides the path to the binary to launch. + BinaryPath string `json:"-"` + // Arguments provides the arugments to the process. Arguments []Argument `json:"arguments,omitempty"` } @@ -125,13 +128,16 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri // GenerateArguments intreprets the arguments in the process configuration and // generates a command invocation. func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) { - results := make([]string, len(configuration.Arguments)) - for indexOfArgument, argument := range configuration.Arguments { + results := make([]string, 0, len(configuration.Arguments)+1) + if configuration.BinaryPath != "" { + results = append(results, configuration.BinaryPath) + } + for _, argument := range configuration.Arguments { result, err := argument.GenerateArgument(processNumber, env) if err != nil { return nil, err } - results[indexOfArgument] = result + results = append(results, result) } return results, nil } diff --git a/fdbkubernetesmonitor/config_test.go b/fdbkubernetesmonitor/config_test.go index 0820fe29ff..d0ea625807 100644 --- a/fdbkubernetesmonitor/config_test.go +++ b/fdbkubernetesmonitor/config_test.go @@ -24,9 +24,6 @@ import ( "os" "reflect" "testing" - - "github.com/go-logr/zapr" - "go.uber.org/zap" ) func loadConfigFromFile(path string) (*ProcessConfiguration, error) { @@ -74,6 +71,33 @@ func TestGeneratingArgumentsForDefaultConfig(t *testing.T) { t.Logf("Expected arguments %v, but got arguments %v", expectedArguments, arguments) t.Fail() } + + config.BinaryPath = "/usr/bin/fdbserver" + + arguments, err = config.GenerateArguments(1, map[string]string{ + "FDB_PUBLIC_IP": "10.0.0.1", + "FDB_POD_IP": "192.168.0.1", + "FDB_ZONE_ID": "zone1", + "FDB_INSTANCE_ID": "storage-1", + }) + if err != nil { + t.Error(err) + return + } + + expectedArguments = []string{ + "/usr/bin/fdbserver", + "--cluster_file", ".testdata/fdb.cluster", + "--public_address", "10.0.0.1:4501", "--listen_address", "192.168.0.1:4501", + "--datadir", ".testdata/data/1", "--class", "storage", + "--locality_zoneid", "zone1", "--locality_instance-id", "storage-1", + "--locality_process-id", "storage-1-1", + } + + if !reflect.DeepEqual(arguments, expectedArguments) { + t.Logf("Expected arguments %v, but got arguments %v", expectedArguments, arguments) + t.Fail() + } } func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) { @@ -102,13 +126,4 @@ func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) { t.Fail() return } - - zapLogger, err := zap.NewDevelopment() - if err != nil { - panic(err) - } - - log := zapr.NewLogger(zapLogger) - log.Info("JPB test", "key", "value") - t.Fail() } diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go new file mode 100644 index 0000000000..80074bc0f1 --- /dev/null +++ b/fdbkubernetesmonitor/copy.go @@ -0,0 +1,100 @@ +// copy.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "fmt" + "io" + "os" + "path" + + "github.com/go-logr/logr" +) + +const ( + bufferSize = 1024 +) + +// copyFile copies a file into the output directory. +func copyFile(logger logr.Logger, inputPath string, outputPath string, required bool) error { + logger.Info("Copying file", "inputPath", inputPath, "outputPath", outputPath) + inputFile, err := os.Open(inputPath) + if err != nil { + logger.Error(err, "Error opening file", "path", inputPath) + return err + } + defer inputFile.Close() + + inputInfo, err := inputFile.Stat() + if err != nil { + logger.Error(err, "Error getting stats for file", "path", inputPath) + return err + } + + if required && inputInfo.Size() == 0 { + return fmt.Errorf("File %s is empty", inputPath) + } + + outputFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY, inputInfo.Mode()) + if err != nil { + return err + } + defer outputFile.Close() + + var buffer = make([]byte, bufferSize) + for { + readLength, readError := inputFile.Read(buffer) + if readError == io.EOF { + break + } + if readError != nil { + logger.Error(readError, "Error reading file", "path", inputPath) + return readError + } + + _, writeError := outputFile.Write(buffer[:readLength]) + if writeError != nil { + logger.Error(writeError, "Error writing file", "path", outputPath) + return writeError + } + } + return nil +} + +// CopyFiles copies a list of files into the output directory. +func CopyFiles(logger logr.Logger, outputDir string, copyDetails map[string]string, requiredCopies map[string]bool) error { + for inputPath, outputSubpath := range copyDetails { + if outputSubpath == "" { + outputSubpath = path.Base(inputPath) + } + outputPath := path.Join(outputDir, outputSubpath) + err := os.MkdirAll(path.Dir(outputPath), os.ModeDir|os.ModePerm) + if err != nil { + return err + } + + required := requiredCopies[inputPath] + err = copyFile(logger, inputPath, outputPath, required) + if err != nil { + return err + } + } + return nil +} diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go index 1cb922d2e2..48e11361fa 100644 --- a/fdbkubernetesmonitor/main.go +++ b/fdbkubernetesmonitor/main.go @@ -21,6 +21,9 @@ package main import ( "fmt" + "os" + "path" + "strings" "github.com/go-logr/zapr" "github.com/spf13/pflag" @@ -28,17 +31,48 @@ import ( ) var ( - inputDir string - fdbserverPath string - monitorConfFile string - logPath string + inputDir string + fdbserverPath string + versionFilePath string + sharedBinaryDir string + monitorConfFile string + logPath string + executionModeString string + outputDir string + copyFiles []string + copyBinaries []string + binaryOutputDirectory string + copyLibraries []string + copyPrimaryLibrary string + requiredCopyFiles []string + mainContainerVersion string + currentContainerVersion string +) + +type executionMode string + +const ( + executionModeLauncher executionMode = "launcher" + executionModeInit executionMode = "init" + executionModeSidecar executionMode = "sidecar" ) func main() { + pflag.StringVar(&executionModeString, "mode", "launcher", "Execution mode. Valid options are launcher, sidecar, and init") pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary") pflag.StringVar(&inputDir, "input-dir", ".", "Directory containing input files") pflag.StringVar(&monitorConfFile, "input-monitor-conf", "config.json", "Name of the file in the input directory that contains the monitor configuration") pflag.StringVar(&logPath, "log-path", "", "Name of a file to send logs to. Logs will be sent to stdout in addition the file you pass in this argument. If this is blank, logs will only by sent to stdout") + pflag.StringVar(&outputDir, "output-dir", ".", "Directory to copy files into") + pflag.StringArrayVar(©Files, "copy-file", nil, "A list of files to copy") + pflag.StringArrayVar(©Binaries, "copy-binary", nil, "A list of binaries to copy from /usr/bin") + pflag.StringVar(&versionFilePath, "version-file", "/var/fdb/version", "Path to a file containing the current FDB version") + pflag.StringVar(&sharedBinaryDir, "shared-binary-dir", "/var/fdb/shared-binaries/bin", "A directory containing binaries that are copied from a sidecar process") + pflag.StringVar(&binaryOutputDirectory, "binary-output-dir", "", "A subdirectory within $(output-dir)/bin to store binaries in. This defaults to the value in /var/fdb/version") + pflag.StringArrayVar(©Libraries, "copy-library", nil, "A list of libraries to copy from /usr/lib/fdb/multiversion to $(output-dir)/lib/multiversion") + pflag.StringVar(©PrimaryLibrary, "copy-primary-library", "", "A library to copy from /usr/lib/fdb/multiversion to $(output-dir)/lib. This file will be renamed to libfdb_c.so") + pflag.StringArrayVar(&requiredCopyFiles, "require-not-empty", nil, "When copying this file, exit with an error if the file is empty") + pflag.StringVar(&mainContainerVersion, "main-container-version", "", "For sidecar mode, this specifies the version of the main container. If this is equal to the current container version, no files will be copied") pflag.Parse() zapConfig := zap.NewProductionConfig() @@ -50,6 +84,72 @@ func main() { panic(err) } + versionBytes, err := os.ReadFile(versionFilePath) + if err != nil { + panic(err) + } + currentContainerVersion = strings.TrimSpace(string(versionBytes)) + logger := zapr.NewLogger(zapLogger) - StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) + copyDetails, requiredCopies, err := getCopyDetails() + if err != nil { + logger.Error(err, "Error getting list of files to copy") + os.Exit(1) + } + + mode := executionMode(executionModeString) + if mode == executionModeLauncher { + StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) + } else if mode == executionModeInit { + err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) + if err != nil { + logger.Error(err, "Error copying files") + os.Exit(1) + } + } else if mode == executionModeSidecar { + if mainContainerVersion != currentContainerVersion { + err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) + if err != nil { + logger.Error(err, "Error copying files") + os.Exit(1) + } + done := make(chan bool) + <-done + } + } else { + logger.Error(nil, "Unknown execution mode", "mode", mode) + os.Exit(1) + } +} + +func getCopyDetails() (map[string]string, map[string]bool, error) { + copyDetails := make(map[string]string, len(copyFiles)+len(copyBinaries)) + + for _, filePath := range copyFiles { + copyDetails[path.Join(inputDir, filePath)] = "" + } + if copyBinaries != nil { + if binaryOutputDirectory == "" { + binaryOutputDirectory = currentContainerVersion + } + for _, copyBinary := range copyBinaries { + copyDetails[fmt.Sprintf("/usr/bin/%s", copyBinary)] = path.Join("bin", binaryOutputDirectory, copyBinary) + } + } + for _, library := range copyLibraries { + copyDetails[fmt.Sprintf("/usr/lib/fdb/multiversion/libfdb_c_%s.so", library)] = path.Join("lib", "multiversion", fmt.Sprintf("libfdb_c_%s.so", library)) + } + if copyPrimaryLibrary != "" { + copyDetails[fmt.Sprintf("/usr/lib/fdb/multiversion/libfdb_c_%s.so", copyPrimaryLibrary)] = path.Join("lib", "libfdb_c.so") + } + requiredCopyMap := make(map[string]bool, len(requiredCopyFiles)) + for _, filePath := range requiredCopyFiles { + fullFilePath := path.Join(inputDir, filePath) + _, present := copyDetails[fullFilePath] + if !present { + return nil, nil, fmt.Errorf("File %s is required, but is not in the --copy-file list", filePath) + } + requiredCopyMap[fullFilePath] = true + } + return copyDetails, requiredCopyMap, nil } diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index 99d8e9c698..32d68d176c 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -26,6 +26,7 @@ import ( "os" "os/exec" "os/signal" + "path" "sync" "syscall" "time" @@ -113,9 +114,25 @@ func (monitor *Monitor) LoadConfiguration() { return } + if currentContainerVersion == configuration.Version { + configuration.BinaryPath = monitor.FDBServerPath + } else { + configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver") + } + + binaryStat, err := os.Stat(configuration.BinaryPath) + if err != nil { + monitor.Logger.Error(err, "Error checking binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) + return + } + if binaryStat.Mode()&0o100 == 0 { + monitor.Logger.Error(nil, "New binary path is not executable", "configuration", configuration, "binaryPath", configuration.BinaryPath) + return + } + _, err = configuration.GenerateArguments(1, nil) if err != nil { - monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration) + monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) return } @@ -165,7 +182,6 @@ func (monitor *Monitor) RunProcess(processNumber int) { monitor.Mutex.Unlock() arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, nil) - arguments = append([]string{monitor.FDBServerPath}, arguments...) if err != nil { logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration) time.Sleep(errorBackoffSeconds * time.Second) diff --git a/packaging/docker/kubernetes/Dockerfile b/packaging/docker/kubernetes/Dockerfile index bee0369a29..2f6d4ca026 100644 --- a/packaging/docker/kubernetes/Dockerfile +++ b/packaging/docker/kubernetes/Dockerfile @@ -57,11 +57,12 @@ COPY packaging/docker/website /mnt/website/ RUN mkdir -p /var/fdb/logs && mkdir -p /var/fdb/tmp && \ curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin + mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ + echo ${FDB_VERSION} > /var/fdb/version # Install additional FoundationDB Client Libraries -ADD packaging/docker/release/download_multiversion_libraries.bash /var/fdb/tmp -RUN bash /var/fdb/tmp/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_LIBRARY_VERSIONS +RUN mkdir -p /usr/lib/fdb/multiversion && \ + for version in $FDB_LIBRARY_VERSIONS; do curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; done # Clean up temporary directories RUN rm -rf /mnt/website && rm -r /var/fdb/tmp diff --git a/packaging/docker/kubernetes/test_config.yaml b/packaging/docker/kubernetes/test_config.yaml index 2034f6282d..d96f883648 100644 --- a/packaging/docker/kubernetes/test_config.yaml +++ b/packaging/docker/kubernetes/test_config.yaml @@ -84,10 +84,33 @@ spec: volumeMounts: - name: dynamic-conf mountPath: /var/fdb/dynamic-conf + - name: shared-binaries + mountPath: /var/fdb/shared-binaries - name: data mountPath: /var/fdb/data - name: logs mountPath: /var/fdb/logs + - name: foundationdb-sidecar + image: foundationdb/foundationdb-kubernetes:latest-sidecar + imagePullPolicy: IfNotPresent + args: + - --mode + - sidecar + - --main-container-version + - 6.3.13 + - --output-dir + - /var/fdb/shared-binaries + - --copy-binary + - fdbserver + - --copy-binary + - fdbcli + - --log-path + - /var/fdb/logs/sidecar.log + volumeMounts: + - name: shared-binaries + mountPath: /var/fdb/shared-binaries + - name: logs + mountPath: /var/fdb/logs serviceAccountName: fdb-kubernetes-example volumes: - name: dynamic-conf @@ -95,6 +118,8 @@ spec: name: fdb-kubernetes-example-config - name: logs emptyDir: {} + - name: shared-binaries + emptyDir: {} volumeClaimTemplates: - metadata: name: data @@ -114,7 +139,7 @@ data: config.json: | { "serverCount": 0, - "version": "6.3.15", + "version": "6.3.13", "arguments": [ {"value": "--cluster_file"}, {"value": "/var/fdb/data/fdb.cluster"}, @@ -186,3 +211,75 @@ roleRef: subjects: - kind: ServiceAccount name: fdb-kubernetes-example +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fdb-kubernetes-example-client +spec: + replicas: 2 + revisionHistoryLimit: 1 + selector: + matchLabels: + app: fdb-kubernetes-example-client + template: + metadata: + labels: + app: fdb-kubernetes-example-client + name: fdb-kubernetes-example-client + spec: + volumes: + - name: config-map + configMap: + name: fdb-kubernetes-example-config + items: + - key: fdb.cluster + path: fdb.cluster + - name: dynamic-conf + emptyDir: {} + initContainers: + - name: foundationdb-kubernetes-init + image: foundationdb/foundationdb-kubernetes:latest + imagePullPolicy: IfNotPresent + args: + - "--mode" + - "init" + - "--input-dir" + - "/var/input-files" + - "--output-dir" + - "/var/output-files" + - "--copy-file" + - "fdb.cluster" + - "--require-not-empty" + - "fdb.cluster" + - "--copy-library" + - "6.1" + - "--copy-library" + - "6.2" + - "--copy-primary-library" + - "6.3" + - "--copy-binary" + - "fdbcli" + volumeMounts: + - name: config-map + mountPath: /var/input-files + - name: dynamic-conf + mountPath: /var/output-files + containers: + - name: client + image: foundationdb/foundationdb-sample-python-app:latest + imagePullPolicy: Always + env: + - name: FDB_CLUSTER_FILE + value: /var/dynamic-conf/fdb.cluster + - name: FDB_API_VERSION + value: "610" + - name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP + value: fdb-kubernetes-example-client + - name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY + value: /var/dynamic-conf/lib/multiversion + - name: LD_LIBRARY_PATH + value: /var/dynamic-conf/lib + volumeMounts: + - name: dynamic-conf + mountPath: /var/dynamic-conf From f8ec3cc27d5ca62a06e432aac63b4052b80285e7 Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Mon, 23 Aug 2021 01:11:25 -0700 Subject: [PATCH 032/338] Add an option to load an additional file of environment variables in fdb-kubernetes-monitor. --- .../.testdata/default_config.json | 2 +- fdbkubernetesmonitor/config.go | 3 +- fdbkubernetesmonitor/main.go | 35 ++++++++++++++++++- fdbkubernetesmonitor/monitor.go | 21 +++++------ 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/fdbkubernetesmonitor/.testdata/default_config.json b/fdbkubernetesmonitor/.testdata/default_config.json index 86cb836164..60d337c4c5 100644 --- a/fdbkubernetesmonitor/.testdata/default_config.json +++ b/fdbkubernetesmonitor/.testdata/default_config.json @@ -1,5 +1,5 @@ { - "version": "6.3.0", + "version": "6.3.15", "serverCount": 1, "arguments": [ {"value": "--cluster_file"}, diff --git a/fdbkubernetesmonitor/config.go b/fdbkubernetesmonitor/config.go index 0899a145d9..ac388f10dd 100644 --- a/fdbkubernetesmonitor/config.go +++ b/fdbkubernetesmonitor/config.go @@ -113,7 +113,8 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri var present bool if env != nil { value, present = env[argument.Source] - } else { + } + if !present { value, present = os.LookupEnv(argument.Source) } if !present { diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go index 48e11361fa..1237ffa81b 100644 --- a/fdbkubernetesmonitor/main.go +++ b/fdbkubernetesmonitor/main.go @@ -20,11 +20,14 @@ package main import ( + "bufio" "fmt" "os" "path" + "regexp" "strings" + "github.com/go-logr/logr" "github.com/go-logr/zapr" "github.com/spf13/pflag" "go.uber.org/zap" @@ -47,6 +50,7 @@ var ( requiredCopyFiles []string mainContainerVersion string currentContainerVersion string + additionalEnvFile string ) type executionMode string @@ -73,6 +77,7 @@ func main() { pflag.StringVar(©PrimaryLibrary, "copy-primary-library", "", "A library to copy from /usr/lib/fdb/multiversion to $(output-dir)/lib. This file will be renamed to libfdb_c.so") pflag.StringArrayVar(&requiredCopyFiles, "require-not-empty", nil, "When copying this file, exit with an error if the file is empty") pflag.StringVar(&mainContainerVersion, "main-container-version", "", "For sidecar mode, this specifies the version of the main container. If this is equal to the current container version, no files will be copied") + pflag.StringVar(&additionalEnvFile, "additional-env-file", "", "A file with additional environment variables to use when interpreting the monitor configuration") pflag.Parse() zapConfig := zap.NewProductionConfig() @@ -99,7 +104,12 @@ func main() { mode := executionMode(executionModeString) if mode == executionModeLauncher { - StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), fdbserverPath) + customEnvironment, err := loadAdditionalEnvironment(logger) + if err != nil { + logger.Error(err, "Error loading additional environment") + os.Exit(1) + } + StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment) } else if mode == executionModeInit { err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) if err != nil { @@ -153,3 +163,26 @@ func getCopyDetails() (map[string]string, map[string]bool, error) { } return copyDetails, requiredCopyMap, nil } + +func loadAdditionalEnvironment(logger logr.Logger) (map[string]string, error) { + var customEnvironment = make(map[string]string) + environmentPattern := regexp.MustCompile(`export ([A-Za-z0-9_]+)=([^\n]*)`) + if additionalEnvFile != "" { + file, err := os.Open(additionalEnvFile) + if err != nil { + return nil, err + } + + envScanner := bufio.NewScanner(file) + for envScanner.Scan() { + envLine := envScanner.Text() + matches := environmentPattern.FindStringSubmatch(envLine) + if matches == nil || envLine == "" { + logger.Error(nil, "Environment file contains line that we cannot parse", "line", envLine, "environmentPattern", environmentPattern) + continue + } + customEnvironment[matches[1]] = matches[2] + } + } + return customEnvironment, nil +} diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index 32d68d176c..a9649d68e9 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -46,8 +46,9 @@ type Monitor struct { // ConfigFile defines the path to the config file to load. ConfigFile string - // FDBServerPath defines the path to the fdbserver binary. - FDBServerPath string + // CustomEnvironment defines the custom environment variables to use when + // interpreting the monitor configuration. + CustomEnvironment map[string]string // ActiveConfiguration defines the active process configuration. ActiveConfiguration *ProcessConfiguration @@ -78,17 +79,17 @@ type Monitor struct { } // StartMonitor starts the monitor loop. -func StartMonitor(logger logr.Logger, configFile string, fdbserverPath string) { +func StartMonitor(logger logr.Logger, configFile string, customEnvironment map[string]string) { podClient, err := CreatePodClient() if err != nil { panic(err) } monitor := &Monitor{ - ConfigFile: configFile, - FDBServerPath: fdbserverPath, - PodClient: podClient, - Logger: logger, + ConfigFile: configFile, + PodClient: podClient, + Logger: logger, + CustomEnvironment: customEnvironment, } go func() { monitor.WatchPodTimestamps() }() @@ -115,7 +116,7 @@ func (monitor *Monitor) LoadConfiguration() { } if currentContainerVersion == configuration.Version { - configuration.BinaryPath = monitor.FDBServerPath + configuration.BinaryPath = fdbserverPath } else { configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver") } @@ -130,7 +131,7 @@ func (monitor *Monitor) LoadConfiguration() { return } - _, err = configuration.GenerateArguments(1, nil) + _, err = configuration.GenerateArguments(1, monitor.CustomEnvironment) if err != nil { monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) return @@ -181,7 +182,7 @@ func (monitor *Monitor) RunProcess(processNumber int) { } monitor.Mutex.Unlock() - arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, nil) + arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment) if err != nil { logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration) time.Sleep(errorBackoffSeconds * time.Second) From 1a5069a0471d6e14a247465a6bf4e93daa57d2e0 Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Fri, 17 Sep 2021 16:26:05 -0700 Subject: [PATCH 033/338] Use an write-and-rename pattern when copying files for atomicity. Restructure the usage of the mutex in the monitor class. --- fdbkubernetesmonitor/README.md | 6 ++- fdbkubernetesmonitor/copy.go | 47 +++++++++--------- fdbkubernetesmonitor/monitor.go | 51 ++++++++++++++------ packaging/docker/kubernetes/test_config.yaml | 4 +- 4 files changed, 67 insertions(+), 41 deletions(-) diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md index 95a54866b6..5f85436636 100644 --- a/fdbkubernetesmonitor/README.md +++ b/fdbkubernetesmonitor/README.md @@ -4,8 +4,8 @@ This package provides a launcher program for running FoundationDB in Kubernetes. To test this, run the following commands from the root of the FoundationDB repository: - docker build -t foundationdb/foundationdb-kubernetes:latest --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . - docker build -t foundationdb/foundationdb-kubernetes:latest-sidecar --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . + docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . + docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . kubectl apply -f packaging/docker/kubernetes/test_config.yaml # Wait for the pods to become ready ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') @@ -14,6 +14,8 @@ repository: # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" +This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. + You can then make changes to the data in the config map and update the fdbserver processes: cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go index 80074bc0f1..bf91d4cede 100644 --- a/fdbkubernetesmonitor/copy.go +++ b/fdbkubernetesmonitor/copy.go @@ -21,17 +21,13 @@ package main import ( "fmt" - "io" + "io/ioutil" "os" "path" "github.com/go-logr/logr" ) -const ( - bufferSize = 1024 -) - // copyFile copies a file into the output directory. func copyFile(logger logr.Logger, inputPath string, outputPath string, required bool) error { logger.Info("Copying file", "inputPath", inputPath, "outputPath", outputPath) @@ -52,29 +48,34 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required return fmt.Errorf("File %s is empty", inputPath) } - outputFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY, inputInfo.Mode()) + outputDir := path.Dir(outputPath) + + tempFile, err := ioutil.TempFile(outputDir, "") if err != nil { return err } - defer outputFile.Close() + defer tempFile.Close() - var buffer = make([]byte, bufferSize) - for { - readLength, readError := inputFile.Read(buffer) - if readError == io.EOF { - break - } - if readError != nil { - logger.Error(readError, "Error reading file", "path", inputPath) - return readError - } - - _, writeError := outputFile.Write(buffer[:readLength]) - if writeError != nil { - logger.Error(writeError, "Error writing file", "path", outputPath) - return writeError - } + _, err = tempFile.ReadFrom(inputFile) + if err != nil { + return err } + + err = tempFile.Close() + if err != nil { + return err + } + + err = os.Chmod(tempFile.Name(), inputInfo.Mode()) + if err != nil { + return err + } + + err = os.Rename(tempFile.Name(), outputPath) + if err != nil { + return err + } + return nil } diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index a9649d68e9..d366a6a2bd 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -68,6 +68,8 @@ type Monitor struct { ProcessIDs []int // Mutex defines a mutex around working with configuration. + // This is used to synchronize access to local state like the active + // configuration and the process IDs from multiple goroutines. Mutex sync.Mutex // PodClient is a client for posting updates about this pod to @@ -137,9 +139,15 @@ func (monitor *Monitor) LoadConfiguration() { return } - monitor.Logger.Info("Received new configuration file", "configuration", configuration) + monitor.acceptConfiguration(configuration, configurationBytes) +} + +// acceptConfiguration is called when the monitor process parses and accepts +// a configuration from the local config file. +func (monitor *Monitor) acceptConfiguration(configuration *ProcessConfiguration, configurationBytes []byte) { monitor.Mutex.Lock() defer monitor.Mutex.Unlock() + monitor.Logger.Info("Received new configuration file", "configuration", configuration) if monitor.ProcessIDs == nil { monitor.ProcessIDs = make([]int, configuration.ServerCount+1) @@ -161,7 +169,7 @@ func (monitor *Monitor) LoadConfiguration() { } } - err = monitor.PodClient.UpdateAnnotations(monitor) + err := monitor.PodClient.UpdateAnnotations(monitor) if err != nil { monitor.Logger.Error(err, "Error updating pod annotations") } @@ -173,14 +181,9 @@ func (monitor *Monitor) RunProcess(processNumber int) { logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "RunProcess") logger.Info("Starting run loop") for { - monitor.Mutex.Lock() - if monitor.ActiveConfiguration.ServerCount < processNumber { - logger.Info("Terminating run loop") - monitor.ProcessIDs[processNumber] = 0 - monitor.Mutex.Unlock() + if !monitor.checkProcessRequired(processNumber) { return } - monitor.Mutex.Unlock() arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment) if err != nil { @@ -220,9 +223,7 @@ func (monitor *Monitor) RunProcess(processNumber int) { startTime := time.Now() logger.Info("Subprocess started", "PID", pid) - monitor.Mutex.Lock() - monitor.ProcessIDs[processNumber] = pid - monitor.Mutex.Unlock() + monitor.updateProcessID(processNumber, pid) if stdout != nil { stdoutScanner := bufio.NewScanner(stdout) @@ -254,9 +255,7 @@ func (monitor *Monitor) RunProcess(processNumber int) { logger.Info("Subprocess terminated", "exitCode", exitCode, "PID", pid) endTime := time.Now() - monitor.Mutex.Lock() - monitor.ProcessIDs[processNumber] = -1 - monitor.Mutex.Unlock() + monitor.updateProcessID(processNumber, -1) processDuration := endTime.Sub(startTime) if processDuration.Seconds() < errorBackoffSeconds { @@ -266,6 +265,30 @@ func (monitor *Monitor) RunProcess(processNumber int) { } } +// checkProcessRequired determines if the latest configuration requires that a +// process stay running. +// If the process is no longer desired, this will remove it from the process ID +// list and return false. If the process is still desired, this will return +// true. +func (monitor *Monitor) checkProcessRequired(processNumber int) bool { + monitor.Mutex.Lock() + defer monitor.Mutex.Unlock() + logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "checkProcessRequired") + if monitor.ActiveConfiguration.ServerCount < processNumber { + logger.Info("Terminating run loop") + monitor.ProcessIDs[processNumber] = 0 + return false + } + return true +} + +// updateProcessID records a new Process ID from a newly launched process. +func (monitor *Monitor) updateProcessID(processNumber int, pid int) { + monitor.Mutex.Lock() + defer monitor.Mutex.Unlock() + monitor.ProcessIDs[processNumber] = pid +} + // WatchConfiguration detects changes to the monitor configuration file. func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) { for { diff --git a/packaging/docker/kubernetes/test_config.yaml b/packaging/docker/kubernetes/test_config.yaml index d96f883648..1f43b7dd3e 100644 --- a/packaging/docker/kubernetes/test_config.yaml +++ b/packaging/docker/kubernetes/test_config.yaml @@ -43,7 +43,7 @@ spec: spec: containers: - name: foundationdb - image: foundationdb/foundationdb-kubernetes:latest + image: foundationdb/foundationdb-kubernetes:6.3.13-local imagePullPolicy: IfNotPresent args: - --input-dir @@ -91,7 +91,7 @@ spec: - name: logs mountPath: /var/fdb/logs - name: foundationdb-sidecar - image: foundationdb/foundationdb-kubernetes:latest-sidecar + image: foundationdb/foundationdb-kubernetes:6.3.15-local imagePullPolicy: IfNotPresent args: - --mode From a6b903e7f8737294fef0b0e62a7818550f5b00fa Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Tue, 21 Sep 2021 12:12:43 -0700 Subject: [PATCH 034/338] Move the new Kubernetes image to centos 7. --- fdbkubernetesmonitor/README.md | 38 +++++++++++++++----------- fdbkubernetesmonitor/main.go | 9 +++--- fdbkubernetesmonitor/monitor.go | 24 +++++++++++----- packaging/docker/kubernetes/Dockerfile | 32 ++++++++++++---------- 4 files changed, 61 insertions(+), 42 deletions(-) diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md index 5f85436636..b8a68a03ac 100644 --- a/fdbkubernetesmonitor/README.md +++ b/fdbkubernetesmonitor/README.md @@ -4,28 +4,34 @@ This package provides a launcher program for running FoundationDB in Kubernetes. To test this, run the following commands from the root of the FoundationDB repository: - docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . - docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . - kubectl apply -f packaging/docker/kubernetes/test_config.yaml - # Wait for the pods to become ready - ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') - cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - - kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite - # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. - kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" +```bash +docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . +docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . +kubectl apply -f packaging/docker/kubernetes/test_config.yaml +# Wait for the pods to become ready +ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') +sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f - +kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite +# Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. +kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" +``` -This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. +This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. Note: This assumes that you are running Docker Desktop on your local machine, with Kubernetes configured through Docker Desktop. You can then make changes to the data in the config map and update the fdbserver processes: - cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - +```bash +sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f - - # You can apply an annotation to speed up the propagation of config - kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite +# You can apply an annotation to speed up the propagation of config +kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite - # Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce. - kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status" +# Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce. +kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status" +``` Once you are done, you can tear down the example with the following command: - kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example +```bash +kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example +``` diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go index 1237ffa81b..821ee13b50 100644 --- a/fdbkubernetesmonitor/main.go +++ b/fdbkubernetesmonitor/main.go @@ -103,20 +103,21 @@ func main() { } mode := executionMode(executionModeString) - if mode == executionModeLauncher { + switch mode { + case executionModeLauncher: customEnvironment, err := loadAdditionalEnvironment(logger) if err != nil { logger.Error(err, "Error loading additional environment") os.Exit(1) } StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment) - } else if mode == executionModeInit { + case executionModeInit: err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) if err != nil { logger.Error(err, "Error copying files") os.Exit(1) } - } else if mode == executionModeSidecar { + case executionModeSidecar: if mainContainerVersion != currentContainerVersion { err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) if err != nil { @@ -126,7 +127,7 @@ func main() { done := make(chan bool) <-done } - } else { + default: logger.Error(nil, "Unknown execution mode", "mode", mode) os.Exit(1) } diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index d366a6a2bd..2db0a469c9 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -22,6 +22,7 @@ package main import ( "bufio" "encoding/json" + "fmt" "io" "os" "os/exec" @@ -123,13 +124,9 @@ func (monitor *Monitor) LoadConfiguration() { configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver") } - binaryStat, err := os.Stat(configuration.BinaryPath) + err = checkOwnerExecutable(configuration.BinaryPath) if err != nil { - monitor.Logger.Error(err, "Error checking binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) - return - } - if binaryStat.Mode()&0o100 == 0 { - monitor.Logger.Error(nil, "New binary path is not executable", "configuration", configuration, "binaryPath", configuration.BinaryPath) + monitor.Logger.Error(err, "Error with binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) return } @@ -142,6 +139,19 @@ func (monitor *Monitor) LoadConfiguration() { monitor.acceptConfiguration(configuration, configurationBytes) } +// checkOwnerExecutable validates that a path is a file that exists and is +// executable by its owner. +func checkOwnerExecutable(path string) error { + binaryStat, err := os.Stat(path) + if err != nil { + return err + } + if binaryStat.Mode()&0o100 == 0 { + return fmt.Errorf("Binary is not executable") + } + return nil +} + // acceptConfiguration is called when the monitor process parses and accepts // a configuration from the local config file. func (monitor *Monitor) acceptConfiguration(configuration *ProcessConfiguration, configurationBytes []byte) { @@ -217,7 +227,7 @@ func (monitor *Monitor) RunProcess(processNumber int) { if cmd.Process != nil { pid = cmd.Process.Pid } else { - logger.Error(nil, "No Process information availale for subprocess") + logger.Error(nil, "No Process information available for subprocess") } startTime := time.Now() diff --git a/packaging/docker/kubernetes/Dockerfile b/packaging/docker/kubernetes/Dockerfile index 2f6d4ca026..58ef7e998a 100644 --- a/packaging/docker/kubernetes/Dockerfile +++ b/packaging/docker/kubernetes/Dockerfile @@ -30,22 +30,24 @@ RUN go build -o /fdb-kubernetes-monitor ./... # Build the main image -FROM ubuntu:18.04 +FROM centos:7.9.2009 -RUN apt-get update && \ - apt-get install -y curl>=7.58.0-2ubuntu3.6 \ - dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \ - lsof>=4.89+dfsg-0.1 \ - tcptraceroute>=1.5beta7+debian-4build1 \ - telnet>=0.17-41 \ - netcat>=1.10-41.1 \ - strace>=4.21-1ubuntu1 \ - tcpdump>=4.9.3-0ubuntu0.18.04.1 \ - less>=487-0.1 \ - vim>=2:8.0.1453-1ubuntu1.4 \ - net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \ - jq>=1.5+dfsg-2 && \ - rm -rf /var/lib/apt/lists/* +RUN yum install -y \ + binutils-2.27-44.base.el7 \ + bind-utils-9.11.4-26.P2.el7_9.7 \ + curl-7.29.0-59.el7_9.1 \ + less-458-9.el7 \ + lsof-4.87-6.el7 \ + nano-2.3.1-10.el7 \ + nmap-ncat-6.40-19.el7 \ + net-tools-2.0-0.25.20131004git.el7 \ + strace-4.24-6.el7 \ + tar-1.26-35.el7 \ + telnet-0.17-66.el7 \ + traceroute-2.0.22-2.el7 \ + tcpdump-4.9.2-4.el7_7.1 \ + vim-enhanced-7.4.629-8.el7_9 \ + && yum clean all ARG FDB_VERSION ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" From ee292e2df7f2cf21595796cc7cb855dde3c53234 Mon Sep 17 00:00:00 2001 From: John Brownlee Date: Wed, 29 Sep 2021 15:56:56 -0700 Subject: [PATCH 035/338] Update based on PR feedback. --- fdbkubernetesmonitor/config.go | 3 ++- fdbkubernetesmonitor/copy.go | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdbkubernetesmonitor/config.go b/fdbkubernetesmonitor/config.go index ac388f10dd..2815dbd235 100644 --- a/fdbkubernetesmonitor/config.go +++ b/fdbkubernetesmonitor/config.go @@ -22,6 +22,7 @@ package main import ( "fmt" "os" + "strconv" ) // ProcessConfiguration models the configuration for starting a FoundationDB @@ -107,7 +108,7 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri number = number * argument.Multiplier } number = number + argument.Offset - return fmt.Sprintf("%d", number), nil + return strconv.Itoa(number), nil case EnvironmentArgumentType: var value string var present bool diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go index bf91d4cede..2414a8cf7d 100644 --- a/fdbkubernetesmonitor/copy.go +++ b/fdbkubernetesmonitor/copy.go @@ -21,7 +21,6 @@ package main import ( "fmt" - "io/ioutil" "os" "path" @@ -50,7 +49,7 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required outputDir := path.Dir(outputPath) - tempFile, err := ioutil.TempFile(outputDir, "") + tempFile, err := os.CreateTemp(outputDir, "") if err != nil { return err } From a4d784a3dc58a3556e58e81e99cf00c561110175 Mon Sep 17 00:00:00 2001 From: Leonidas Tsampros Date: Fri, 1 Oct 2021 11:17:39 +0100 Subject: [PATCH 036/338] packaging: apt doesn't support >= and fix tini installation --- packaging/docker/misc/tini-amd64.sha256sum | 1 - packaging/docker/release/Dockerfile | 36 ++++++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) delete mode 100644 packaging/docker/misc/tini-amd64.sha256sum diff --git a/packaging/docker/misc/tini-amd64.sha256sum b/packaging/docker/misc/tini-amd64.sha256sum deleted file mode 100644 index 3cb1f9f635..0000000000 --- a/packaging/docker/misc/tini-amd64.sha256sum +++ /dev/null @@ -1 +0,0 @@ -93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini-amd64 diff --git a/packaging/docker/release/Dockerfile b/packaging/docker/release/Dockerfile index 7df65e63c6..8bdfcd2109 100644 --- a/packaging/docker/release/Dockerfile +++ b/packaging/docker/release/Dockerfile @@ -20,28 +20,30 @@ FROM ubuntu:18.04 RUN apt-get update && \ - apt-get install -y curl>=7.58.0-2ubuntu3.6 \ - dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \ - lsof>=4.89+dfsg-0.1 \ - tcptraceroute>=1.5beta7+debian-4build1 \ - telnet>=0.17-41 \ - netcat>=1.10-41.1 \ - strace>=4.21-1ubuntu1 \ - tcpdump>=4.9.3-0ubuntu0.18.04.1 \ - less>=487-0.1 \ - vim>=2:8.0.1453-1ubuntu1.4 \ - net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \ - jq>=1.5+dfsg-2 \ - openssl>=1.1.1-1ubuntu2.1~18.04.9 && \ + apt-get install -y curl \ + dnsutils \ + lsof \ + tcptraceroute \ + telnet \ + netcat \ + strace \ + tcpdump \ + less \ + vim \ + net-tools \ + jq \ + openssl && \ rm -rf /var/lib/apt/lists/* COPY misc/tini-amd64.sha256sum /tmp/ # Adding tini as PID 1 https://github.com/krallin/tini ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini +RUN curl -o /tmp/tini-amd64 -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ + curl -o /tmp/tini-amd64.sha256sum -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64.sha256sum && \ + cd tmp && sha256sum -c /tmp/tini-amd64.sha256sum && \ + mv /tmp/tini-amd64 /usr/bin/tini && \ + chmod +x /usr/bin/tini + ARG FDB_VERSION ARG FDB_ADDITIONAL_VERSIONS="5.1.7" From 504f08a102a5df1927e7def5363e0d70c0fbbb04 Mon Sep 17 00:00:00 2001 From: Aaron Molitor Date: Thu, 28 Oct 2021 23:12:22 -0700 Subject: [PATCH 037/338] consolidate docker stuff, add perf and flamegraph parts to release image --- packaging/docker/Dockerfile.eks | 161 +++++++++++------- packaging/docker/misc/flamegraph.sha256sum | 2 - packaging/docker/release/Dockerfile | 92 +++++----- .../docker/release/create_cluster_file.bash | 52 ------ .../release/create_server_environment.bash | 43 ----- .../download_multiversion_libraries.bash | 31 ---- packaging/docker/release/fdb.bash | 51 +++++- packaging/docker/sidecar/Dockerfile | 56 +++--- packaging/docker/sidecar/entrypoint.bash | 2 +- packaging/docker/sidecar/requirements.txt | 1 - 10 files changed, 224 insertions(+), 267 deletions(-) delete mode 100644 packaging/docker/misc/flamegraph.sha256sum delete mode 100755 packaging/docker/release/create_cluster_file.bash delete mode 100755 packaging/docker/release/create_server_environment.bash delete mode 100755 packaging/docker/release/download_multiversion_libraries.bash diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks index bc05b4d5a6..b2aaf7b4f4 100644 --- a/packaging/docker/Dockerfile.eks +++ b/packaging/docker/Dockerfile.eks @@ -1,76 +1,108 @@ +# Dockerfile +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM amazonlinux:2.0.20210326.0 as base RUN yum install -y \ - binutils \ - bind-utils \ - curl \ - gdb \ - jq \ - less \ - lsof \ - nc \ - net-tools \ - perf \ - perl \ - procps \ - python38 \ - python3-pip \ - strace \ - tar \ - traceroute \ - telnet \ - tcpdump \ - unzip \ - vim + binutils \ + bind-utils \ + curl \ + gdb \ + jq \ + less \ + lsof \ + nc \ + net-tools \ + perf \ + perl \ + procps \ + python38 \ + python3-pip \ + strace \ + tar \ + traceroute \ + telnet \ + tcpdump \ + unzip \ + vim && \ + yum clean all && \ + rm -rf /var/cache/yum -#todo: nload, iperf, numademo +# TODO: nload, iperf, numademo -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip" -o "awscliv2.zip" \ - && unzip awscliv2.zip && ./aws/install && rm -rf aws +RUN curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ + echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + sha256sum -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* -COPY misc/tini-amd64.sha256sum /tmp/ -COPY misc/flamegraph.sha256sum /tmp/ # Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini - -COPY sidecar/requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* # Install flamegraph -RUN curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ - curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ - sha256sum -c /tmp/flamegraph.sha256sum && \ - chmod +x stackcollapse-perf.pl flamegraph.pl && \ - mv stackcollapse-perf.pl flamegraph.pl /usr/bin - -# TODO: Only used by sidecar -RUN groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb +RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ + curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ + echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ + echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ + sha256sum -c flamegraph.sha256sum && \ + chmod +x stackcollapse-perf.pl flamegraph.pl && \ + mv stackcollapse-perf.pl flamegraph.pl /usr/bin ARG FDB_VERSION +ARG FDB_ADDITIONAL_VERSIONS="6.3.12 6.2.30 6.1.13 5.1.7" +ARG FDB_WEBSITE=https://www.foundationdb.org + +# Install additional FoundationDB Client Libraries +RUN mkdir -p /usr/lib/fdb/multiversion && \ + for version in $FDB_ADDITIONAL_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_$version.so; \ + done && \ + rm -rf /mnt/website -# These are the output of the current build (not stripped) COPY --chown=root bin /usr/bin/ COPY --chown=root lib/libfdb_c.so /var/fdb/lib/ RUN mv /var/fdb/lib/libfdb_c.so /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so RUN ln -s /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so /var/fdb/lib/libfdb_c.so -# ------------------------------------------------- + +# =========================== END OF LAYER: base =============================== FROM base as foundationdb -COPY release/*.bash /var/fdb/scripts/ -RUN mkdir -p /var/fdb/logs +ARG FDB_VERSION -# TODO: FDB_ADDITIONAL_VERSIONS -RUN mkdir -p /usr/lib/fdb/multiversion +WORKDIR / + +# Set Up Runtime Scripts and Directories +ADD release/fdb.bash /var/fdb/scripts/ +RUN chmod a+x /var/fdb/scripts/fdb.bash + +RUN mkdir -p /var/fdb/logs VOLUME /var/fdb/data # Runtime Configuration Options + ENV FDB_PORT 4500 ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster ENV FDB_NETWORKING_MODE container @@ -79,27 +111,34 @@ ENV FDB_COORDINATOR_PORT 4500 ENV FDB_CLUSTER_FILE_CONTENTS "" ENV FDB_PROCESS_CLASS unset -ENTRYPOINT ["/usr/bin/tini", "-g", "--"] -CMD /var/fdb/scripts/fdb.bash +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] -# ------------------------------------------------- +# =========================== END OF LAYER: foundationdb =============================== FROM base AS sidecar +WORKDIR / -COPY sidecar/entrypoint.bash / -COPY sidecar/sidecar.py / -RUN chmod a+x /sidecar.py /entrypoint.bash +ARG FDB_VERSION + +# Set Up Runtime Scripts and Directories + +ADD sidecar/entrypoint.bash sidecar/sidecar.py / +RUN chmod a+x /entrypoint.bash /sidecar.py +RUN pip3 install watchdog==0.9.0 + +RUN echo ${FDB_VERSION} > /var/fdb/version && \ + mkdir -p /var/fdb/lib && \ + groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb VOLUME /var/input-files VOLUME /var/output-files -ARG FDB_VERSION +USER fdb -RUN echo ${FDB_VERSION} ; echo ${FDB_VERSION}> /var/fdb/version -RUN mkdir -p /var/fdb/lib +# Runtime Configuration Options ENV LISTEN_PORT 8080 -USER fdb - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] +# =========================== END OF LAYER: sidecar =============================== diff --git a/packaging/docker/misc/flamegraph.sha256sum b/packaging/docker/misc/flamegraph.sha256sum deleted file mode 100644 index bb435ced8b..0000000000 --- a/packaging/docker/misc/flamegraph.sha256sum +++ /dev/null @@ -1,2 +0,0 @@ -a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl -5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl diff --git a/packaging/docker/release/Dockerfile b/packaging/docker/release/Dockerfile index 8bdfcd2109..fc58f64d49 100644 --- a/packaging/docker/release/Dockerfile +++ b/packaging/docker/release/Dockerfile @@ -20,63 +20,70 @@ FROM ubuntu:18.04 RUN apt-get update && \ - apt-get install -y curl \ - dnsutils \ - lsof \ - tcptraceroute \ - telnet \ - netcat \ - strace \ - tcpdump \ - less \ - vim \ - net-tools \ - jq \ - openssl && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y \ + curl \ + dnsutils \ + jq \ + less \ + linux-tools-generic \ + lsof \ + net-tools \ + netcat \ + openssl \ + perl \ + strace \ + tcpdump \ + tcptraceroute \ + telnet \ + vim && \ + rm -rf /var/lib/apt/lists/* -COPY misc/tini-amd64.sha256sum /tmp/ +WORKDIR /tmp # Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -o /tmp/tini-amd64 -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - curl -o /tmp/tini-amd64.sha256sum -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64.sha256sum && \ - cd tmp && sha256sum -c /tmp/tini-amd64.sha256sum && \ - mv /tmp/tini-amd64 /usr/bin/tini && \ - chmod +x /usr/bin/tini +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* +# Install flamegraph +RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ + curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ + echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ + echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ + sha256sum -c flamegraph.sha256sum && \ + chmod +x stackcollapse-perf.pl flamegraph.pl && \ + mv stackcollapse-perf.pl flamegraph.pl /usr/bin ARG FDB_VERSION ARG FDB_ADDITIONAL_VERSIONS="5.1.7" ARG FDB_WEBSITE=https://www.foundationdb.org -WORKDIR /var/fdb/tmp COPY website /mnt/website/ -# Install FoundationDB Binaries -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -r /var/fdb/tmp - -WORKDIR / - -## TODO: Can unify everything above this line -## TODO: we can almost unify the additional client library download, -## but sidecar.py expects them in a different location, -## with a different naming convention. - RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so -# Set Up Runtime Scripts and Directories -ADD release/*.bash /var/fdb/scripts/ -RUN chmod a+x /var/fdb/scripts/*.bash +# Install FoundationDB Binaries +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ + mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ + rm -rf /tmp/* # Install additional FoundationDB Client Libraries -RUN /var/fdb/scripts/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_ADDITIONAL_VERSIONS +RUN mkdir -p /usr/lib/fdb/multiversion && \ + for version in $FDB_ADDITIONAL_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_$version.so; \ + done && \ + rm -rf /mnt/website -RUN rm -rf /mnt/website +WORKDIR / -RUN mkdir -p /var/fdb/logs +# Set Up Runtime Scripts and Directories +ADD release/fdb.bash /var/fdb/scripts/ +RUN chmod a+x /var/fdb/scripts/fdb.bash + +RUN mkdir -p /var/fdb/logs VOLUME /var/fdb/data @@ -90,5 +97,4 @@ ENV FDB_COORDINATOR_PORT 4500 ENV FDB_CLUSTER_FILE_CONTENTS "" ENV FDB_PROCESS_CLASS unset -ENTRYPOINT ["/usr/bin/tini", "-g", "--"] -CMD /var/fdb/scripts/fdb.bash +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] diff --git a/packaging/docker/release/create_cluster_file.bash b/packaging/docker/release/create_cluster_file.bash deleted file mode 100755 index c1bb959b8e..0000000000 --- a/packaging/docker/release/create_cluster_file.bash +++ /dev/null @@ -1,52 +0,0 @@ -#! /bin/bash - -# -# create_cluster_file.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This script creates a cluster file for a server or client. -# This takes the cluster file path from the FDB_CLUSTER_FILE -# environment variable, with a default of /etc/foundationdb/fdb.cluster -# -# The name of the coordinator must be defined in the FDB_COORDINATOR environment -# variable, and it must be a name that can be resolved through DNS. - -function create_cluster_file() { - FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster} - mkdir -p $(dirname $FDB_CLUSTER_FILE) - - if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then - echo "$FDB_CLUSTER_FILE_CONTENTS" > $FDB_CLUSTER_FILE - elif [[ -n $FDB_COORDINATOR ]]; then - coordinator_ip=$(dig +short $FDB_COORDINATOR) - if [[ -z "$coordinator_ip" ]]; then - echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2 - exit 1 - fi - coordinator_port=${FDB_COORDINATOR_PORT:-4500} - echo "docker:docker@$coordinator_ip:$coordinator_port" > $FDB_CLUSTER_FILE - else - echo "FDB_COORDINATOR environment variable not defined" 1>&2 - exit 1 - fi -} - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - create_cluster_file "$@" -fi diff --git a/packaging/docker/release/create_server_environment.bash b/packaging/docker/release/create_server_environment.bash deleted file mode 100755 index 51a782f991..0000000000 --- a/packaging/docker/release/create_server_environment.bash +++ /dev/null @@ -1,43 +0,0 @@ -#! /bin/bash - -# -# create_server_environment.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -source /var/fdb/scripts/create_cluster_file.bash - -function create_server_environment() { - env_file=/var/fdb/.fdbenv - - if [[ "$FDB_NETWORKING_MODE" == "host" ]]; then - public_ip=127.0.0.1 - elif [[ "$FDB_NETWORKING_MODE" == "container" ]]; then - public_ip=$(hostname -i | awk '{print $1}') - else - echo "Unknown FDB Networking mode \"$FDB_NETWORKING_MODE\"" 1>&2 - exit 1 - fi - - echo "export PUBLIC_IP=$public_ip" > $env_file - if [[ -z $FDB_COORDINATOR && -z "$FDB_CLUSTER_FILE_CONTENTS" ]]; then - FDB_CLUSTER_FILE_CONTENTS="docker:docker@$public_ip:$FDB_PORT" - fi - - create_cluster_file -} diff --git a/packaging/docker/release/download_multiversion_libraries.bash b/packaging/docker/release/download_multiversion_libraries.bash deleted file mode 100755 index 1cd5770ff3..0000000000 --- a/packaging/docker/release/download_multiversion_libraries.bash +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash - -# -# download_multiversion_libraries.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -mkdir -p /usr/lib/fdb/multiversion -website=$1 -shift -for version in $*; do - origin=$website/downloads/$version/linux/libfdb_c_$version.so - destination=/usr/lib/fdb/multiversion/libfdb_c_$version.so - echo "Downloading $origin to $destination" - curl $origin -o $destination -done diff --git a/packaging/docker/release/fdb.bash b/packaging/docker/release/fdb.bash index 943c8ed58b..5d23fc4133 100755 --- a/packaging/docker/release/fdb.bash +++ b/packaging/docker/release/fdb.bash @@ -1,11 +1,11 @@ -#! /bin/bash +#!/bin/bash # # fdb.bash # # This source file is part of the FoundationDB open source project # -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,10 +20,49 @@ # limitations under the License. # -source /var/fdb/scripts/create_server_environment.bash +function create_cluster_file() { + FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster} + mkdir -p "$(dirname $FDB_CLUSTER_FILE)" + + if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then + echo "$FDB_CLUSTER_FILE_CONTENTS" > "$FDB_CLUSTER_FILE" + elif [[ -n $FDB_COORDINATOR ]]; then + coordinator_ip=$(dig +short "$FDB_COORDINATOR") + if [[ -z "$coordinator_ip" ]]; then + echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2 + exit 1 + fi + coordinator_port=${FDB_COORDINATOR_PORT:-4500} + echo "docker:docker@$coordinator_ip:$coordinator_port" > "$FDB_CLUSTER_FILE" + else + echo "FDB_COORDINATOR environment variable not defined" 1>&2 + exit 1 + fi +} + +function create_server_environment() { + env_file=/var/fdb/.fdbenv + + if [[ "$FDB_NETWORKING_MODE" == "host" ]]; then + public_ip=127.0.0.1 + elif [[ "$FDB_NETWORKING_MODE" == "container" ]]; then + public_ip=$(hostname -i | awk '{print $1}') + else + echo "Unknown FDB Networking mode \"$FDB_NETWORKING_MODE\"" 1>&2 + exit 1 + fi + + echo "export PUBLIC_IP=$public_ip" > $env_file + if [[ -z $FDB_COORDINATOR && -z "$FDB_CLUSTER_FILE_CONTENTS" ]]; then + FDB_CLUSTER_FILE_CONTENTS="docker:docker@$public_ip:$FDB_PORT" + fi + + create_cluster_file +} + create_server_environment source /var/fdb/.fdbenv echo "Starting FDB server on $PUBLIC_IP:$FDB_PORT" -fdbserver --listen_address 0.0.0.0:$FDB_PORT --public_address $PUBLIC_IP:$FDB_PORT \ - --datadir /var/fdb/data --logdir /var/fdb/logs \ - --locality_zoneid="$(hostname)" --locality_machineid="$(hostname)" --class $FDB_PROCESS_CLASS +fdbserver --listen_address 0.0.0.0:"$FDB_PORT" --public_address "$PUBLIC_IP:$FDB_PORT" \ + --datadir /var/fdb/data --logdir /var/fdb/logs \ + --locality_zoneid="$(hostname)" --locality_machineid="$(hostname)" --class "$FDB_PROCESS_CLASS" diff --git a/packaging/docker/sidecar/Dockerfile b/packaging/docker/sidecar/Dockerfile index b2d76693ec..3c281f8987 100644 --- a/packaging/docker/sidecar/Dockerfile +++ b/packaging/docker/sidecar/Dockerfile @@ -2,7 +2,7 @@ # # This source file is part of the FoundationDB open source project # -# Copyright 2018-2019 Apple Inc. and the FoundationDB project authors +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,32 +20,38 @@ FROM python:3.9-slim RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ - rm -rf /var/lub/apt/lists/* + apt-get install -y --no-install-recommends \ + curl && \ + pip install watchdog==0.9.0 && \ + rm -rf /var/lib/apt/lists/* -COPY misc/tini-amd64.sha256sum /tmp/ +WORKDIR /tmp # Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* -COPY sidecar/requirements.txt /tmp -RUN pip install -r tmp/requirements.txt - -ARG FDB_VERSION= +ARG FDB_VERSION ARG FDB_ADDITIONAL_VERSIONS="6.2.30 6.1.13" ARG FDB_WEBSITE=https://www.foundationdb.org -WORKDIR /var/fdb/tmp COPY website /mnt/website/ # Install FoundationDB Binaries RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -r /var/fdb/tmp + chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ + mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ + rm -rf /tmp/* + +# Install additional FoundationDB Client Libraries +RUN mkdir -p /var/fdb/lib && \ + for version in $FDB_ADDITIONAL_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ + done && \ + rm -rf /mnt/website WORKDIR / @@ -53,16 +59,10 @@ WORKDIR / ADD sidecar/entrypoint.bash sidecar/sidecar.py / RUN chmod a+x /entrypoint.bash /sidecar.py -# Install additional FoundationDB Client Libraries -RUN mkdir -p /var/fdb/lib && \ - for version in $FDB_ADDITIONAL_VERSIONS; do curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; done - -RUN rm -rf /mnt/website - -RUN echo ${FDB_VERSION} > /var/fdb/version && \ - mkdir -p /var/fdb/lib && \ - groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb +RUN echo ${FDB_VERSION} > /var/fdb/version && \ + mkdir -p /var/fdb/lib && \ + groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb VOLUME /var/input-files @@ -70,6 +70,8 @@ VOLUME /var/output-files USER fdb +# Runtime Configuration Options + ENV LISTEN_PORT 8080 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] diff --git a/packaging/docker/sidecar/entrypoint.bash b/packaging/docker/sidecar/entrypoint.bash index b6678fc831..165f11bce4 100755 --- a/packaging/docker/sidecar/entrypoint.bash +++ b/packaging/docker/sidecar/entrypoint.bash @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # entrypoint.bash # diff --git a/packaging/docker/sidecar/requirements.txt b/packaging/docker/sidecar/requirements.txt index c7fcc8bac8..e69de29bb2 100644 --- a/packaging/docker/sidecar/requirements.txt +++ b/packaging/docker/sidecar/requirements.txt @@ -1 +0,0 @@ -watchdog==0.9.0 \ No newline at end of file From 13613ab0f1b71dec6c233bcf7f22ff9fc6531a2c Mon Sep 17 00:00:00 2001 From: QA Hoang Date: Thu, 28 Oct 2021 15:00:08 -0700 Subject: [PATCH 038/338] fixed mako bug and added comment --- bindings/c/test/mako/mako.c | 5 ++++- bindings/c/test/mako/mako.h | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index f2027c4217..3cbbd7d50f 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -1297,12 +1297,15 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi if (args->client_threads_per_version > 0) { err = fdb_network_set_option( - FDB_NET_OPTION_CLIENT_THREADS_PER_VERSION, (uint8_t*)&args->client_threads_per_version, sizeof(uint32_t)); + FDB_NET_OPTION_CLIENT_THREADS_PER_VERSION, (uint8_t*)&args->client_threads_per_version, sizeof(int64_t)); if (err) { fprintf(stderr, "ERROR: fdb_network_set_option (FDB_NET_OPTION_CLIENT_THREADS_PER_VERSION) (%d): %s\n", (uint8_t*)&args->client_threads_per_version, fdb_get_error(err)); + // let's exit here since we do not want to confuse users + // that mako is running with multi-threaded client enabled + return -1; } } diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index 66a8039dcf..2af3a7059b 100644 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -143,7 +143,7 @@ typedef struct { int txntagging; char txntagging_prefix[TAGPREFIXLENGTH_MAX]; FDBStreamingMode streaming_mode; - uint32_t client_threads_per_version; + int client_threads_per_version; int disable_ryw; char json_output_path[PATH_MAX]; } mako_args_t; From 13bb7838aa32f20c15f435c72b32ef8ad449d221 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 30 Oct 2021 21:07:38 -0700 Subject: [PATCH 039/338] Enable clang -Wformat warning --- bindings/c/test/mako/mako.c | 70 +++++++++---------- bindings/c/test/txn_size_test.c | 8 +-- cmake/ConfigureCompiler.cmake | 1 - fdbcli/ChangeFeedCommand.actor.cpp | 2 +- fdbcli/SetClassCommand.actor.cpp | 2 +- fdbclient/BlobGranuleReader.actor.cpp | 6 +- fdbclient/NativeAPI.actor.cpp | 10 +-- fdbserver/BlobManager.actor.cpp | 20 +++--- fdbserver/BlobWorker.actor.cpp | 66 ++++++++--------- fdbserver/SimulatedCluster.actor.cpp | 6 +- fdbserver/VersionedBTree.actor.cpp | 24 +++---- fdbserver/fdbserver.actor.cpp | 4 +- fdbserver/networktest.actor.cpp | 2 +- fdbserver/tester.actor.cpp | 2 +- .../workloads/BlobGranuleVerifier.actor.cpp | 24 +++---- fdbserver/workloads/RyowCorrectness.actor.cpp | 4 +- 16 files changed, 126 insertions(+), 125 deletions(-) diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index 3cbbd7d50f..37deaed08c 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -943,7 +943,7 @@ int run_workload(FDBTransaction* transaction, if (tracetimer == dotrace) { fdb_error_t err; tracetimer = 0; - snprintf(traceid, 32, "makotrace%019lld", total_xacts); + snprintf(traceid, 32, "makotrace%019ld", total_xacts); fprintf(debugme, "DEBUG: txn tracing %s\n", traceid); err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, @@ -1101,7 +1101,7 @@ void* worker_thread(void* thread_args) { } fprintf(debugme, - "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%d (tid:%lld)\n", + "DEBUG: worker_id:%d (%d) thread_id:%d (%d) database_index:%lu (tid:%lu)\n", worker_id, args->num_processes, thread_id, @@ -1301,7 +1301,7 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi if (err) { fprintf(stderr, "ERROR: fdb_network_set_option (FDB_NET_OPTION_CLIENT_THREADS_PER_VERSION) (%d): %s\n", - (uint8_t*)&args->client_threads_per_version, + args->client_threads_per_version, fdb_get_error(err)); // let's exit here since we do not want to confuse users // that mako is running with multi-threaded client enabled @@ -2038,9 +2038,9 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0) { uint64_t ops_total_diff = ops_total[op] - ops_total_prev[op]; - printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total_diff); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", ops_total_diff); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), ops_total_diff); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), ops_total_diff); } errors_diff[op] = errors_total[op] - errors_total_prev[op]; print_err = (errors_diff[op] > 0); @@ -2068,7 +2068,7 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors"); for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", errors_diff[op]); if (fp) { fprintf(fp, "\"errors\": %.2f", conflicts_diff); } @@ -2213,10 +2213,10 @@ void print_report(mako_args_t* args, break; } } - printf("Total Xacts: %8lld\n", totalxacts); - printf("Total Conflicts: %8lld\n", conflicts); - printf("Total Errors: %8lld\n", totalerrors); - printf("Overall TPS: %8lld\n\n", totalxacts * 1000000000 / duration_nsec); + printf("Total Xacts: %8lu\n", totalxacts); + printf("Total Conflicts: %8lu\n", conflicts); + printf("Total Errors: %8lu\n", totalerrors); + printf("Overall TPS: %8lu\n\n", totalxacts * 1000000000 / duration_nsec); if (fp) { fprintf(fp, "\"results\": {"); @@ -2224,10 +2224,10 @@ void print_report(mako_args_t* args, fprintf(fp, "\"totalProcesses\": %d,", args->num_processes); fprintf(fp, "\"totalThreads\": %d,", args->num_threads); fprintf(fp, "\"targetTPS\": %d,", args->tpsmax); - fprintf(fp, "\"totalXacts\": %lld,", totalxacts); - fprintf(fp, "\"totalConflicts\": %lld,", conflicts); - fprintf(fp, "\"totalErrors\": %lld,", totalerrors); - fprintf(fp, "\"overallTPS\": %lld,", totalxacts * 1000000000 / duration_nsec); + fprintf(fp, "\"totalXacts\": %lu,", totalxacts); + fprintf(fp, "\"totalConflicts\": %lu,", conflicts); + fprintf(fp, "\"totalErrors\": %lu,", totalerrors); + fprintf(fp, "\"overallTPS\": %lu,", totalxacts * 1000000000 / duration_nsec); } /* per-op stats */ @@ -2240,9 +2240,9 @@ void print_report(mako_args_t* args, } for (op = 0; op < MAX_OP; op++) { if ((args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", ops_total[op]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), ops_total[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), ops_total[op]); } } } @@ -2263,9 +2263,9 @@ void print_report(mako_args_t* args, printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors"); for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", errors_total[op]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), errors_total[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), errors_total[op]); } } } @@ -2282,12 +2282,12 @@ void print_report(mako_args_t* args, for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) { if (lat_total[op]) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_samples[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_samples[op]); } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); } if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_samples[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), lat_samples[op]); } } } @@ -2303,9 +2303,9 @@ void print_report(mako_args_t* args, if (lat_min[op] == -1) { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_min[op]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_min[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), lat_min[op]); } } } @@ -2320,9 +2320,9 @@ void print_report(mako_args_t* args, for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) { if (lat_total[op]) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_total[op] / lat_samples[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_total[op] / lat_samples[op]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_total[op] / lat_samples[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), lat_total[op] / lat_samples[op]); } } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); @@ -2341,9 +2341,9 @@ void print_report(mako_args_t* args, if (lat_max[op] == 0) { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", lat_max[op]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_max[op]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), lat_max[op]); } } } @@ -2393,9 +2393,9 @@ void print_report(mako_args_t* args, } else { median = (dataPoints[op][num_points[op] / 2] + dataPoints[op][num_points[op] / 2 - 1]) >> 1; } - printf("%" STR(STATS_FIELD_WIDTH) "lld ", median); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", median); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), median); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), median); } } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); @@ -2417,9 +2417,9 @@ void print_report(mako_args_t* args, } if (lat_total[op]) { point_95pct = ((float)(num_points[op]) * 0.95) - 1; - printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_95pct]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_95pct]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_95pct]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), dataPoints[op][point_95pct]); } } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); @@ -2441,9 +2441,9 @@ void print_report(mako_args_t* args, } if (lat_total[op]) { point_99pct = ((float)(num_points[op]) * 0.99) - 1; - printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_99pct]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_99pct]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_99pct]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), dataPoints[op][point_99pct]); } } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); @@ -2465,9 +2465,9 @@ void print_report(mako_args_t* args, } if (lat_total[op]) { point_99_9pct = ((float)(num_points[op]) * 0.999) - 1; - printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_99_9pct]); + printf("%" STR(STATS_FIELD_WIDTH) "lu ", dataPoints[op][point_99_9pct]); if (fp) { - fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_99_9pct]); + fprintf(fp, "\"%s\": %lu,", get_ops_name(op), dataPoints[op][point_99_9pct]); } } else { printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); @@ -2529,7 +2529,7 @@ int stats_process_main(mako_args_t* args, fprintf(fp, "\"value_length\": %d,", args->value_length); fprintf(fp, "\"commit_get\": %d,", args->commit_get); fprintf(fp, "\"verbose\": %d,", args->verbose); - fprintf(fp, "\"cluster_files\": \"%s\",", args->cluster_files); + fprintf(fp, "\"cluster_files\": \"%s\",", args->cluster_files[0]); fprintf(fp, "\"log_group\": \"%s\",", args->log_group); fprintf(fp, "\"prefixpadding\": %d,", args->prefixpadding); fprintf(fp, "\"trace\": %d,", args->trace); diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index f1c90cd720..b8be90ceb1 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -67,25 +67,25 @@ void runTests(struct ResultSet* rs) { fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); - printf("size %d: %u\n", i, sizes[i]); + printf("size %d: %ld\n", i, sizes[i]); i++; fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); - printf("size %d: %u\n", i, sizes[i]); + printf("size %d: %ld\n", i, sizes[i]); i++; fdb_transaction_clear(tr, keys[i], KEY_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); - printf("size %d: %u\n", i, sizes[i]); + printf("size %d: %ld\n", i, sizes[i]); i++; fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); - printf("size %d: %u\n", i, sizes[i]); + printf("size %d: %ld\n", i, sizes[i]); i++; for (j = 0; j + 1 < i; j++) { diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 6379f7bf14..dccfbcc7ee 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -284,7 +284,6 @@ else() # Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 11 -Wno-comment -Wno-delete-non-virtual-dtor - -Wno-format -Wno-mismatched-tags -Wno-missing-field-initializers -Wno-sign-compare diff --git a/fdbcli/ChangeFeedCommand.actor.cpp b/fdbcli/ChangeFeedCommand.actor.cpp index d28d96c367..b5e7a79ff1 100644 --- a/fdbcli/ChangeFeedCommand.actor.cpp +++ b/fdbcli/ChangeFeedCommand.actor.cpp @@ -127,7 +127,7 @@ ACTOR Future changeFeedCommandActor(Database localDb, std::vector> res = waitNext(feedResults.getFuture())) { for (auto& it : res) { for (auto& it2 : it.mutations) { - printf("%lld %s\n", it.version, it2.toString().c_str()); + printf("%ld %s\n", it.version, it2.toString().c_str()); } } } diff --git a/fdbcli/SetClassCommand.actor.cpp b/fdbcli/SetClassCommand.actor.cpp index ee3ebfe454..bec2103287 100644 --- a/fdbcli/SetClassCommand.actor.cpp +++ b/fdbcli/SetClassCommand.actor.cpp @@ -48,7 +48,7 @@ ACTOR Future printProcessClass(Reference db) { ASSERT(processSourceList.size() == processTypeList.size()); if (!processTypeList.size()) printf("No processes are registered in the database.\n"); - printf("There are currently %zu processes in the database:\n", processTypeList.size()); + printf("There are currently %d processes in the database:\n", processTypeList.size()); for (int index = 0; index < processTypeList.size(); index++) { std::string address = processTypeList[index].key.removePrefix(fdb_cli::processClassTypeSpecialKeyRange.begin).toString(); diff --git a/fdbclient/BlobGranuleReader.actor.cpp b/fdbclient/BlobGranuleReader.actor.cpp index 9211124ad5..0c0ac42edf 100644 --- a/fdbclient/BlobGranuleReader.actor.cpp +++ b/fdbclient/BlobGranuleReader.actor.cpp @@ -97,7 +97,7 @@ ACTOR Future readSnapshotFile(Reference bstore } }*/ if (BG_READ_DEBUG) { - printf("Started with %d rows from snapshot file %s after pruning to [%s - %s)\n", + printf("Started with %lu rows from snapshot file %s after pruning to [%s - %s)\n", dataMap->size(), f.toString().c_str(), keyRange.begin.printable().c_str(), @@ -143,7 +143,7 @@ ACTOR Future> readDeltaFile(Reference result[i + 1].version) { - printf("BG VERSION ORDER VIOLATION IN DELTA FILE: '%lld', '%lld'\n", + printf("BG VERSION ORDER VIOLATION IN DELTA FILE: '%ld', '%ld'\n", result[i].version, result[i + 1].version); } @@ -313,7 +313,7 @@ ACTOR Future readBlobGranule(BlobGranuleChunkRef chunk, arena.dependsOn(snapshotArena); if (BG_READ_DEBUG) { - printf("Applying %d delta files\n", readDeltaFutures.size()); + printf("Applying %lu delta files\n", readDeltaFutures.size()); } for (Future> deltaFuture : readDeltaFutures) { Standalone result = wait(deltaFuture); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 117524a43a..badb03415d 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -7201,7 +7201,7 @@ ACTOR Future readBlobGranulesStreamActor(Reference db, blobGranuleMapping = _bgMapping; if (blobGranuleMapping.more) { if (BG_REQUEST_DEBUG) { - printf("BG Mapping for [%s - %s) too large!\n"); + // printf("BG Mapping for [%s - %s) too large!\n"); } throw unsupported_operation(); } @@ -7215,7 +7215,7 @@ ACTOR Future readBlobGranulesStreamActor(Reference db, } if (BG_REQUEST_DEBUG) { - printf("Doing blob granule request @ %lld\n", endVersion); + printf("Doing blob granule request @ %ld\n", endVersion); printf("blob worker assignments:\n"); } @@ -7290,7 +7290,7 @@ ACTOR Future readBlobGranulesStreamActor(Reference db, nullptr)); if (BG_REQUEST_DEBUG) { - printf("Blob granule request for [%s - %s) @ %lld - %lld got reply from %s:\n", + printf("Blob granule request for [%s - %s) @ %ld - %ld got reply from %s:\n", granuleStartKey.printable().c_str(), granuleEndKey.printable().c_str(), begin, @@ -7311,11 +7311,11 @@ ACTOR Future readBlobGranulesStreamActor(Reference db, } printf(" Deltas: (%d)", chunk.newDeltas.size()); if (chunk.newDeltas.size() > 0) { - printf(" with version [%lld - %lld]", + printf(" with version [%ld - %ld]", chunk.newDeltas[0].version, chunk.newDeltas[chunk.newDeltas.size() - 1].version); } - printf(" IncludedVersion: %lld\n", chunk.includedVersion); + printf(" IncludedVersion: %ld\n", chunk.includedVersion); printf("\n\n"); } Arena a; diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index 557352f11c..a312f248c9 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -235,7 +235,7 @@ ACTOR Future>> splitRange(ReferencegetTransaction().getStorageMetrics(range, CLIENT_KNOBS->TOO_MANY)); if (BM_DEBUG) { - printf("Estimated bytes for [%s - %s): %lld\n", + printf("Estimated bytes for [%s - %s): %ld\n", range.begin.printable().c_str(), range.end.printable().c_str(), estimated.bytes); @@ -300,7 +300,7 @@ static UID pickWorkerForAssign(BlobManagerData* bmData) { ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment assignment, UID workerID, int64_t seqNo) { if (BM_DEBUG) { - printf("BM %s %s range [%s - %s) @ (%lld, %lld)\n", + printf("BM %s %s range [%s - %s) @ (%ld, %ld)\n", bmData->id.toString().c_str(), assignment.isAssign ? "assigning" : "revoking", assignment.keyRange.begin.printable().c_str(), @@ -379,7 +379,7 @@ ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment as // FIXME: improvement would be to add history of failed workers to assignment so it can try other ones first } else { if (BM_DEBUG) { - printf("BM got error revoking range [%s - %s) from worker %s", + printf("BM got error revoking range [%s - %s) from worker", assignment.keyRange.begin.printable().c_str(), assignment.keyRange.end.printable().c_str()); } @@ -472,7 +472,7 @@ ACTOR Future checkManagerLock(Reference tr, Blo ASSERT(currentEpoch > bmData->epoch); if (BM_DEBUG) { - printf("BM %s found new epoch %d > %d in lock check\n", + printf("BM %s found new epoch %ld > %ld in lock check\n", bmData->id.toString().c_str(), currentEpoch, bmData->epoch); @@ -625,7 +625,7 @@ ACTOR Future maybeSplitRange(BlobManagerData* bmData, std::tuple prevGranuleLock = decodeBlobGranuleLockValue(lockValue.get()); if (std::get<0>(prevGranuleLock) > bmData->epoch) { if (BM_DEBUG) { - printf("BM %s found a higher epoch %d than %d for granule lock of [%s - %s)\n", + printf("BM %s found a higher epoch %ld than %ld for granule lock of [%s - %s)\n", bmData->id.toString().c_str(), std::get<0>(prevGranuleLock), bmData->epoch, @@ -770,7 +770,7 @@ ACTOR Future monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerIn GranuleStatusReply rep = waitNext(statusStream.getFuture()); if (BM_DEBUG) { - printf("BM %lld got status of [%s - %s) @ (%lld, %lld) from BW %s: %s\n", + printf("BM %ld got status of [%s - %s) @ (%ld, %ld) from BW %s: %s\n", bmData->epoch, rep.granuleRange.begin.printable().c_str(), rep.granuleRange.end.printable().c_str(), @@ -806,14 +806,14 @@ ACTOR Future monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerIn rep.granuleRange.end == lastReqForGranule.end() && rep.epoch == lastReqForGranule.value().first && rep.seqno == lastReqForGranule.value().second) { if (BM_DEBUG) { - printf("Manager %lld received repeat status for the same granule [%s - %s) @ %lld, ignoring.", + printf("Manager %ld received repeat status for the same granule [%s - %s), ignoring.", bmData->epoch, rep.granuleRange.begin.printable().c_str(), rep.granuleRange.end.printable().c_str()); } } else { if (BM_DEBUG) { - printf("Manager %lld evaluating [%s - %s) for split\n", + printf("Manager %ld evaluating [%s - %s) for split\n", bmData->epoch, rep.granuleRange.begin.printable().c_str(), rep.granuleRange.end.printable().c_str()); @@ -858,7 +858,7 @@ ACTOR Future monitorBlobWorker(BlobManagerData* bmData, BlobWorkerInterfac choose { when(wait(waitFailure)) { if (BM_DEBUG) { - printf("BM %lld detected BW %s is dead\n", bmData->epoch, bwInterf.id().toString().c_str()); + printf("BM %ld detected BW %s is dead\n", bmData->epoch, bwInterf.id().toString().c_str()); } TraceEvent("BlobWorkerFailed", bmData->id).detail("BlobWorkerID", bwInterf.id()); } @@ -1115,7 +1115,7 @@ ACTOR Future blobManager(BlobManagerInterface bmInterf, } if (BM_DEBUG) { - printf("Blob manager acquired lock at epoch %lld\n", epoch); + printf("Blob manager acquired lock at epoch %ld\n", epoch); } // needed to pick up changes to dbinfo in case new CC comes along diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index d85d113e99..35cf40241d 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -193,7 +193,7 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { bool managerEpochOk(int64_t epoch) { if (epoch < currentManagerEpoch) { if (BW_DEBUG) { - printf("BW %s got request from old epoch %lld, notifying manager it is out of date\n", + printf("BW %s got request from old epoch %ld, notifying manager it is out of date\n", id.toString().c_str(), epoch); } @@ -202,7 +202,7 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { if (epoch > currentManagerEpoch) { currentManagerEpoch = epoch; if (BW_DEBUG) { - printf("BW %s found new manager epoch %lld\n", id.toString().c_str(), currentManagerEpoch); + printf("BW %s found new manager epoch %ld\n", id.toString().c_str(), currentManagerEpoch); } } @@ -216,7 +216,7 @@ static void acquireGranuleLock(int64_t epoch, int64_t seqno, int64_t prevOwnerEp // returns true if our lock (E, S) >= (Eprev, Sprev) if (epoch < prevOwnerEpoch || (epoch == prevOwnerEpoch && seqno < prevOwnerSeqno)) { if (BW_DEBUG) { - printf("Lock acquire check failed. Proposed (%lld, %lld) < previous (%lld, %lld)\n", + printf("Lock acquire check failed. Proposed (%ld, %ld) < previous (%ld, %ld)\n", epoch, seqno, prevOwnerEpoch, @@ -239,7 +239,7 @@ static void checkGranuleLock(int64_t epoch, int64_t seqno, int64_t ownerEpoch, i // returns true if we still own the lock, false if someone else does if (epoch != ownerEpoch || seqno != ownerSeqno) { if (BW_DEBUG) { - printf("Lock assignment check failed. Expected (%lld, %lld), got (%lld, %lld)\n", + printf("Lock assignment check failed. Expected (%ld, %ld), got (%ld, %ld)\n", epoch, seqno, ownerEpoch, @@ -303,7 +303,7 @@ ACTOR Future readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, } } if (BW_DEBUG) { - printf("Loaded %d snapshot and %d delta files for %s\n", + printf("Loaded %lu snapshot and %lu delta files for %s\n", files->snapshotFiles.size(), files->deltaFiles.size(), granuleID.toString().c_str()); @@ -546,7 +546,7 @@ ACTOR Future writeDeltaFile(Reference bwData, wait(tr->commit()); if (BW_DEBUG) { - printf("Granule %s [%s - %s) updated fdb with delta file %s of size %d at version %lld, cv=%lld\n", + printf("Granule %s [%s - %s) updated fdb with delta file %s of size %d at version %ld, cv=%ld\n", granuleID.toString().c_str(), keyRange.begin.printable().c_str(), keyRange.end.printable().c_str(), @@ -812,7 +812,7 @@ ACTOR Future compactFromBlob(Reference bwData, chunk.includedVersion = version; if (BW_DEBUG) { - printf("Re-snapshotting [%s - %s) @ %lld from blob\n", + printf("Re-snapshotting [%s - %s) @ %ld from blob\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), version); @@ -911,7 +911,7 @@ ACTOR Future handleCompletedDeltaFile(Reference bwData, if (completedDeltaFile.version > cfStartVersion) { if (BW_DEBUG) { - printf("Popping change feed %s at %lld\n", cfKey.printable().c_str(), completedDeltaFile.version); + printf("Popping change feed %s at %ld\n", cfKey.printable().c_str(), completedDeltaFile.version); } // FIXME: for a write-hot shard, we could potentially batch these and only pop the largest one after several // have completed @@ -968,7 +968,7 @@ static Version doGranuleRollback(Reference metadata, metadata->bytesInNewDeltaFiles -= df.bytes; toPop++; if (BW_DEBUG) { - printf("[%s - %s) rollback cancelling delta file @ %lld\n", + printf("[%s - %s) rollback cancelling delta file @ %ld\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), df.version); @@ -1013,7 +1013,7 @@ static Version doGranuleRollback(Reference metadata, } mIdx++; if (BW_DEBUG) { - printf("[%s - %s) rollback discarding %d in-memory mutations, %d mutations and %lld bytes left\n", + printf("[%s - %s) rollback discarding %d in-memory mutations, %d mutations and %ld bytes left\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), metadata->currentDeltas.size() - mIdx, @@ -1030,7 +1030,7 @@ static Version doGranuleRollback(Reference metadata, } if (BW_DEBUG) { - printf("[%s - %s) finishing rollback to %lld\n", + printf("[%s - %s) finishing rollback to %ld\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), cfRollbackVersion); @@ -1093,8 +1093,8 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str()); printf(" CFID: %s\n", startState.granuleID.toString().c_str()); - printf(" CF Start Version: %lld\n", startState.changeFeedStartVersion); - printf(" Previous Durable Version: %lld\n", startState.previousDurableVersion); + printf(" CF Start Version: %ld\n", startState.changeFeedStartVersion); + printf(" Previous Durable Version: %ld\n", startState.previousDurableVersion); printf(" doSnapshot=%s\n", startState.doSnapshot ? "T" : "F"); printf(" Prev CFID: %s\n", startState.parentGranule.present() ? startState.parentGranule.get().second.toString().c_str() : ""); @@ -1259,7 +1259,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, if (metadata->bufferedDeltaBytes >= SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES && deltas.version > lastVersion) { if (BW_DEBUG) { - printf("Granule [%s - %s) flushing delta file after %d bytes @ %lld %lld%s\n", + printf("Granule [%s - %s) flushing delta file after %lu bytes @ %ld %ld%s\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), metadata->bufferedDeltaBytes, @@ -1321,7 +1321,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, if (snapshotEligible && metadata->bytesInNewDeltaFiles >= SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT && !readOldChangeFeed) { if (BW_DEBUG && (inFlightBlobSnapshot.isValid() || !inFlightDeltaFiles.empty())) { - printf("Granule [%s - %s) ready to re-snapshot, waiting for outstanding %d snapshot and %d " + printf("Granule [%s - %s) ready to re-snapshot, waiting for outstanding %d snapshot and %lu " "deltas to " "finish\n", metadata->keyRange.begin.printable().c_str(), @@ -1350,7 +1350,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, inFlightDeltaFiles.clear(); if (BW_DEBUG) { - printf("Granule [%s - %s) checking with BM for re-snapshot after %d bytes\n", + printf("Granule [%s - %s) checking with BM for re-snapshot after %lu bytes\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), metadata->bytesInNewDeltaFiles); @@ -1400,7 +1400,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, } if (BW_DEBUG) { - printf("Granule [%s - %s) re-snapshotting after %d bytes\n", + printf("Granule [%s - %s) re-snapshotting after %lu bytes\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), metadata->bytesInNewDeltaFiles); @@ -1467,7 +1467,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, if (!rollbacksInProgress.empty()) { ASSERT(rollbacksInProgress.front().first == rollbackVersion); ASSERT(rollbacksInProgress.front().second == deltas.version); - printf("Passed rollback %lld -> %lld\n", deltas.version, rollbackVersion); + printf("Passed rollback %ld -> %ld\n", deltas.version, rollbackVersion); rollbacksCompleted.push_back(rollbacksInProgress.front()); rollbacksInProgress.pop_front(); } else { @@ -1479,13 +1479,13 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, metadata->currentDeltas.back().version <= rollbackVersion)) { if (BW_DEBUG) { - printf("BW skipping rollback %lld -> %lld completely\n", + printf("BW skipping rollback %ld -> %ld completely\n", deltas.version, rollbackVersion); } } else { if (BW_DEBUG) { - printf("BW [%s - %s) ROLLBACK @ %lld -> %lld\n", + printf("BW [%s - %s) ROLLBACK @ %ld -> %ld\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), deltas.version, @@ -1527,7 +1527,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, } else if (!rollbacksInProgress.empty() && rollbacksInProgress.front().first < deltas.version && rollbacksInProgress.front().second > deltas.version) { if (BW_DEBUG) { - printf("Skipping mutations @ %lld b/c prior rollback\n", deltas.version); + printf("Skipping mutations @ %ld b/c prior rollback\n", deltas.version); } } else { for (auto& delta : deltas.mutations) { @@ -1555,7 +1555,7 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, ASSERT(startState.parentGranule.present()); oldChangeFeedDataComplete = startState.parentGranule.get(); if (BW_DEBUG) { - printf("Granule [%s - %s) switching to new change feed %s @ %lld\n", + printf("Granule [%s - %s) switching to new change feed %s @ %ld\n", metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), startState.granuleID.toString().c_str(), @@ -1676,7 +1676,7 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, } if (BW_DEBUG) { - printf("Loaded %d history entries for granule [%s - %s) (%d skipped)\n", + printf("Loaded %lu history entries for granule [%s - %s) (%d skipped)\n", historyEntryStack.size(), metadata->keyRange.begin.printable().c_str(), metadata->keyRange.end.printable().c_str(), @@ -1855,7 +1855,7 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData } if (BW_REQUEST_DEBUG) { - printf("[%s - %s) @ %lld time traveled back to %s [%s - %s) @ [%lld - %lld)\n", + printf("[%s - %s) @ %ld time traveled back to %s [%s - %s) @ [%ld - %ld)\n", req.keyRange.begin.printable().c_str(), req.keyRange.end.printable().c_str(), req.readVersion, @@ -1894,7 +1894,7 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData if (rollbackCount == metadata->rollbackCount.get()) { break; } else if (BW_REQUEST_DEBUG) { - printf("[%s - %s) @ %lld hit rollback, restarting waitForVersion\n", + printf("[%s - %s) @ %ld hit rollback, restarting waitForVersion\n", req.keyRange.begin.printable().c_str(), req.keyRange.end.printable().c_str(), req.readVersion); @@ -2222,7 +2222,7 @@ ACTOR Future changeBlobRange(Reference bwData, bool disposeOnCleanup, bool selfReassign) { if (BW_DEBUG) { - printf("%s range for [%s - %s): %s @ (%lld, %lld)\n", + printf("%s range for [%s - %s): %s @ (%ld, %ld)\n", selfReassign ? "Re-assigning" : "Changing", keyRange.begin.printable().c_str(), keyRange.end.printable().c_str(), @@ -2273,7 +2273,7 @@ ACTOR Future changeBlobRange(Reference bwData, if (r.value().activeMetadata.isValid() && thisAssignmentNewer) { // cancel actors for old range and clear reference if (BW_DEBUG) { - printf(" [%s - %s): @ (%lld, %lld) (cancelling)\n", + printf(" [%s - %s): @ (%ld, %ld) (cancelling)\n", r.begin().printable().c_str(), r.end().printable().c_str(), r.value().lastEpoch, @@ -2298,7 +2298,7 @@ ACTOR Future changeBlobRange(Reference bwData, bwData->granuleMetadata.insert(keyRange, newMetadata); if (BW_DEBUG) { - printf("Inserting new range [%s - %s): %s @ (%lld, %lld)\n", + printf("Inserting new range [%s - %s): %s @ (%ld, %ld)\n", keyRange.begin.printable().c_str(), keyRange.end.printable().c_str(), newMetadata.activeMetadata.isValid() ? "T" : "F", @@ -2308,7 +2308,7 @@ ACTOR Future changeBlobRange(Reference bwData, for (auto& it : newerRanges) { if (BW_DEBUG) { - printf("Re-inserting newer range [%s - %s): %s @ (%lld, %lld)\n", + printf("Re-inserting newer range [%s - %s): %s @ (%ld, %ld)\n", it.first.begin.printable().c_str(), it.first.end.printable().c_str(), it.second.activeMetadata.isValid() ? "T" : "F", @@ -2332,8 +2332,8 @@ static bool resumeBlobRange(Reference bwData, KeyRange keyRange, !existingRange.value().activeMetadata.isValid()) { if (BW_DEBUG) { - printf("BW %s got out of date resume range for [%s - %s) @ (%lld, %lld). Currently [%s - %s) @ (%lld, " - "%lld): %s\n", + printf("BW %s got out of date resume range for [%s - %s) @ (%ld, %ld). Currently [%s - %s) @ (%ld, " + "%ld): %s\n", bwData->id.toString().c_str(), existingRange.begin().printable().c_str(), existingRange.end().printable().c_str(), @@ -2555,7 +2555,7 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, --self->stats.numRangesAssigned; state AssignBlobRangeRequest assignReq = _req; if (BW_DEBUG) { - printf("Worker %s assigned range [%s - %s) @ (%lld, %lld):\n continue=%s\n", + printf("Worker %s assigned range [%s - %s) @ (%ld, %ld):\n continue=%s\n", self->id.toString().c_str(), assignReq.keyRange.begin.printable().c_str(), assignReq.keyRange.end.printable().c_str(), @@ -2574,7 +2574,7 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, state RevokeBlobRangeRequest revokeReq = _req; --self->stats.numRangesAssigned; if (BW_DEBUG) { - printf("Worker %s revoked range [%s - %s) @ (%lld, %lld):\n dispose=%s\n", + printf("Worker %s revoked range [%s - %s) @ (%ld, %ld):\n dispose=%s\n", self->id.toString().c_str(), revokeReq.keyRange.begin.printable().c_str(), revokeReq.keyRange.end.printable().c_str(), diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 9062c20b58..2c812eca2a 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -221,7 +221,9 @@ class TestConfig { } if (attrib == "configureLocked") { - sscanf(value.c_str(), "%d", &configureLocked); + int configureLockedInt; + sscanf(value.c_str(), "%d", &configureLockedInt); + configureLocked = (configureLockedInt != 0); } if (attrib == "startIncompatibleProcess") { @@ -2301,4 +2303,4 @@ ACTOR void setupAndRun(std::string dataFolder, destructed = true; wait(Never()); ASSERT(false); -} \ No newline at end of file +} diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index bec413db23..4462e1a1d5 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -7929,7 +7929,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { ASSERT(RedwoodRecordRef::Delta::LengthFormatSizes[2] == 6); ASSERT(RedwoodRecordRef::Delta::LengthFormatSizes[3] == 8); - printf("sizeof(RedwoodRecordRef) = %d\n", sizeof(RedwoodRecordRef)); + printf("sizeof(RedwoodRecordRef) = %lu\n", sizeof(RedwoodRecordRef)); // Test pageID stuff. { @@ -8862,7 +8862,7 @@ TEST_CASE("/redwood/correctness/unit/deltaTree/IntIntPair") { pos = newPos; } double elapsed = timer() - start; - printf("Seek/skip test, count=%d jumpMax=%d, items=%d, oldSeek=%d useHint=%d: Elapsed %f seconds %.2f M/s\n", + printf("Seek/skip test, count=%d jumpMax=%d, items=%lu, oldSeek=%d useHint=%d: Elapsed %f seconds %.2f M/s\n", count, jumpMax, items.size(), @@ -8905,7 +8905,7 @@ TEST_CASE("/redwood/correctness/unit/deltaTree/IntIntPair") { pos = newPos; } double elapsed = timer() - start; - printf("DeltaTree2 Seek/skip test, count=%d jumpMax=%d, items=%d, oldSeek=%d useHint=%d: Elapsed %f seconds " + printf("DeltaTree2 Seek/skip test, count=%d jumpMax=%d, items=%lu, oldSeek=%d useHint=%d: Elapsed %f seconds " "%.2f M/s\n", count, jumpMax, @@ -8983,7 +8983,7 @@ TEST_CASE(":/redwood/performance/mutationBuffer") { strings.push_back(randomString(arena, 5)); } - printf("Inserting and then finding each string...\n", count); + printf("Inserting %d elements and then finding each string...\n", count); double start = timer(); VersionedBTree::MutationBuffer m; for (int i = 0; i < count; ++i) { @@ -9254,7 +9254,7 @@ TEST_CASE("/redwood/correctness/btree") { commit = map(btree->commit(version), [=, &ops = totalPageOps, v = version](Void) { // Update pager ops before clearing metrics ops += g_redwoodMetrics.pageOps(); - printf("Committed %s PageOps %" PRId64 "/%" PRId64 " (%.2f%%) VerificationMapEntries %d/%d (%.2f%%)\n", + printf("Committed %s PageOps %" PRId64 "/%" PRId64 " (%.2f%%) VerificationMapEntries %lu/%d (%.2f%%)\n", toString(v).c_str(), ops, targetPageOps, @@ -9508,7 +9508,7 @@ TEST_CASE(":/redwood/performance/extentQueue") { for (v = 1; v <= numEntries; ++v) { // Sometimes do a commit if (currentCommitSize >= targetCommitSize) { - printf("currentCommitSize: %d, cumulativeCommitSize: %d, pageCacheCount: %d\n", + printf("currentCommitSize: %d, cumulativeCommitSize: %ld, pageCacheCount: %ld\n", currentCommitSize, cumulativeCommitSize, pager->getPageCacheCount()); @@ -9531,7 +9531,7 @@ TEST_CASE(":/redwood/performance/extentQueue") { } cumulativeCommitSize += currentCommitSize; printf( - "Final cumulativeCommitSize: %d, pageCacheCount: %d\n", cumulativeCommitSize, pager->getPageCacheCount()); + "Final cumulativeCommitSize: %ld, pageCacheCount: %ld\n", cumulativeCommitSize, pager->getPageCacheCount()); wait(m_extentQueue.flush()); extentQueueState = m_extentQueue.getState(); printf("Commit ExtentQueue getState(): %s\n", extentQueueState.toString().c_str()); @@ -9592,7 +9592,7 @@ TEST_CASE(":/redwood/performance/extentQueue") { entriesRead, cumulativeCommitSize / elapsed / 1e6); - printf("pageCacheCount: %d extentCacheCount: %d\n", pager->getPageCacheCount(), pager->getExtentCacheCount()); + printf("pageCacheCount: %ld extentCacheCount: %ld\n", pager->getPageCacheCount(), pager->getExtentCacheCount()); pager->extentCacheClear(); m_extentQueue.resetHeadReader(); @@ -9985,7 +9985,7 @@ ACTOR Future prefixClusteredInsert(IKeyValueStore* kvs, state int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize; state int recordsPerPrefix = recordCountTarget / source.numPrefixes(); - printf("\nstoreType: %d\n", kvs->getType()); + printf("\nstoreType: %d\n", static_cast(kvs->getType())); printf("commitTarget: %d\n", commitTarget); printf("prefixSource: %s\n", source.toString().c_str()); printf("usePrefixesInOrder: %d\n", usePrefixesInOrder); @@ -10074,7 +10074,7 @@ ACTOR Future sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valu state int recordSize = source.prefixLen + sizeof(uint64_t) + valueSize; state int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize; - printf("\nstoreType: %d\n", kvs->getType()); + printf("\nstoreType: %d\n", static_cast(kvs->getType())); printf("commitTarget: %d\n", commitTarget); printf("valueSize: %d\n", valueSize); printf("recordSize: %d\n", recordSize); @@ -10208,7 +10208,7 @@ ACTOR Future randomRangeScans(IKeyValueStore* kvs, int recordCountTarget, bool singlePrefix, int rowLimit) { - printf("\nstoreType: %d\n", kvs->getType()); + printf("\nstoreType: %d\n", static_cast(kvs->getType())); printf("prefixSource: %s\n", source.toString().c_str()); printf("suffixSize: %d\n", suffixSize); printf("recordCountTarget: %d\n", recordCountTarget); @@ -10224,7 +10224,7 @@ ACTOR Future randomRangeScans(IKeyValueStore* kvs, state double start = timer(); state std::function stats = [&]() { double elapsed = timer() - start; - printf("Cumulative stats: %.2f seconds %d queries %.2f MB %d records %.2f qps %.2f MB/s %.2f rec/s\r\n", + printf("Cumulative stats: %.2f seconds %d queries %.2f MB %ld records %.2f qps %.2f MB/s %.2f rec/s\r\n", elapsed, queries, bytesRead / 1e6, diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 1889b9525b..41f01b91c9 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -529,7 +529,7 @@ static void printOptionUsage(std::string option, std::string description) { std::stringstream sstream(description); if (sstream.eof()) { - printf(result.c_str()); + printf("%s", result.c_str()); return; } @@ -552,7 +552,7 @@ static void printOptionUsage(std::string option, std::string description) { } result += currLine + '\n'; - printf(result.c_str()); + printf("%s", result.c_str()); } static void printUsage(const char* name, bool devhelp) { diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp index 9149d6ec8a..654cf617f4 100644 --- a/fdbserver/networktest.actor.cpp +++ b/fdbserver/networktest.actor.cpp @@ -584,7 +584,7 @@ struct P2PNetworkTest { self->startTime = now(); - printf("%d listeners, %d remotes, %d outgoing connections\n", + printf("%lu listeners, %lu remotes, %d outgoing connections\n", self->listeners.size(), self->remotes.size(), self->connectionsOut); diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 8c98b3cf54..f60dac24c4 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -422,7 +422,7 @@ void printSimulatedTopology() { printf("%smachineId: %s\n", indent.c_str(), p->locality.describeMachineId().c_str()); } indent += " "; - printf("%sAddress: %s\n", indent.c_str(), p->address.toString().c_str(), p->name); + printf("%sAddress: %s\n", indent.c_str(), p->address.toString().c_str()); indent += " "; printf("%sClass: %s\n", indent.c_str(), p->startingClass.toString().c_str()); printf("%sName: %s\n", indent.c_str(), p->name); diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index 5b451f3e3e..cb3812074f 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -237,7 +237,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { .detail("BlobSize", blob.first.size()); if (BGV_DEBUG) { - printf("\nMismatch for [%s - %s) @ %lld (%s). F(%d) B(%d):\n", + printf("\nMismatch for [%s - %s) @ %ld (%s). F(%d) B(%d):\n", range.begin.printable().c_str(), range.end.printable().c_str(), v, @@ -291,11 +291,11 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } printf(" Deltas: (%d)", chunk.newDeltas.size()); if (chunk.newDeltas.size() > 0) { - printf(" with version [%lld - %lld]", + printf(" with version [%ld - %ld]", chunk.newDeltas[0].version, chunk.newDeltas[chunk.newDeltas.size() - 1].version); } - printf(" IncludedVersion: %lld\n", chunk.includedVersion); + printf(" IncludedVersion: %ld\n", chunk.includedVersion); } printf("\n"); } @@ -416,7 +416,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { state KeyRange r = range; state PromiseStream> chunkStream; if (BGV_DEBUG) { - printf("Final availability check [%s - %s) @ %lld\n", + printf("Final availability check [%s - %s) @ %ld\n", r.begin.printable().c_str(), r.end.printable().c_str(), readVersion); @@ -435,7 +435,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { break; } if (BGV_DEBUG) { - printf("BG Verifier failed final availability check for [%s - %s) @ %lld with error %s. Last " + printf("BG Verifier failed final availability check for [%s - %s) @ %ld with error %s. Last " "Success=[%s - %s)\n", r.begin.printable().c_str(), r.end.printable().c_str(), @@ -452,13 +452,13 @@ struct BlobGranuleVerifierWorkload : TestWorkload { printf("Blob Granule Verifier finished with:\n"); printf(" %d successful final granule checks\n", checks); printf(" %d failed final granule checks\n", availabilityPassed ? 0 : 1); - printf(" %lld mismatches\n", self->mismatches); - printf(" %lld time travel too old\n", self->timeTravelTooOld); - printf(" %lld errors\n", self->errors); - printf(" %lld initial reads\n", self->initialReads); - printf(" %lld time travel reads\n", self->timeTravelReads); - printf(" %lld rows\n", self->rowsRead); - printf(" %lld bytes\n", self->bytesRead); + printf(" %ld mismatches\n", self->mismatches); + printf(" %ld time travel too old\n", self->timeTravelTooOld); + printf(" %ld errors\n", self->errors); + printf(" %ld initial reads\n", self->initialReads); + printf(" %ld time travel reads\n", self->timeTravelReads); + printf(" %ld rows\n", self->rowsRead); + printf(" %ld bytes\n", self->bytesRead); // FIXME: add above as details TraceEvent("BlobGranuleVerifierChecked"); return availabilityPassed && self->mismatches == 0 && checks > 0 && self->timeTravelTooOld == 0; diff --git a/fdbserver/workloads/RyowCorrectness.actor.cpp b/fdbserver/workloads/RyowCorrectness.actor.cpp index 2d905b230c..eb68fea0c1 100644 --- a/fdbserver/workloads/RyowCorrectness.actor.cpp +++ b/fdbserver/workloads/RyowCorrectness.actor.cpp @@ -299,14 +299,14 @@ struct RyowCorrectnessWorkload : ApiWorkload { printable(op.beginKey).c_str(), printable(op.endKey).c_str(), op.limit, - op.reverse); + static_cast(op.reverse)); break; case Operation::GET_RANGE_SELECTOR: printf("Operation GET_RANGE_SELECTOR failed: begin = %s, end = %s, limit = %d, reverse = %d\n", op.beginSelector.toString().c_str(), op.endSelector.toString().c_str(), op.limit, - op.reverse); + static_cast(op.reverse)); break; case Operation::GET_KEY: printf("Operation GET_KEY failed: selector = %s\n", op.beginSelector.toString().c_str()); From 78e36e75902a904d2cf3fb28e90f65f3e146d4ea Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 29 Oct 2021 11:18:47 -0700 Subject: [PATCH 040/338] fix: simulation only validation could throw errors which would impact the behavior of the cluster controller --- fdbserver/ClusterController.actor.cpp | 245 ++++++++++++++------------ 1 file changed, 129 insertions(+), 116 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 1456319a3f..e529b023c2 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1214,40 +1214,44 @@ public: exclusionWorkerIds); if (g_network->isSimulated()) { - auto testWorkers = getWorkersForTlogsBackup( - conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); - RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); - RoleFitness fitness(workers, ProcessClass::TLog, id_used); + try { + auto testWorkers = getWorkersForTlogsBackup( + conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); + RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); + RoleFitness fitness(workers, ProcessClass::TLog, id_used); - std::map>, int> field_count; - std::set>> zones; - for (auto& worker : testWorkers) { - if (!zones.count(worker.interf.locality.zoneId())) { - field_count[worker.interf.locality.get(pa1->attributeKey())]++; - zones.insert(worker.interf.locality.zoneId()); + std::map>, int> field_count; + std::set>> zones; + for (auto& worker : testWorkers) { + if (!zones.count(worker.interf.locality.zoneId())) { + field_count[worker.interf.locality.get(pa1->attributeKey())]++; + zones.insert(worker.interf.locality.zoneId()); + } } - } - // backup recruitment is not required to use degraded processes that have better fitness - // so we cannot compare degraded between the two methods - testFitness.degraded = fitness.degraded; + // backup recruitment is not required to use degraded processes that have better fitness + // so we cannot compare degraded between the two methods + testFitness.degraded = fitness.degraded; - int minField = 100; + int minField = 100; - for (auto& f : field_count) { - minField = std::min(minField, f.second); - } - - if (fitness > testFitness && minField > 1) { - for (auto& w : testWorkers) { - TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + for (auto& f : field_count) { + minField = std::min(minField, f.second); } - for (auto& w : workers) { - TraceEvent("RealTLogs").detail("Interf", w.interf.address()); + + if (fitness > testFitness && minField > 1) { + for (auto& w : testWorkers) { + TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + } + for (auto& w : workers) { + TraceEvent("RealTLogs").detail("Interf", w.interf.address()); + } + TraceEvent("FitnessCompare") + .detail("TestF", testFitness.toString()) + .detail("RealF", fitness.toString()); + ASSERT(false); } - TraceEvent("FitnessCompare") - .detail("TestF", testFitness.toString()) - .detail("RealF", fitness.toString()); - ASSERT(false); + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors } } @@ -1267,25 +1271,29 @@ public: getWorkersForTlogsSimple(conf, required, desired, id_used, checkStable, dcIds, exclusionWorkerIds); if (g_network->isSimulated()) { - auto testWorkers = getWorkersForTlogsBackup( - conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); - RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); - RoleFitness fitness(workers, ProcessClass::TLog, id_used); - // backup recruitment is not required to use degraded processes that have better fitness - // so we cannot compare degraded between the two methods - testFitness.degraded = fitness.degraded; + try { + auto testWorkers = getWorkersForTlogsBackup( + conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); + RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); + RoleFitness fitness(workers, ProcessClass::TLog, id_used); + // backup recruitment is not required to use degraded processes that have better fitness + // so we cannot compare degraded between the two methods + testFitness.degraded = fitness.degraded; - if (fitness > testFitness) { - for (auto& w : testWorkers) { - TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + if (fitness > testFitness) { + for (auto& w : testWorkers) { + TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + } + for (auto& w : workers) { + TraceEvent("RealTLogs").detail("Interf", w.interf.address()); + } + TraceEvent("FitnessCompare") + .detail("TestF", testFitness.toString()) + .detail("RealF", fitness.toString()); + ASSERT(false); } - for (auto& w : workers) { - TraceEvent("RealTLogs").detail("Interf", w.interf.address()); - } - TraceEvent("FitnessCompare") - .detail("TestF", testFitness.toString()) - .detail("RealF", fitness.toString()); - ASSERT(false); + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors } } return workers; @@ -2119,82 +2127,87 @@ public: RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req); if (g_network->isSimulated()) { - // FIXME: The logic to pick a satellite in a remote region is not - // deterministic and can therefore break this nondeterminism check. - // Since satellites will generally be in the primary region, - // disable the determinism check for remote region satellites. - bool remoteDCUsedAsSatellite = false; - if (req.configuration.regions.size() > 1) { - auto [region, remoteRegion] = - getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId); - for (const auto& satellite : region.satellites) { - if (satellite.dcId == remoteRegion.dcId) { - remoteDCUsedAsSatellite = true; + try { + // FIXME: The logic to pick a satellite in a remote region is not + // deterministic and can therefore break this nondeterminism check. + // Since satellites will generally be in the primary region, + // disable the determinism check for remote region satellites. + bool remoteDCUsedAsSatellite = false; + if (req.configuration.regions.size() > 1) { + auto [region, remoteRegion] = + getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId); + for (const auto& satellite : region.satellites) { + if (satellite.dcId == remoteRegion.dcId) { + remoteDCUsedAsSatellite = true; + } } } - } - if (!remoteDCUsedAsSatellite) { - RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req); + if (!remoteDCUsedAsSatellite) { + RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req); - std::map>, int> firstUsed; - std::map>, int> secondUsed; - updateKnownIds(&firstUsed); - updateKnownIds(&secondUsed); + std::map>, int> firstUsed; + std::map>, int> secondUsed; + updateKnownIds(&firstUsed); + updateKnownIds(&secondUsed); - // auto mworker = id_worker.find(masterProcessId); - //TraceEvent("CompareAddressesMaster") - // .detail("Master", - // mworker != id_worker.end() ? mworker->second.details.interf.address() : NetworkAddress()); + // auto mworker = id_worker.find(masterProcessId); + //TraceEvent("CompareAddressesMaster") + // .detail("Master", + // mworker != id_worker.end() ? mworker->second.details.interf.address() : + // NetworkAddress()); - updateIdUsed(rep.tLogs, firstUsed); - updateIdUsed(compare.tLogs, secondUsed); - compareWorkers( - req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); - updateIdUsed(rep.satelliteTLogs, firstUsed); - updateIdUsed(compare.satelliteTLogs, secondUsed); - compareWorkers(req.configuration, - rep.satelliteTLogs, - firstUsed, - compare.satelliteTLogs, - secondUsed, - ProcessClass::TLog, - "Satellite"); - updateIdUsed(rep.commitProxies, firstUsed); - updateIdUsed(compare.commitProxies, secondUsed); - updateIdUsed(rep.grvProxies, firstUsed); - updateIdUsed(compare.grvProxies, secondUsed); - updateIdUsed(rep.resolvers, firstUsed); - updateIdUsed(compare.resolvers, secondUsed); - compareWorkers(req.configuration, - rep.commitProxies, - firstUsed, - compare.commitProxies, - secondUsed, - ProcessClass::CommitProxy, - "CommitProxy"); - compareWorkers(req.configuration, - rep.grvProxies, - firstUsed, - compare.grvProxies, - secondUsed, - ProcessClass::GrvProxy, - "GrvProxy"); - compareWorkers(req.configuration, - rep.resolvers, - firstUsed, - compare.resolvers, - secondUsed, - ProcessClass::Resolver, - "Resolver"); - updateIdUsed(rep.backupWorkers, firstUsed); - updateIdUsed(compare.backupWorkers, secondUsed); - compareWorkers(req.configuration, - rep.backupWorkers, - firstUsed, - compare.backupWorkers, - secondUsed, - ProcessClass::Backup, - "Backup"); + updateIdUsed(rep.tLogs, firstUsed); + updateIdUsed(compare.tLogs, secondUsed); + compareWorkers( + req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); + updateIdUsed(rep.satelliteTLogs, firstUsed); + updateIdUsed(compare.satelliteTLogs, secondUsed); + compareWorkers(req.configuration, + rep.satelliteTLogs, + firstUsed, + compare.satelliteTLogs, + secondUsed, + ProcessClass::TLog, + "Satellite"); + updateIdUsed(rep.commitProxies, firstUsed); + updateIdUsed(compare.commitProxies, secondUsed); + updateIdUsed(rep.grvProxies, firstUsed); + updateIdUsed(compare.grvProxies, secondUsed); + updateIdUsed(rep.resolvers, firstUsed); + updateIdUsed(compare.resolvers, secondUsed); + compareWorkers(req.configuration, + rep.commitProxies, + firstUsed, + compare.commitProxies, + secondUsed, + ProcessClass::CommitProxy, + "CommitProxy"); + compareWorkers(req.configuration, + rep.grvProxies, + firstUsed, + compare.grvProxies, + secondUsed, + ProcessClass::GrvProxy, + "GrvProxy"); + compareWorkers(req.configuration, + rep.resolvers, + firstUsed, + compare.resolvers, + secondUsed, + ProcessClass::Resolver, + "Resolver"); + updateIdUsed(rep.backupWorkers, firstUsed); + updateIdUsed(compare.backupWorkers, secondUsed); + compareWorkers(req.configuration, + rep.backupWorkers, + firstUsed, + compare.backupWorkers, + secondUsed, + ProcessClass::Backup, + "Backup"); + } + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors } } return rep; From ee00135a6b5c26cd3c821b2e38995c92663b8b9e Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 29 Oct 2021 16:42:48 -0700 Subject: [PATCH 041/338] skip good recruitment errors when doing simulation only validation --- fdbserver/ClusterController.actor.cpp | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index e529b023c2..8691e11d73 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1671,7 +1671,8 @@ public: } ErrorOr findWorkersForConfigurationFromDC(RecruitFromConfigurationRequest const& req, - Optional dcId) { + Optional dcId, + bool checkGoodRecruitment) { RecruitFromConfigurationReply result; std::map>, int> id_used; updateKnownIds(&id_used); @@ -1782,7 +1783,7 @@ public: [](const WorkerDetails& w) { return w.interf; }); } - if (!goodRecruitmentTime.isReady() && + if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && (RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1 && @@ -1808,7 +1809,8 @@ public: return result; } - RecruitFromConfigurationReply findWorkersForConfigurationDispatch(RecruitFromConfigurationRequest const& req) { + RecruitFromConfigurationReply findWorkersForConfigurationDispatch(RecruitFromConfigurationRequest const& req, + bool checkGoodRecruitment) { if (req.configuration.regions.size() > 1) { std::vector regions = req.configuration.regions; if (regions[0].priority == regions[1].priority && regions[1].dcId == clusterControllerDcId.get()) { @@ -1845,7 +1847,7 @@ public: bool setPrimaryDesired = false; try { - auto reply = findWorkersForConfigurationFromDC(req, regions[0].dcId); + auto reply = findWorkersForConfigurationFromDC(req, regions[0].dcId, checkGoodRecruitment); setPrimaryDesired = true; std::vector> dcPriority; dcPriority.push_back(regions[0].dcId); @@ -1862,7 +1864,8 @@ public: .detail("RecruitedTxnSystemDcId", regions[0].dcId); throw no_more_servers(); } catch (Error& e) { - if (!goodRemoteRecruitmentTime.isReady() && regions[1].dcId != clusterControllerDcId.get()) { + if (!goodRemoteRecruitmentTime.isReady() && regions[1].dcId != clusterControllerDcId.get() && + checkGoodRecruitment) { throw operation_failed(); } @@ -1872,7 +1875,7 @@ public: TraceEvent(SevWarn, "AttemptingRecruitmentInRemoteDc", id) .detail("SetPrimaryDesired", setPrimaryDesired) .error(e); - auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId); + auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId, checkGoodRecruitment); if (!setPrimaryDesired) { std::vector> dcPriority; dcPriority.push_back(regions[1].dcId); @@ -1890,7 +1893,8 @@ public: std::vector> dcPriority; dcPriority.push_back(req.configuration.regions[0].dcId); desiredDcIds.set(dcPriority); - auto reply = findWorkersForConfigurationFromDC(req, req.configuration.regions[0].dcId); + auto reply = + findWorkersForConfigurationFromDC(req, req.configuration.regions[0].dcId, checkGoodRecruitment); if (reply.isError()) { throw reply.getError(); } else if (req.configuration.regions[0].dcId == clusterControllerDcId.get()) { @@ -2059,7 +2063,7 @@ public: .detail("DesiredResolvers", req.configuration.getDesiredResolvers()) .detail("ActualResolvers", result.resolvers.size()); - if (!goodRecruitmentTime.isReady() && + if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && (RoleFitness( SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || @@ -2125,7 +2129,7 @@ public: } RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { - RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req); + RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req, true); if (g_network->isSimulated()) { try { // FIXME: The logic to pick a satellite in a remote region is not @@ -2143,7 +2147,7 @@ public: } } if (!remoteDCUsedAsSatellite) { - RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req); + RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req, false); std::map>, int> firstUsed; std::map>, int> secondUsed; From b0cec2984946b44251ab4c08423a89b6f35bf7ac Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 11:53:53 -0700 Subject: [PATCH 042/338] Enable unused-local-typedef clang warning --- cmake/ConfigureCompiler.cmake | 1 - fdbclient/json_spirit/json_spirit_writer_template.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 6379f7bf14..d49a2befe4 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -293,7 +293,6 @@ else() -Wno-unknown-pragmas -Wno-unknown-warning-option -Wno-unused-function - -Wno-unused-local-typedef -Wno-unused-parameter ) if (USE_CCACHE) diff --git a/fdbclient/json_spirit/json_spirit_writer_template.h b/fdbclient/json_spirit/json_spirit_writer_template.h index 1422ee272c..0bf3f5d2e5 100644 --- a/fdbclient/json_spirit/json_spirit_writer_template.h +++ b/fdbclient/json_spirit/json_spirit_writer_template.h @@ -32,8 +32,6 @@ inline char to_hex_char(unsigned int c) { template String_type non_printable_to_string(unsigned int c) { - typedef typename String_type::value_type Char_type; - String_type result(6, '\\'); result[1] = 'u'; From 25257f6f87155644c8a20a50db3a525313b4b891 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 12:42:24 -0700 Subject: [PATCH 043/338] Enable unused-function warning for clang --- cmake/ConfigureCompiler.cmake | 1 - fdbclient/ClientLibManagement.actor.cpp | 4 +- fdbclient/DatabaseConfiguration.cpp | 3 - fdbclient/FDBTypes.cpp | 8 ++ fdbclient/FDBTypes.h | 8 +- fdbserver/DeltaTree.h | 4 +- fdbserver/OldTLogServer_6_2.actor.cpp | 2 +- fdbserver/RestoreController.actor.cpp | 126 +++++++++++++----------- fdbserver/SimulatedCluster.actor.cpp | 6 +- flow/Tracing.actor.cpp | 28 +++--- flow/crc32c.cpp | 4 +- 11 files changed, 101 insertions(+), 93 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index d49a2befe4..400cead811 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -292,7 +292,6 @@ else() -Wno-undefined-var-template -Wno-unknown-pragmas -Wno-unknown-warning-option - -Wno-unused-function -Wno-unused-parameter ) if (USE_CCACHE) diff --git a/fdbclient/ClientLibManagement.actor.cpp b/fdbclient/ClientLibManagement.actor.cpp index 8b24956ee3..9ca0571e32 100644 --- a/fdbclient/ClientLibManagement.actor.cpp +++ b/fdbclient/ClientLibManagement.actor.cpp @@ -198,7 +198,7 @@ KeyRef chunkKeyFromNo(StringRef clientLibBinPrefix, size_t chunkNo, Arena& arena return clientLibBinPrefix.withSuffix(format("%06zu", chunkNo), arena); } -ClientLibPlatform getCurrentClientPlatform() { +[[maybe_unused]] ClientLibPlatform getCurrentClientPlatform() { #ifdef __x86_64__ #if defined(_WIN32) return ClientLibPlatform::X86_64_WINDOWS; @@ -707,4 +707,4 @@ ACTOR Future>> listClientLibraries(Database db, return result; } -} // namespace ClientLibManagement \ No newline at end of file +} // namespace ClientLibManagement diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index c2cb04bb2f..d778b35845 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -578,9 +578,6 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { return true; // All of the above options currently require recovery to take effect } -static KeyValueRef* lower_bound(VectorRef& config, KeyRef const& key) { - return std::lower_bound(config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey()); -} static KeyValueRef const* lower_bound(VectorRef const& config, KeyRef const& key) { return std::lower_bound(config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey()); } diff --git a/fdbclient/FDBTypes.cpp b/fdbclient/FDBTypes.cpp index 3639776e2d..8ada7f2c08 100644 --- a/fdbclient/FDBTypes.cpp +++ b/fdbclient/FDBTypes.cpp @@ -65,3 +65,11 @@ std::string KeySelectorRef::toString() const { return format("%d+lastLessThan(%s)", offset, printable(key).c_str()); } } + +std::string describe(const std::string& s) { + return s; +} + +std::string describe(UID const& item) { + return item.shortString(); +} diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 17ad22b93e..d0eb4c0d0b 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -188,18 +188,14 @@ inline std::string describe(const int item) { } // Allows describeList to work on a vector of std::string -static std::string describe(const std::string& s) { - return s; -} +std::string describe(const std::string& s); template std::string describe(Reference const& item) { return item->toString(); } -static std::string describe(UID const& item) { - return item.shortString(); -} +std::string describe(UID const& item); template std::string describe(T const& item) { diff --git a/fdbserver/DeltaTree.h b/fdbserver/DeltaTree.h index c1219bd71a..2c1f7ea91f 100644 --- a/fdbserver/DeltaTree.h +++ b/fdbserver/DeltaTree.h @@ -94,7 +94,7 @@ static int __lessOrEqualPowerOfTwo(unsigned int n) { } */ -static int perfectSubtreeSplitPoint(int subtree_size) { +static inline int perfectSubtreeSplitPoint(int subtree_size) { // return the inorder index of the root node in a subtree of the given size // consistent with the resulting binary search tree being "perfect" (having minimal height // and all missing nodes as far right as possible). @@ -103,7 +103,7 @@ static int perfectSubtreeSplitPoint(int subtree_size) { return std::min(s * 2 + 1, subtree_size - s - 1); } -static int perfectSubtreeSplitPointCached(int subtree_size) { +static inline int perfectSubtreeSplitPointCached(int subtree_size) { static uint16_t* points = nullptr; static const int max = 500; if (points == nullptr) { diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp index e25b80f881..4893a5da03 100644 --- a/fdbserver/OldTLogServer_6_2.actor.cpp +++ b/fdbserver/OldTLogServer_6_2.actor.cpp @@ -267,7 +267,7 @@ static StringRef stripTagMessagesKey(StringRef key) { return key.substr(sizeof(UID) + sizeof(Tag) + persistTagMessagesKeys.begin.size()); } -static StringRef stripTagMessageRefsKey(StringRef key) { +[[maybe_unused]] static StringRef stripTagMessageRefsKey(StringRef key) { return key.substr(sizeof(UID) + sizeof(Tag) + persistTagMessageRefsKeys.begin.size()); } diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index 860e6b2e56..a9d0444b74 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -37,7 +37,8 @@ #include "flow/Platform.h" #include "flow/actorcompiler.h" // This must be the last #include. -ACTOR static Future clearDB(Database cx); +// TODO: Support [[maybe_unused]] attribute for actors +// ACTOR static Future clearDB(Database cx); ACTOR static Future collectBackupFiles(Reference bc, std::vector* rangeFiles, std::vector* logFiles, @@ -76,7 +77,8 @@ ACTOR static Future notifyLoadersVersionBatchFinished(std::map notifyRestoreCompleted(Reference self, bool terminate); ACTOR static Future signalRestoreCompleted(Reference self, Database cx); -ACTOR static Future updateHeartbeatTime(Reference self); +// TODO: Support [[maybe_unused]] attribute for actors +// ACTOR static Future updateHeartbeatTime(Reference self); ACTOR static Future checkRolesLiveness(Reference self); void splitKeyRangeForAppliers(Reference batchData, @@ -900,16 +902,18 @@ ACTOR static Future buildRangeVersions(KeyRangeMap* pRangeVersion return Void(); } +/* ACTOR static Future clearDB(Database cx) { - wait(runRYWTransaction(cx, [](Reference tr) -> Future { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - tr->clear(normalKeys); - return Void(); - })); + wait(runRYWTransaction(cx, [](Reference tr) -> Future { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->clear(normalKeys); + return Void(); + })); - return Void(); + return Void(); } +*/ ACTOR static Future initializeVersionBatch(std::map appliersInterf, std::map loadersInterf, @@ -1135,67 +1139,69 @@ ACTOR static Future signalRestoreCompleted(Reference updateHeartbeatTime(Reference self) { - wait(self->recruitedRoles.getFuture()); + wait(self->recruitedRoles.getFuture()); - int numRoles = self->loadersInterf.size() + self->appliersInterf.size(); - state std::map::iterator loader = self->loadersInterf.begin(); - state std::map::iterator applier = self->appliersInterf.begin(); - state std::vector> fReplies(numRoles, Never()); // TODO: Reserve memory for this vector - state std::vector nodes; - state int index = 0; - state Future fTimeout = Void(); + int numRoles = self->loadersInterf.size() + self->appliersInterf.size(); + state std::map::iterator loader = self->loadersInterf.begin(); + state std::map::iterator applier = self->appliersInterf.begin(); + state std::vector> fReplies(numRoles, Never()); // TODO: Reserve memory for this vector + state std::vector nodes; + state int index = 0; + state Future fTimeout = Void(); - // Initialize nodes only once - std::transform(self->loadersInterf.begin(), - self->loadersInterf.end(), - std::back_inserter(nodes), - [](const std::pair& in) { return in.first; }); - std::transform(self->appliersInterf.begin(), - self->appliersInterf.end(), - std::back_inserter(nodes), - [](const std::pair& in) { return in.first; }); + // Initialize nodes only once + std::transform(self->loadersInterf.begin(), + self->loadersInterf.end(), + std::back_inserter(nodes), + [](const std::pair& in) { return in.first; }); + std::transform(self->appliersInterf.begin(), + self->appliersInterf.end(), + std::back_inserter(nodes), + [](const std::pair& in) { return in.first; }); - loop { - loader = self->loadersInterf.begin(); - applier = self->appliersInterf.begin(); - index = 0; - std::fill(fReplies.begin(), fReplies.end(), Never()); - // ping loaders and appliers - while (loader != self->loadersInterf.end()) { - fReplies[index] = loader->second.heartbeat.getReply(RestoreSimpleRequest()); - loader++; - index++; - } - while (applier != self->appliersInterf.end()) { - fReplies[index] = applier->second.heartbeat.getReply(RestoreSimpleRequest()); - applier++; - index++; - } + loop { + loader = self->loadersInterf.begin(); + applier = self->appliersInterf.begin(); + index = 0; + std::fill(fReplies.begin(), fReplies.end(), Never()); + // ping loaders and appliers + while (loader != self->loadersInterf.end()) { + fReplies[index] = loader->second.heartbeat.getReply(RestoreSimpleRequest()); + loader++; + index++; + } + while (applier != self->appliersInterf.end()) { + fReplies[index] = applier->second.heartbeat.getReply(RestoreSimpleRequest()); + applier++; + index++; + } - fTimeout = delay(SERVER_KNOBS->FASTRESTORE_HEARTBEAT_DELAY); + fTimeout = delay(SERVER_KNOBS->FASTRESTORE_HEARTBEAT_DELAY); - // Here we have to handle error, otherwise controller worker will fail and exit. - try { - wait(waitForAll(fReplies) || fTimeout); - } catch (Error& e) { - // This should be an ignorable error. - TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "FastRestoreUpdateHeartbeatError").error(e); - } + // Here we have to handle error, otherwise controller worker will fail and exit. + try { + wait(waitForAll(fReplies) || fTimeout); + } catch (Error& e) { + // This should be an ignorable error. + TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "FastRestoreUpdateHeartbeatError").error(e); + } - // Update the most recent heart beat time for each role - for (int i = 0; i < fReplies.size(); ++i) { - if (!fReplies[i].isError() && fReplies[i].isReady()) { - double currentTime = now(); - auto item = self->rolesHeartBeatTime.emplace(nodes[i], currentTime); - item.first->second = currentTime; - } - } - wait(fTimeout); // Ensure not updating heartbeat too quickly - } + // Update the most recent heart beat time for each role + for (int i = 0; i < fReplies.size(); ++i) { + if (!fReplies[i].isError() && fReplies[i].isReady()) { + double currentTime = now(); + auto item = self->rolesHeartBeatTime.emplace(nodes[i], currentTime); + item.first->second = currentTime; + } + } + wait(fTimeout); // Ensure not updating heartbeat too quickly + } } +*/ // Check if a restore role dies or disconnected ACTOR static Future checkRolesLiveness(Reference self) { diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 9062c20b58..62d1f71db9 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -1223,7 +1223,7 @@ void SimulationConfig::set_config(std::string config) { db.set(kv.first, kv.second); } -StringRef StringRefOf(const char* s) { +[[maybe_unused]] StringRef StringRefOf(const char* s) { return StringRef((uint8_t*)s, strlen(s)); } @@ -2188,7 +2188,7 @@ bool rocksDBEnabled = false; #endif // Populates the TestConfig fields according to what is found in the test file. -void checkTestConf(const char* testFile, TestConfig* testConfig) {} +[[maybe_unused]] void checkTestConf(const char* testFile, TestConfig* testConfig) {} } // namespace @@ -2301,4 +2301,4 @@ ACTOR void setupAndRun(std::string dataFolder, destructed = true; wait(Never()); ASSERT(false); -} \ No newline at end of file +} diff --git a/flow/Tracing.actor.cpp b/flow/Tracing.actor.cpp index 4cb35bc117..173fbe1196 100644 --- a/flow/Tracing.actor.cpp +++ b/flow/Tracing.actor.cpp @@ -122,26 +122,28 @@ ACTOR Future simulationStartServer() { } } +/* // Runs on an interval, printing debug information and performing other // connection tasks. ACTOR Future traceLog(int* pendingMessages, bool* sendError) { - state bool sendErrorReset = false; + state bool sendErrorReset = false; - loop { - TraceEvent("TracingSpanQueueSize").detail("PendingMessages", *pendingMessages); + loop { + TraceEvent("TracingSpanQueueSize").detail("PendingMessages", *pendingMessages); - // Wait at least one full loop before attempting to send messages - // again. - if (sendErrorReset) { - sendErrorReset = false; - *sendError = false; - } else if (*sendError) { - sendErrorReset = true; - } + // Wait at least one full loop before attempting to send messages + // again. + if (sendErrorReset) { + sendErrorReset = false; + *sendError = false; + } else if (*sendError) { + sendErrorReset = true; + } - wait(delay(kQueueSizeLogInterval)); - } + wait(delay(kQueueSizeLogInterval)); + } } +*/ struct UDPTracer : public ITracer { protected: diff --git a/flow/crc32c.cpp b/flow/crc32c.cpp index 759bfd31ef..e9339333a5 100644 --- a/flow/crc32c.cpp +++ b/flow/crc32c.cpp @@ -37,7 +37,7 @@ #include "flow/Platform.h" #include "crc32c-generated-constants.cpp" -static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length) { +[[maybe_unused]] static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length) { for (size_t i = 0; i < length; ++i) { crc = crc ^ input[i]; for (int j = 0; j < 8; j++) @@ -49,7 +49,7 @@ static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length /* Table-driven software version as a fall-back. This is about 15 times slower than using the hardware instructions. This assumes little-endian integers, as is the case on Intel processors that the assembler code here is for. */ -static uint32_t append_adler_table(uint32_t crci, const uint8_t* input, size_t length) { +[[maybe_unused]] static uint32_t append_adler_table(uint32_t crci, const uint8_t* input, size_t length) { const uint8_t* next = input; uint64_t crc; From 168e75bb1e88894a77865d9482fec0bd988fb10a Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 13:25:16 -0700 Subject: [PATCH 044/338] Remove unused shouldNotHaveFriends* functions --- fdbrpc/FlowTests.actor.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fdbrpc/FlowTests.actor.cpp b/fdbrpc/FlowTests.actor.cpp index 3d67b25e0e..65d3d36019 100644 --- a/fdbrpc/FlowTests.actor.cpp +++ b/fdbrpc/FlowTests.actor.cpp @@ -1287,8 +1287,6 @@ TEST_CASE("/fdbrpc/flow/wait_expression_after_cancel") { template struct ShouldNotGoIntoClassContextStack; -ACTOR static Future shouldNotHaveFriends(); - class Foo1 { public: explicit Foo1(int x) : x(x) {} @@ -1363,8 +1361,6 @@ ACTOR Future Outer::Foo5::fooActor(Outer::Foo5* self) { return self->x; } -ACTOR static Future shouldNotHaveFriends2(); - // Meant to be run with -fsanitize=undefined TEST_CASE("/flow/DeterministicRandom/SignedOverflow") { deterministicRandom()->randomInt(std::numeric_limits::min(), 0); From ebcc023b6f2d1a157080d3e71ebc03f44db222a3 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 13:52:52 -0700 Subject: [PATCH 045/338] Enable missing-field-initializers clang warning --- bindings/java/JavaWorkload.cpp | 6 +++--- cmake/ConfigureCompiler.cmake | 1 - fdbclient/FileBackupAgent.actor.cpp | 8 ++++---- fdbclient/SpecialKeySpace.actor.cpp | 2 +- fdbrpc/FlowTransport.h | 4 ++-- fdbserver/VersionedBTree.actor.cpp | 4 ++-- flow/network.h | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 555a6cb434..1bf6c7ff4f 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -176,9 +176,9 @@ void promiseSend(JNIEnv, jclass, jlong self, jboolean value) { struct JNIError { JNIEnv* env; - jthrowable throwable = nullptr; - const char* file; - int line; + jthrowable throwable{ nullptr }; + const char* file{ nullptr }; + int line{ 0 }; std::string location() const { if (file == nullptr) { diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 400cead811..61d800c30c 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -286,7 +286,6 @@ else() -Wno-delete-non-virtual-dtor -Wno-format -Wno-mismatched-tags - -Wno-missing-field-initializers -Wno-sign-compare -Wno-tautological-pointer-compare -Wno-undefined-var-template diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index b42b192435..3c6dba50e2 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -193,10 +193,10 @@ public: struct RestoreFile { Version version; std::string fileName; - bool isRange; // false for log file - int64_t blockSize; - int64_t fileSize; - Version endVersion; // not meaningful for range files + bool isRange{ false }; // false for log file + int64_t blockSize{ 0 }; + int64_t fileSize{ 0 }; + Version endVersion{ ::invalidVersion }; // not meaningful for range files Tuple pack() const { return Tuple() diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index bea850dea5..cd10cd9304 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1961,7 +1961,7 @@ void parse(StringRef& val, WaitState& w) { } void parse(StringRef& val, time_t& t) { - struct tm tm = { 0 }; + struct tm tm; #ifdef _WIN32 std::istringstream s(val.toString()); s.imbue(std::locale(setlocale(LC_TIME, nullptr))); diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h index 78f91b29f2..abc42e9d70 100644 --- a/fdbrpc/FlowTransport.h +++ b/fdbrpc/FlowTransport.h @@ -39,9 +39,9 @@ public: // Endpoint represents a particular service (e.g. a serialized Promise or PromiseStream) // An endpoint is either "local" (used for receiving data) or "remote" (used for sending data) constexpr static FileIdentifier file_identifier = 10618805; - typedef UID Token; + using Token = UID; NetworkAddressList addresses; - Token token; + Token token{}; Endpoint() {} Endpoint(const NetworkAddressList& addresses, Token token) : addresses(addresses), token(token) { diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index bec413db23..8f011f5a99 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -3873,8 +3873,8 @@ struct SplitStringRef { struct const_iterator { const uint8_t* ptr; - const uint8_t* end; - const uint8_t* next; + const uint8_t* end{ nullptr }; + const uint8_t* next{ nullptr }; inline bool operator==(const const_iterator& rhs) const { return ptr == rhs.ptr; } inline bool operator!=(const const_iterator& rhs) const { return !(*this == rhs); } diff --git a/flow/network.h b/flow/network.h index 8af923b197..60a190c0af 100644 --- a/flow/network.h +++ b/flow/network.h @@ -283,7 +283,7 @@ struct hash { struct NetworkAddressList { NetworkAddress address; - Optional secondaryAddress; + Optional secondaryAddress{}; bool operator==(NetworkAddressList const& r) const { return address == r.address && secondaryAddress == r.secondaryAddress; From d0c9cf4fb0e39a969d0878be5f9c0029feeb72de Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 14:16:14 -0700 Subject: [PATCH 046/338] Enable mismatched-tags clang warning --- cmake/ConfigureCompiler.cmake | 1 - fdbclient/ActorLineageProfiler.h | 2 +- fdbclient/ConfigTransactionInterface.h | 10 +++++----- fdbrpc/FlowTransport.h | 2 +- fdbserver/DataDistribution.actor.cpp | 2 +- fdbserver/DeltaTree.h | 5 +++-- flow/Trace.h | 4 ++-- flow/flow.h | 3 ++- flow/network.h | 2 +- flow/serialize.h | 3 ++- flow/singleton.h | 2 +- 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 61d800c30c..9343ee402f 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -285,7 +285,6 @@ else() -Wno-comment -Wno-delete-non-virtual-dtor -Wno-format - -Wno-mismatched-tags -Wno-sign-compare -Wno-tautological-pointer-compare -Wno-undefined-var-template diff --git a/fdbclient/ActorLineageProfiler.h b/fdbclient/ActorLineageProfiler.h index a55d1541e1..07b7c30966 100644 --- a/fdbclient/ActorLineageProfiler.h +++ b/fdbclient/ActorLineageProfiler.h @@ -96,7 +96,7 @@ struct ConfigError { class ProfilerConfigT { private: // private types using Lock = std::unique_lock; - friend class crossbow::create_static; + friend struct crossbow::create_static; private: // members std::shared_ptr ingestor = std::make_shared(); diff --git a/fdbclient/ConfigTransactionInterface.h b/fdbclient/ConfigTransactionInterface.h index 6e71b72457..c6f2aa920f 100644 --- a/fdbclient/ConfigTransactionInterface.h +++ b/fdbclient/ConfigTransactionInterface.h @@ -188,11 +188,11 @@ struct ConfigTransactionInterface { public: static constexpr FileIdentifier file_identifier = 982485; - struct RequestStream getGeneration; - struct RequestStream get; - struct RequestStream getClasses; - struct RequestStream getKnobs; - struct RequestStream commit; + class RequestStream getGeneration; + class RequestStream get; + class RequestStream getClasses; + class RequestStream getKnobs; + class RequestStream commit; ConfigTransactionInterface(); void setupWellKnownEndpoints(); diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h index abc42e9d70..24daae400a 100644 --- a/fdbrpc/FlowTransport.h +++ b/fdbrpc/FlowTransport.h @@ -134,7 +134,7 @@ public: } }; -struct TransportData; +class TransportData; struct Peer : public ReferenceCounted { TransportData* transport; diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 7249934267..21697dedc1 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -47,7 +47,7 @@ #include "flow/serialize.h" class TCTeamInfo; -struct TCMachineInfo; +class TCMachineInfo; class TCMachineTeamInfo; namespace { diff --git a/fdbserver/DeltaTree.h b/fdbserver/DeltaTree.h index 2c1f7ea91f..4a2f77c52f 100644 --- a/fdbserver/DeltaTree.h +++ b/fdbserver/DeltaTree.h @@ -349,7 +349,7 @@ public: } }; - struct Cursor; + class Cursor; // A Mirror is an accessor for a DeltaTree which allows insertion and reading. Both operations are done // using cursors which point to and share nodes in an tree that is built on-demand and mirrors the compressed @@ -515,7 +515,8 @@ public: // Cursor provides a way to seek into a DeltaTree and iterate over its contents // All Cursors from a Mirror share the same decoded node 'cache' (tree of DecodedNodes) - struct Cursor { + class Cursor { + public: Cursor() : mirror(nullptr), node(nullptr) {} Cursor(Mirror* r) : mirror(r), node(mirror->root) {} diff --git a/flow/Trace.h b/flow/Trace.h index aeaabb4373..52dc94aab7 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -588,8 +588,8 @@ void removeTraceRole(std::string const& role); void retrieveTraceLogIssues(std::set& out); void setTraceLogGroup(const std::string& role); template -struct Future; -struct Void; +class Future; +class Void; Future pingTraceLogWriterThread(); enum trace_clock_t { TRACE_CLOCK_NOW, TRACE_CLOCK_REALTIME }; diff --git a/flow/flow.h b/flow/flow.h index 366ac01175..b331107ee5 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -445,7 +445,8 @@ struct LineageProperties : LineagePropertiesBase { } }; -struct ActorLineage : ThreadSafeReferenceCounted { +class ActorLineage : public ThreadSafeReferenceCounted { +public: friend class LineageReference; struct Property { diff --git a/flow/network.h b/flow/network.h index 60a190c0af..a0710f5ce4 100644 --- a/flow/network.h +++ b/flow/network.h @@ -407,7 +407,7 @@ public: }; // forward declare SendBuffer, declared in serialize.h -struct SendBuffer; +class SendBuffer; class IConnection { public: diff --git a/flow/serialize.h b/flow/serialize.h index 07f70b1f24..ac48fa9740 100644 --- a/flow/serialize.h +++ b/flow/serialize.h @@ -851,7 +851,8 @@ struct ISerializeSource { }; template -struct MakeSerializeSource : ISerializeSource { +class MakeSerializeSource : public ISerializeSource { +public: using value_type = V; void serializePacketWriter(PacketWriter& w) const override { ObjectWriter writer([&](size_t size) { return w.writeBytes(size); }, AssumeVersion(w.protocolVersion())); diff --git a/flow/singleton.h b/flow/singleton.h index b565604674..7d193e8dde 100644 --- a/flow/singleton.h +++ b/flow/singleton.h @@ -272,4 +272,4 @@ typename singleton::pointer singleton::instance_ = nullp template M singleton::mutex_; -} // namespace crossbow \ No newline at end of file +} // namespace crossbow From 8a69aa08a2b8045ee60c8444261d03f951cbb81f Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 14:25:37 -0700 Subject: [PATCH 047/338] Enable tautological-pointer-compare clang warning --- cmake/ConfigureCompiler.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 9343ee402f..fe055612a0 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -286,7 +286,6 @@ else() -Wno-delete-non-virtual-dtor -Wno-format -Wno-sign-compare - -Wno-tautological-pointer-compare -Wno-undefined-var-template -Wno-unknown-pragmas -Wno-unknown-warning-option From c7b28abaf0abfe6575980095d1eef45331b77c45 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 16:53:37 -0700 Subject: [PATCH 048/338] Enable unknown-pragmas warning for clang --- cmake/ConfigureCompiler.cmake | 1 - fdbclient/VersionedMap.h | 2 ++ flow/actorcompiler.h | 2 ++ flow/flow.h | 2 ++ flow/genericactors.actor.h | 3 +++ flow/serialize.h | 2 ++ 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index fe055612a0..f85cc84f75 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -287,7 +287,6 @@ else() -Wno-format -Wno-sign-compare -Wno-undefined-var-template - -Wno-unknown-pragmas -Wno-unknown-warning-option -Wno-unused-parameter ) diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h index 32371689a2..1cd78a1828 100644 --- a/fdbclient/VersionedMap.h +++ b/fdbclient/VersionedMap.h @@ -38,7 +38,9 @@ // PTree also supports efficient finger searches. namespace PTreeImpl { +#ifdef _MSC_VER #pragma warning(disable : 4800) +#endif template struct PTree : public ReferenceCounted>, FastAllocated>, NonCopyable { diff --git a/flow/actorcompiler.h b/flow/actorcompiler.h index a20faf408d..e783900e45 100644 --- a/flow/actorcompiler.h +++ b/flow/actorcompiler.h @@ -70,4 +70,6 @@ T waitNext(const FutureStream&); #define THIS_ADDR uintptr_t(nullptr) #endif +#ifdef _MSC_VER #pragma warning(disable : 4355) // 'this' : used in base member initializer list +#endif diff --git a/flow/flow.h b/flow/flow.h index b331107ee5..a3d0e5d7a7 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -24,10 +24,12 @@ #include "flow/FastRef.h" #pragma once +#ifdef _MSC_VER #pragma warning(disable : 4244 4267) // SOMEDAY: Carefully check for integer overflow issues (e.g. size_t to int // conversions like this suppresses) #pragma warning(disable : 4345) #pragma warning(error : 4239) +#endif #include #include diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h index 583648ff07..73df375c39 100644 --- a/flow/genericactors.actor.h +++ b/flow/genericactors.actor.h @@ -37,7 +37,10 @@ #include "flow/Util.h" #include "flow/IndexedSet.h" #include "flow/actorcompiler.h" // This must be the last #include. + +#ifdef _MSC_VER #pragma warning(disable : 4355) // 'this' : used in base member initializer list +#endif ACTOR template Future traceAfter(Future what, const char* type, const char* key, X value, bool traceErrors = false) { diff --git a/flow/serialize.h b/flow/serialize.h index ac48fa9740..9e0abbdab1 100644 --- a/flow/serialize.h +++ b/flow/serialize.h @@ -268,7 +268,9 @@ inline void load(Archive& ar, std::map& value) { ASSERT(ar.protocolVersion().isValid()); } +#ifdef _MSC_VER #pragma intrinsic(memcpy) +#endif #if VALGRIND static bool valgrindCheck(const void* data, int bytes, const char* context) { From 27db99a77f28ee6231b13b4fb2e0473a4d85755a Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 22:03:09 -0700 Subject: [PATCH 049/338] Enable clang comment warnings --- cmake/ConfigureCompiler.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index f85cc84f75..0890dcf0d1 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -282,7 +282,6 @@ else() -Woverloaded-virtual -Wshift-sign-overflow # Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 11 - -Wno-comment -Wno-delete-non-virtual-dtor -Wno-format -Wno-sign-compare From 7f09bdbda450cbfdb8e4db0d61a5888347e1efb9 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Thu, 28 Oct 2021 22:34:20 -0700 Subject: [PATCH 050/338] Remove -Wclass-memaccess for clang --- cmake/ConfigureCompiler.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 0890dcf0d1..160c46f5ae 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -313,7 +313,7 @@ else() -fvisibility=hidden -Wreturn-type -fPIC) - if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86") + if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86" AND NOT CLANG) add_compile_options($<$:-Wclass-memaccess>) endif() if (GPERFTOOLS_FOUND AND GCC) From af51de902f1482a5c39d9133d90979f04f381fd9 Mon Sep 17 00:00:00 2001 From: Yao Xiao Date: Wed, 22 Sep 2021 12:55:37 -0700 Subject: [PATCH 051/338] Add documation about network options. --- .../sphinx/source/api-common.rst.inc | 45 ++++++++++++++++++ documentation/sphinx/source/api-python.rst | 46 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index f70e16a5d6..df2b819f79 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -588,3 +588,48 @@ .. |locality-get-addresses-for-key-blurb| replace:: Returns a list of public network addresses as strings, one for each of the storage servers responsible for storing ``key`` and its associated value. + +.. |option-knob| replace:: + + Sets internal tuning or debugging knobs. + +.. |option-tls-verify-peers| replace:: + + Sets the peer certificate field verification criteria. + +.. |option-tls-ca-bytes| replace:: + + Sets the certificate authority bundle. + +.. |option-tls-ca-path| replace:: + + Sets the file from which to load the certificate authority bundle. + +.. |option-tls-password| replace:: + + Sets the passphrase for encrypted private key. Password should be set before setting the key for the password to be used. + +.. |option-disable-multi-version-client-api| replace:: + + Disables the multi-version client API and instead uses the local client directly. Must be set before setting up the network. + +.. |option-set-disable-local-client| replace:: + + Prevents connections through the local client, allowing only connections through externally loaded client libraries. + +.. |option-set-client-threads-per-version| replace:: + + Spawns multiple worker threads for each version of the client that is loaded. Setting this to a number greater than one implies disable_local_client. + +.. |option-disable-client-statistics-logging| replace:: + + Disables logging of client statistics, such as sampled transaction activity. + +.. |option-enable-run-loop-profiling| replace:: + + Enables debugging feature to perform run loop profiling. Requires trace logging to be enabled. WARNING: this feature is not recommended for use in production. + + +.. |option-set-distributed-client-tracer| replace:: + + Sets a tracer to run on the client. Should be set to the same value as the tracer set on the server. \ No newline at end of file diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 2897be0153..300f1d1d63 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -125,6 +125,10 @@ After importing the ``fdb`` module and selecting an API version, you probably wa .. note:: |network-options-warning| + .. method :: fdb.options.set_knob("knob_name=value") + + |option-knob| + .. method :: fdb.options.set_trace_enable( output_directory=None ) |option-trace-enable-blurb| @@ -188,6 +192,48 @@ After importing the ``fdb`` module and selecting an API version, you probably wa .. method :: fdb.options.set_tls_key_bytes(bytes) |option-tls-key-bytes| + + .. method :: fdb.options.set_tls_verify_peers(verification_pattern) + + |option-tls-verify-peers| + + .. method :: fdb.options.set_tls_ca_bytes(ca_bundle) + + |option-tls-ca-bytes| + + .. method :: fdb.options.set_tls_ca_path(path) + + |option-tls-ca-path| + + .. method :: fdb.options.set_tls_password(password) + + |option-tls-password| + + .. method :: fdb.options.set_disable_multi_version_client_api() + + |option-disable-multi-version-client-api| + + .. method :: fdb.options.set_disable_local_client() + + |option-set-disable-local-client| + + .. method :: fdb.options.set_ client_threads_per_version(number) + + |option-set-client-threads-per-version| + + .. method :: fdb.options.set_disable_client_statistics_logging() + + |option-disable-client-statistics-logging| + + .. method :: fdb.options.set_enable_run_loop_profiling() + + |option-enable-run-loop-profiling| + + .. method :: fdb.options.set_distributed_client_tracer(tracer_type) + + |option-set-distributed-client-tracer| + + Please refer to fdboptions.py (generated) for a comprehensive list of options. .. _api-python-keys: From 90b231e96ea1037ece4f7b7d84d4162b2803c460 Mon Sep 17 00:00:00 2001 From: Yao Xiao Date: Fri, 1 Oct 2021 11:05:15 -0700 Subject: [PATCH 052/338] Add link to client knobs. --- documentation/sphinx/source/api-common.rst.inc | 2 +- documentation/sphinx/source/api-python.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index df2b819f79..2cba7718dd 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -591,7 +591,7 @@ .. |option-knob| replace:: - Sets internal tuning or debugging knobs. + Sets internal tuning or debugging knobs. Available knobs could be found at https://github.com/apple/foundationdb/blob/master/fdbclient/ClientKnobs.h. .. |option-tls-verify-peers| replace:: diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 300f1d1d63..7f2b9c4af0 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -217,7 +217,7 @@ After importing the ``fdb`` module and selecting an API version, you probably wa |option-set-disable-local-client| - .. method :: fdb.options.set_ client_threads_per_version(number) + .. method :: fdb.options.set_client_threads_per_version(number) |option-set-client-threads-per-version| From 648bd336b5bf36a0423978f528ac95ffbc8448aa Mon Sep 17 00:00:00 2001 From: Yao Xiao Date: Mon, 1 Nov 2021 14:32:24 -0700 Subject: [PATCH 053/338] resolve comments --- documentation/sphinx/source/api-common.rst.inc | 2 +- documentation/sphinx/source/api-python.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 2cba7718dd..70378d524c 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -591,7 +591,7 @@ .. |option-knob| replace:: - Sets internal tuning or debugging knobs. Available knobs could be found at https://github.com/apple/foundationdb/blob/master/fdbclient/ClientKnobs.h. + Sets internal tuning or debugging knobs. The argument to this function should be a string representing the knob name and the value, e.g. "transaction_size_limit=1000". .. |option-tls-verify-peers| replace:: diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 7f2b9c4af0..18fbd01adb 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -125,7 +125,7 @@ After importing the ``fdb`` module and selecting an API version, you probably wa .. note:: |network-options-warning| - .. method :: fdb.options.set_knob("knob_name=value") + .. method :: fdb.options.set_knob(knob) |option-knob| From e08721c7f4c8cac9caa03e7931f1de69d69e19be Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 31 Oct 2021 15:49:54 -0700 Subject: [PATCH 054/338] Added flow/serialize/Downgrade unit tests --- flow/serialize.cpp | 89 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/flow/serialize.cpp b/flow/serialize.cpp index 7b1e03b262..a4570721b9 100644 --- a/flow/serialize.cpp +++ b/flow/serialize.cpp @@ -18,8 +18,9 @@ * limitations under the License. */ -#include "flow/serialize.h" #include "flow/network.h" +#include "flow/serialize.h" +#include "flow/UnitTest.h" _AssumeVersion::_AssumeVersion(ProtocolVersion version) : v(version) { if (!version.isValid()) { @@ -38,3 +39,89 @@ const void* BinaryReader::readBytes(int bytes) { begin = e; return b; } + +namespace { + +auto const oldKey = "oldKey"_sr; +auto const newKey = "newKey"_sr; + +struct _Struct { + static constexpr FileIdentifier file_identifier = 2340487; + int oldField{ 0 }; +}; + +struct OldStruct : public _Struct { + void setFields() { oldField = 1; } + bool isSet() const { return oldField == 1; } + + template + void serialize(Archive& ar) { + serializer(ar, oldField); + } +}; + +struct NewStruct : public _Struct { + int newField{ 0 }; + + bool isSet() const { return oldField == 1 && newField == 2; } + void setFields() { + oldField = 1; + newField = 2; + } + + template + void serialize(Archive& ar) { + serializer(ar, oldField, newField); + } +}; + +void verifyData(StringRef value, int numObjects) { + { + // use BinaryReader + BinaryReader reader(value, IncludeVersion()); + std::vector data; + reader >> data; + ASSERT_EQ(data.size(), numObjects); + for (const auto& object : data) { + ASSERT(object.isSet()); + } + } + { + // use ArenaReader + ArenaReader reader(Arena(), value, IncludeVersion()); + std::vector data; + reader >> data; + ASSERT_EQ(data.size(), numObjects); + for (const auto& oldObject : data) { + ASSERT(oldObject.isSet()); + } + } +} + +} // namespace + +TEST_CASE("flow/serialize/Downgrade/WriteOld") { + BinaryWriter writer(IncludeVersion(g_network->protocolVersion())); + auto const numObjects = deterministicRandom()->randomInt(1, 101); + std::vector data(numObjects); + for (auto& oldObject : data) { + oldObject.setFields(); + } + writer << data; + verifyData(writer.toValue(), numObjects); + return Void(); +} + +TEST_CASE("flow/serialize/Downgrade/WriteNew") { + auto protocolVersion = g_network->protocolVersion(); + protocolVersion.addObjectSerializerFlag(); + ObjectWriter writer(IncludeVersion(protocolVersion)); + auto const numObjects = deterministicRandom()->randomInt(1, 101); + std::vector data(numObjects); + for (auto& newObject : data) { + newObject.setFields(); + } + writer.serialize(data); + verifyData(writer.toStringRef(), numObjects); + return Void(); +} From 45cff017c242cc85eef3c95226558c32b1a165af Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 31 Oct 2021 15:50:48 -0700 Subject: [PATCH 055/338] Remove Downgrade workload --- fdbserver/CMakeLists.txt | 1 - fdbserver/workloads/Downgrade.actor.cpp | 168 ------------------------ 2 files changed, 169 deletions(-) delete mode 100644 fdbserver/workloads/Downgrade.actor.cpp diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index f9fa0aad46..ca7d7d6db5 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -182,7 +182,6 @@ set(FDBSERVER_SRCS workloads/DDMetricsExclude.actor.cpp workloads/DiskDurability.actor.cpp workloads/DiskDurabilityTest.actor.cpp - workloads/Downgrade.actor.cpp workloads/DummyWorkload.actor.cpp workloads/ExternalWorkload.actor.cpp workloads/FastTriggeredWatches.actor.cpp diff --git a/fdbserver/workloads/Downgrade.actor.cpp b/fdbserver/workloads/Downgrade.actor.cpp deleted file mode 100644 index e5157bcf8d..0000000000 --- a/fdbserver/workloads/Downgrade.actor.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Downgrade.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbclient/NativeAPI.actor.h" -#include "fdbserver/TesterInterface.actor.h" -#include "fdbserver/workloads/workloads.actor.h" -#include "flow/serialize.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -struct DowngradeWorkload : TestWorkload { - - static constexpr const char* NAME = "Downgrade"; - Key oldKey, newKey; - int numObjects; - - DowngradeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - oldKey = getOption(options, LiteralStringRef("oldKey"), LiteralStringRef("oldKey")); - newKey = getOption(options, LiteralStringRef("newKey"), LiteralStringRef("newKey")); - numObjects = getOption(options, LiteralStringRef("numOptions"), deterministicRandom()->randomInt(0, 100)); - } - - struct _Struct { - static constexpr FileIdentifier file_identifier = 2340487; - int oldField = 0; - }; - - struct OldStruct : public _Struct { - void setFields() { oldField = 1; } - bool isSet() const { return oldField == 1; } - - template - void serialize(Archive& ar) { - serializer(ar, oldField); - } - }; - - struct NewStruct : public _Struct { - int newField = 0; - - bool isSet() const { return oldField == 1 && newField == 2; } - void setFields() { - oldField = 1; - newField = 2; - } - - template - void serialize(Archive& ar) { - serializer(ar, oldField, newField); - } - }; - - ACTOR static Future writeOld(Database cx, int numObjects, Key key) { - BinaryWriter writer(IncludeVersion(g_network->protocolVersion())); - std::vector data(numObjects); - for (auto& oldObject : data) { - oldObject.setFields(); - } - writer << data; - state Value value = writer.toValue(); - - state Transaction tr(cx); - loop { - try { - tr.set(key, value); - wait(tr.commit()); - return Void(); - } catch (Error& e) { - wait(tr.onError(e)); - } - } - } - - ACTOR static Future writeNew(Database cx, int numObjects, Key key) { - ProtocolVersion protocolVersion = g_network->protocolVersion(); - protocolVersion.addObjectSerializerFlag(); - ObjectWriter writer(IncludeVersion(protocolVersion)); - std::vector data(numObjects); - for (auto& newObject : data) { - newObject.setFields(); - } - writer.serialize(data); - state Value value = writer.toStringRef(); - - state Transaction tr(cx); - loop { - try { - tr.set(key, value); - wait(tr.commit()); - return Void(); - } catch (Error& e) { - wait(tr.onError(e)); - } - } - } - - ACTOR static Future readData(Database cx, int numObjects, Key key) { - state Transaction tr(cx); - state Value value; - - loop { - try { - Optional _value = wait(tr.get(key)); - ASSERT(_value.present()); - value = _value.get(); - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - - { - // use BinaryReader - BinaryReader reader(value, IncludeVersion()); - std::vector data; - reader >> data; - ASSERT(data.size() == numObjects); - for (const auto& oldObject : data) { - ASSERT(oldObject.isSet()); - } - } - { - // use ArenaReader - ArenaReader reader(Arena(), value, IncludeVersion()); - std::vector data; - reader >> data; - ASSERT(data.size() == numObjects); - for (const auto& oldObject : data) { - ASSERT(oldObject.isSet()); - } - } - return Void(); - } - - std::string description() const override { return NAME; } - - Future setup(Database const& cx) override { - return clientId ? Void() : (writeOld(cx, numObjects, oldKey) && writeNew(cx, numObjects, newKey)); - } - - Future start(Database const& cx) override { - return clientId ? Void() : (readData(cx, numObjects, oldKey) && readData(cx, numObjects, newKey)); - } - - Future check(Database const& cx) override { - // Failures are checked with assertions - return true; - } - void getMetrics(std::vector& m) override {} -}; - -WorkloadFactory DowngradeWorkloadFactory(DowngradeWorkload::NAME); From 2e3f3ea2afd5e09a43de1e29ab2b84689bfa1f7b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 31 Oct 2021 15:52:39 -0700 Subject: [PATCH 056/338] Remove unused constants from serialize.cpp --- flow/serialize.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/flow/serialize.cpp b/flow/serialize.cpp index a4570721b9..7316b995a2 100644 --- a/flow/serialize.cpp +++ b/flow/serialize.cpp @@ -42,9 +42,6 @@ const void* BinaryReader::readBytes(int bytes) { namespace { -auto const oldKey = "oldKey"_sr; -auto const newKey = "newKey"_sr; - struct _Struct { static constexpr FileIdentifier file_identifier = 2340487; int oldField{ 0 }; From cf3c9dd5201051a95f3a2176ece105a6f15fa8fb Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 31 Oct 2021 21:00:18 -0700 Subject: [PATCH 057/338] Remove reference to deleted Downgrade.toml file --- tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 56d12d77a5..22c77e091d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -181,7 +181,6 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/ConflictRangeRYOWCheck.toml) add_fdb_test(TEST_FILES rare/CycleRollbackClogged.toml) add_fdb_test(TEST_FILES rare/CycleWithKills.toml) - add_fdb_test(TEST_FILES rare/Downgrade.toml) add_fdb_test(TEST_FILES rare/FuzzTest.toml) add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml) add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml) From 70b5ee35b941b90e4e32681dc5f652cd23a2b92e Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Mon, 1 Nov 2021 13:48:38 -0700 Subject: [PATCH 058/338] Add comment to flow/serialize/Downgrade/WriteNew unit test Co-authored-by: Andrew Noyes --- flow/serialize.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/flow/serialize.cpp b/flow/serialize.cpp index 7316b995a2..04d8b460cd 100644 --- a/flow/serialize.cpp +++ b/flow/serialize.cpp @@ -109,6 +109,7 @@ TEST_CASE("flow/serialize/Downgrade/WriteOld") { return Void(); } +// Verify that old code will still be able to read the values of the struct it knows about, even if we add a new field and write a message with new code. TEST_CASE("flow/serialize/Downgrade/WriteNew") { auto protocolVersion = g_network->protocolVersion(); protocolVersion.addObjectSerializerFlag(); From 841e6b211be3e914a59d4adc70b083b84531812d Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 1 Nov 2021 14:01:29 -0700 Subject: [PATCH 059/338] Run clang-format on flow/serialize.cpp --- flow/serialize.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flow/serialize.cpp b/flow/serialize.cpp index 04d8b460cd..a39461d3dd 100644 --- a/flow/serialize.cpp +++ b/flow/serialize.cpp @@ -109,7 +109,8 @@ TEST_CASE("flow/serialize/Downgrade/WriteOld") { return Void(); } -// Verify that old code will still be able to read the values of the struct it knows about, even if we add a new field and write a message with new code. +// Verify that old code will still be able to read the values of the struct it knows about, even if we add a new field +// and write a message with new code. TEST_CASE("flow/serialize/Downgrade/WriteNew") { auto protocolVersion = g_network->protocolVersion(); protocolVersion.addObjectSerializerFlag(); From 3a6a9bdba57a62b733bda4654f7c9d34d2a607b0 Mon Sep 17 00:00:00 2001 From: Yao Xiao <87789492+yao-xiao-github@users.noreply.github.com> Date: Mon, 1 Nov 2021 16:07:36 -0700 Subject: [PATCH 060/338] Update documentation/sphinx/source/api-common.rst.inc apply fix Co-authored-by: A.J. Beamon --- documentation/sphinx/source/api-common.rst.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 70378d524c..b98b7c5461 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -629,7 +629,6 @@ Enables debugging feature to perform run loop profiling. Requires trace logging to be enabled. WARNING: this feature is not recommended for use in production. - .. |option-set-distributed-client-tracer| replace:: Sets a tracer to run on the client. Should be set to the same value as the tracer set on the server. \ No newline at end of file From 7ce29dd153af94f85d3d9618985b5c99f4b9876a Mon Sep 17 00:00:00 2001 From: Yao Xiao Date: Mon, 1 Nov 2021 16:26:37 -0700 Subject: [PATCH 061/338] fix CI error --- documentation/sphinx/source/api-common.rst.inc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index b98b7c5461..704e224496 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -609,10 +609,6 @@ Sets the passphrase for encrypted private key. Password should be set before setting the key for the password to be used. -.. |option-disable-multi-version-client-api| replace:: - - Disables the multi-version client API and instead uses the local client directly. Must be set before setting up the network. - .. |option-set-disable-local-client| replace:: Prevents connections through the local client, allowing only connections through externally loaded client libraries. From d6a31078fe69dacc24e15ef0c3bc926b5b9f89c5 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Tue, 2 Nov 2021 10:01:23 -0500 Subject: [PATCH 062/338] C API for blob granules --- bindings/c/fdb_c.cpp | 37 +++++++++ bindings/c/foundationdb/fdb_c.h | 32 ++++++++ bindings/c/test/unit/fdb_api.cpp | 28 +++++++ bindings/c/test/unit/fdb_api.hpp | 21 ++++++ fdbclient/DatabaseContext.h | 5 +- fdbclient/FDBTypes.h | 8 ++ fdbclient/IClientApi.h | 7 ++ fdbclient/MultiVersionTransaction.actor.cpp | 75 +++++++++++++++++++ fdbclient/MultiVersionTransaction.h | 43 +++++++++++ fdbclient/NativeAPI.actor.cpp | 50 ++++++++++--- fdbclient/ThreadSafeTransaction.cpp | 67 +++++++++++++++++ fdbclient/ThreadSafeTransaction.h | 7 ++ .../workloads/BlobGranuleVerifier.actor.cpp | 22 +----- 13 files changed, 374 insertions(+), 28 deletions(-) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index ecb78e4df7..9e611c1af1 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -260,6 +260,14 @@ extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const *out_count = na.size();); } +extern "C" DLLEXPORT fdb_error_t fdb_future_get_keyrange_array(FDBFuture* f, + FDBKeyRange const** out_ranges, + int* out_count) { + CATCH_AND_RETURN(Standalone> na = TSAV(Standalone>, f)->get(); + *out_ranges = (FDBKeyRange*)na.begin(); + *out_count = na.size();); +} + extern "C" DLLEXPORT fdb_error_t fdb_future_get_key_array(FDBFuture* f, FDBKey const** out_key_array, int* out_count) { CATCH_AND_RETURN(Standalone> na = TSAV(Standalone>, f)->get(); *out_key_array = (FDBKey*)na.begin(); @@ -381,6 +389,35 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db }).extractPtr()); } +extern "C" DLLEXPORT FDBFuture* fdb_database_get_blob_granule_ranges(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + return (FDBFuture*)(DB(db)->getBlobGranuleRanges(range).extractPtr()); +} + +extern "C" DLLEXPORT FDBFuture* fdb_database_read_blob_granules(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + int64_t beginVersion, + int64_t endVersion, + FDBReadBlobGranuleContext granule_context) { + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + + // FIXME: better way to convert? + ReadBlobGranuleContext context; + context.userContext = granule_context.userContext; + context.start_load_f = granule_context.start_load_f; + context.get_load_f = granule_context.get_load_f; + context.free_load_f = granule_context.free_load_f; + + return (FDBFuture*)(DB(db)->readBlobGranules(range, beginVersion, endVersion, context).extractPtr()); +} + extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) { try { TXN(tr)->delref(); diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 81bf10d8a8..2b98d639a1 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -112,8 +112,21 @@ typedef struct keyvalue { int value_length; } FDBKeyValue; #endif +typedef struct keyrange { + const uint8_t* begin_key; + int begin_key_length; + const uint8_t* end_key; + int end_key_length; +} FDBKeyRange; #pragma pack(pop) +typedef struct readgranulecontext { + void* userContext; + int64_t (*start_load_f)(const char*, int, int64_t, int64_t, void*); + uint8_t (*get_load_f)(int64_t, void*); + void (*free_load_f)(uint8_t*); +} FDBReadBlobGranuleContext; + DLLEXPORT void fdb_future_cancel(FDBFuture* f); DLLEXPORT void fdb_future_release_memory(FDBFuture* f); @@ -159,6 +172,10 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* const char*** out_strings, int* out_count); +DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_keyrange_array(FDBFuture* f, + FDBKeyRange const** out_ranges, + int* out_count); + DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database); DLLEXPORT void fdb_database_destroy(FDBDatabase* d); @@ -191,6 +208,21 @@ DLLEXPORT WARN_UNUSED_RESULT double fdb_database_get_main_thread_busyness(FDBDat DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db, uint64_t expected_version); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_blob_granule_ranges(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_read_blob_granules(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + int64_t beginVersion, + int64_t endVersion, + FDBReadBlobGranuleContext granuleContext); + DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr); DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr); diff --git a/bindings/c/test/unit/fdb_api.cpp b/bindings/c/test/unit/fdb_api.cpp index e59085eeb9..2ab48c6d47 100644 --- a/bindings/c/test/unit/fdb_api.cpp +++ b/bindings/c/test/unit/fdb_api.cpp @@ -78,6 +78,12 @@ void Future::cancel() { return fdb_future_get_string_array(future_, out_strings, out_count); } +// KeyRangeArrayFuture + +[[nodiscard]] fdb_error_t KeyRangeArrayFuture::get(const FDBKeyRange** out_keyranges, int* out_count) { + return fdb_future_get_keyrange_array(future_, out_keyranges, out_count); +} + // KeyValueArrayFuture [[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) { @@ -105,6 +111,28 @@ EmptyFuture Database::create_snapshot(FDBDatabase* db, return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length)); } +KeyRangeArrayFuture Database::get_blob_granule_ranges(FDBDatabase* db, + std::string_view begin_key, + std::string_view end_key) { + return KeyRangeArrayFuture(fdb_database_get_blob_granule_ranges( + db, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size())); +} +KeyValueArrayFuture Database::read_blob_granules(FDBDatabase* db, + std::string_view begin_key, + std::string_view end_key, + int64_t beginVersion, + int64_t endVersion, + FDBReadBlobGranuleContext granuleContext) { + return KeyValueArrayFuture(fdb_database_read_blob_granules(db, + (const uint8_t*)begin_key.data(), + begin_key.size(), + (const uint8_t*)end_key.data(), + end_key.size(), + beginVersion, + endVersion, + granuleContext)); +} + // Transaction Transaction::Transaction(FDBDatabase* db) { diff --git a/bindings/c/test/unit/fdb_api.hpp b/bindings/c/test/unit/fdb_api.hpp index 17f25d55ee..4c1acb9629 100644 --- a/bindings/c/test/unit/fdb_api.hpp +++ b/bindings/c/test/unit/fdb_api.hpp @@ -132,9 +132,22 @@ public: private: friend class Transaction; + friend class Database; KeyValueArrayFuture(FDBFuture* f) : Future(f) {} }; +class KeyRangeArrayFuture : public Future { +public: + // Call this function instead of fdb_future_get_keyrange_array when using + // the KeyRangeArrayFuture type. It's behavior is identical to + // fdb_future_get_keyrange_array. + fdb_error_t get(const FDBKeyRange** out_keyranges, int* out_count); + +private: + friend class Database; + KeyRangeArrayFuture(FDBFuture* f) : Future(f) {} +}; + class EmptyFuture : public Future { private: friend class Transaction; @@ -156,6 +169,14 @@ public: int uid_length, const uint8_t* snap_command, int snap_command_length); + + KeyRangeArrayFuture get_blob_granule_ranges(FDBDatabase* db, std::string_view begin_key, std::string_view end_key); + KeyValueArrayFuture read_blob_granules(FDBDatabase* db, + std::string_view begin_key, + std::string_view end_key, + int64_t beginVersion, + int64_t endVersion, + FDBReadBlobGranuleContext granule_context); }; // Wrapper around FDBTransaction, providing the same set of calls as the C API. diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 837d4ec793..2c4d7fa512 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -261,7 +261,10 @@ public: Future> getOverlappingChangeFeeds(KeyRangeRef ranges, Version minVersion); Future popChangeFeedMutations(Key rangeID, Version version); - Future getBlobGranuleRangesStream(const PromiseStream& results, KeyRange range); + Future>> getBlobGranuleRanges(KeyRange range); + Future>> readBlobGranules(KeyRange range, + Version begin, + Optional end); Future readBlobGranulesStream(const PromiseStream>& results, KeyRange range, Version begin, diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 17ad22b93e..f9f871245d 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -1177,4 +1177,12 @@ inline bool isValidPerpetualStorageWiggleLocality(std::string locality) { return ((pos > 0 && pos < locality.size() - 1) || locality == "0"); } +// matches what's in fdb_c.h +struct ReadBlobGranuleContext { + void* userContext; + int64_t (*start_load_f)(const char*, int, int64_t, int64_t, void*); + uint8_t (*get_load_f)(int64_t, void*); + void (*free_load_f)(uint8_t*); +}; + #endif diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index cf304202bb..7eb1ef40f8 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -124,6 +124,13 @@ public: // Management API, create snapshot virtual ThreadFuture createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0; + virtual ThreadFuture>> getBlobGranuleRanges(const KeyRangeRef& keyRange) = 0; + + virtual ThreadFuture readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granuleContext) = 0; + // used in template functions as the Transaction type that can be created through createTransaction() using TransactionT = ITransaction; }; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 9d701439d9..8314d3468c 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -359,6 +359,58 @@ double DLDatabase::getMainThreadBusyness() { return 0; } +ThreadFuture>> DLDatabase::getBlobGranuleRanges(const KeyRangeRef& keyRange) { + if (!api->databaseGetBlobGranuleRanges) { + return unsupported_operation(); + } + + FdbCApi::FDBFuture* f = api->databaseGetBlobGranuleRanges( + db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size()); + return toThreadFuture>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + const FdbCApi::FDBKeyRange* keyRanges; + int keyRangesLength; + FdbCApi::fdb_error_t error = api->futureGetKeyRangeArray(f, &keyRanges, &keyRangesLength); + ASSERT(!error); + return Standalone>(VectorRef((KeyRangeRef*)keyRanges, keyRangesLength), + Arena()); + }); +} + +ThreadFuture DLDatabase::readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granuleContext) { + if (!api->databaseReadBlobGranules) { + return unsupported_operation(); + } + + // FIXME: better way to convert here? + FdbCApi::FDBReadBlobGranuleContext context; + context.userContext = granuleContext.userContext; + context.start_load_f = granuleContext.start_load_f; + context.get_load_f = granuleContext.get_load_f; + context.free_load_f = granuleContext.free_load_f; + + FdbCApi::FDBFuture* f = api->databaseReadBlobGranules(db, + keyRange.begin.begin(), + keyRange.begin.size(), + keyRange.end.begin(), + keyRange.end.size(), + beginVersion, + endVersion, + context); + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + const FdbCApi::FDBKeyValue* kvs; + int count; + FdbCApi::fdb_bool_t more; + FdbCApi::fdb_error_t error = api->futureGetKeyValueArray(f, &kvs, &count, &more); + ASSERT(!error); + + // The memory for this is stored in the FDBFuture and is released when the future gets destroyed + return RangeResult(RangeResultRef(VectorRef((KeyValueRef*)kvs, count), more), Arena()); + }); +} + // Returns the protocol version reported by the coordinator this client is connected to // If an expected version is given, the future won't return until the protocol version is different than expected // Note: this will never return if the server is running a protocol from FDB 5.0 or older @@ -434,6 +486,13 @@ void DLApi::init() { headerVersion >= 700); loadClientFunction( &api->databaseGetServerProtocol, lib, fdbCPath, "fdb_database_get_server_protocol", headerVersion >= 700); + loadClientFunction(&api->databaseGetBlobGranuleRanges, + lib, + fdbCPath, + "fdb_database_get_blob_granule_ranges", + headerVersion >= 710); + loadClientFunction( + &api->databaseReadBlobGranules, lib, fdbCPath, "fdb_database_read_blob_granules", headerVersion >= 710); loadClientFunction(&api->databaseDestroy, lib, fdbCPath, "fdb_database_destroy"); loadClientFunction(&api->databaseRebootWorker, lib, fdbCPath, "fdb_database_reboot_worker", headerVersion >= 700); loadClientFunction(&api->databaseForceRecoveryWithDataLoss, @@ -488,6 +547,8 @@ void DLApi::init() { loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); loadClientFunction(&api->futureGetStringArray, lib, fdbCPath, "fdb_future_get_string_array"); + loadClientFunction( + &api->futureGetKeyRangeArray, lib, fdbCPath, "fdb_future_get_keyrange_array", headerVersion >= 710); loadClientFunction(&api->futureGetKeyArray, lib, fdbCPath, "fdb_future_get_key_array", headerVersion >= 700); loadClientFunction(&api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array"); loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback"); @@ -1107,6 +1168,20 @@ double MultiVersionDatabase::getMainThreadBusyness() { return localClientBusyness; } +ThreadFuture>> MultiVersionDatabase::getBlobGranuleRanges( + const KeyRangeRef& keyRange) { + // FIXME: what to do if not set?.. + return dbState->db->getBlobGranuleRanges(keyRange); +} + +ThreadFuture MultiVersionDatabase::readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granuleContext) { + // FIXME: what to do if not set?.. + return dbState->db->readBlobGranules(keyRange, beginVersion, endVersion, granuleContext); +} + // Returns the protocol version reported by the coordinator this client is connected to // If an expected version is given, the future won't return until the protocol version is different than expected // Note: this will never return if the server is running a protocol from FDB 5.0 or older diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index 95d9a8b14c..c83746ef4b 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -48,8 +48,21 @@ struct FdbCApi : public ThreadSafeReferenceCounted { const void* value; int valueLength; } FDBKeyValue; + typedef struct keyrange { + const void* beginKey; + int beginKeyLength; + const void* endKey; + int endKeyLength; + } FDBKeyRange; #pragma pack(pop) + typedef struct readgranulecontext { + void* userContext; + int64_t (*start_load_f)(const char*, int, int64_t, int64_t, void*); + uint8_t (*get_load_f)(int64_t, void*); + void (*free_load_f)(uint8_t*); + } FDBReadBlobGranuleContext; + typedef int fdb_error_t; typedef int fdb_bool_t; @@ -85,6 +98,21 @@ struct FdbCApi : public ThreadSafeReferenceCounted { double (*databaseGetMainThreadBusyness)(FDBDatabase* database); FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion); + FDBFuture* (*databaseGetBlobGranuleRanges)(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + + FDBFuture* (*databaseReadBlobGranules)(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + int64_t beginVersion, + int64_t endVersion, + FDBReadBlobGranuleContext granule_context); + // Transaction fdb_error_t (*transactionSetOption)(FDBTransaction* tr, FDBTransactionOption option, @@ -175,6 +203,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { fdb_error_t (*futureGetKey)(FDBFuture* f, uint8_t const** outKey, int* outKeyLength); fdb_error_t (*futureGetValue)(FDBFuture* f, fdb_bool_t* outPresent, uint8_t const** outValue, int* outValueLength); fdb_error_t (*futureGetStringArray)(FDBFuture* f, const char*** outStrings, int* outCount); + fdb_error_t (*futureGetKeyRangeArray)(FDBFuture* f, const FDBKeyRange** out_keyranges, int* outCount); fdb_error_t (*futureGetKeyArray)(FDBFuture* f, FDBKey const** outKeys, int* outCount); fdb_error_t (*futureGetKeyValueArray)(FDBFuture* f, FDBKeyValue const** outKV, int* outCount, fdb_bool_t* outMore); fdb_error_t (*futureSetCallback)(FDBFuture* f, FDBCallback callback, void* callback_parameter); @@ -285,6 +314,13 @@ public: ThreadFuture forceRecoveryWithDataLoss(const StringRef& dcid) override; ThreadFuture createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override; + ThreadFuture>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override; + + ThreadFuture readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granule_context) override; + private: const Reference api; FdbCApi::FDBDatabase* @@ -496,6 +532,13 @@ public: ThreadFuture forceRecoveryWithDataLoss(const StringRef& dcid) override; ThreadFuture createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override; + ThreadFuture>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override; + + ThreadFuture readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granuleContext) override; + // private: struct LegacyVersionMonitor; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 117524a43a..ee97f8c64e 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -7109,15 +7109,15 @@ Future DatabaseContext::popChangeFeedMutations(Key rangeID, Version versio return popChangeFeedMutationsActor(Reference::addRef(this), rangeID, version); } -#define BG_REQUEST_DEBUG false +#define BG_REQUEST_DEBUG true -ACTOR Future getBlobGranuleRangesStreamActor(Reference db, - PromiseStream results, - KeyRange keyRange) { +ACTOR Future>> getBlobGranuleRangesActor(Reference db, + KeyRange keyRange) { // FIXME: use streaming range read state Database cx(db); state Reference tr = makeReference(cx); state KeyRange currentRange = keyRange; + state Standalone> results; if (BG_REQUEST_DEBUG) { printf("Getting Blob Granules for [%s - %s)\n", keyRange.begin.printable().c_str(), @@ -7131,14 +7131,14 @@ ACTOR Future getBlobGranuleRangesStreamActor(Reference db for (int i = 0; i < blobGranuleMapping.size() - 1; i++) { if (blobGranuleMapping[i].value.size()) { - results.send(KeyRangeRef(blobGranuleMapping[i].key, blobGranuleMapping[i + 1].key)); + results.push_back(results.arena(), + KeyRangeRef(blobGranuleMapping[i].key, blobGranuleMapping[i + 1].key)); } } if (blobGranuleMapping.more) { currentRange = KeyRangeRef(blobGranuleMapping.back().key, currentRange.end); } else { - results.sendError(end_of_stream()); - return Void(); + return results; } } catch (Error& e) { wait(tr->onError(e)); @@ -7146,11 +7146,11 @@ ACTOR Future getBlobGranuleRangesStreamActor(Reference db } } -Future DatabaseContext::getBlobGranuleRangesStream(const PromiseStream& results, KeyRange range) { +Future>> DatabaseContext::getBlobGranuleRanges(KeyRange range) { if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { throw client_invalid_operation(); } - return getBlobGranuleRangesStreamActor(Reference::addRef(this), results, range); + return getBlobGranuleRangesActor(Reference::addRef(this), range); } // hack (for now) to get blob worker interface into load balance @@ -7355,6 +7355,38 @@ Future DatabaseContext::readBlobGranulesStream(const PromiseStream::addRef(this), results, range, begin, end); } +ACTOR Future>> readBlobGranulesActor(DatabaseContext* self, + KeyRange range, + Version begin, + Optional end) { + state PromiseStream> chunks; + state Standalone> results; + state Future reader = self->readBlobGranulesStream(chunks, range, begin, end); + loop { + try { + Standalone chunk = waitNext(chunks.getFuture()); + results.arena().dependsOn(chunk.arena()); + results.push_back(results.arena(), chunk); + } catch (Error& e) { + if (e.code() == error_code_end_of_stream) { + break; + } + throw; + } + } + wait(reader); + return results; +} + +Future>> DatabaseContext::readBlobGranules(KeyRange range, + Version begin, + Optional end) { + if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + throw client_invalid_operation(); + } + return readBlobGranulesActor(this, range, begin, end); +} + ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) { state ReadYourWritesTransaction tr(cx); loop { diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 3810d08191..13f4c51e15 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -110,6 +110,73 @@ ThreadFuture ThreadSafeDatabase::getServerProtocol(Optional Future { return db->getClusterProtocol(expectedVersion); }); } +ThreadFuture>> ThreadSafeDatabase::getBlobGranuleRanges(const KeyRangeRef& keyRange) { + DatabaseContext* db = this->db; + KeyRange r = keyRange; + + return onMainThread( + [db, r]() -> Future>> { return db->getBlobGranuleRanges(r); }); +} + +ThreadFuture ThreadSafeDatabase::readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granule_context) { + // In V1 of api this is required, field is just for forward compatibility + ASSERT(beginVersion == 0); + + DatabaseContext* db = this->db; + KeyRange r = keyRange; + + ThreadFuture>> getFilesFuture = + onMainThread([db, r, beginVersion, endVersion]() -> Future>> { + return db->readBlobGranules(r, beginVersion, endVersion); + }); + + // FIXME: can this safely avoid another main thread jump? + getFilesFuture.blockUntilReadyCheckOnMainThread(); + Standalone> files = getFilesFuture.get(); + + // FIXME: could submit multiple chunks to start_load_f in parallel? + RangeResult results; + + int chunkIdx = 0; + for (BlobGranuleChunkRef& chunk : files) { + printf("TSD::readBlobGranules chunk %d\n", chunkIdx++); + // In V1 of api this is required, optional is just for forward compatibility + ASSERT(chunk.snapshotFile.present()); + std::string snapshotFname = chunk.snapshotFile.get().filename.toString(); + int64_t snapshotLoadId = granule_context.start_load_f(snapshotFname.c_str(), + snapshotFname.size(), + chunk.snapshotFile.get().offset, + chunk.snapshotFile.get().length, + granule_context.userContext); + printf(" S_ID=%lld\n", snapshotLoadId); + + int64_t deltaLoadIds[chunk.deltaFiles.size()]; + for (int deltaFileIdx = 0; deltaFileIdx < chunk.deltaFiles.size(); deltaFileIdx++) { + std::string deltaFName = chunk.deltaFiles[deltaFileIdx].filename.toString(); + deltaLoadIds[deltaFileIdx] = granule_context.start_load_f(deltaFName.c_str(), + deltaFName.size(), + chunk.deltaFiles[deltaFileIdx].offset, + chunk.deltaFiles[deltaFileIdx].length, + granule_context.userContext); + printf(" D_ID=%lld\n", deltaLoadIds[deltaFileIdx]); + } + + RangeResult chunkRows; + + // FIXME: actually implement materialization! For now just printing stuff out + + results.arena().dependsOn(chunkRows.arena()); + results.append(results.arena(), chunkRows.begin(), chunkRows.size()); + + // FIXME: free with granule_context + } + + return results; +} + ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) { ClusterConnectionFile* connFile = new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first); diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index 75faa67745..ceafbd0927 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -57,6 +57,13 @@ public: ThreadFuture forceRecoveryWithDataLoss(const StringRef& dcid) override; ThreadFuture createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override; + ThreadFuture>> getBlobGranuleRanges(const KeyRangeRef& keyRange) override; + + ThreadFuture readBlobGranules(const KeyRangeRef& keyRange, + Version beginVersion, + Version endVersion, + ReadBlobGranuleContext granuleContext) override; + private: friend class ThreadSafeTransaction; bool isConfigDB{ false }; diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index 5b451f3e3e..cc57c77ced 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -60,7 +60,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { std::vector> clients; Reference bstore; - AsyncVar> granuleRanges; + AsyncVar>> granuleRanges; BlobGranuleVerifierWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { doSetup = !clientId; // only do this on the "first" client @@ -144,22 +144,8 @@ struct BlobGranuleVerifierWorkload : TestWorkload { // updates the current set of granules in the database, but on a delay, so there can be some mismatch if ranges // change loop { - state std::vector allGranules; - state Transaction tr(cx); - state PromiseStream stream; - state Future reader = cx->getBlobGranuleRangesStream(stream, normalKeys); - loop { - try { - KeyRange r = waitNext(stream.getFuture()); - allGranules.push_back(r); - } catch (Error& e) { - if (e.code() == error_code_end_of_stream) { - break; - } - throw e; - } - } - wait(reader); + Standalone> allGranules = wait(cx->getBlobGranuleRanges(normalKeys)); + // printf("BG find granules found %d granules\n", allGranules.size()); self->granuleRanges.set(allGranules); @@ -411,7 +397,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { state int checks = 0; state bool availabilityPassed = true; - state std::vector allRanges = self->granuleRanges.get(); + state Standalone> allRanges = self->granuleRanges.get(); for (auto& range : allRanges) { state KeyRange r = range; state PromiseStream> chunkStream; From 382882f1c124484bb459ec049d4ee08056d53f74 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Tue, 2 Nov 2021 13:29:49 -0500 Subject: [PATCH 063/338] mako successfully calls read_blob_granules and gets stuff back --- bindings/c/test/mako/mako.c | 62 +++++++++++++++++++-- bindings/c/test/mako/mako.h | 1 + fdbclient/ClientKnobs.cpp | 1 + fdbclient/DatabaseContext.h | 1 + fdbclient/MultiVersionTransaction.actor.cpp | 4 ++ fdbclient/NativeAPI.actor.cpp | 7 ++- fdbclient/ThreadSafeTransaction.cpp | 56 ++++++++++--------- fdbclient/vexillographer/fdb.options | 2 + fdbserver/BlobWorker.actor.cpp | 1 - 9 files changed, 103 insertions(+), 32 deletions(-) diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index f2027c4217..3b1c377763 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -541,8 +541,40 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2) return FDB_SUCCESS; } +int run_op_read_blob_granules(FDBDatabase* database, char* keystr, char* keystr2, int64_t readVersion) { + FDBFuture* f; + fdb_error_t err; + FDBKeyValue const* out_kv; + int out_count; + int out_more; + + // Not used currently! FIXME: fix warning + FDBReadBlobGranuleContext context; + + f = fdb_database_read_blob_granules(database, + (uint8_t*)keystr, + strlen(keystr), + (uint8_t*)keystr2, + strlen(keystr2), + 0 /* beginVersion*/, + readVersion, + context); + + wait_future(f); + + err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); + if (err) { + fprintf(stderr, "ERROR: fdb_future_get_keyvalue_array: %s\n", fdb_get_error(err)); + fdb_future_destroy(f); + return FDB_ERROR_RETRY; + } + fdb_future_destroy(f); + return FDB_SUCCESS; +} + /* run one transaction */ -int run_one_transaction(FDBTransaction* transaction, +int run_one_transaction(FDBDatabase* database, + FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr, @@ -785,6 +817,10 @@ retryTxn: rc = run_op_clearrange(transaction, keystr2, keystr); docommit = 1; break; + case OP_READ_BG: + // Requires that there is an explicit grv before bg + rc = run_op_read_blob_granules(database, keystr, keystr2, readversion); + break; default: fprintf(stderr, "ERROR: Unknown Operation %d\n", i); break; @@ -863,7 +899,8 @@ retryTxn: return 0; } -int run_workload(FDBTransaction* transaction, +int run_workload(FDBDatabase* database, + FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor, @@ -980,7 +1017,7 @@ int run_workload(FDBTransaction* transaction, } rc = run_one_transaction( - transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated); + database, transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated); if (rc) { /* FIXME: run_one_transaction should return something meaningful */ fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc); @@ -1058,6 +1095,9 @@ void get_stats_file_name(char filename[], int worker_id, int thread_id, int op) case OP_TRANSACTION: strcat(filename, "TRANSACTION"); break; + case OP_READ_BG: + strcat(filename, "READBLOBGRANULES"); + break; } } @@ -1147,7 +1187,8 @@ void* worker_thread(void* thread_args) { /* run the workload */ else if (args->mode == MODE_RUN) { - rc = run_workload(transaction, + rc = run_workload(database, + transaction, args, thread_tps, throttle_factor, @@ -1350,6 +1391,8 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi if (args->disable_ryw) { fdb_database_set_option(process.databases[i], FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE, (uint8_t*)NULL, 0); } + // always do not materialize blob granules + fdb_database_set_option(process.databases[i], FDB_DB_OPTION_TEST_BG_NO_MATERIALIZE, (uint8_t*)NULL, 0); } #endif @@ -1538,6 +1581,15 @@ int parse_transaction(mako_args_t* args, char* optarg) { } else if (strncmp(ptr, "sc", 2) == 0) { op = OP_SETCLEAR; ptr += 2; + } else if (strncmp(ptr, "bg", 2) == 0) { + if (!args->txnspec.ops[OP_GETREADVERSION][OP_COUNT]) { + fprintf(debugme, "Error: bg requires explicit grv first!\n", ptr); + error = 1; + break; + } + op = OP_READ_BG; + rangeop = 1; + ptr += 2; } else { fprintf(debugme, "Error: Invalid transaction spec: %s\n", ptr); error = 1; @@ -1928,6 +1980,8 @@ char* get_ops_name(int ops_code) { return "COMMIT"; case OP_TRANSACTION: return "TRANSACTION"; + case OP_READ_BG: + return "READBLOBGRANULE"; default: return ""; } diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index 66a8039dcf..e6a0d7e35e 100644 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -51,6 +51,7 @@ enum Operations { OP_SETCLEARRANGE, OP_COMMIT, OP_TRANSACTION, /* pseudo-operation - cumulative time for the operation + commit */ + OP_READ_BG, MAX_OP /* must be the last item */ }; diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index 66c4390c4d..2d9161863b 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -259,6 +259,7 @@ void ClientKnobs::initialize(Randomize randomize) { init( MVC_CLIENTLIB_CHUNKS_PER_TRANSACTION, 32 ); // blob granules + // TODO CHANGE BACK BEFORE MERGE! init( ENABLE_BLOB_GRANULES, false ); // clang-format on diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 2c4d7fa512..b5c1f1458e 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -409,6 +409,7 @@ public: int transactionTracingEnabled; double verifyCausalReadsProp = 0.0; + bool blobGranuleNoMaterialize = false; Future logger; Future throttleExpirer; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 8314d3468c..f8fb9586b8 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -380,9 +380,12 @@ ThreadFuture DLDatabase::readBlobGranules(const KeyRangeRef& keyRan Version beginVersion, Version endVersion, ReadBlobGranuleContext granuleContext) { + + printf(" DLDatabase::readBlobGranules\n"); if (!api->databaseReadBlobGranules) { return unsupported_operation(); } + printf(" DLDatabase::readBlobGranules 2\n"); // FIXME: better way to convert here? FdbCApi::FDBReadBlobGranuleContext context; @@ -1179,6 +1182,7 @@ ThreadFuture MultiVersionDatabase::readBlobGranules(const KeyRangeR Version endVersion, ReadBlobGranuleContext granuleContext) { // FIXME: what to do if not set?.. + printf(" MultiVersionDatabase::readBlobGranules. DB set=%d\n", dbState->db ? 1 : 0); return dbState->db->readBlobGranules(keyRange, beginVersion, endVersion, granuleContext); } diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index ee97f8c64e..ac50af5c7e 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -1681,6 +1681,11 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional readBlobGranulesStreamActor(Reference db, Arena a; a.dependsOn(rep.arena); results.send(Standalone(chunk, a)); - keyRange = KeyRangeRef(chunk.keyRange.end, keyRange.end); + keyRange = KeyRangeRef(std::min(chunk.keyRange.end, keyRange.end), keyRange.end); } } results.sendError(end_of_stream()); diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 13f4c51e15..73a49570c8 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -137,41 +137,45 @@ ThreadFuture ThreadSafeDatabase::readBlobGranules(const KeyRangeRef getFilesFuture.blockUntilReadyCheckOnMainThread(); Standalone> files = getFilesFuture.get(); - // FIXME: could submit multiple chunks to start_load_f in parallel? RangeResult results; - int chunkIdx = 0; + + // FIXME: could submit multiple chunks to start_load_f in parallel? for (BlobGranuleChunkRef& chunk : files) { - printf("TSD::readBlobGranules chunk %d\n", chunkIdx++); - // In V1 of api this is required, optional is just for forward compatibility - ASSERT(chunk.snapshotFile.present()); - std::string snapshotFname = chunk.snapshotFile.get().filename.toString(); - int64_t snapshotLoadId = granule_context.start_load_f(snapshotFname.c_str(), - snapshotFname.size(), - chunk.snapshotFile.get().offset, - chunk.snapshotFile.get().length, - granule_context.userContext); - printf(" S_ID=%lld\n", snapshotLoadId); - - int64_t deltaLoadIds[chunk.deltaFiles.size()]; - for (int deltaFileIdx = 0; deltaFileIdx < chunk.deltaFiles.size(); deltaFileIdx++) { - std::string deltaFName = chunk.deltaFiles[deltaFileIdx].filename.toString(); - deltaLoadIds[deltaFileIdx] = granule_context.start_load_f(deltaFName.c_str(), - deltaFName.size(), - chunk.deltaFiles[deltaFileIdx].offset, - chunk.deltaFiles[deltaFileIdx].length, - granule_context.userContext); - printf(" D_ID=%lld\n", deltaLoadIds[deltaFileIdx]); - } - RangeResult chunkRows; - // FIXME: actually implement materialization! For now just printing stuff out + if (!db->blobGranuleNoMaterialize) { + // FIXME: actually implement file loading and materialization! For now just printing stuff out + + // Start load process for all files in chunk + // In V1 of api snapshot is required, optional is just for forward compatibility + ASSERT(chunk.snapshotFile.present()); + std::string snapshotFname = chunk.snapshotFile.get().filename.toString(); + int64_t snapshotLoadId = granule_context.start_load_f(snapshotFname.c_str(), + snapshotFname.size(), + chunk.snapshotFile.get().offset, + chunk.snapshotFile.get().length, + granule_context.userContext); + printf(" S_ID=%lld\n", snapshotLoadId); // TODO REMOVE + + int64_t deltaLoadIds[chunk.deltaFiles.size()]; + for (int deltaFileIdx = 0; deltaFileIdx < chunk.deltaFiles.size(); deltaFileIdx++) { + std::string deltaFName = chunk.deltaFiles[deltaFileIdx].filename.toString(); + deltaLoadIds[deltaFileIdx] = granule_context.start_load_f(deltaFName.c_str(), + deltaFName.size(), + chunk.deltaFiles[deltaFileIdx].offset, + chunk.deltaFiles[deltaFileIdx].length, + granule_context.userContext); + printf(" D_ID=%lld\n", deltaLoadIds[deltaFileIdx]); // TODO REMOVE + } + } results.arena().dependsOn(chunkRows.arena()); results.append(results.arena(), chunkRows.begin(), chunkRows.size()); - // FIXME: free with granule_context + if (!db->blobGranuleNoMaterialize) { + // FIXME: free with granule_context + } } return results; diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index 996fae6dc7..c18b294073 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -202,6 +202,8 @@ description is not currently required but encouraged. description="Use configuration database." />