diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.actor.cpp index 95e6099346..0bb5953e2b 100644 --- a/fdbserver/workloads/DiskFailureInjection.actor.cpp +++ b/fdbserver/workloads/DiskFailureInjection.actor.cpp @@ -23,7 +23,6 @@ #include "fdbserver/workloads/workloads.actor.h" #include "fdbrpc/simulator.h" #include "fdbserver/WorkerInterface.actor.h" -#include "fdbserver/ServerDBInfo.h" #include "fdbserver/QuietDatabase.h" #include "fdbserver/Status.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -43,7 +42,6 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload { double percentBitFlips = 10; double periodicBroadcastInterval = 5.0; std::vector chosenWorkers; - std::vector> clients; // Verification Mode: We run the workload indefinitely in this mode. // The idea is to keep going until we get a non-zero chaosMetric to ensure // that we haven't lost the chaos event. testDuration is ignored in this mode @@ -76,23 +74,20 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload { // 2. Starting the actor that injects failures on chosen storage servers Future start(Database const& cx) override { if (enabled) { - clients.push_back(timeout(diskFailureInjectionClient(cx, this), testDuration, Void())); - // In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that - // a non-zero chaosMetric was found. + auto result = diskFailureInjectionClient(cx, this); + // In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that + // a non-zero chaosMetric was found. if (verificationMode) { - clients.push_back(periodicEventBroadcast(this)); - } else + return (periodicEventBroadcast(this) && delay(testDuration)) || result; + } else { // Else we honor the testDuration - clients.push_back(timeout(periodicEventBroadcast(this), testDuration, Void())); - return waitForAll(clients); + return timeout(periodicEventBroadcast(this) && result, testDuration, Void()); + } } else return Void(); } - Future check(Database const& cx) override { - clients.clear(); - return true; - } + Future check(Database const& cx) override { return true; } void getMetrics(std::vector& m) override {} @@ -160,6 +155,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload { } catch (Error& e) { // If we failed to get a complete list of storage servers, we can't inject failure events // But don't throw the error in that case + TraceEvent("ChaosCouldNotGetStorages").error(e); continue; } auto machine = deterministicRandom()->randomChoice(machines); @@ -192,11 +188,20 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload { for (auto worker : workers) { workersMap[worker.interf.address()] = worker.interf; } + TraceEvent("ResendChaos") + .detail("ChosenWorkersSize", self->chosenWorkers.size()) + .detail("FoundWorkers", workersMap.size()) + .detail( + "ResendToNumber", + std::count_if(self->chosenWorkers.begin(), + self->chosenWorkers.end(), + [&map = std::as_const(workersMap)](auto const& addr) { return map.count(addr) > 0; })); for (auto& workerAddress : self->chosenWorkers) { auto itr = workersMap.find(workerAddress); if (itr != workersMap.end()) { - if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) + if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) { self->injectDiskDelays(itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod); + } if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) { if (g_simulator == g_network) g_simulator->corruptWorkerMap[workerAddress] = true;