wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't kill failure injection before verification succeeded

This commit is contained in:
Markus Pilman 2022-10-27 11:31:27 -06:00
parent 83c86a9453
commit fca217de55
1 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
fdbserver/workloads/DiskFailureInjection.actor.cpp
View File

@ -23,7 +23,6 @@
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/QuietDatabase.h"
#include "fdbserver/Status.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -43,7 +42,6 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
double percentBitFlips = 10;
double periodicBroadcastInterval = 5.0;
std::vector<NetworkAddress> chosenWorkers;
std::vector<Future<Void>> clients;
// Verification Mode: We run the workload indefinitely in this mode.
// The idea is to keep going until we get a non-zero chaosMetric to ensure
// that we haven't lost the chaos event. testDuration is ignored in this mode
@ -76,23 +74,20 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
// 2. Starting the actor that injects failures on chosen storage servers
Future<Void> start(Database const& cx) override {
if (enabled) {
clients.push_back(timeout(diskFailureInjectionClient<WorkerInterface>(cx, this), testDuration, Void()));
// In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that
// a non-zero chaosMetric was found.
auto result = diskFailureInjectionClient<WorkerInterface>(cx, this);
// In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that
// a non-zero chaosMetric was found.
if (verificationMode) {
clients.push_back(periodicEventBroadcast(this));
} else
return (periodicEventBroadcast(this) && delay(testDuration)) || result;
} else {
// Else we honor the testDuration
clients.push_back(timeout(periodicEventBroadcast(this), testDuration, Void()));
return waitForAll(clients);
return timeout(periodicEventBroadcast(this) && result, testDuration, Void());
}
} else
return Void();
}
Future<bool> check(Database const& cx) override {
clients.clear();
return true;
}
Future<bool> check(Database const& cx) override { return true; }
void getMetrics(std::vector<PerfMetric>& m) override {}
@ -160,6 +155,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
} catch (Error& e) {
// If we failed to get a complete list of storage servers, we can't inject failure events
// But don't throw the error in that case
TraceEvent("ChaosCouldNotGetStorages").error(e);
continue;
}
auto machine = deterministicRandom()->randomChoice(machines);
@ -192,11 +188,20 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
for (auto worker : workers) {
workersMap[worker.interf.address()] = worker.interf;
}
TraceEvent("ResendChaos")
.detail("ChosenWorkersSize", self->chosenWorkers.size())
.detail("FoundWorkers", workersMap.size())
.detail(
"ResendToNumber",
std::count_if(self->chosenWorkers.begin(),
self->chosenWorkers.end(),
[&map = std::as_const(workersMap)](auto const& addr) { return map.count(addr) > 0; }));
for (auto& workerAddress : self->chosenWorkers) {
auto itr = workersMap.find(workerAddress);
if (itr != workersMap.end()) {
if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle))
if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) {
self->injectDiskDelays(itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod);
}
if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) {
if (g_simulator == g_network)
g_simulator->corruptWorkerMap[workerAddress] = true;