wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbserver/tester/TesterServer.cpp

628 lines
23 KiB
C++
Raw Blame History

/*
* TesterServer.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include <cstdio>
#include <map>
#include <string>
#include <vector>
#include "flow/ActorCollection.h"
#include "flow/CoroUtils.h"
#include "flow/DeterministicRandom.h"
#include "flow/Platform.h"
#include "flow/Trace.h"
#include "fdbrpc/Locality.h"
#include "fdbrpc/SimulatorProcessInfo.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/core/FDBSimulatorProcessInfo.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/core/ServerDBInfo.h"
#include "fdbserver/core/WorkerInterface.h"
#include "TesterServer.h"
#include "fdbserver/tester/workloads.h"
namespace {
Standalone<VectorRef<KeyValueRef>> checkAllOptionsConsumed(VectorRef<KeyValueRef> options) {
static StringRef nothing = ""_sr;
Standalone<VectorRef<KeyValueRef>> unconsumed;
for (int i = 0; i < options.size(); i++) {
if (!(options[i].value == nothing)) {
TraceEvent(SevError, "OptionNotConsumed")
.detail("Key", options[i].key.toString().c_str())
.detail("Value", options[i].value.toString().c_str());
unconsumed.push_back_deep(unconsumed.arena(), options[i]);
}
}
return unconsumed;
}
Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
Reference<IClusterConnectionRecord> ccr,
VectorRef<KeyValueRef> options,
Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
Reference<TestWorkload> workload;
Value testName = getOption(options, "testName"_sr, "no-test-specified"_sr);
WorkloadContext wcx;
wcx.clientId = work.clientId;
wcx.clientCount = work.clientCount;
wcx.ccr = ccr;
wcx.dbInfo = dbInfo;
wcx.options = options;
wcx.sharedRandomNumber = work.sharedRandomNumber;
wcx.rangesToCheck = work.rangesToCheck;
workload = IWorkloadFactory::create(testName.toString(), wcx);
if (workload) {
co_await workload->initialized();
}
auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
if (!workload || !unconsumedOptions.empty()) {
TraceEvent evt(SevError, "TestCreationError");
evt.detail("TestName", testName);
if (!workload) {
evt.detail("Reason", "Null workload");
fprintf(stderr,
"ERROR: Workload could not be created, perhaps testName (%s) is not a valid workload\n",
printable(testName).c_str());
} else {
evt.detail("Reason", "Not all options consumed");
fprintf(stderr, "ERROR: Workload had invalid options. The following were unrecognized:\n");
for (int i = 0; i < unconsumedOptions.size(); i++) {
fprintf(stderr,
" '%s' = '%s'\n",
unconsumedOptions[i].key.toString().c_str(),
unconsumedOptions[i].value.toString().c_str());
}
}
throw test_specification_invalid();
}
co_return workload;
}
Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
Reference<IClusterConnectionRecord> ccr,
Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
WorkloadContext wcx;
std::vector<Future<Reference<TestWorkload>>> ifaces;
if (work.options.size() < 1) {
TraceEvent(SevError, "TestCreationError").detail("Reason", "No options provided");
fprintf(stderr, "ERROR: No options were provided for workload.\n");
throw test_specification_invalid();
}
wcx.clientId = work.clientId;
wcx.clientCount = work.clientCount;
wcx.sharedRandomNumber = work.sharedRandomNumber;
wcx.ccr = ccr;
wcx.dbInfo = dbInfo;
wcx.rangesToCheck = work.rangesToCheck;
// FIXME: Other stuff not filled in; why isn't this constructed here and passed down to the other
// getWorkloadIface()?
for (int i = 0; i < work.options.size(); i++) {
ifaces.push_back(getWorkloadIface(work, ccr, work.options[i], dbInfo));
}
co_await waitForAll(ifaces);
auto compound = makeReference<CompoundWorkload>(wcx);
for (int i = 0; i < work.options.size(); i++) {
compound->add(ifaces[i].getValue());
}
compound->addFailureInjection(work);
co_return compound;
}
} // namespace
/**
* Only works in simulation. This method prints all simulated processes in a human readable form to stdout. It groups
* processes by data center, data hall, zone, and machine (in this order).
*/
void printSimulatedTopology() {
if (!g_network->isSimulated()) {
return;
}
auto processes = g_simulator->getAllProcesses();
std::sort(processes.begin(), processes.end(), [](ISimulator::ProcessInfo* lhs, ISimulator::ProcessInfo* rhs) {
auto l = lhs->locality;
auto r = rhs->locality;
if (l.dcId() != r.dcId()) {
return l.dcId() < r.dcId();
}
if (l.dataHallId() != r.dataHallId()) {
return l.dataHallId() < r.dataHallId();
}
if (l.zoneId() != r.zoneId()) {
return l.zoneId() < r.zoneId();
}
if (l.machineId() != r.machineId()) {
return l.machineId() < r.machineId();
}
return lhs->address < rhs->address;
});
printf("Simulated Cluster Topology:\n");
printf("===========================\n");
Optional<Standalone<StringRef>> dcId, dataHallId, zoneId, machineId;
for (auto p : processes) {
std::string indent = "";
if (dcId != p->locality.dcId()) {
dcId = p->locality.dcId();
printf("%sdcId: %s\n", indent.c_str(), p->locality.describeDcId().c_str());
}
indent += " ";
if (dataHallId != p->locality.dataHallId()) {
dataHallId = p->locality.dataHallId();
printf("%sdataHallId: %s\n", indent.c_str(), p->locality.describeDataHall().c_str());
}
indent += " ";
if (zoneId != p->locality.zoneId()) {
zoneId = p->locality.zoneId();
printf("%szoneId: %s\n", indent.c_str(), p->locality.describeZone().c_str());
}
indent += " ";
if (machineId != p->locality.machineId()) {
machineId = p->locality.machineId();
printf("%smachineId: %s\n", indent.c_str(), p->locality.describeMachineId().c_str());
}
indent += " ";
printf("%sAddress: %s\n", indent.c_str(), p->address.toString().c_str());
indent += " ";
printf("%sClass: %s\n", indent.c_str(), getSimulatorProcessClass(p).toString().c_str());
printf("%sName: %s\n", indent.c_str(), p->name.c_str());
}
}
Future<Void> databaseWarmer(Database cx) {
while (true) {
Transaction tr(cx);
co_await tr.getReadVersion();
co_await delay(0.25);
}
co_return; // Unreachable but required for coroutine
}
namespace {
// Tries indefinitely to commit a simple, self conflicting transaction
Future<Void> pingDatabase(Database cx) {
Transaction tr(cx);
while (true) {
bool needsErrorHandling = false;
Error caughtError;
try {
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> v =
co_await tr.get(StringRef("/Liveness/" + deterministicRandom()->randomUniqueID().toString()));
tr.makeSelfConflicting();
co_await tr.commit();
co_return;
} catch (Error& e) {
TraceEvent("PingingDatabaseTransactionError").error(e);
caughtError = e;
needsErrorHandling = true;
}
if (needsErrorHandling) {
co_await tr.onError(caughtError);
}
}
}
} // namespace
Future<Void> testDatabaseLiveness(Database cx, double databasePingDelay, std::string context, double startDelay) {
co_await delay(startDelay);
while (true) {
Error caughtError;
try {
double start = now();
auto traceMsg = "PingingDatabaseLiveness_" + context;
TraceEvent(traceMsg.c_str()).log();
co_await timeoutError(pingDatabase(cx), databasePingDelay);
double pingTime = now() - start;
ASSERT(pingTime > 0);
TraceEvent(("PingingDatabaseLivenessDone_" + context).c_str()).detail("TimeTaken", pingTime);
co_await delay(databasePingDelay - pingTime);
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled) {
TraceEvent(SevError, ("PingingDatabaseLivenessError_" + context).c_str())
.error(e)
.detail("PingDelay", databasePingDelay);
}
throw;
}
}
}
namespace {
template <class T>
void sendResult(ReplyPromise<T>& reply, Optional<ErrorOr<T>> const& result) {
auto& res = result.get();
if (res.isError())
reply.sendError(res.getError());
else
reply.send(res.get());
}
Future<Reference<TestWorkload>> getConsistencyCheckUrgentWorkloadIface(WorkloadRequest work,
Reference<IClusterConnectionRecord> ccr,
Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
WorkloadContext wcx;
wcx.clientId = work.clientId;
wcx.clientCount = work.clientCount;
wcx.sharedRandomNumber = work.sharedRandomNumber;
wcx.ccr = ccr;
wcx.dbInfo = dbInfo;
wcx.rangesToCheck = work.rangesToCheck;
Reference<TestWorkload> iface = co_await getWorkloadIface(work, ccr, work.options[0], dbInfo);
co_return iface;
}
Future<Void> runConsistencyCheckUrgentWorkloadAsync(Database cx,
WorkloadInterface workIface,
Reference<TestWorkload> workload) {
ReplyPromise<Void> jobReq = co_await workIface.start.getFuture();
try {
TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadReceived", workIface.id())
.detail("WorkloadName", workload->description())
.detail("ClientCount", workload->clientCount)
.detail("ClientId", workload->clientId);
co_await workload->start(cx);
jobReq.send(Void());
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadError", workIface.id())
.errorUnsuppressed(e)
.detail("WorkloadName", workload->description())
.detail("ClientCount", workload->clientCount)
.detail("ClientId", workload->clientId);
jobReq.sendError(consistency_check_urgent_task_failed());
}
co_return;
}
Future<Void> testerServerConsistencyCheckerUrgentWorkload(WorkloadRequest work,
Reference<IClusterConnectionRecord> ccr,
Reference<AsyncVar<struct ServerDBInfo> const> dbInfo) {
WorkloadInterface workIface;
bool replied = false;
try {
Database cx = openDBOnServer(dbInfo);
co_await delay(1.0);
Reference<TestWorkload> workload = co_await getConsistencyCheckUrgentWorkloadIface(work, ccr, dbInfo);
Future<Void> test = runConsistencyCheckUrgentWorkloadAsync(cx, workIface, workload);
work.reply.send(workIface);
replied = true;
co_await test;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterRunWorkloadFailed").errorUnsuppressed(e);
if (!replied) {
work.reply.sendError(e);
}
}
co_return;
}
Future<Void> runWorkloadAsync(Database cx,
WorkloadInterface workIface,
Reference<TestWorkload> workload,
double databasePingDelay) {
Optional<ErrorOr<Void>> setupResult;
Optional<ErrorOr<Void>> startResult;
Optional<ErrorOr<CheckReply>> checkResult;
ReplyPromise<Void> setupReq;
ReplyPromise<Void> startReq;
ReplyPromise<CheckReply> checkReq;
ReplyPromise<std::vector<PerfMetric>> metricsReq;
TraceEvent("TestBeginAsync", workIface.id())
.detail("Workload", workload->description())
.detail("DatabasePingDelay", databasePingDelay);
Future<Void> databaseError =
databasePingDelay == 0.0 ? Never() : testDatabaseLiveness(cx, databasePingDelay, "RunWorkloadAsync");
ReplyPromise<Void> stopReq;
while (true) {
int action = 0;
co_await Choose()
.When(workIface.setup.getFuture(),
[&](ReplyPromise<Void> const& req) {
setupReq = req;
action = 1;
})
.When(workIface.start.getFuture(),
[&](ReplyPromise<Void> const& req) {
startReq = req;
action = 2;
})
.When(workIface.check.getFuture(),
[&](ReplyPromise<CheckReply> const& req) {
checkReq = req;
action = 3;
})
.When(workIface.metrics.getFuture(),
[&](ReplyPromise<std::vector<PerfMetric>> const& req) {
metricsReq = req;
action = 4;
})
.When(workIface.stop.getFuture(),
[&](ReplyPromise<Void> const& r) {
stopReq = r;
action = 5;
})
.run();
if (action == 5) {
stopReq.send(Void());
break;
}
if (action == 1) {
// setup
printf("Test received trigger for setup...\n");
TraceEvent("TestSetupBeginning", workIface.id()).detail("Workload", workload->description());
if (!setupResult.present()) {
try {
co_await (workload->setup(cx) || databaseError);
TraceEvent("TestSetupComplete", workIface.id()).detail("Workload", workload->description());
setupResult = Void();
} catch (Error& e) {
setupResult = operation_failed();
TraceEvent(SevError, "TestSetupError", workIface.id())
.error(e)
.detail("Workload", workload->description());
if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
throw;
}
}
sendResult(setupReq, setupResult);
} else if (action == 2) {
// start
if (!startResult.present()) {
try {
TraceEvent("TestStarting", workIface.id()).detail("Workload", workload->description());
co_await (workload->start(cx) || databaseError);
startResult = Void();
} catch (Error& e) {
startResult = operation_failed();
if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
throw;
TraceEvent(SevError, "TestFailure", workIface.id())
.errorUnsuppressed(e)
.detail("Reason", "Error starting workload")
.detail("Workload", workload->description());
}
TraceEvent("TestComplete", workIface.id())
.detail("Workload", workload->description())
.detail("OK", !startResult.get().isError());
printf("%s complete\n", workload->description().c_str());
}
sendResult(startReq, startResult);
} else if (action == 3) {
// check
if (!checkResult.present()) {
try {
TraceEvent("TestChecking", workIface.id()).detail("Workload", workload->description());
bool check = co_await timeoutError(workload->check(cx), workload->getCheckTimeout());
checkResult = CheckReply{ (!startResult.present() || !startResult.get().isError()) && check };
} catch (Error& e) {
checkResult = operation_failed();
if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
throw;
TraceEvent(SevError, "TestFailure", workIface.id())
.error(e)
.detail("Reason", "Error checking workload")
.detail("Workload", workload->description());
}
TraceEvent("TestCheckComplete", workIface.id()).detail("Workload", workload->description());
}
sendResult(checkReq, checkResult);
} else if (action == 4) {
// metrics
try {
std::vector<PerfMetric> m = co_await workload->getMetrics();
TraceEvent("WorkloadSendMetrics", workIface.id()).detail("Count", m.size());
metricsReq.send(m);
} catch (Error& e) {
if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
throw;
TraceEvent(SevError, "WorkloadSendMetrics", workIface.id()).error(e);
metricsReq.sendError(operation_failed());
}
}
}
co_return;
}
Future<Void> testerServerWorkload(WorkloadRequest work,
Reference<IClusterConnectionRecord> ccr,
Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
LocalityData locality) {
WorkloadInterface workIface;
bool replied = false;
Database cx;
try {
std::map<std::string, std::string> details;
details["WorkloadTitle"] = printable(work.title);
details["ClientId"] = format("%d", work.clientId);
details["ClientCount"] = format("%d", work.clientCount);
details["WorkloadTimeout"] = format("%d", work.timeout);
startRole(Role::TESTER, workIface.id(), UID(), details);
if (work.useDatabase) {
cx = Database::createDatabase(ccr, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
co_await delay(1.0);
}
// add test for "done" ?
TraceEvent("WorkloadReceived", workIface.id()).detail("Title", work.title);
Reference<TestWorkload> workload = co_await getWorkloadIface(work, ccr, dbInfo);
if (!workload) {
TraceEvent("TestCreationError").detail("Reason", "Workload could not be created");
fprintf(stderr, "ERROR: The workload could not be created.\n");
throw test_specification_invalid();
}
Future<Void> test = runWorkloadAsync(cx, workIface, workload, work.databasePingDelay) ||
traceRole(Role::TESTER, workIface.id());
work.reply.send(workIface);
replied = true;
if (work.timeout > 0) {
test = timeoutError(test, work.timeout);
}
co_await test;
endRole(Role::TESTER, workIface.id(), "Complete");
} catch (Error& e) {
TraceEvent(SevDebug, "TesterWorkloadFailed").errorUnsuppressed(e);
if (!replied) {
if (e.code() == error_code_test_specification_invalid)
work.reply.sendError(e);
else
work.reply.sendError(operation_failed());
}
bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete ||
e.code() == error_code_actor_cancelled;
endRole(Role::TESTER, workIface.id(), "Error", ok, e);
if (e.code() != error_code_test_specification_invalid && e.code() != error_code_timed_out) {
throw; // fatal errors will kill the testerServer as well
}
}
co_return;
}
} // namespace
Future<Void> testerServerCore(TesterInterface const& interf,
Reference<IClusterConnectionRecord> const& ccr,
Reference<AsyncVar<struct ServerDBInfo> const> const& dbInfo,
LocalityData const& locality,
Optional<std::string> const& expectedWorkLoad) {
// C++20 coroutine safety: const& parameters only store the reference in the coroutine frame,
// not the object. The referred-to object may be destroyed after the coroutine suspends
// (e.g. local variables in a caller's if-block, or temporaries from default arguments).
// Copy all const& parameters to ensure they survive across suspend points.
TesterInterface interfCopy = interf;
Reference<IClusterConnectionRecord> ccrCopy = ccr;
Reference<AsyncVar<struct ServerDBInfo> const> dbInfoCopy = dbInfo;
LocalityData localityCopy = locality;
Optional<std::string> expectedWorkLoadCopy = expectedWorkLoad;
PromiseStream<Future<Void>> addWorkload;
Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
// Dedicated to consistencyCheckerUrgent
// At any time, we only allow at most 1 consistency checker workload on a server
std::pair<int64_t, Future<Void>> consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>());
TraceEvent("StartingTesterServerCore", interfCopy.id())
.detail("ExpectedWorkload", expectedWorkLoadCopy.present() ? expectedWorkLoadCopy.get() : "[Unset]");
while (true) {
int action = 0;
WorkloadRequest workReq;
Future<Void> checkerDone =
consistencyCheckerUrgentTester.second.isValid() ? consistencyCheckerUrgentTester.second : Never();
co_await Choose()
.When(workerFatalError, [&](Void const&) { action = 1; })
.When(checkerDone, [&](Void const&) { action = 2; })
.When(interfCopy.recruitments.getFuture(),
[&](WorkloadRequest const& work) {
workReq = work;
action = 3;
})
.run();
if (action == 1) {
break;
}
if (action == 2) {
ASSERT(consistencyCheckerUrgentTester.first != 0);
TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadEnd", interfCopy.id())
.detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first);
consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>()); // reset
continue;
}
if (action == 3) {
if (expectedWorkLoadCopy.present() && expectedWorkLoadCopy.get() != workReq.title) {
TraceEvent(SevError, "StartingTesterServerCoreUnexpectedWorkload", interfCopy.id())
.detail("ClientId", workReq.clientId)
.detail("ClientCount", workReq.clientCount)
.detail("ExpectedWorkLoad", expectedWorkLoadCopy.get())
.detail("WorkLoad", workReq.title);
// Drop the workload
} else if (workReq.title == "ConsistencyCheckUrgent") {
// The workload is a consistency checker urgent workload
if (workReq.sharedRandomNumber == consistencyCheckerUrgentTester.first) {
// A single req can be sent for multiple times. In this case, the sharedRandomNumber is same as
// the existing one. For this scenario, we reply an error. This case should be rare.
TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterDuplicatedRequest", interfCopy.id())
.detail("ConsistencyCheckerId", workReq.sharedRandomNumber)
.detail("ClientId", workReq.clientId)
.detail("ClientCount", workReq.clientCount);
workReq.reply.sendError(consistency_check_urgent_duplicate_request());
} else {
// When the req.sharedRandomNumber is different from the existing one, the cluster has muiltiple
// consistencycheckurgent roles at the same time. Evenutally, the cluster will have only one
// consistencycheckurgent role in a stable state. So, in this case, we simply let the new request to
// overwrite the old request. After the work is destroyed, the broken_promise will be replied.
if (consistencyCheckerUrgentTester.second.isValid() &&
!consistencyCheckerUrgentTester.second.isReady()) {
TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_TesterWorkloadConflict", interfCopy.id())
.detail("ExistingConsistencyCheckerId", consistencyCheckerUrgentTester.first)
.detail("ArrivingConsistencyCheckerId", workReq.sharedRandomNumber)
.detail("ClientId", workReq.clientId)
.detail("ClientCount", workReq.clientCount);
}
consistencyCheckerUrgentTester =
std::make_pair(workReq.sharedRandomNumber,
testerServerConsistencyCheckerUrgentWorkload(workReq, ccrCopy, dbInfoCopy));
TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadInitialized", interfCopy.id())
.detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
.detail("ClientId", workReq.clientId)
.detail("ClientCount", workReq.clientCount);
}
} else {
addWorkload.send(testerServerWorkload(workReq, ccrCopy, dbInfoCopy, localityCopy));
}
}
}
co_return;
}
Reference in New Issue View Git Blame Copy Permalink