wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add some basic support for running multiple extra clusters in simulation. Use this to simulate a metacluster in some tests.

This commit is contained in:
A.J. Beamon 2022-06-10 10:08:18 -07:00
parent 739fc9ce6b
commit 986dd67278
35 changed files with 415 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
fdbrpc/sim_validation.cpp
View File

@ -51,13 +51,13 @@ void debug_advanceVersion(UID id, int64_t version, const char* suffix) {
}
void debug_advanceMinCommittedVersion(UID id, int64_t version) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return;
debug_advanceVersion(id, version, "min");
}
void debug_advanceMaxCommittedVersion(UID id, int64_t version) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return;
debug_advanceVersion(id, version, "max");
}
@ -67,7 +67,7 @@ bool debug_checkPartRestoredVersion(UID id,
std::string context,
std::string minormax,
Severity sev = SevError) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
if (disabledMachines.count(id))
return false;
@ -88,33 +88,33 @@ bool debug_checkPartRestoredVersion(UID id,
}
bool debug_checkRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
return debug_checkPartRestoredVersion(id, version, context, "min", sev) ||
debug_checkPartRestoredVersion(id, version, context, "max", sev);
}
void debug_removeVersions(UID id) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return;
validationData.erase(id.toString() + "min");
validationData.erase(id.toString() + "max");
}
bool debug_versionsExist(UID id) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
return validationData.count(id.toString() + "min") != 0 || validationData.count(id.toString() + "max") != 0;
}
bool debug_checkMinRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
return debug_checkPartRestoredVersion(id, version, context, "min", sev);
}
bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
return debug_checkPartRestoredVersion(id, version, context, "max", sev);
}
@ -129,13 +129,13 @@ void debug_setCheckRelocationDuration(bool check) {
checkRelocationDuration = check;
}
void debug_advanceVersionTimestamp(int64_t version, double t) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return;
timedVersionsValidationData[version] = t;
}
bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
if (!g_network->isSimulated() || g_simulator.extraDB)
if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty())
return false;
if (!timedVersionsValidationData.count(version)) {
TraceEvent(SevWarn, (context + "UnknownTime").c_str())

23
fdbrpc/simulator.h
View File

@ -54,6 +54,25 @@ public:
enum TSSMode { Disabled, EnabledNormal, EnabledAddDelay, EnabledDropMutations };
enum class BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
enum class ExtraDatabaseMode { Disabled, LocalOrSingle, Single, Local, Multiple };
static ExtraDatabaseMode stringToExtraDatabaseMode(std::string databaseMode) {
if (databaseMode == "Disabled") {
return ExtraDatabaseMode::Disabled;
} else if (databaseMode == "LocalOrSingle") {
return ExtraDatabaseMode::LocalOrSingle;
} else if (databaseMode == "Single") {
return ExtraDatabaseMode::Single;
} else if (databaseMode == "Local") {
return ExtraDatabaseMode::Local;
} else if (databaseMode == "Multiple") {
return ExtraDatabaseMode::Multiple;
} else {
TraceEvent(SevError, "UnknownExtraDatabaseMode").detail("DatabaseMode", databaseMode);
ASSERT(false);
throw internal_error();
}
};
// Subclasses may subclass ProcessInfo as well
struct MachineInfo;
@ -392,7 +411,7 @@ public:
allSwapsDisabled = false;
}
bool canSwapToMachine(Optional<Standalone<StringRef>> zoneId) const {
return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && !extraDB;
return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && extraDatabases.empty();
}
void enableSwapsToAll() {
swapsDisabled.clear();
@ -419,7 +438,7 @@ public:
int listenersPerProcess;
std::set<NetworkAddress> protectedAddresses;
std::map<NetworkAddress, ProcessInfo*> currentlyRebootingProcesses;
std::unique_ptr<class ClusterConnectionString> extraDB;
std::vector<class ClusterConnectionString> extraDatabases;
Reference<IReplicationPolicy> storagePolicy;
Reference<IReplicationPolicy> tLogPolicy;
int32_t tLogWriteAntiQuorum;

145
fdbserver/SimulatedCluster.actor.cpp
View File

@ -220,8 +220,12 @@ class TestConfig {
std::string attrib = removeWhitespace(line.substr(0, found));
std::string value = removeWhitespace(line.substr(found + 1));
if (attrib == "extraDB") {
sscanf(value.c_str(), "%d", &extraDB);
if (attrib == "extraDatabaseMode") {
extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(value);
}
if (attrib == "extraDatabaseCount") {
sscanf(value.c_str(), "%d", &extraDatabaseCount);
}
if (attrib == "minimumReplication") {
@ -291,7 +295,9 @@ class TestConfig {
ConfigDBType configDBType{ ConfigDBType::DISABLED };
public:
int extraDB = 0;
ISimulator::ExtraDatabaseMode extraDatabaseMode = ISimulator::ExtraDatabaseMode::Disabled;
// The number of extra database used if the database mode is MULTIPLE
int extraDatabaseCount = 1;
int minimumReplication = 0;
int minimumRegions = 0;
bool configureLocked = false;
@ -354,7 +360,9 @@ public:
return;
}
ConfigBuilder builder;
builder.add("extraDB", &extraDB)
std::string extraDatabaseModeStr;
builder.add("extraDatabaseMode", &extraDatabaseModeStr)
.add("extraDatabaseCount", &extraDatabaseCount)
.add("minimumReplication", &minimumReplication)
.add("minimumRegions", &minimumRegions)
.add("configureLocked", &configureLocked)
@ -406,6 +414,9 @@ public:
if (!isFirstTestInRestart) {
isFirstTestInRestart = tomlKeyPresent(file, "restartInfoLocation");
}
if (!extraDatabaseModeStr.empty()) {
extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(extraDatabaseModeStr);
}
} catch (std::exception& e) {
std::cerr << e.what() << std::endl;
TraceEvent("TOMLParseError").detail("Error", printable(e.what()));
@ -466,22 +477,23 @@ ACTOR Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
}
if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
ASSERT(g_simulator.extraDatabases.size() == 1);
Database cx = Database::createDatabase(connRecord, -1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
state Database extraDB = Database::createDatabase(extraFile, -1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
state Database drDatabase = Database::createDatabase(extraFile, -1);
TraceEvent("StartingDrAgents")
.detail("ConnectionString", connRecord->getConnectionString().toString())
.detail("ExtraString", extraFile->getConnectionString().toString());
state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB);
state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(drDatabase);
auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
agentFutures.push_back(dbAgent.run(extraDB, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
agentFutures.push_back(dbAgent.run(drDatabase, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
while (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
wait(delay(1.0));
@ -1087,10 +1099,10 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
if (tssModeStr != nullptr) {
g_simulator.tssMode = (ISimulator::TSSMode)atoi(tssModeStr);
}
bool enableExtraDB = (testConfig.extraDB == 3);
ClusterConnectionString conn(ini.GetValue("META", "connectionString"));
if (enableExtraDB) {
g_simulator.extraDB = std::make_unique<ClusterConnectionString>(ini.GetValue("META", "connectionString"));
if (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local) {
g_simulator.extraDatabases.clear();
g_simulator.extraDatabases.push_back(conn);
}
if (!testConfig.disableHostname) {
auto mockDNSStr = ini.GetValue("META", "mockDNS");
@ -1251,7 +1263,8 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
// Configuration details compiled in a structure used when setting up a simulated cluster
struct SimulationConfig {
explicit SimulationConfig(const TestConfig& testConfig);
int extraDB;
ISimulator::ExtraDatabaseMode extraDatabaseMode;
int extraDatabaseCount;
bool generateFearless;
DatabaseConfiguration db;
@ -1280,7 +1293,8 @@ private:
void generateNormalConfig(const TestConfig& testConfig);
};
SimulationConfig::SimulationConfig(const TestConfig& testConfig) : extraDB(testConfig.extraDB) {
SimulationConfig::SimulationConfig(const TestConfig& testConfig)
: extraDatabaseMode(testConfig.extraDatabaseMode), extraDatabaseCount(testConfig.extraDatabaseCount) {
generateNormalConfig(testConfig);
}
@ -1726,7 +1740,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
machine_count = std::max(datacenters + 2,
((db.minDatacentersRequired() > 0) ? datacenters : 1) *
std::max(3, db.minZonesRequiredPerDatacenter()));
machine_count = deterministicRandom()->randomInt(machine_count, std::max(machine_count + 1, extraDB ? 6 : 10));
machine_count = deterministicRandom()->randomInt(
machine_count,
std::max(machine_count + 1, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
// generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that
// if we do not remove the surplus server and machine teams, the simulation test will report error.
// This is needed to make sure the number of server (and machine) teams is no larger than the desired number.
@ -1736,7 +1752,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
// while the max possible machine team number is 10.
// If machine_count > 5, we can still test the effectivenss of machine teams
// Note: machine_count may be much larger than 5 because we may have a big replication factor
machine_count = std::max(machine_count, deterministicRandom()->randomInt(5, extraDB ? 6 : 10));
machine_count = std::max(machine_count,
deterministicRandom()->randomInt(
5, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
}
}
machine_count += datacenters * testConfig.extraMachineCountDC;
@ -1763,7 +1781,8 @@ void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) {
} else if (generateFearless) {
processes_per_machine = 1;
} else {
processes_per_machine = deterministicRandom()->randomInt(1, (extraDB ? 14 : 28) / machine_count + 2);
processes_per_machine = deterministicRandom()->randomInt(
1, (extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 28 : 14) / machine_count + 2);
}
}
@ -2100,26 +2119,23 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
conn = ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr);
}
// If extraDB==0, leave g_simulator.extraDB as null because the test does not use DR.
if (testConfig.extraDB == 1) {
// The DR database can be either a new database or itself
g_simulator.extraDB =
BUGGIFY
? (useHostname ? std::make_unique<ClusterConnectionString>(coordinatorHostnames, "TestCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(coordinatorAddresses, "TestCluster:0"_sr))
: (useHostname
? std::make_unique<ClusterConnectionString>(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(extraCoordinatorAddresses, "ExtraCluster:0"_sr));
} else if (testConfig.extraDB == 2) {
// The DR database is a new database
g_simulator.extraDB =
useHostname ? std::make_unique<ClusterConnectionString>(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(extraCoordinatorAddresses, "ExtraCluster:0"_sr);
} else if (testConfig.extraDB == 3) {
// The DR database is the same database
g_simulator.extraDB = useHostname
? std::make_unique<ClusterConnectionString>(coordinatorHostnames, "TestCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(coordinatorAddresses, "TestCluster:0"_sr);
bool useLocalDatabase = (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::LocalOrSingle && BUGGIFY) ||
testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local;
if (useLocalDatabase) {
g_simulator.extraDatabases.push_back(useHostname
? ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr)
: ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr));
} else if (testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
int extraDatabaseCount =
testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Multiple && testConfig.extraDatabaseCount > 0
? testConfig.extraDatabaseCount
: 1;
for (int i = 0; i < extraDatabaseCount; ++i) {
g_simulator.extraDatabases.push_back(
useHostname
? ClusterConnectionString(extraCoordinatorHostnames, StringRef(format("ExtraCluster%04d:0", i)))
: ClusterConnectionString(extraCoordinatorAddresses, StringRef(format("ExtraCluster%04d:0", i))));
}
}
*pConnString = conn;
@ -2128,7 +2144,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
.detail("String", conn.toString())
.detail("ConfigString", startingConfigString);
bool requiresExtraDBMachines = testConfig.extraDB && g_simulator.extraDB->toString() != conn.toString();
bool requiresExtraDBMachines = !g_simulator.extraDatabases.empty() && !useLocalDatabase;
int assignedMachines = 0, nonVersatileMachines = 0;
bool gradualMigrationPossible = true;
std::vector<ProcessClass::ClassType> processClassesSubSet = { ProcessClass::UnsetClass,
@ -2236,31 +2252,35 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
"SimulatedMachine"));
if (requiresExtraDBMachines) {
std::vector<IPAddress> extraIps;
extraIps.reserve(processesPerMachine);
for (int i = 0; i < processesPerMachine; i++) {
extraIps.push_back(
makeIPAddressForSim(useIPv6, { 4, dc, deterministicRandom()->randomInt(1, i + 2), machine }));
int cluster = 4;
for (auto extraDatabase : g_simulator.extraDatabases) {
std::vector<IPAddress> extraIps;
extraIps.reserve(processesPerMachine);
for (int i = 0; i < processesPerMachine; i++) {
extraIps.push_back(makeIPAddressForSim(
useIPv6, { cluster, dc, deterministicRandom()->randomInt(1, i + 2), machine }));
}
Standalone<StringRef> newMachineId(deterministicRandom()->randomUniqueID().toString());
LocalityData localities(Optional<Standalone<StringRef>>(), newZoneId, newMachineId, dcUID);
localities.set("data_hall"_sr, dcUID);
systemActors->push_back(reportErrors(simulatedMachine(extraDatabase,
extraIps,
sslEnabled,
localities,
processClass,
baseFolder,
false,
machine == useSeedForMachine,
AgentNone,
sslOnly,
whitelistBinPaths,
protocolVersion,
configDBType),
"SimulatedMachine"));
}
Standalone<StringRef> newMachineId(deterministicRandom()->randomUniqueID().toString());
LocalityData localities(Optional<Standalone<StringRef>>(), newZoneId, newMachineId, dcUID);
localities.set("data_hall"_sr, dcUID);
systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB,
extraIps,
sslEnabled,
localities,
processClass,
baseFolder,
false,
machine == useSeedForMachine,
AgentNone,
sslOnly,
whitelistBinPaths,
protocolVersion,
configDBType),
"SimulatedMachine"));
++cluster;
}
assignedMachines++;
@ -2386,7 +2406,8 @@ ACTOR void setupAndRun(std::string dataFolder,
// Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate
// the tenant map and related state.
// TODO: reenable when backup/DR or BlobGranule supports tenants.
if (std::string_view(testFile).find("Backup") != std::string_view::npos || testConfig.extraDB != 0) {
if (std::string_view(testFile).find("Backup") != std::string_view::npos ||
testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
allowDefaultTenant = false;
}

5
fdbserver/tester.actor.cpp
View File

@ -1141,7 +1141,10 @@ ACTOR Future<bool> runTest(Database cx,
std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKeys = {
// These are read by SimulatedCluster and used before testers exist. Thus, they must
// be recognized and accepted, but there's no point in placing them into a testSpec.
{ "extraDB", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDB", ""); } },
{ "extraDatabaseMode",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } },
{ "extraDatabaseCount",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseCount", ""); } },
{ "configureLocked",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); } },
{ "minimumReplication",

5
fdbserver/workloads/ApiWorkload.h
View File

@ -285,9 +285,10 @@ struct ApiWorkload : TestWorkload {
minValueLength = getOption(options, LiteralStringRef("minValueLength"), 1);
maxValueLength = getOption(options, LiteralStringRef("maxValueLength"), 10000);
useExtraDB = g_network->isSimulated() && g_simulator.extraDB != nullptr;
useExtraDB = g_network->isSimulated() && !g_simulator.extraDatabases.empty();
if (useExtraDB) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
}
}

3
fdbserver/workloads/AtomicSwitchover.actor.cpp
View File

@ -39,7 +39,8 @@ struct AtomicSwitchoverWorkload : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
}

3
fdbserver/workloads/BackupToDBAbort.actor.cpp
View File

@ -36,7 +36,8 @@ struct BackupToDBAbort : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
lockid = UID(0xbeeffeed, 0xdecaf00d);

3
fdbserver/workloads/BackupToDBCorrectness.actor.cpp
View File

@ -128,7 +128,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
}
}
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
TraceEvent("BARW_Start").detail("Locked", locked);

3
fdbserver/workloads/BackupToDBUpgrade.actor.cpp
View File

@ -76,7 +76,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
}
}
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
TraceEvent("DRU_Start").log();

64
fdbserver/workloads/ChangeConfig.actor.cpp
View File

@ -53,37 +53,45 @@ struct ChangeConfigWorkload : TestWorkload {
void getMetrics(std::vector<PerfMetric>& m) override {}
// When simulated two clusters for DR tests, this actor sets the starting configuration
// for the extra cluster.
ACTOR Future<Void> extraDatabaseConfigure(ChangeConfigWorkload* self) {
if (g_network->isSimulated() && g_simulator.extraDB) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
state Database extraDB = Database::createDatabase(extraFile, -1);
wait(delay(5 * deterministicRandom()->random01()));
if (self->configMode.size()) {
if (g_simulator.startingDisabledConfiguration != "") {
// It is not safe to allow automatic failover to a region which is not fully replicated,
// so wait for both regions to be fully replicated before enabling failover
wait(success(ManagementAPI::changeConfig(
extraDB.getReference(), g_simulator.startingDisabledConfiguration, true)));
TraceEvent("WaitForReplicasExtra").log();
wait(waitForFullReplication(extraDB));
TraceEvent("WaitForReplicasExtraEnd").log();
}
wait(success(ManagementAPI::changeConfig(extraDB.getReference(), self->configMode, true)));
ACTOR Future<Void> configureExtraDatabase(ChangeConfigWorkload* self, Database db) {
wait(delay(5 * deterministicRandom()->random01()));
if (self->configMode.size()) {
if (g_simulator.startingDisabledConfiguration != "") {
// It is not safe to allow automatic failover to a region which is not fully replicated,
// so wait for both regions to be fully replicated before enabling failover
wait(success(
ManagementAPI::changeConfig(db.getReference(), g_simulator.startingDisabledConfiguration, true)));
TraceEvent("WaitForReplicasExtra").log();
wait(waitForFullReplication(db));
TraceEvent("WaitForReplicasExtraEnd").log();
}
if (self->networkAddresses.size()) {
if (self->networkAddresses == "auto")
wait(CoordinatorsChangeActor(extraDB, self, true));
else
wait(CoordinatorsChangeActor(extraDB, self));
}
wait(delay(5 * deterministicRandom()->random01()));
wait(success(ManagementAPI::changeConfig(db.getReference(), self->configMode, true)));
}
if (self->networkAddresses.size()) {
if (self->networkAddresses == "auto")
wait(CoordinatorsChangeActor(db, self, true));
else
wait(CoordinatorsChangeActor(db, self));
}
wait(delay(5 * deterministicRandom()->random01()));
return Void();
}
// When simulating multiple clusters, this actor sets the starting configuration
// for the extra clusters.
Future<Void> configureExtraDatabases(ChangeConfigWorkload* self) {
std::vector<Future<Void>> futures;
if (g_network->isSimulated()) {
for (auto extraDatabase : g_simulator.extraDatabases) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(extraDatabase);
Database db = Database::createDatabase(extraFile, -1);
futures.push_back(configureExtraDatabase(self, db));
}
}
return waitForAll(futures);
}
// Either changes the database configuration, or changes the coordinators based on the parameters
// of the workload.
ACTOR Future<Void> ChangeConfigClient(Database cx, ChangeConfigWorkload* self) {
@ -93,7 +101,7 @@ struct ChangeConfigWorkload : TestWorkload {
state bool extraConfigureBefore = deterministicRandom()->random01() < 0.5;
if (extraConfigureBefore) {
wait(self->extraDatabaseConfigure(self));
wait(self->configureExtraDatabases(self));
}
if (self->configMode.size()) {
@ -116,7 +124,7 @@ struct ChangeConfigWorkload : TestWorkload {
}
if (!extraConfigureBefore) {
wait(self->extraDatabaseConfigure(self));
wait(self->configureExtraDatabases(self));
}
return Void();

3
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -2037,8 +2037,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
}
ACTOR Future<bool> checkWorkerList(Database cx, ConsistencyCheckWorkload* self) {
if (g_simulator.extraDB)
if (!g_simulator.extraDatabases.empty()) {
return true;
}
std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
std::set<NetworkAddress> workerAddresses;

4
fdbserver/workloads/DifferentClustersSameRV.actor.cpp
View File

@ -37,8 +37,8 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
bool switchComplete = false;
DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
ASSERT(g_simulator.extraDB != nullptr);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
testDuration = getOption(options, LiteralStringRef("testDuration"), 100.0);
switchAfter = getOption(options, LiteralStringRef("switchAfter"), 50.0);

68
fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
View File

@ -35,6 +35,7 @@
struct TenantManagementConcurrencyWorkload : TestWorkload {
const TenantName tenantNamePrefix = "tenant_management_concurrency_workload_"_sr;
const Key testParametersKey = "test_parameters"_sr;
int maxTenants;
int maxTenantGroups;
@ -49,22 +50,50 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
maxTenantGroups = std::min<int>(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20));
testDuration = getOption(options, "testDuration"_sr, 60.0);
if (clientId == 0) {
useMetacluster = deterministicRandom()->coinflip();
} else {
// Other clients read the metacluster state from the database
useMetacluster = false;
}
}
std::string description() const override { return "TenantManagementConcurrency"; }
struct TestParameters {
constexpr static FileIdentifier file_identifier = 14350843;
bool useMetacluster = false;
TestParameters() {}
TestParameters(bool useMetacluster) : useMetacluster(useMetacluster) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, useMetacluster);
}
Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); }
static TestParameters decode(ValueRef const& value) {
TestParameters params;
ObjectReader reader(value.begin(), Unversioned());
reader.deserialize(params);
return params;
}
};
Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
ACTOR Future<Void> _setup(Database cx, TenantManagementConcurrencyWorkload* self) {
Reference<IDatabase> threadSafeHandle =
wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
TraceEvent("CreatedThreadSafeHandle");
MultiVersionApi::api->selectApiVersion(cx->apiVersion);
self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
if (self->useMetacluster) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
self->dataDb = Database::createDatabase(extraFile, -1);
if (self->clientId == 0) {
@ -72,12 +101,45 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, *g_simulator.extraDB, entry));
wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry));
}
} else {
self->dataDb = cx;
}
state Transaction tr(cx);
if (self->clientId == 0) {
// Send test parameters to the other clients
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.set(self->testParametersKey, TestParameters(self->useMetacluster).encode());
wait(tr.commit());
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
} else {
// Read the tenant subspace chosen and saved by client 0
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(tr.get(self->testParametersKey));
if (val.present()) {
TestParameters params = TestParameters::decode(val.get());
self->useMetacluster = params.useMetacluster;
break;
}
wait(delay(1.0));
tr.reset();
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
return Void();
}

99
fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -49,7 +49,7 @@ struct TenantManagementWorkload : TestWorkload {
Key tenantSubspace;
const Key keyName = "key"_sr;
const Key tenantSubspaceKey = "tenant_subspace"_sr;
const Key testParametersKey = "test_parameters"_sr;
const Value noTenantValue = "no_tenant"_sr;
const TenantName tenantNamePrefix = "tenant_management_workload_"_sr;
TenantName localTenantNamePrefix;
@ -98,36 +98,59 @@ struct TenantManagementWorkload : TestWorkload {
testDuration = getOption(options, "testDuration"_sr, 60.0);
localTenantNamePrefix = format("%stenant_%d_", tenantNamePrefix.toString().c_str(), clientId);
useMetacluster = false;
bool defaultUseMetacluster = false;
if (clientId == 0 && g_network->isSimulated() && !g_simulator.extraDatabases.empty()) {
defaultUseMetacluster = deterministicRandom()->coinflip();
}
useMetacluster = getOption(options, "useMetacluster"_sr, defaultUseMetacluster);
}
std::string description() const override { return "TenantManagement"; }
struct TestParameters {
constexpr static FileIdentifier file_identifier = 1527576;
Key tenantSubspace;
bool useMetacluster = false;
TestParameters() {}
TestParameters(Key tenantSubspace, bool useMetacluster)
: tenantSubspace(tenantSubspace), useMetacluster(useMetacluster) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, tenantSubspace, useMetacluster);
}
Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); }
static TestParameters decode(ValueRef const& value) {
TestParameters params;
ObjectReader reader(value.begin(), Unversioned());
reader.deserialize(params);
return params;
}
};
Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
ACTOR Future<Void> _setup(Database cx, TenantManagementWorkload* self) {
Reference<IDatabase> threadSafeHandle =
wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
TraceEvent("CreatedThreadSafeHandle");
MultiVersionApi::api->selectApiVersion(cx->apiVersion);
self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
if (self->useMetacluster) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
self->dataDb = Database::createDatabase(extraFile, -1);
if (self->clientId == 0) {
if (self->useMetacluster && self->clientId == 0) {
wait(success(ManagementAPI::changeConfig(cx.getReference(), "tenant_mode=management", true)));
DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, *g_simulator.extraDB, entry));
wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry));
}
} else {
self->dataDb = cx;
}
state Transaction tr(self->dataDb);
state Transaction tr(cx);
if (self->clientId == 0) {
// Configure the tenant subspace prefix that is applied to all tenants
// This feature isn't supported in a metacluster, so we skip it if doing a metacluster test.
@ -143,13 +166,12 @@ struct TenantManagementWorkload : TestWorkload {
}
}
// Set a key outside of all tenants to make sure that our tenants aren't writing to the regular key-space
// Also communicates the chosen tenant subspace to all other clients by storing it in a key
// Communicates test parameters to all other clients by storing it in a key
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.set(self->keyName, self->noTenantValue);
tr.set(self->tenantSubspaceKey, self->tenantSubspace);
tr.set(self->testParametersKey,
TestParameters(self->tenantSubspace, self->useMetacluster).encode());
tr.set(tenantDataPrefixKey, self->tenantSubspace);
wait(tr.commit());
break;
@ -157,14 +179,17 @@ struct TenantManagementWorkload : TestWorkload {
wait(tr.onError(e));
}
}
} else {
// Read the tenant subspace chosen and saved by client 0
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<Value> val = wait(tr.get(self->tenantSubspaceKey));
Optional<Value> val = wait(tr.get(self->testParametersKey));
if (val.present()) {
self->tenantSubspace = val.get();
TestParameters params = TestParameters::decode(val.get());
self->tenantSubspace = params.tenantSubspace;
self->useMetacluster = params.useMetacluster;
break;
}
@ -176,6 +201,29 @@ struct TenantManagementWorkload : TestWorkload {
}
}
if (self->useMetacluster) {
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
self->dataDb = Database::createDatabase(extraFile, -1);
} else {
self->dataDb = cx;
}
if (self->clientId == 0) {
// Set a key outside of all tenants to make sure that our tenants aren't writing to the regular key-space
state Transaction dataTr(self->dataDb);
loop {
try {
dataTr.setOption(FDBTransactionOptions::RAW_ACCESS);
dataTr.set(self->keyName, self->noTenantValue);
wait(dataTr.commit());
break;
} catch (Error& e) {
wait(dataTr.onError(e));
}
}
}
return Void();
}
@ -281,7 +329,8 @@ struct TenantManagementWorkload : TestWorkload {
loop {
try {
Optional<Void> result =
wait(timeout(self->createImpl(cx, tr, tenantsToCreate, operationType, self), 30));
wait(timeout(self->createImpl(cx, tr, tenantsToCreate, operationType, self),
deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
// Database operations shouldn't get here if the tenant already exists
@ -570,8 +619,9 @@ struct TenantManagementWorkload : TestWorkload {
state bool retried = false;
loop {
try {
Optional<Void> result = wait(timeout(
self->deleteImpl(cx, tr, beginTenant, endTenant, tenants, operationType, self), 30));
Optional<Void> result =
wait(timeout(self->deleteImpl(cx, tr, beginTenant, endTenant, tenants, operationType, self),
deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
// Database operations shouldn't get here if the tenant didn't exist
@ -608,9 +658,6 @@ struct TenantManagementWorkload : TestWorkload {
} else {
ASSERT(resultEntry.get().tenantState == TenantState::READY);
}
} else if (deterministicRandom()->coinflip()) {
// Nothing to delete, so randomly retry
break;
}
}

10
fdbserver/workloads/VersionStamp.actor.cpp
View File

@ -155,7 +155,8 @@ struct VersionStampWorkload : TestWorkload {
ACTOR Future<bool> _check(Database cx, VersionStampWorkload* self) {
if (self->validateExtraDB) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
cx = Database::createDatabase(extraFile, -1);
}
state ReadYourWritesTransaction tr(cx);
@ -312,8 +313,9 @@ struct VersionStampWorkload : TestWorkload {
state double lastTime = now();
state Database extraDB;
if (g_simulator.extraDB != nullptr) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
if (!g_simulator.extraDatabases.empty()) {
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
}
@ -380,7 +382,7 @@ struct VersionStampWorkload : TestWorkload {
} catch (Error& e) {
err = e;
if (err.code() == error_code_database_locked && g_simulator.extraDB != nullptr) {
if (err.code() == error_code_database_locked && !g_simulator.extraDatabases.empty()) {
//TraceEvent("VST_CommitDatabaseLocked");
cx_is_primary = !cx_is_primary;
tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB);

5
fdbserver/workloads/WriteDuringRead.actor.cpp
View File

@ -87,9 +87,10 @@ struct WriteDuringReadWorkload : TestWorkload {
TEST(adjacentKeys &&
(nodes + minNode) > CLIENT_KNOBS->KEY_SIZE_LIMIT); // WriteDuringReadWorkload testing large keys
useExtraDB = g_simulator.extraDB != nullptr;
useExtraDB = !g_simulator.extraDatabases.empty();
if (useExtraDB) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
ASSERT(g_simulator.extraDatabases.size() == 1);
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(g_simulator.extraDatabases[0]);
extraDB = Database::createDatabase(extraFile, -1);
useSystemKeys = false;
}

10
tests/CMakeLists.txt
View File

@ -250,12 +250,12 @@ if(WITH_PYTHON)
add_fdb_test(
TEST_FILES restarting/from_6.3.13/CycleTestRestart-1.txt
restarting/from_6.3.13/CycleTestRestart-2.txt)
add_fdb_test(
TEST_FILES restarting/from_6.3.13/DrUpgradeRestart-1.txt
restarting/from_6.3.13/DrUpgradeRestart-2.txt)
add_fdb_test(
TEST_FILES restarting/from_6.3.13/StorefrontTestRestart-1.txt
restarting/from_6.3.13/StorefrontTestRestart-2.txt)
add_fdb_test(
TEST_FILES restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt)
add_fdb_test(
TEST_FILES restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml)
@ -286,6 +286,9 @@ if(WITH_PYTHON)
add_fdb_test(
TEST_FILES restarting/from_7.1.0/VersionVectorEnableRestart-1.toml
restarting/from_7.1.0/VersionVectorEnableRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.txt
restarting/from_7.2.0/DrUpgradeRestart-2.txt)
add_fdb_test(TEST_FILES slow/ApiCorrectness.toml)
@ -323,6 +326,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapse.toml)
add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapseIncrement.toml)
add_fdb_test(TEST_FILES slow/SwizzledTenantManagement.toml)
add_fdb_test(TEST_FILES slow/SwizzledTenantManagementMetacluster.toml)
add_fdb_test(TEST_FILES slow/TenantManagement.toml)
add_fdb_test(TEST_FILES slow/TenantManagementConcurrency.toml)
add_fdb_test(TEST_FILES slow/VersionStampBackupToDB.toml)

2
tests/fast/AtomicBackupToDBCorrectness.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 1
extraDatabaseMode = 'LocalOrSingle'
[[test]]
testTitle = 'BackupAndRestore'

2
tests/fast/BackupToDBCorrectness.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 1
extraDatabaseMode = 'LocalOrSingle'
[[test]]
testTitle = 'BackupAndRestore'

2
tests/fast/BackupToDBCorrectnessClean.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 1
extraDatabaseMode = 'LocalOrSingle'
[[test]]
testTitle = 'BackupAndRestore'

2
tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt
View File

@ -1,4 +1,4 @@
extraDB=3
extraDatabaseMode=Local
testTitle=DrUpgrade
runSetup=false

0
tests/restarting/from_6.3.13/DrUpgradeRestart-1.txt → tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
View File

2
tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt → tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt
View File

@ -1,4 +1,4 @@
extraDB=3
extraDatabaseMode=Local
testTitle=DrUpgrade
runSetup=false

21
tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt Normal file
View File

@ -0,0 +1,21 @@
storageEngineExcludeTypes=3
extraDatabaseMode=Local
testTitle=DrUpgrade
clearAfterTest=false
simBackupAgents=BackupToDB
testName=Cycle
nodeCount=30000
transactionsPerSecond=2500.0
testDuration=30.0
expectedRate=0
testName=BackupToDBUpgrade
backupAfter=10.0
stopDifferentialAfter=50.0
backupRangesCount=-1
testName=SaveAndKill
restartInfoLocation=simfdb/restartInfo.ini
testDuration=40.0

18
tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt Normal file
View File

@ -0,0 +1,18 @@
extraDatabaseMode=Local
testTitle=DrUpgrade
runSetup=false
clearAfterTest=false
simBackupAgents=BackupToDB
waitForQuiescenceBegin=false
testName=Cycle
nodeCount=30000
transactionsPerSecond=2500.0
testDuration=30.0
expectedRate=0
testName=BackupToDBUpgrade
backupAfter=10.0
backupRangesCount=-1
stopDifferentialAfter=70.0

2
tests/slow/ApiCorrectnessSwitchover.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'ApiCorrectnessTest'

2
tests/slow/DifferentClustersSameRV.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'DifferentClustersSameRV'

2
tests/slow/SharedBackupCorrectness.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 1
extraDatabaseMode = 'LocalOrSingle'
[[test]]
testTitle = 'BackupAndRestore'

2
tests/slow/SharedBackupToDBCorrectness.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 1
extraDatabaseMode = 'LocalOrSingle'
[[test]]
testTitle = 'BackupAndRestore'

39
tests/slow/SwizzledTenantManagementMetacluster.toml Normal file
View File

@ -0,0 +1,39 @@
[configuration]
allowDefaultTenant = false
allowDisablingTenants = false
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'TenantManagementTest'
clearAfterTest = true
timeout = 2100
runSetup = true
[[test.workload]]
testName = 'TenantManagement'
maxTenants = 1000
testDuration = 60
[[test.workload]]
testName = 'RandomClogging'
testDuration = 120.0
swizzle = 1
[[test.workload]]
testName = 'Rollback'
testDuration = 120.0
meanDelay = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 120.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 120.0

1
tests/slow/TenantManagement.toml
View File

@ -1,6 +1,7 @@
[configuration]
allowDefaultTenant = false
allowDisablingTenants = false
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'TenantManagementTest'

2
tests/slow/TenantManagementConcurrency.toml
View File

@ -1,7 +1,7 @@
[configuration]
allowDefaultTenant = false
allowDisablingTenants = false
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'TenantManagementConcurrencyTest'

2
tests/slow/VersionStampBackupToDB.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'VersionStampBackupToDB'

2
tests/slow/VersionStampSwitchover.toml
View File

@ -1,5 +1,5 @@
[configuration]
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'VersionStampCorrectnessTest'

2
tests/slow/WriteDuringReadSwitchover.toml
View File

@ -1,6 +1,6 @@
[configuration]
StderrSeverity = 30
extraDB = 2
extraDatabaseMode = 'Single'
[[test]]
testTitle = 'WriteDuringReadTest'