diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp index 292c5255f3..111000ba61 100644 --- a/fdbrpc/sim_validation.cpp +++ b/fdbrpc/sim_validation.cpp @@ -51,13 +51,13 @@ void debug_advanceVersion(UID id, int64_t version, const char* suffix) { } void debug_advanceMinCommittedVersion(UID id, int64_t version) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return; debug_advanceVersion(id, version, "min"); } void debug_advanceMaxCommittedVersion(UID id, int64_t version) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return; debug_advanceVersion(id, version, "max"); } @@ -67,7 +67,7 @@ bool debug_checkPartRestoredVersion(UID id, std::string context, std::string minormax, Severity sev = SevError) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; if (disabledMachines.count(id)) return false; @@ -88,33 +88,33 @@ bool debug_checkPartRestoredVersion(UID id, } bool debug_checkRestoredVersion(UID id, int64_t version, std::string context, Severity sev) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; return debug_checkPartRestoredVersion(id, version, context, "min", sev) || debug_checkPartRestoredVersion(id, version, context, "max", sev); } void debug_removeVersions(UID id) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return; validationData.erase(id.toString() + "min"); validationData.erase(id.toString() + "max"); } bool debug_versionsExist(UID id) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; return validationData.count(id.toString() + "min") != 0 || validationData.count(id.toString() + "max") != 0; } bool debug_checkMinRestoredVersion(UID id, int64_t version, std::string context, Severity sev) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; return debug_checkPartRestoredVersion(id, version, context, "min", sev); } bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context, Severity sev) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; return debug_checkPartRestoredVersion(id, version, context, "max", sev); } @@ -129,13 +129,13 @@ void debug_setCheckRelocationDuration(bool check) { checkRelocationDuration = check; } void debug_advanceVersionTimestamp(int64_t version, double t) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return; timedVersionsValidationData[version] = t; } bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) { - if (!g_network->isSimulated() || g_simulator.extraDB) + if (!g_network->isSimulated() || !g_simulator.extraDatabases.empty()) return false; if (!timedVersionsValidationData.count(version)) { TraceEvent(SevWarn, (context + "UnknownTime").c_str()) diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index eb7b4d65ec..feb1a26334 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -54,6 +54,25 @@ public: enum TSSMode { Disabled, EnabledNormal, EnabledAddDelay, EnabledDropMutations }; enum class BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB }; + enum class ExtraDatabaseMode { Disabled, LocalOrSingle, Single, Local, Multiple }; + + static ExtraDatabaseMode stringToExtraDatabaseMode(std::string databaseMode) { + if (databaseMode == "Disabled") { + return ExtraDatabaseMode::Disabled; + } else if (databaseMode == "LocalOrSingle") { + return ExtraDatabaseMode::LocalOrSingle; + } else if (databaseMode == "Single") { + return ExtraDatabaseMode::Single; + } else if (databaseMode == "Local") { + return ExtraDatabaseMode::Local; + } else if (databaseMode == "Multiple") { + return ExtraDatabaseMode::Multiple; + } else { + TraceEvent(SevError, "UnknownExtraDatabaseMode").detail("DatabaseMode", databaseMode); + ASSERT(false); + throw internal_error(); + } + }; // Subclasses may subclass ProcessInfo as well struct MachineInfo; @@ -392,7 +411,7 @@ public: allSwapsDisabled = false; } bool canSwapToMachine(Optional> zoneId) const { - return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && !extraDB; + return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && extraDatabases.empty(); } void enableSwapsToAll() { swapsDisabled.clear(); @@ -419,7 +438,7 @@ public: int listenersPerProcess; std::set protectedAddresses; std::map currentlyRebootingProcesses; - std::unique_ptr extraDB; + std::vector extraDatabases; Reference storagePolicy; Reference tLogPolicy; int32_t tLogWriteAntiQuorum; diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 445abea918..72d638eb7c 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -220,8 +220,12 @@ class TestConfig { std::string attrib = removeWhitespace(line.substr(0, found)); std::string value = removeWhitespace(line.substr(found + 1)); - if (attrib == "extraDB") { - sscanf(value.c_str(), "%d", &extraDB); + if (attrib == "extraDatabaseMode") { + extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(value); + } + + if (attrib == "extraDatabaseCount") { + sscanf(value.c_str(), "%d", &extraDatabaseCount); } if (attrib == "minimumReplication") { @@ -291,7 +295,9 @@ class TestConfig { ConfigDBType configDBType{ ConfigDBType::DISABLED }; public: - int extraDB = 0; + ISimulator::ExtraDatabaseMode extraDatabaseMode = ISimulator::ExtraDatabaseMode::Disabled; + // The number of extra database used if the database mode is MULTIPLE + int extraDatabaseCount = 1; int minimumReplication = 0; int minimumRegions = 0; bool configureLocked = false; @@ -354,7 +360,9 @@ public: return; } ConfigBuilder builder; - builder.add("extraDB", &extraDB) + std::string extraDatabaseModeStr; + builder.add("extraDatabaseMode", &extraDatabaseModeStr) + .add("extraDatabaseCount", &extraDatabaseCount) .add("minimumReplication", &minimumReplication) .add("minimumRegions", &minimumRegions) .add("configureLocked", &configureLocked) @@ -406,6 +414,9 @@ public: if (!isFirstTestInRestart) { isFirstTestInRestart = tomlKeyPresent(file, "restartInfoLocation"); } + if (!extraDatabaseModeStr.empty()) { + extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(extraDatabaseModeStr); + } } catch (std::exception& e) { std::cerr << e.what() << std::endl; TraceEvent("TOMLParseError").detail("Error", printable(e.what())); @@ -466,22 +477,23 @@ ACTOR Future runDr(Reference connRecord) { } if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) { + ASSERT(g_simulator.extraDatabases.size() == 1); Database cx = Database::createDatabase(connRecord, -1); - auto extraFile = makeReference(*g_simulator.extraDB); - state Database extraDB = Database::createDatabase(extraFile, -1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); + state Database drDatabase = Database::createDatabase(extraFile, -1); TraceEvent("StartingDrAgents") .detail("ConnectionString", connRecord->getConnectionString().toString()) .detail("ExtraString", extraFile->getConnectionString().toString()); state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx); - state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB); + state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(drDatabase); auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); - agentFutures.push_back(dbAgent.run(extraDB, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); + agentFutures.push_back(dbAgent.run(drDatabase, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); while (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) { wait(delay(1.0)); @@ -1087,10 +1099,10 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor if (tssModeStr != nullptr) { g_simulator.tssMode = (ISimulator::TSSMode)atoi(tssModeStr); } - bool enableExtraDB = (testConfig.extraDB == 3); ClusterConnectionString conn(ini.GetValue("META", "connectionString")); - if (enableExtraDB) { - g_simulator.extraDB = std::make_unique(ini.GetValue("META", "connectionString")); + if (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local) { + g_simulator.extraDatabases.clear(); + g_simulator.extraDatabases.push_back(conn); } if (!testConfig.disableHostname) { auto mockDNSStr = ini.GetValue("META", "mockDNS"); @@ -1251,7 +1263,8 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor // Configuration details compiled in a structure used when setting up a simulated cluster struct SimulationConfig { explicit SimulationConfig(const TestConfig& testConfig); - int extraDB; + ISimulator::ExtraDatabaseMode extraDatabaseMode; + int extraDatabaseCount; bool generateFearless; DatabaseConfiguration db; @@ -1280,7 +1293,8 @@ private: void generateNormalConfig(const TestConfig& testConfig); }; -SimulationConfig::SimulationConfig(const TestConfig& testConfig) : extraDB(testConfig.extraDB) { +SimulationConfig::SimulationConfig(const TestConfig& testConfig) + : extraDatabaseMode(testConfig.extraDatabaseMode), extraDatabaseCount(testConfig.extraDatabaseCount) { generateNormalConfig(testConfig); } @@ -1726,7 +1740,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) { machine_count = std::max(datacenters + 2, ((db.minDatacentersRequired() > 0) ? datacenters : 1) * std::max(3, db.minZonesRequiredPerDatacenter())); - machine_count = deterministicRandom()->randomInt(machine_count, std::max(machine_count + 1, extraDB ? 6 : 10)); + machine_count = deterministicRandom()->randomInt( + machine_count, + std::max(machine_count + 1, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6)); // generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that // if we do not remove the surplus server and machine teams, the simulation test will report error. // This is needed to make sure the number of server (and machine) teams is no larger than the desired number. @@ -1736,7 +1752,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) { // while the max possible machine team number is 10. // If machine_count > 5, we can still test the effectivenss of machine teams // Note: machine_count may be much larger than 5 because we may have a big replication factor - machine_count = std::max(machine_count, deterministicRandom()->randomInt(5, extraDB ? 6 : 10)); + machine_count = std::max(machine_count, + deterministicRandom()->randomInt( + 5, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6)); } } machine_count += datacenters * testConfig.extraMachineCountDC; @@ -1763,7 +1781,8 @@ void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) { } else if (generateFearless) { processes_per_machine = 1; } else { - processes_per_machine = deterministicRandom()->randomInt(1, (extraDB ? 14 : 28) / machine_count + 2); + processes_per_machine = deterministicRandom()->randomInt( + 1, (extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 28 : 14) / machine_count + 2); } } @@ -2100,26 +2119,23 @@ void setupSimulatedSystem(std::vector>* systemActors, conn = ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr); } - // If extraDB==0, leave g_simulator.extraDB as null because the test does not use DR. - if (testConfig.extraDB == 1) { - // The DR database can be either a new database or itself - g_simulator.extraDB = - BUGGIFY - ? (useHostname ? std::make_unique(coordinatorHostnames, "TestCluster:0"_sr) - : std::make_unique(coordinatorAddresses, "TestCluster:0"_sr)) - : (useHostname - ? std::make_unique(extraCoordinatorHostnames, "ExtraCluster:0"_sr) - : std::make_unique(extraCoordinatorAddresses, "ExtraCluster:0"_sr)); - } else if (testConfig.extraDB == 2) { - // The DR database is a new database - g_simulator.extraDB = - useHostname ? std::make_unique(extraCoordinatorHostnames, "ExtraCluster:0"_sr) - : std::make_unique(extraCoordinatorAddresses, "ExtraCluster:0"_sr); - } else if (testConfig.extraDB == 3) { - // The DR database is the same database - g_simulator.extraDB = useHostname - ? std::make_unique(coordinatorHostnames, "TestCluster:0"_sr) - : std::make_unique(coordinatorAddresses, "TestCluster:0"_sr); + bool useLocalDatabase = (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::LocalOrSingle && BUGGIFY) || + testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local; + if (useLocalDatabase) { + g_simulator.extraDatabases.push_back(useHostname + ? ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr) + : ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr)); + } else if (testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) { + int extraDatabaseCount = + testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Multiple && testConfig.extraDatabaseCount > 0 + ? testConfig.extraDatabaseCount + : 1; + for (int i = 0; i < extraDatabaseCount; ++i) { + g_simulator.extraDatabases.push_back( + useHostname + ? ClusterConnectionString(extraCoordinatorHostnames, StringRef(format("ExtraCluster%04d:0", i))) + : ClusterConnectionString(extraCoordinatorAddresses, StringRef(format("ExtraCluster%04d:0", i)))); + } } *pConnString = conn; @@ -2128,7 +2144,7 @@ void setupSimulatedSystem(std::vector>* systemActors, .detail("String", conn.toString()) .detail("ConfigString", startingConfigString); - bool requiresExtraDBMachines = testConfig.extraDB && g_simulator.extraDB->toString() != conn.toString(); + bool requiresExtraDBMachines = !g_simulator.extraDatabases.empty() && !useLocalDatabase; int assignedMachines = 0, nonVersatileMachines = 0; bool gradualMigrationPossible = true; std::vector processClassesSubSet = { ProcessClass::UnsetClass, @@ -2236,31 +2252,35 @@ void setupSimulatedSystem(std::vector>* systemActors, "SimulatedMachine")); if (requiresExtraDBMachines) { - std::vector extraIps; - extraIps.reserve(processesPerMachine); - for (int i = 0; i < processesPerMachine; i++) { - extraIps.push_back( - makeIPAddressForSim(useIPv6, { 4, dc, deterministicRandom()->randomInt(1, i + 2), machine })); + int cluster = 4; + for (auto extraDatabase : g_simulator.extraDatabases) { + std::vector extraIps; + extraIps.reserve(processesPerMachine); + for (int i = 0; i < processesPerMachine; i++) { + extraIps.push_back(makeIPAddressForSim( + useIPv6, { cluster, dc, deterministicRandom()->randomInt(1, i + 2), machine })); + } + + Standalone newMachineId(deterministicRandom()->randomUniqueID().toString()); + + LocalityData localities(Optional>(), newZoneId, newMachineId, dcUID); + localities.set("data_hall"_sr, dcUID); + systemActors->push_back(reportErrors(simulatedMachine(extraDatabase, + extraIps, + sslEnabled, + localities, + processClass, + baseFolder, + false, + machine == useSeedForMachine, + AgentNone, + sslOnly, + whitelistBinPaths, + protocolVersion, + configDBType), + "SimulatedMachine")); } - - Standalone newMachineId(deterministicRandom()->randomUniqueID().toString()); - - LocalityData localities(Optional>(), newZoneId, newMachineId, dcUID); - localities.set("data_hall"_sr, dcUID); - systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB, - extraIps, - sslEnabled, - localities, - processClass, - baseFolder, - false, - machine == useSeedForMachine, - AgentNone, - sslOnly, - whitelistBinPaths, - protocolVersion, - configDBType), - "SimulatedMachine")); + ++cluster; } assignedMachines++; @@ -2386,7 +2406,8 @@ ACTOR void setupAndRun(std::string dataFolder, // Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate // the tenant map and related state. // TODO: reenable when backup/DR or BlobGranule supports tenants. - if (std::string_view(testFile).find("Backup") != std::string_view::npos || testConfig.extraDB != 0) { + if (std::string_view(testFile).find("Backup") != std::string_view::npos || + testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) { allowDefaultTenant = false; } diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index d56538741b..3eb4991979 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1141,7 +1141,10 @@ ACTOR Future runTest(Database cx, std::map> testSpecGlobalKeys = { // These are read by SimulatedCluster and used before testers exist. Thus, they must // be recognized and accepted, but there's no point in placing them into a testSpec. - { "extraDB", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDB", ""); } }, + { "extraDatabaseMode", + [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } }, + { "extraDatabaseCount", + [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseCount", ""); } }, { "configureLocked", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); } }, { "minimumReplication", diff --git a/fdbserver/workloads/ApiWorkload.h b/fdbserver/workloads/ApiWorkload.h index f61f1b5c5d..600d753fca 100644 --- a/fdbserver/workloads/ApiWorkload.h +++ b/fdbserver/workloads/ApiWorkload.h @@ -285,9 +285,10 @@ struct ApiWorkload : TestWorkload { minValueLength = getOption(options, LiteralStringRef("minValueLength"), 1); maxValueLength = getOption(options, LiteralStringRef("maxValueLength"), 10000); - useExtraDB = g_network->isSimulated() && g_simulator.extraDB != nullptr; + useExtraDB = g_network->isSimulated() && !g_simulator.extraDatabases.empty(); if (useExtraDB) { - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); } } diff --git a/fdbserver/workloads/AtomicSwitchover.actor.cpp b/fdbserver/workloads/AtomicSwitchover.actor.cpp index 0337da51b4..66e1981275 100644 --- a/fdbserver/workloads/AtomicSwitchover.actor.cpp +++ b/fdbserver/workloads/AtomicSwitchover.actor.cpp @@ -39,7 +39,8 @@ struct AtomicSwitchoverWorkload : TestWorkload { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); } diff --git a/fdbserver/workloads/BackupToDBAbort.actor.cpp b/fdbserver/workloads/BackupToDBAbort.actor.cpp index a505d8e02e..ec663138bd 100644 --- a/fdbserver/workloads/BackupToDBAbort.actor.cpp +++ b/fdbserver/workloads/BackupToDBAbort.actor.cpp @@ -36,7 +36,8 @@ struct BackupToDBAbort : TestWorkload { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); lockid = UID(0xbeeffeed, 0xdecaf00d); diff --git a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp index 415744a677..70f7ea1e1f 100644 --- a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp @@ -128,7 +128,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { } } - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); TraceEvent("BARW_Start").detail("Locked", locked); diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp index 6ccb691c9d..c49be5384f 100644 --- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp +++ b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp @@ -76,7 +76,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload { } } - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); TraceEvent("DRU_Start").log(); diff --git a/fdbserver/workloads/ChangeConfig.actor.cpp b/fdbserver/workloads/ChangeConfig.actor.cpp index 8f9949ec52..63c0b1effa 100644 --- a/fdbserver/workloads/ChangeConfig.actor.cpp +++ b/fdbserver/workloads/ChangeConfig.actor.cpp @@ -53,37 +53,45 @@ struct ChangeConfigWorkload : TestWorkload { void getMetrics(std::vector& m) override {} - // When simulated two clusters for DR tests, this actor sets the starting configuration - // for the extra cluster. - ACTOR Future extraDatabaseConfigure(ChangeConfigWorkload* self) { - if (g_network->isSimulated() && g_simulator.extraDB) { - auto extraFile = makeReference(*g_simulator.extraDB); - state Database extraDB = Database::createDatabase(extraFile, -1); - - wait(delay(5 * deterministicRandom()->random01())); - if (self->configMode.size()) { - if (g_simulator.startingDisabledConfiguration != "") { - // It is not safe to allow automatic failover to a region which is not fully replicated, - // so wait for both regions to be fully replicated before enabling failover - wait(success(ManagementAPI::changeConfig( - extraDB.getReference(), g_simulator.startingDisabledConfiguration, true))); - TraceEvent("WaitForReplicasExtra").log(); - wait(waitForFullReplication(extraDB)); - TraceEvent("WaitForReplicasExtraEnd").log(); - } - wait(success(ManagementAPI::changeConfig(extraDB.getReference(), self->configMode, true))); + ACTOR Future configureExtraDatabase(ChangeConfigWorkload* self, Database db) { + wait(delay(5 * deterministicRandom()->random01())); + if (self->configMode.size()) { + if (g_simulator.startingDisabledConfiguration != "") { + // It is not safe to allow automatic failover to a region which is not fully replicated, + // so wait for both regions to be fully replicated before enabling failover + wait(success( + ManagementAPI::changeConfig(db.getReference(), g_simulator.startingDisabledConfiguration, true))); + TraceEvent("WaitForReplicasExtra").log(); + wait(waitForFullReplication(db)); + TraceEvent("WaitForReplicasExtraEnd").log(); } - if (self->networkAddresses.size()) { - if (self->networkAddresses == "auto") - wait(CoordinatorsChangeActor(extraDB, self, true)); - else - wait(CoordinatorsChangeActor(extraDB, self)); - } - wait(delay(5 * deterministicRandom()->random01())); + wait(success(ManagementAPI::changeConfig(db.getReference(), self->configMode, true))); } + if (self->networkAddresses.size()) { + if (self->networkAddresses == "auto") + wait(CoordinatorsChangeActor(db, self, true)); + else + wait(CoordinatorsChangeActor(db, self)); + } + + wait(delay(5 * deterministicRandom()->random01())); return Void(); } + // When simulating multiple clusters, this actor sets the starting configuration + // for the extra clusters. + Future configureExtraDatabases(ChangeConfigWorkload* self) { + std::vector> futures; + if (g_network->isSimulated()) { + for (auto extraDatabase : g_simulator.extraDatabases) { + auto extraFile = makeReference(extraDatabase); + Database db = Database::createDatabase(extraFile, -1); + futures.push_back(configureExtraDatabase(self, db)); + } + } + return waitForAll(futures); + } + // Either changes the database configuration, or changes the coordinators based on the parameters // of the workload. ACTOR Future ChangeConfigClient(Database cx, ChangeConfigWorkload* self) { @@ -93,7 +101,7 @@ struct ChangeConfigWorkload : TestWorkload { state bool extraConfigureBefore = deterministicRandom()->random01() < 0.5; if (extraConfigureBefore) { - wait(self->extraDatabaseConfigure(self)); + wait(self->configureExtraDatabases(self)); } if (self->configMode.size()) { @@ -116,7 +124,7 @@ struct ChangeConfigWorkload : TestWorkload { } if (!extraConfigureBefore) { - wait(self->extraDatabaseConfigure(self)); + wait(self->configureExtraDatabases(self)); } return Void(); diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp index 4cf52624f1..b64c814009 100644 --- a/fdbserver/workloads/ConsistencyCheck.actor.cpp +++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp @@ -2037,8 +2037,9 @@ struct ConsistencyCheckWorkload : TestWorkload { } ACTOR Future checkWorkerList(Database cx, ConsistencyCheckWorkload* self) { - if (g_simulator.extraDB) + if (!g_simulator.extraDatabases.empty()) { return true; + } std::vector workers = wait(getWorkers(self->dbInfo)); std::set workerAddresses; diff --git a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp index 9598607830..e058ada295 100644 --- a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp +++ b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp @@ -37,8 +37,8 @@ struct DifferentClustersSameRVWorkload : TestWorkload { bool switchComplete = false; DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - ASSERT(g_simulator.extraDB != nullptr); - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); testDuration = getOption(options, LiteralStringRef("testDuration"), 100.0); switchAfter = getOption(options, LiteralStringRef("switchAfter"), 50.0); diff --git a/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp index 844c470eb2..c8fd161838 100644 --- a/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp +++ b/fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp @@ -35,6 +35,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload { const TenantName tenantNamePrefix = "tenant_management_concurrency_workload_"_sr; + const Key testParametersKey = "test_parameters"_sr; int maxTenants; int maxTenantGroups; @@ -49,22 +50,50 @@ struct TenantManagementConcurrencyWorkload : TestWorkload { maxTenantGroups = std::min(2 * maxTenants, getOption(options, "maxTenantGroups"_sr, 20)); testDuration = getOption(options, "testDuration"_sr, 60.0); + if (clientId == 0) { + useMetacluster = deterministicRandom()->coinflip(); + } else { + // Other clients read the metacluster state from the database useMetacluster = false; } + } std::string description() const override { return "TenantManagementConcurrency"; } + struct TestParameters { + constexpr static FileIdentifier file_identifier = 14350843; + + bool useMetacluster = false; + + TestParameters() {} + TestParameters(bool useMetacluster) : useMetacluster(useMetacluster) {} + + template + void serialize(Ar& ar) { + serializer(ar, useMetacluster); + } + + Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); } + + static TestParameters decode(ValueRef const& value) { + TestParameters params; + ObjectReader reader(value.begin(), Unversioned()); + reader.deserialize(params); + return params; + } + }; + Future setup(Database const& cx) override { return _setup(cx, this); } ACTOR Future _setup(Database cx, TenantManagementConcurrencyWorkload* self) { Reference threadSafeHandle = wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx))); - TraceEvent("CreatedThreadSafeHandle"); MultiVersionApi::api->selectApiVersion(cx->apiVersion); self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle); if (self->useMetacluster) { - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); self->dataDb = Database::createDatabase(extraFile, -1); if (self->clientId == 0) { @@ -72,12 +101,45 @@ struct TenantManagementConcurrencyWorkload : TestWorkload { DataClusterEntry entry; entry.capacity.numTenantGroups = 1e9; - wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, *g_simulator.extraDB, entry)); + wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry)); } } else { self->dataDb = cx; } + state Transaction tr(cx); + if (self->clientId == 0) { + // Send test parameters to the other clients + loop { + try { + tr.setOption(FDBTransactionOptions::RAW_ACCESS); + tr.set(self->testParametersKey, TestParameters(self->useMetacluster).encode()); + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } else { + // Read the tenant subspace chosen and saved by client 0 + loop { + try { + tr.setOption(FDBTransactionOptions::RAW_ACCESS); + Optional val = wait(tr.get(self->testParametersKey)); + if (val.present()) { + TestParameters params = TestParameters::decode(val.get()); + self->useMetacluster = params.useMetacluster; + break; + } + + wait(delay(1.0)); + tr.reset(); + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + return Void(); } diff --git a/fdbserver/workloads/TenantManagementWorkload.actor.cpp b/fdbserver/workloads/TenantManagementWorkload.actor.cpp index e3fda8e32b..5bc1ade14b 100644 --- a/fdbserver/workloads/TenantManagementWorkload.actor.cpp +++ b/fdbserver/workloads/TenantManagementWorkload.actor.cpp @@ -49,7 +49,7 @@ struct TenantManagementWorkload : TestWorkload { Key tenantSubspace; const Key keyName = "key"_sr; - const Key tenantSubspaceKey = "tenant_subspace"_sr; + const Key testParametersKey = "test_parameters"_sr; const Value noTenantValue = "no_tenant"_sr; const TenantName tenantNamePrefix = "tenant_management_workload_"_sr; TenantName localTenantNamePrefix; @@ -98,36 +98,59 @@ struct TenantManagementWorkload : TestWorkload { testDuration = getOption(options, "testDuration"_sr, 60.0); localTenantNamePrefix = format("%stenant_%d_", tenantNamePrefix.toString().c_str(), clientId); - useMetacluster = false; + + bool defaultUseMetacluster = false; + if (clientId == 0 && g_network->isSimulated() && !g_simulator.extraDatabases.empty()) { + defaultUseMetacluster = deterministicRandom()->coinflip(); + } + + useMetacluster = getOption(options, "useMetacluster"_sr, defaultUseMetacluster); } std::string description() const override { return "TenantManagement"; } + struct TestParameters { + constexpr static FileIdentifier file_identifier = 1527576; + + Key tenantSubspace; + bool useMetacluster = false; + + TestParameters() {} + TestParameters(Key tenantSubspace, bool useMetacluster) + : tenantSubspace(tenantSubspace), useMetacluster(useMetacluster) {} + + template + void serialize(Ar& ar) { + serializer(ar, tenantSubspace, useMetacluster); + } + + Value encode() const { return ObjectWriter::toValue(*this, Unversioned()); } + + static TestParameters decode(ValueRef const& value) { + TestParameters params; + ObjectReader reader(value.begin(), Unversioned()); + reader.deserialize(params); + return params; + } + }; + Future setup(Database const& cx) override { return _setup(cx, this); } ACTOR Future _setup(Database cx, TenantManagementWorkload* self) { Reference threadSafeHandle = wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx))); - TraceEvent("CreatedThreadSafeHandle"); MultiVersionApi::api->selectApiVersion(cx->apiVersion); self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle); - if (self->useMetacluster) { - - auto extraFile = makeReference(*g_simulator.extraDB); - self->dataDb = Database::createDatabase(extraFile, -1); - - if (self->clientId == 0) { + if (self->useMetacluster && self->clientId == 0) { wait(success(ManagementAPI::changeConfig(cx.getReference(), "tenant_mode=management", true))); DataClusterEntry entry; entry.capacity.numTenantGroups = 1e9; - wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, *g_simulator.extraDB, entry)); + wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry)); } - } else { - self->dataDb = cx; - } - state Transaction tr(self->dataDb); + + state Transaction tr(cx); if (self->clientId == 0) { // Configure the tenant subspace prefix that is applied to all tenants // This feature isn't supported in a metacluster, so we skip it if doing a metacluster test. @@ -143,13 +166,12 @@ struct TenantManagementWorkload : TestWorkload { } } - // Set a key outside of all tenants to make sure that our tenants aren't writing to the regular key-space - // Also communicates the chosen tenant subspace to all other clients by storing it in a key + // Communicates test parameters to all other clients by storing it in a key loop { try { tr.setOption(FDBTransactionOptions::RAW_ACCESS); - tr.set(self->keyName, self->noTenantValue); - tr.set(self->tenantSubspaceKey, self->tenantSubspace); + tr.set(self->testParametersKey, + TestParameters(self->tenantSubspace, self->useMetacluster).encode()); tr.set(tenantDataPrefixKey, self->tenantSubspace); wait(tr.commit()); break; @@ -157,14 +179,17 @@ struct TenantManagementWorkload : TestWorkload { wait(tr.onError(e)); } } + } else { // Read the tenant subspace chosen and saved by client 0 loop { try { tr.setOption(FDBTransactionOptions::RAW_ACCESS); - Optional val = wait(tr.get(self->tenantSubspaceKey)); + Optional val = wait(tr.get(self->testParametersKey)); if (val.present()) { - self->tenantSubspace = val.get(); + TestParameters params = TestParameters::decode(val.get()); + self->tenantSubspace = params.tenantSubspace; + self->useMetacluster = params.useMetacluster; break; } @@ -176,6 +201,29 @@ struct TenantManagementWorkload : TestWorkload { } } + if (self->useMetacluster) { + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); + self->dataDb = Database::createDatabase(extraFile, -1); + } else { + self->dataDb = cx; + } + + if (self->clientId == 0) { + // Set a key outside of all tenants to make sure that our tenants aren't writing to the regular key-space + state Transaction dataTr(self->dataDb); + loop { + try { + dataTr.setOption(FDBTransactionOptions::RAW_ACCESS); + dataTr.set(self->keyName, self->noTenantValue); + wait(dataTr.commit()); + break; + } catch (Error& e) { + wait(dataTr.onError(e)); + } + } + } + return Void(); } @@ -281,7 +329,8 @@ struct TenantManagementWorkload : TestWorkload { loop { try { Optional result = - wait(timeout(self->createImpl(cx, tr, tenantsToCreate, operationType, self), 30)); + wait(timeout(self->createImpl(cx, tr, tenantsToCreate, operationType, self), + deterministicRandom()->randomInt(1, 30))); if (result.present()) { // Database operations shouldn't get here if the tenant already exists @@ -570,8 +619,9 @@ struct TenantManagementWorkload : TestWorkload { state bool retried = false; loop { try { - Optional result = wait(timeout( - self->deleteImpl(cx, tr, beginTenant, endTenant, tenants, operationType, self), 30)); + Optional result = + wait(timeout(self->deleteImpl(cx, tr, beginTenant, endTenant, tenants, operationType, self), + deterministicRandom()->randomInt(1, 30))); if (result.present()) { // Database operations shouldn't get here if the tenant didn't exist @@ -608,9 +658,6 @@ struct TenantManagementWorkload : TestWorkload { } else { ASSERT(resultEntry.get().tenantState == TenantState::READY); } - } else if (deterministicRandom()->coinflip()) { - // Nothing to delete, so randomly retry - break; } } diff --git a/fdbserver/workloads/VersionStamp.actor.cpp b/fdbserver/workloads/VersionStamp.actor.cpp index 89ee83c254..e9bd796cba 100644 --- a/fdbserver/workloads/VersionStamp.actor.cpp +++ b/fdbserver/workloads/VersionStamp.actor.cpp @@ -155,7 +155,8 @@ struct VersionStampWorkload : TestWorkload { ACTOR Future _check(Database cx, VersionStampWorkload* self) { if (self->validateExtraDB) { - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); cx = Database::createDatabase(extraFile, -1); } state ReadYourWritesTransaction tr(cx); @@ -312,8 +313,9 @@ struct VersionStampWorkload : TestWorkload { state double lastTime = now(); state Database extraDB; - if (g_simulator.extraDB != nullptr) { - auto extraFile = makeReference(*g_simulator.extraDB); + if (!g_simulator.extraDatabases.empty()) { + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); } @@ -380,7 +382,7 @@ struct VersionStampWorkload : TestWorkload { } catch (Error& e) { err = e; - if (err.code() == error_code_database_locked && g_simulator.extraDB != nullptr) { + if (err.code() == error_code_database_locked && !g_simulator.extraDatabases.empty()) { //TraceEvent("VST_CommitDatabaseLocked"); cx_is_primary = !cx_is_primary; tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB); diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp index a462877aba..292c3ac47c 100644 --- a/fdbserver/workloads/WriteDuringRead.actor.cpp +++ b/fdbserver/workloads/WriteDuringRead.actor.cpp @@ -87,9 +87,10 @@ struct WriteDuringReadWorkload : TestWorkload { TEST(adjacentKeys && (nodes + minNode) > CLIENT_KNOBS->KEY_SIZE_LIMIT); // WriteDuringReadWorkload testing large keys - useExtraDB = g_simulator.extraDB != nullptr; + useExtraDB = !g_simulator.extraDatabases.empty(); if (useExtraDB) { - auto extraFile = makeReference(*g_simulator.extraDB); + ASSERT(g_simulator.extraDatabases.size() == 1); + auto extraFile = makeReference(g_simulator.extraDatabases[0]); extraDB = Database::createDatabase(extraFile, -1); useSystemKeys = false; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9025048781..033fb92ece 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,12 +250,12 @@ if(WITH_PYTHON) add_fdb_test( TEST_FILES restarting/from_6.3.13/CycleTestRestart-1.txt restarting/from_6.3.13/CycleTestRestart-2.txt) - add_fdb_test( - TEST_FILES restarting/from_6.3.13/DrUpgradeRestart-1.txt - restarting/from_6.3.13/DrUpgradeRestart-2.txt) add_fdb_test( TEST_FILES restarting/from_6.3.13/StorefrontTestRestart-1.txt restarting/from_6.3.13/StorefrontTestRestart-2.txt) + add_fdb_test( + TEST_FILES restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt + restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt) add_fdb_test( TEST_FILES restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml) @@ -286,6 +286,9 @@ if(WITH_PYTHON) add_fdb_test( TEST_FILES restarting/from_7.1.0/VersionVectorEnableRestart-1.toml restarting/from_7.1.0/VersionVectorEnableRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.txt + restarting/from_7.2.0/DrUpgradeRestart-2.txt) add_fdb_test(TEST_FILES slow/ApiCorrectness.toml) @@ -323,6 +326,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapse.toml) add_fdb_test(TEST_FILES slow/SwizzledRollbackTimeLapseIncrement.toml) add_fdb_test(TEST_FILES slow/SwizzledTenantManagement.toml) + add_fdb_test(TEST_FILES slow/SwizzledTenantManagementMetacluster.toml) add_fdb_test(TEST_FILES slow/TenantManagement.toml) add_fdb_test(TEST_FILES slow/TenantManagementConcurrency.toml) add_fdb_test(TEST_FILES slow/VersionStampBackupToDB.toml) diff --git a/tests/fast/AtomicBackupToDBCorrectness.toml b/tests/fast/AtomicBackupToDBCorrectness.toml index 1b601da923..84214337d9 100644 --- a/tests/fast/AtomicBackupToDBCorrectness.toml +++ b/tests/fast/AtomicBackupToDBCorrectness.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 1 +extraDatabaseMode = 'LocalOrSingle' [[test]] testTitle = 'BackupAndRestore' diff --git a/tests/fast/BackupToDBCorrectness.toml b/tests/fast/BackupToDBCorrectness.toml index cf50093657..a5e80590c4 100644 --- a/tests/fast/BackupToDBCorrectness.toml +++ b/tests/fast/BackupToDBCorrectness.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 1 +extraDatabaseMode = 'LocalOrSingle' [[test]] testTitle = 'BackupAndRestore' diff --git a/tests/fast/BackupToDBCorrectnessClean.toml b/tests/fast/BackupToDBCorrectnessClean.toml index 9c2e9135e5..4ef236bc01 100644 --- a/tests/fast/BackupToDBCorrectnessClean.toml +++ b/tests/fast/BackupToDBCorrectnessClean.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 1 +extraDatabaseMode = 'LocalOrSingle' [[test]] testTitle = 'BackupAndRestore' diff --git a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt index 439ccedb5e..366d8bc2a3 100644 --- a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt +++ b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt @@ -1,4 +1,4 @@ -extraDB=3 +extraDatabaseMode=Local testTitle=DrUpgrade runSetup=false diff --git a/tests/restarting/from_6.3.13/DrUpgradeRestart-1.txt b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt similarity index 100% rename from tests/restarting/from_6.3.13/DrUpgradeRestart-1.txt rename to tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt diff --git a/tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt similarity index 93% rename from tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt rename to tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt index 439ccedb5e..366d8bc2a3 100644 --- a/tests/restarting/from_6.3.13/DrUpgradeRestart-2.txt +++ b/tests/restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt @@ -1,4 +1,4 @@ -extraDB=3 +extraDatabaseMode=Local testTitle=DrUpgrade runSetup=false diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt b/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt new file mode 100644 index 0000000000..81f932bada --- /dev/null +++ b/tests/restarting/from_7.2.0/DrUpgradeRestart-1.txt @@ -0,0 +1,21 @@ +storageEngineExcludeTypes=3 +extraDatabaseMode=Local + +testTitle=DrUpgrade +clearAfterTest=false +simBackupAgents=BackupToDB + + testName=Cycle + nodeCount=30000 + transactionsPerSecond=2500.0 + testDuration=30.0 + expectedRate=0 + + testName=BackupToDBUpgrade + backupAfter=10.0 + stopDifferentialAfter=50.0 + backupRangesCount=-1 + + testName=SaveAndKill + restartInfoLocation=simfdb/restartInfo.ini + testDuration=40.0 diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt b/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt new file mode 100644 index 0000000000..366d8bc2a3 --- /dev/null +++ b/tests/restarting/from_7.2.0/DrUpgradeRestart-2.txt @@ -0,0 +1,18 @@ +extraDatabaseMode=Local + +testTitle=DrUpgrade +runSetup=false +clearAfterTest=false +simBackupAgents=BackupToDB +waitForQuiescenceBegin=false + + testName=Cycle + nodeCount=30000 + transactionsPerSecond=2500.0 + testDuration=30.0 + expectedRate=0 + + testName=BackupToDBUpgrade + backupAfter=10.0 + backupRangesCount=-1 + stopDifferentialAfter=70.0 diff --git a/tests/slow/ApiCorrectnessSwitchover.toml b/tests/slow/ApiCorrectnessSwitchover.toml index 3b4bfd5680..42679e9445 100644 --- a/tests/slow/ApiCorrectnessSwitchover.toml +++ b/tests/slow/ApiCorrectnessSwitchover.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'ApiCorrectnessTest' diff --git a/tests/slow/DifferentClustersSameRV.toml b/tests/slow/DifferentClustersSameRV.toml index add7a2377e..31121b5ccc 100644 --- a/tests/slow/DifferentClustersSameRV.toml +++ b/tests/slow/DifferentClustersSameRV.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'DifferentClustersSameRV' diff --git a/tests/slow/SharedBackupCorrectness.toml b/tests/slow/SharedBackupCorrectness.toml index c03b89831a..f202cae840 100644 --- a/tests/slow/SharedBackupCorrectness.toml +++ b/tests/slow/SharedBackupCorrectness.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 1 +extraDatabaseMode = 'LocalOrSingle' [[test]] testTitle = 'BackupAndRestore' diff --git a/tests/slow/SharedBackupToDBCorrectness.toml b/tests/slow/SharedBackupToDBCorrectness.toml index 3a3a07dfbd..cebee48a2e 100644 --- a/tests/slow/SharedBackupToDBCorrectness.toml +++ b/tests/slow/SharedBackupToDBCorrectness.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 1 +extraDatabaseMode = 'LocalOrSingle' [[test]] testTitle = 'BackupAndRestore' diff --git a/tests/slow/SwizzledTenantManagementMetacluster.toml b/tests/slow/SwizzledTenantManagementMetacluster.toml new file mode 100644 index 0000000000..f3a727ea4f --- /dev/null +++ b/tests/slow/SwizzledTenantManagementMetacluster.toml @@ -0,0 +1,39 @@ +[configuration] +allowDefaultTenant = false +allowDisablingTenants = false +extraDatabaseMode = 'Single' + +[[test]] +testTitle = 'TenantManagementTest' +clearAfterTest = true +timeout = 2100 +runSetup = true + + [[test.workload]] + testName = 'TenantManagement' + maxTenants = 1000 + testDuration = 60 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 120.0 + swizzle = 1 + + [[test.workload]] + testName = 'Rollback' + testDuration = 120.0 + meanDelay = 10.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 120.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 120.0 diff --git a/tests/slow/TenantManagement.toml b/tests/slow/TenantManagement.toml index 5848bdf4e3..d097d40742 100644 --- a/tests/slow/TenantManagement.toml +++ b/tests/slow/TenantManagement.toml @@ -1,6 +1,7 @@ [configuration] allowDefaultTenant = false allowDisablingTenants = false +extraDatabaseMode = 'Single' [[test]] testTitle = 'TenantManagementTest' diff --git a/tests/slow/TenantManagementConcurrency.toml b/tests/slow/TenantManagementConcurrency.toml index 2dba86d7ea..f89ccf27df 100644 --- a/tests/slow/TenantManagementConcurrency.toml +++ b/tests/slow/TenantManagementConcurrency.toml @@ -1,7 +1,7 @@ [configuration] allowDefaultTenant = false allowDisablingTenants = false -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'TenantManagementConcurrencyTest' diff --git a/tests/slow/VersionStampBackupToDB.toml b/tests/slow/VersionStampBackupToDB.toml index 4b36182dd0..88e2182cb6 100644 --- a/tests/slow/VersionStampBackupToDB.toml +++ b/tests/slow/VersionStampBackupToDB.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'VersionStampBackupToDB' diff --git a/tests/slow/VersionStampSwitchover.toml b/tests/slow/VersionStampSwitchover.toml index 328c199b93..16603193be 100644 --- a/tests/slow/VersionStampSwitchover.toml +++ b/tests/slow/VersionStampSwitchover.toml @@ -1,5 +1,5 @@ [configuration] -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'VersionStampCorrectnessTest' diff --git a/tests/slow/WriteDuringReadSwitchover.toml b/tests/slow/WriteDuringReadSwitchover.toml index b5232c9119..f7eb0cf01f 100644 --- a/tests/slow/WriteDuringReadSwitchover.toml +++ b/tests/slow/WriteDuringReadSwitchover.toml @@ -1,6 +1,6 @@ [configuration] StderrSeverity = 30 -extraDB = 2 +extraDatabaseMode = 'Single' [[test]] testTitle = 'WriteDuringReadTest'