From 936bf5336a8657c205a88ae3cc1d73da21aaa076 Mon Sep 17 00:00:00 2001 From: Ata E Husain Bohra Date: Thu, 6 Jan 2022 12:15:51 -0800 Subject: [PATCH] Revert "Revert "Refactor: ClusterController driving cluster-recovery state machine" (#6191) * Revert "Revert "Refactor: ClusterController driving cluster-recovery state machine"" Major changes includes: 1. Re-revert Sequencer refactor commits listed below (in listed order): 1.a. This reverts commit bb17e194d9c9888e203421290959bd7f2c075d7f. 1.b. This reverts commit d174bb2e06bff01157d16c652073536c54d17f7f. 1.c. This reverts commit 30b05b469c87d9b526b427751c211fb5cf7ff9cd. 2. Update Status.actor to track ClusterController interface to track recovery status. 3. Introduce a ServerKnob to define "cluster recovery trace event" prefix; for now keeping it as "Master", however, it should allow smooth transition to "Cluster" prefix as it seems more appropriate. --- design/recovery-internals.md | 5 +- fdbclient/Schemas.cpp | 1 + fdbclient/ServerKnobs.cpp | 3 + fdbclient/ServerKnobs.h | 3 + fdbserver/BackupWorker.actor.cpp | 2 +- fdbserver/CMakeLists.txt | 3 + fdbserver/ClusterController.actor.cpp | 3557 +------------------ fdbserver/ClusterController.actor.h | 3286 +++++++++++++++++ fdbserver/ClusterRecovery.actor.cpp | 1888 ++++++++++ fdbserver/ClusterRecovery.actor.h | 314 ++ fdbserver/CommitProxyServer.actor.cpp | 40 +- fdbserver/GrvProxyServer.actor.cpp | 19 +- fdbserver/MasterInterface.h | 110 +- fdbserver/OldTLogServer_4_6.actor.cpp | 35 +- fdbserver/OldTLogServer_6_0.actor.cpp | 39 +- fdbserver/OldTLogServer_6_2.actor.cpp | 39 +- fdbserver/Status.actor.cpp | 36 +- fdbserver/TLogServer.actor.cpp | 61 +- fdbserver/TagPartitionedLogSystem.actor.cpp | 28 +- fdbserver/WorkerInterface.actor.h | 73 +- fdbserver/masterserver.actor.cpp | 1827 +--------- fdbserver/worker.actor.cpp | 8 +- flow/error_definitions.h | 11 +- flow/network.h | 1 + 24 files changed, 6019 insertions(+), 5370 deletions(-) create mode 100644 fdbserver/ClusterController.actor.h create mode 100644 fdbserver/ClusterRecovery.actor.cpp create mode 100644 fdbserver/ClusterRecovery.actor.h diff --git a/design/recovery-internals.md b/design/recovery-internals.md index 69eb8b92ee..75927de285 100644 --- a/design/recovery-internals.md +++ b/design/recovery-internals.md @@ -55,7 +55,7 @@ Recovery has 9 phases, which are defined as the 9 states in the source code: REA The recovery process is like a state machine, changing from one state to the next state. We will describe in the rest of this document what each phase does to drive the recovery to the next state. -Recovery tracks the information of each recovery phase in `MasterRecoveryState` trace event. By checking the message, we can find which phase the recovery is stuck at. The status used in the `MasterRecoveryState` trace event is defined as `RecoveryStatus` structure in `RecoveryState.h`. The status, instead of the name of the 9 phases, is typically used in diagnosing production issues. +Recovery tracks the information of each recovery phase in `RecoveryState` trace event. By checking the message, we can find which phase the recovery is stuck at. The status used in the `RecoveryState` trace event is defined as `RecoveryStatus` structure in `RecoveryState.h`. The status, instead of the name of the 9 phases, is typically used in diagnosing production issues. ## Phase 1: READING_CSTATE @@ -139,8 +139,9 @@ However, reading the txnStateStore can be slow because it needs to read from dis **Recruiting roles step.** There are cases where the recovery can get stuck at recruiting enough roles for the txn system configuration. For example, if a cluster with replica factor equal to three has only three tLogs and one of them dies during the recovery, the cluster will not succeed in recruiting 3 tLogs and the recovery will get stuck. Another example is when a new database is created and the cluster does not have a valid txnStateStore. To get out of this situation, the master will use an emergency transaction to forcibly change the configuration such that the recruitment can succeed. This configuration change may temporarily violate the contract of the desired configuration, but it is only temporary. +ServerKnob::CLUSTER_RECOVERY_EVENT_NAME_PREFIX defines the prefix for cluster recovery trace events. Hereafter, refered as 'RecoveryEventPrefix' in this document. -We can use the trace event `MasterRecoveredConfig`, which dumps the information of the new transaction system’s configuration, to diagnose why the recovery is blocked in this phase. +We can use the trace event `RecoveredConfig`, which dumps the information of the new transaction system’s configuration, to diagnose why the recovery is blocked in this phase. ## Phase 4: RECOVERY_TRANSACTION diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 10f77d7638..d2592721c9 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -493,6 +493,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "name":{ "$enum":[ "unreachable_master_worker", + "unreachable_cluster_controller_worker", "unreachable_dataDistributor_worker", "unreachable_ratekeeper_worker", "unreachable_blobManager_worker", diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 8422ac3272..f276a7b577 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -781,6 +781,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( LATENCY_SAMPLE_SIZE, 100000 ); init( LATENCY_METRICS_LOGGING_INTERVAL, 60.0 ); + // Cluster recovery + init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master"); + // Blob granlues init( BG_URL, "" ); // TODO: store in system key space, eventually init( BG_SNAPSHOT_FILE_TARGET_BYTES, 10000000 ); if( randomize && BUGGIFY ) { deterministicRandom()->random01() < 0.1 ? BG_SNAPSHOT_FILE_TARGET_BYTES /= 100 : BG_SNAPSHOT_FILE_TARGET_BYTES /= 10; } diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index d38ae0b126..3dafbbed52 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -736,6 +736,9 @@ public: int LATENCY_SAMPLE_SIZE; double LATENCY_METRICS_LOGGING_INTERVAL; + // Cluster recovery + std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX; + // blob granule stuff // FIXME: configure url with database configuration instead of knob eventually std::string BG_URL; diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index ca426b7569..c8045bb0b5 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -1086,7 +1086,7 @@ ACTOR Future backupWorker(BackupInterface interf, TraceEvent("BackupWorkerDone", self.myId).detail("BackupEpoch", self.backupEpoch); // Notify master so that this worker can be removed from log system, then this // worker (for an old epoch's unfinished work) can safely exit. - wait(brokenPromiseToNever(db->get().master.notifyBackupWorkerDone.getReply( + wait(brokenPromiseToNever(db->get().clusterInterface.notifyBackupWorkerDone.getReply( BackupWorkerDoneRequest(self.myId, self.backupEpoch)))); break; } diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index a9d958bb40..111ef074cb 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -8,7 +8,10 @@ set(FDBSERVER_SRCS BlobManager.actor.cpp BlobManagerInterface.h BlobWorker.actor.cpp + ClusterController.actor.h ClusterController.actor.cpp + ClusterRecovery.actor.h + ClusterRecovery.actor.cpp ConfigBroadcaster.actor.cpp ConfigBroadcaster.h ConfigDatabaseUnitTests.actor.cpp diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 88a3e90d2f..e8eaaf06e6 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -21,35 +21,42 @@ #include #include #include +#include #include #include #include "fdbrpc/FailureMonitor.h" #include "flow/ActorCollection.h" -#include "flow/SystemMonitor.h" #include "fdbclient/ClusterConnectionMemoryRecord.h" #include "fdbclient/NativeAPI.actor.h" +#include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/BackupInterface.h" -#include "fdbserver/CoordinationInterface.h" +#include "fdbserver/BackupProgress.actor.h" +#include "fdbserver/ConfigBroadcaster.h" +#include "fdbserver/CoordinatedState.h" +#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class +#include "fdbserver/ClusterController.actor.h" +#include "fdbserver/ClusterRecovery.actor.h" #include "fdbserver/DataDistributorInterface.h" -#include "fdbserver/Knobs.h" +#include "fdbserver/DBCoreState.h" #include "fdbserver/ConfigBroadcaster.h" #include "fdbserver/MoveKeys.actor.h" -#include "fdbserver/WorkerInterface.actor.h" #include "fdbserver/LeaderElection.h" +#include "fdbserver/LogSystem.h" #include "fdbserver/LogSystemConfig.h" +#include "fdbserver/LogSystemDiskQueueAdapter.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/RatekeeperInterface.h" #include "fdbserver/BlobManagerInterface.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/Status.h" #include "fdbserver/LatencyBandConfig.h" -#include "fdbclient/DatabaseContext.h" #include "fdbclient/GlobalConfig.actor.h" #include "fdbserver/RecoveryState.h" #include "fdbclient/ReadYourWrites.h" #include "fdbrpc/Replication.h" #include "fdbrpc/ReplicationUtils.h" +#include "fdbrpc/sim_validation.h" #include "fdbclient/KeyBackedTypes.h" #include "flow/Util.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -62,3215 +69,6 @@ void failAfter(Future trigger, Endpoint e); // be increased if we ever have more than 100 singletons (unlikely). static const int PID_USED_AMP_FOR_NON_SINGLETON = 100; -struct WorkerInfo : NonCopyable { - Future watcher; - ReplyPromise reply; - Generation gen; - int reboots; - ProcessClass initialClass; - ClusterControllerPriorityInfo priorityInfo; - WorkerDetails details; - Future haltRatekeeper; - Future haltDistributor; - Future haltBlobManager; - Standalone> issues; - - WorkerInfo() - : gen(-1), reboots(0), - priorityInfo(ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown) {} - WorkerInfo(Future watcher, - ReplyPromise reply, - Generation gen, - WorkerInterface interf, - ProcessClass initialClass, - ProcessClass processClass, - ClusterControllerPriorityInfo priorityInfo, - bool degraded, - Standalone> issues) - : watcher(watcher), reply(reply), gen(gen), reboots(0), initialClass(initialClass), priorityInfo(priorityInfo), - details(interf, processClass, degraded), issues(issues) {} - - WorkerInfo(WorkerInfo&& r) noexcept - : watcher(std::move(r.watcher)), reply(std::move(r.reply)), gen(r.gen), reboots(r.reboots), - initialClass(r.initialClass), priorityInfo(r.priorityInfo), details(std::move(r.details)), - haltRatekeeper(r.haltRatekeeper), haltDistributor(r.haltDistributor), haltBlobManager(r.haltBlobManager), - issues(r.issues) {} - void operator=(WorkerInfo&& r) noexcept { - watcher = std::move(r.watcher); - reply = std::move(r.reply); - gen = r.gen; - reboots = r.reboots; - initialClass = r.initialClass; - priorityInfo = r.priorityInfo; - details = std::move(r.details); - haltRatekeeper = r.haltRatekeeper; - haltDistributor = r.haltDistributor; - haltBlobManager = r.haltBlobManager; - issues = r.issues; - } -}; - -struct WorkerFitnessInfo { - WorkerDetails worker; - ProcessClass::Fitness fitness; - int used; - - WorkerFitnessInfo() : fitness(ProcessClass::NeverAssign), used(0) {} - WorkerFitnessInfo(WorkerDetails worker, ProcessClass::Fitness fitness, int used) - : worker(worker), fitness(fitness), used(used) {} -}; - -class ClusterControllerData { -public: - struct DBInfo { - Reference> clientInfo; - Reference> serverInfo; - std::map incompatibleConnections; - AsyncTrigger forceMasterFailure; - int64_t masterRegistrationCount; - int64_t dbInfoCount; - bool recoveryStalled; - bool forceRecovery; - DatabaseConfiguration config; // Asynchronously updated via master registration - DatabaseConfiguration fullyRecoveredConfig; - Database db; - int unfinishedRecoveries; - int logGenerations; - bool cachePopulated; - std::map> clientStatus; - Future clientCounter; - int clientCount; - - DBInfo() - : clientInfo(new AsyncVar()), serverInfo(new AsyncVar()), - masterRegistrationCount(0), dbInfoCount(0), recoveryStalled(false), forceRecovery(false), - db(DatabaseContext::create(clientInfo, - Future(), - LocalityData(), - EnableLocalityLoadBalance::True, - TaskPriority::DefaultEndpoint, - LockAware::True)), // SOMEDAY: Locality! - unfinishedRecoveries(0), logGenerations(0), cachePopulated(false), clientCount(0) { - clientCounter = countClients(this); - } - - void setDistributor(const DataDistributorInterface& interf) { - auto newInfo = serverInfo->get(); - newInfo.id = deterministicRandom()->randomUniqueID(); - newInfo.infoGeneration = ++dbInfoCount; - newInfo.distributor = interf; - serverInfo->set(newInfo); - } - - void setRatekeeper(const RatekeeperInterface& interf) { - auto newInfo = serverInfo->get(); - newInfo.id = deterministicRandom()->randomUniqueID(); - newInfo.infoGeneration = ++dbInfoCount; - newInfo.ratekeeper = interf; - serverInfo->set(newInfo); - } - - void setBlobManager(const BlobManagerInterface& interf) { - auto newInfo = serverInfo->get(); - newInfo.id = deterministicRandom()->randomUniqueID(); - newInfo.infoGeneration = ++dbInfoCount; - newInfo.blobManager = interf; - serverInfo->set(newInfo); - } - - void clearInterf(ProcessClass::ClassType t) { - auto newInfo = serverInfo->get(); - newInfo.id = deterministicRandom()->randomUniqueID(); - newInfo.infoGeneration = ++dbInfoCount; - if (t == ProcessClass::DataDistributorClass) { - newInfo.distributor = Optional(); - } else if (t == ProcessClass::RatekeeperClass) { - newInfo.ratekeeper = Optional(); - } else if (t == ProcessClass::BlobManagerClass) { - newInfo.blobManager = Optional(); - } - serverInfo->set(newInfo); - } - - ACTOR static Future countClients(DBInfo* self) { - loop { - wait(delay(SERVER_KNOBS->CC_PRUNE_CLIENTS_INTERVAL)); - - self->clientCount = 0; - for (auto itr = self->clientStatus.begin(); itr != self->clientStatus.end();) { - if (now() - itr->second.first < 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) { - self->clientCount += itr->second.second.clientCount; - ++itr; - } else { - itr = self->clientStatus.erase(itr); - } - } - } - } - }; - - struct UpdateWorkerList { - Future init(Database const& db) { return update(this, db); } - - void set(Optional> processID, Optional data) { - delta[processID] = data; - anyDelta.set(true); - } - - private: - std::map>, Optional> delta; - AsyncVar anyDelta; - - ACTOR static Future update(UpdateWorkerList* self, Database db) { - // The Database we are using is based on worker registrations to this cluster controller, which come only - // from master servers that we started, so it shouldn't be possible for multiple cluster controllers to - // fight. - state Transaction tr(db); - loop { - try { - tr.clear(workerListKeys); - wait(tr.commit()); - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - - loop { - tr.reset(); - - // Wait for some changes - while (!self->anyDelta.get()) - wait(self->anyDelta.onChange()); - self->anyDelta.set(false); - - state std::map>, Optional> delta; - delta.swap(self->delta); - - TraceEvent("UpdateWorkerList").detail("DeltaCount", delta.size()); - - // Do a transaction to write the changes - loop { - try { - for (auto w = delta.begin(); w != delta.end(); ++w) { - if (w->second.present()) { - tr.set(workerListKeyFor(w->first.get()), workerListValue(w->second.get())); - } else - tr.clear(workerListKeyFor(w->first.get())); - } - wait(tr.commit()); - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - } - } - }; - - bool workerAvailable(WorkerInfo const& worker, bool checkStable) { - return (now() - startTime < 2 * FLOW_KNOBS->SERVER_REQUEST_INTERVAL) || - (IFailureMonitor::failureMonitor().getState(worker.details.interf.storage.getEndpoint()).isAvailable() && - (!checkStable || worker.reboots < 2)); - } - - bool isLongLivedStateless(Optional const& processId) { - return (db.serverInfo->get().distributor.present() && - db.serverInfo->get().distributor.get().locality.processId() == processId) || - (db.serverInfo->get().ratekeeper.present() && - db.serverInfo->get().ratekeeper.get().locality.processId() == processId) || - (db.serverInfo->get().blobManager.present() && - db.serverInfo->get().blobManager.get().locality.processId() == processId); - } - - WorkerDetails getStorageWorker(RecruitStorageRequest const& req) { - std::set>> excludedMachines(req.excludeMachines.begin(), - req.excludeMachines.end()); - std::set>> includeDCs(req.includeDCs.begin(), req.includeDCs.end()); - std::set excludedAddresses(req.excludeAddresses.begin(), req.excludeAddresses.end()); - - for (auto& it : id_worker) - if (workerAvailable(it.second, false) && - !excludedMachines.count(it.second.details.interf.locality.zoneId()) && - (includeDCs.size() == 0 || includeDCs.count(it.second.details.interf.locality.dcId())) && - !addressExcluded(excludedAddresses, it.second.details.interf.address()) && - (!it.second.details.interf.secondaryAddress().present() || - !addressExcluded(excludedAddresses, it.second.details.interf.secondaryAddress().get())) && - it.second.details.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::UnsetFit) { - return it.second.details; - } - - if (req.criticalRecruitment) { - ProcessClass::Fitness bestFit = ProcessClass::NeverAssign; - Optional bestInfo; - for (auto& it : id_worker) { - ProcessClass::Fitness fit = it.second.details.processClass.machineClassFitness(ProcessClass::Storage); - if (workerAvailable(it.second, false) && - !excludedMachines.count(it.second.details.interf.locality.zoneId()) && - (includeDCs.size() == 0 || includeDCs.count(it.second.details.interf.locality.dcId())) && - !addressExcluded(excludedAddresses, it.second.details.interf.address()) && fit < bestFit) { - bestFit = fit; - bestInfo = it.second.details; - } - } - - if (bestInfo.present()) { - return bestInfo.get(); - } - } - - throw no_more_servers(); - } - - // Returns a worker that can be used by a blob worker - // Note: we restrict the set of possible workers to those in the same DC as the BM/CC - WorkerDetails getBlobWorker(RecruitBlobWorkerRequest const& req) { - std::set excludedAddresses(req.excludeAddresses.begin(), req.excludeAddresses.end()); - for (auto& it : id_worker) { - // the worker must be available, have the same dcID as CC, - // not be one of the excluded addrs from req and have the approriate fitness - if (workerAvailable(it.second, false) && - clusterControllerDcId == it.second.details.interf.locality.dcId() && - !addressExcluded(excludedAddresses, it.second.details.interf.address()) && - (!it.second.details.interf.secondaryAddress().present() || - !addressExcluded(excludedAddresses, it.second.details.interf.secondaryAddress().get())) && - it.second.details.processClass.machineClassFitness(ProcessClass::BlobWorker) == ProcessClass::BestFit) { - return it.second.details; - } - } - - throw no_more_servers(); - } - - std::vector getWorkersForSeedServers( - DatabaseConfiguration const& conf, - Reference const& policy, - Optional>> const& dcId = Optional>>()) { - std::map> fitness_workers; - std::vector results; - Reference logServerSet = Reference(new LocalityMap()); - LocalityMap* logServerMap = (LocalityMap*)logServerSet.getPtr(); - bool bCompleted = false; - - for (auto& it : id_worker) { - auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage); - if (workerAvailable(it.second, false) && !conf.isExcludedServer(it.second.details.interf.addresses()) && - !isExcludedDegradedServer(it.second.details.interf.addresses()) && - fitness != ProcessClass::NeverAssign && - (!dcId.present() || it.second.details.interf.locality.dcId() == dcId.get())) { - fitness_workers[fitness].push_back(it.second.details); - } - } - - for (auto& it : fitness_workers) { - for (auto& worker : it.second) { - logServerMap->add(worker.interf.locality, &worker); - } - - std::vector bestSet; - if (logServerSet->selectReplicas(policy, bestSet)) { - results.reserve(bestSet.size()); - for (auto& entry : bestSet) { - auto object = logServerMap->getObject(entry); - results.push_back(*object); - } - bCompleted = true; - break; - } - } - - logServerSet->clear(); - logServerSet.clear(); - - if (!bCompleted) { - throw no_more_servers(); - } - - return results; - } - - // Adds workers to the result such that each field is used in the result set as evenly as possible, - // with a secondary criteria of minimizing the reuse of zoneIds - // only add workers which have a field which is already in the result set - void addWorkersByLowestField(StringRef field, - int desired, - const std::vector& workers, - std::set& resultSet) { - typedef Optional> Field; - typedef Optional> Zone; - typedef std::tuple FieldCount; - typedef std::pair ZoneCount; - - std::priority_queue, std::greater> fieldQueue; - std::map, std::greater>> - field_zoneQueue; - - std::map> field_count; - std::map> zone_count; - std::map> zone_workers; - - // Count the amount of fields and zones already in the result set - for (auto& worker : resultSet) { - auto thisField = worker.interf.locality.get(field); - auto thisZone = worker.interf.locality.zoneId(); - auto thisDc = worker.interf.locality.dcId(); - - auto& fitness = field_count[thisField]; - fitness.first++; - fitness.second = thisDc == clusterControllerDcId; - - auto& zc = zone_count[thisZone]; - zc.first++; - zc.second = thisField; - } - - for (auto& worker : workers) { - auto thisField = worker.interf.locality.get(field); - auto thisZone = worker.interf.locality.zoneId(); - - if (field_count.count(thisField)) { - zone_workers[thisZone].push_back(worker); - zone_count[thisZone].second = thisField; - } - } - - // try to avoid fields in the cluster controller datacenter if everything else is equal - for (auto& it : field_count) { - fieldQueue.emplace(it.second.first, it.second.second, it.first); - } - - for (auto& it : zone_count) { - field_zoneQueue[it.second.second].emplace(it.second.first, it.first); - } - - // start with the least used field, and try to find a worker with that field - while (fieldQueue.size()) { - auto lowestField = fieldQueue.top(); - auto& lowestZoneQueue = field_zoneQueue[std::get<2>(lowestField)]; - bool added = false; - // start with the least used zoneId, and try and find a worker with that zone - while (lowestZoneQueue.size() && !added) { - auto lowestZone = lowestZoneQueue.top(); - auto& zoneWorkers = zone_workers[lowestZone.second]; - - while (zoneWorkers.size() && !added) { - if (!resultSet.count(zoneWorkers.back())) { - resultSet.insert(zoneWorkers.back()); - if (resultSet.size() == desired) { - return; - } - added = true; - } - zoneWorkers.pop_back(); - } - lowestZoneQueue.pop(); - if (added && zoneWorkers.size()) { - ++lowestZone.first; - lowestZoneQueue.push(lowestZone); - } - } - fieldQueue.pop(); - if (added) { - ++std::get<0>(lowestField); - fieldQueue.push(lowestField); - } - } - } - - // Adds workers to the result which minimize the reuse of zoneIds - void addWorkersByLowestZone(int desired, - const std::vector& workers, - std::set& resultSet) { - typedef Optional> Zone; - typedef std::pair ZoneCount; - - std::map zone_count; - std::map> zone_workers; - std::priority_queue, std::greater> zoneQueue; - - for (const auto& worker : workers) { - auto thisZone = worker.interf.locality.zoneId(); - zone_count[thisZone] = 0; - zone_workers[thisZone].push_back(worker); - } - - for (auto& worker : resultSet) { - auto thisZone = worker.interf.locality.zoneId(); - zone_count[thisZone]++; - } - - for (auto& it : zone_count) { - zoneQueue.emplace(it.second, it.first); - } - - while (zoneQueue.size()) { - auto lowestZone = zoneQueue.top(); - auto& zoneWorkers = zone_workers[lowestZone.second]; - - bool added = false; - while (zoneWorkers.size() && !added) { - if (!resultSet.count(zoneWorkers.back())) { - resultSet.insert(zoneWorkers.back()); - if (resultSet.size() == desired) { - return; - } - added = true; - } - zoneWorkers.pop_back(); - } - zoneQueue.pop(); - if (added && zoneWorkers.size()) { - ++lowestZone.first; - zoneQueue.push(lowestZone); - } - } - } - - // Log the reason why the worker is considered as unavailable. - void logWorkerUnavailable(const Severity severity, - const UID& id, - const std::string& method, - const std::string& reason, - const WorkerDetails& details, - const ProcessClass::Fitness& fitness, - const std::set>& dcIds) { - // Construct the list of DCs where the TLog recruitment is happening. This is mainly for logging purpose. - std::string dcList; - for (const auto& dc : dcIds) { - if (!dcList.empty()) { - dcList += ','; - } - dcList += printable(dc); - } - // Logging every possible options is a lot for every recruitment; logging all of the options with GoodFit or - // BestFit may work because there should only be like 30 tlog class processes. Plus, the recruitment happens - // only during initial database creation and recovery. So these trace events should be sparse. - if (fitness == ProcessClass::GoodFit || fitness == ProcessClass::BestFit || - fitness == ProcessClass::NeverAssign) { - TraceEvent(severity, "GetTLogTeamWorkerUnavailable", id) - .detail("TLogRecruitMethod", method) - .detail("Reason", reason) - .detail("WorkerID", details.interf.id()) - .detail("WorkerDC", details.interf.locality.dcId()) - .detail("Address", details.interf.addresses().toString()) - .detail("Fitness", fitness) - .detail("RecruitmentDcIds", dcList); - } - } - - // A TLog recruitment method specialized for three_data_hall and three_datacenter configurations - // It attempts to evenly recruit processes from across data_halls or datacenters - std::vector getWorkersForTlogsComplex(DatabaseConfiguration const& conf, - int32_t desired, - std::map>, int>& id_used, - StringRef field, - int minFields, - int minPerField, - bool allowDegraded, - bool checkStable, - const std::set>& dcIds, - const std::vector& exclusionWorkerIds) { - std::map, std::vector> fitness_workers; - - // Go through all the workers to list all the workers that can be recruited. - for (const auto& [worker_process_id, worker_info] : id_worker) { - const auto& worker_details = worker_info.details; - auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); - - if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != - exclusionWorkerIds.end()) { - logWorkerUnavailable(SevInfo, id, "complex", "Worker is excluded", worker_details, fitness, dcIds); - continue; - } - if (!workerAvailable(worker_info, checkStable)) { - logWorkerUnavailable(SevInfo, id, "complex", "Worker is not available", worker_details, fitness, dcIds); - continue; - } - if (conf.isExcludedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "complex", - "Worker server is excluded from the cluster", - worker_details, - fitness, - dcIds); - continue; - } - if (isExcludedDegradedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "complex", - "Worker server is excluded from the cluster due to degradation", - worker_details, - fitness, - dcIds); - continue; - } - if (fitness == ProcessClass::NeverAssign) { - logWorkerUnavailable( - SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); - continue; - } - if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { - logWorkerUnavailable( - SevDebug, id, "complex", "Worker is not in the target DC", worker_details, fitness, dcIds); - continue; - } - if (!allowDegraded && worker_details.degraded) { - logWorkerUnavailable( - SevInfo, id, "complex", "Worker is degraded and not allowed", worker_details, fitness, dcIds); - continue; - } - - fitness_workers[std::make_tuple( - fitness, id_used[worker_process_id], isLongLivedStateless(worker_process_id))] - .push_back(worker_details); - } - - auto requiredFitness = ProcessClass::NeverAssign; - int requiredUsed = 1e6; - - typedef Optional> Field; - typedef Optional> Zone; - std::map, std::vector>> field_zones; - std::set fieldsWithMin; - std::map field_count; - std::map> field_fitness; - - // Determine the best required workers by finding the workers with enough unique zoneIds per field - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - deterministicRandom()->randomShuffle(workerIter->second); - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - - for (auto& worker : workerIter->second) { - auto thisField = worker.interf.locality.get(field); - auto& zones = field_zones[thisField]; - if (zones.first.insert(worker.interf.locality.zoneId()).second) { - zones.second.push_back(worker); - if (zones.first.size() == minPerField) { - fieldsWithMin.insert(thisField); - } - } - field_count[thisField]++; - field_fitness.insert( - { thisField, - std::make_tuple(fitness, used, worker.interf.locality.dcId() == clusterControllerDcId) }); - } - if (fieldsWithMin.size() >= minFields) { - requiredFitness = fitness; - requiredUsed = used; - } - } - - if (fieldsWithMin.size() < minFields) { - throw no_more_servers(); - } - - std::set chosenFields; - // If we cannot use all of the fields, use the fields which allow the best workers to be chosen - if (fieldsWithMin.size() * minPerField > desired) { - std::vector> orderedFields; - for (auto& it : fieldsWithMin) { - auto& fitness = field_fitness[it]; - orderedFields.emplace_back( - std::get<0>(fitness), std::get<1>(fitness), std::get<2>(fitness), field_count[it], it); - } - std::sort(orderedFields.begin(), orderedFields.end()); - int totalFields = desired / minPerField; - int maxCount = 0; - for (int i = 0; i < orderedFields.size() && chosenFields.size() < totalFields; i++) { - if (chosenFields.size() == totalFields - 1 && maxCount + std::get<3>(orderedFields[i]) < desired) { - for (int j = i + 1; j < orderedFields.size(); j++) { - if (maxCount + std::get<3>(orderedFields[j]) >= desired) { - chosenFields.insert(std::get<4>(orderedFields[j])); - break; - } - } - } - if (chosenFields.size() < totalFields) { - maxCount += std::get<3>(orderedFields[i]); - chosenFields.insert(std::get<4>(orderedFields[i])); - } - } - } else { - chosenFields = fieldsWithMin; - } - - // Create a result set with fulfills the minField and minPerField requirements before adding more workers - std::set resultSet; - for (auto& it : chosenFields) { - auto& w = field_zones[it].second; - for (int i = 0; i < minPerField; i++) { - resultSet.insert(w[i]); - } - } - - // Continue adding workers to the result set until we reach the desired number of workers - for (auto workerIter = fitness_workers.begin(); - workerIter != fitness_workers.end() && resultSet.size() < desired; - ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - if (workerIter->second.size() + resultSet.size() <= desired) { - for (auto& worker : workerIter->second) { - if (chosenFields.count(worker.interf.locality.get(field))) { - resultSet.insert(worker); - } - } - } else { - addWorkersByLowestField(field, desired, workerIter->second, resultSet); - } - } - - for (auto& result : resultSet) { - id_used[result.interf.locality.processId()]++; - } - - return std::vector(resultSet.begin(), resultSet.end()); - } - - // Attempt to recruit TLogs without degraded processes and see if it improves the configuration - std::vector getWorkersForTlogsComplex(DatabaseConfiguration const& conf, - int32_t desired, - std::map>, int>& id_used, - StringRef field, - int minFields, - int minPerField, - bool checkStable, - const std::set>& dcIds, - const std::vector& exclusionWorkerIds) { - desired = std::max(desired, minFields * minPerField); - std::map>, int> withDegradedUsed = id_used; - auto withDegraded = getWorkersForTlogsComplex(conf, - desired, - withDegradedUsed, - field, - minFields, - minPerField, - true, - checkStable, - dcIds, - exclusionWorkerIds); - RoleFitness withDegradedFitness(withDegraded, ProcessClass::TLog, withDegradedUsed); - ASSERT(withDegraded.size() <= desired); - - bool usedDegraded = false; - for (auto& it : withDegraded) { - if (it.degraded) { - usedDegraded = true; - break; - } - } - - if (!usedDegraded) { - id_used = withDegradedUsed; - return withDegraded; - } - - try { - std::map>, int> withoutDegradedUsed = id_used; - auto withoutDegraded = getWorkersForTlogsComplex(conf, - desired, - withoutDegradedUsed, - field, - minFields, - minPerField, - false, - checkStable, - dcIds, - exclusionWorkerIds); - RoleFitness withoutDegradedFitness(withoutDegraded, ProcessClass::TLog, withoutDegradedUsed); - ASSERT(withoutDegraded.size() <= desired); - - if (withDegradedFitness < withoutDegradedFitness) { - id_used = withDegradedUsed; - return withDegraded; - } - id_used = withoutDegradedUsed; - return withoutDegraded; - } catch (Error& e) { - if (e.code() != error_code_no_more_servers) { - throw; - } - id_used = withDegradedUsed; - return withDegraded; - } - } - - // A TLog recruitment method specialized for single, double, and triple configurations - // It recruits processes from with unique zoneIds until it reaches the desired amount - std::vector getWorkersForTlogsSimple(DatabaseConfiguration const& conf, - int32_t required, - int32_t desired, - std::map>, int>& id_used, - bool checkStable, - const std::set>& dcIds, - const std::vector& exclusionWorkerIds) { - std::map, std::vector> fitness_workers; - - // Go through all the workers to list all the workers that can be recruited. - for (const auto& [worker_process_id, worker_info] : id_worker) { - const auto& worker_details = worker_info.details; - auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); - - if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != - exclusionWorkerIds.end()) { - logWorkerUnavailable(SevInfo, id, "simple", "Worker is excluded", worker_details, fitness, dcIds); - continue; - } - if (!workerAvailable(worker_info, checkStable)) { - logWorkerUnavailable(SevInfo, id, "simple", "Worker is not available", worker_details, fitness, dcIds); - continue; - } - if (conf.isExcludedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "simple", - "Worker server is excluded from the cluster", - worker_details, - fitness, - dcIds); - continue; - } - if (isExcludedDegradedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "simple", - "Worker server is excluded from the cluster due to degradation", - worker_details, - fitness, - dcIds); - continue; - } - if (fitness == ProcessClass::NeverAssign) { - logWorkerUnavailable( - SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); - continue; - } - if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { - logWorkerUnavailable( - SevDebug, id, "simple", "Worker is not in the target DC", worker_details, fitness, dcIds); - continue; - } - - // This worker is a candidate for TLog recruitment. - bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId; - // Prefer recruiting a TransactionClass non-degraded process over a LogClass degraded process - if (worker_details.degraded) { - fitness = std::max(fitness, ProcessClass::GoodFit); - } - - fitness_workers[std::make_tuple(fitness, - id_used[worker_process_id], - worker_details.degraded, - isLongLivedStateless(worker_process_id), - inCCDC)] - .push_back(worker_details); - } - - auto requiredFitness = ProcessClass::BestFit; - int requiredUsed = 0; - - std::set>> zones; - std::set resultSet; - - // Determine the best required workers by finding the workers with enough unique zoneIds - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - deterministicRandom()->randomShuffle(workerIter->second); - for (auto& worker : workerIter->second) { - if (!zones.count(worker.interf.locality.zoneId())) { - zones.insert(worker.interf.locality.zoneId()); - resultSet.insert(worker); - if (resultSet.size() == required) { - break; - } - } - } - if (resultSet.size() == required) { - requiredFitness = fitness; - requiredUsed = used; - break; - } - } - - if (resultSet.size() < required) { - throw no_more_servers(); - } - - // Continue adding workers to the result set until we reach the desired number of workers - for (auto workerIter = fitness_workers.begin(); - workerIter != fitness_workers.end() && resultSet.size() < desired; - ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - if (workerIter->second.size() + resultSet.size() <= desired) { - for (auto& worker : workerIter->second) { - resultSet.insert(worker); - } - } else { - addWorkersByLowestZone(desired, workerIter->second, resultSet); - } - } - - ASSERT(resultSet.size() >= required && resultSet.size() <= desired); - - for (auto& result : resultSet) { - id_used[result.interf.locality.processId()]++; - } - - return std::vector(resultSet.begin(), resultSet.end()); - } - - // A backup method for TLog recruitment that is used for custom policies, but does a worse job - // selecting the best workers. - // conf: the database configuration. - // required: the required number of TLog workers to select. - // desired: the desired number of TLog workers to select. - // policy: the TLog replication policy the selection needs to satisfy. - // id_used: keep track of process IDs of selected workers. - // checkStable: when true, only select from workers that are considered as stable worker (not rebooted more than - // twice recently). - // dcIds: the target data centers the workers are in. The selected workers must all be from these - // data centers: - // exclusionWorkerIds: the workers to be excluded from the selection. - std::vector getWorkersForTlogsBackup( - DatabaseConfiguration const& conf, - int32_t required, - int32_t desired, - Reference const& policy, - std::map>, int>& id_used, - bool checkStable = false, - const std::set>& dcIds = std::set>(), - const std::vector& exclusionWorkerIds = {}) { - std::map, std::vector> fitness_workers; - std::vector results; - Reference logServerSet = Reference(new LocalityMap()); - LocalityMap* logServerMap = (LocalityMap*)logServerSet.getPtr(); - bool bCompleted = false; - desired = std::max(required, desired); - - // Go through all the workers to list all the workers that can be recruited. - for (const auto& [worker_process_id, worker_info] : id_worker) { - const auto& worker_details = worker_info.details; - auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); - - if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != - exclusionWorkerIds.end()) { - logWorkerUnavailable(SevInfo, id, "deprecated", "Worker is excluded", worker_details, fitness, dcIds); - continue; - } - if (!workerAvailable(worker_info, checkStable)) { - logWorkerUnavailable( - SevInfo, id, "deprecated", "Worker is not available", worker_details, fitness, dcIds); - continue; - } - if (conf.isExcludedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "deprecated", - "Worker server is excluded from the cluster", - worker_details, - fitness, - dcIds); - continue; - } - if (isExcludedDegradedServer(worker_details.interf.addresses())) { - logWorkerUnavailable(SevInfo, - id, - "deprecated", - "Worker server is excluded from the cluster due to degradation", - worker_details, - fitness, - dcIds); - continue; - } - if (fitness == ProcessClass::NeverAssign) { - logWorkerUnavailable( - SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); - continue; - } - if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { - logWorkerUnavailable( - SevDebug, id, "deprecated", "Worker is not in the target DC", worker_details, fitness, dcIds); - continue; - } - - // This worker is a candidate for TLog recruitment. - bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId; - // Prefer recruiting a TransactionClass non-degraded process over a LogClass degraded process - if (worker_details.degraded) { - fitness = std::max(fitness, ProcessClass::GoodFit); - } - - fitness_workers[std::make_tuple(fitness, id_used[worker_process_id], worker_details.degraded, inCCDC)] - .push_back(worker_details); - } - - auto requiredFitness = ProcessClass::BestFit; - int requiredUsed = 0; - bool requiredDegraded = false; - bool requiredInCCDC = false; - - // Determine the minimum fitness and used necessary to fulfill the policy - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - if (fitness > requiredFitness || used > requiredUsed) { - if (logServerSet->size() >= required && logServerSet->validate(policy)) { - bCompleted = true; - break; - } - requiredFitness = fitness; - requiredUsed = used; - } - - if (std::get<2>(workerIter->first)) { - requiredDegraded = true; - } - if (std::get<3>(workerIter->first)) { - requiredInCCDC = true; - } - for (auto& worker : workerIter->second) { - logServerMap->add(worker.interf.locality, &worker); - } - } - - if (!bCompleted && !(logServerSet->size() >= required && logServerSet->validate(policy))) { - std::vector tLocalities; - for (auto& object : logServerMap->getObjects()) { - tLocalities.push_back(object->interf.locality); - } - - logServerSet->clear(); - logServerSet.clear(); - throw no_more_servers(); - } - - // If we have less than the desired amount, return all of the processes we have - if (logServerSet->size() <= desired) { - for (auto& object : logServerMap->getObjects()) { - results.push_back(*object); - } - for (auto& result : results) { - id_used[result.interf.locality.processId()]++; - } - return results; - } - - // If we have added any degraded processes, try and remove them to see if we can still - // have the desired amount of processes - if (requiredDegraded) { - logServerMap->clear(); - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - auto addingDegraded = std::get<2>(workerIter->first); - if (addingDegraded) { - continue; - } - for (auto& worker : workerIter->second) { - logServerMap->add(worker.interf.locality, &worker); - } - } - if (logServerSet->size() >= desired && logServerSet->validate(policy)) { - requiredDegraded = false; - } - } - - // If we have added any processes in the CC DC, try and remove them to see if we can still - // have the desired amount of processes - if (requiredInCCDC) { - logServerMap->clear(); - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - auto addingDegraded = std::get<2>(workerIter->first); - auto inCCDC = std::get<3>(workerIter->first); - if (inCCDC || (!requiredDegraded && addingDegraded)) { - continue; - } - for (auto& worker : workerIter->second) { - logServerMap->add(worker.interf.locality, &worker); - } - } - if (logServerSet->size() >= desired && logServerSet->validate(policy)) { - requiredInCCDC = false; - } - } - - logServerMap->clear(); - for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { - auto fitness = std::get<0>(workerIter->first); - auto used = std::get<1>(workerIter->first); - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } - auto addingDegraded = std::get<2>(workerIter->first); - auto inCCDC = std::get<3>(workerIter->first); - if ((!requiredInCCDC && inCCDC) || (!requiredDegraded && addingDegraded)) { - continue; - } - for (auto& worker : workerIter->second) { - logServerMap->add(worker.interf.locality, &worker); - } - } - - if (logServerSet->size() == desired) { - for (auto& object : logServerMap->getObjects()) { - results.push_back(*object); - } - for (auto& result : results) { - id_used[result.interf.locality.processId()]++; - } - return results; - } - - std::vector bestSet; - std::vector tLocalities; - - // We have more than the desired number of processes, so use the policy engine to - // pick a diverse subset of them - bCompleted = findBestPolicySet(bestSet, - logServerSet, - policy, - desired, - SERVER_KNOBS->POLICY_RATING_TESTS, - SERVER_KNOBS->POLICY_GENERATIONS); - ASSERT(bCompleted); - results.reserve(results.size() + bestSet.size()); - for (auto& entry : bestSet) { - auto object = logServerMap->getObject(entry); - ASSERT(object); - results.push_back(*object); - tLocalities.push_back(object->interf.locality); - } - for (auto& result : results) { - id_used[result.interf.locality.processId()]++; - } - TraceEvent("GetTLogTeamDone") - .detail("Policy", policy->info()) - .detail("Results", results.size()) - .detail("Processes", logServerSet->size()) - .detail("Workers", id_worker.size()) - .detail("Required", required) - .detail("Desired", desired) - .detail("Fitness", requiredFitness) - .detail("Used", requiredUsed) - .detail("AddingDegraded", requiredDegraded) - .detail("InCCDC", requiredInCCDC) - .detail("BestCount", bestSet.size()) - .detail("BestZones", ::describeZones(tLocalities)) - .detail("BestDataHalls", ::describeDataHalls(tLocalities)); - return results; - } - - // Selects the best method for TLog recruitment based on the specified policy - std::vector getWorkersForTlogs(DatabaseConfiguration const& conf, - int32_t required, - int32_t desired, - Reference const& policy, - std::map>, int>& id_used, - bool checkStable = false, - const std::set>& dcIds = std::set>(), - const std::vector& exclusionWorkerIds = {}) { - desired = std::max(required, desired); - bool useSimple = false; - if (policy->name() == "Across") { - PolicyAcross* pa1 = (PolicyAcross*)policy.getPtr(); - Reference embedded = pa1->embeddedPolicy(); - if (embedded->name() == "Across") { - PolicyAcross* pa2 = (PolicyAcross*)embedded.getPtr(); - if (pa2->attributeKey() == "zoneid" && pa2->embeddedPolicyName() == "One") { - std::map>, int> testUsed = id_used; - - auto workers = getWorkersForTlogsComplex(conf, - desired, - id_used, - pa1->attributeKey(), - pa1->getCount(), - pa2->getCount(), - checkStable, - dcIds, - exclusionWorkerIds); - - if (g_network->isSimulated()) { - try { - auto testWorkers = getWorkersForTlogsBackup( - conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); - RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); - RoleFitness fitness(workers, ProcessClass::TLog, id_used); - - std::map>, int> field_count; - std::set>> zones; - for (auto& worker : testWorkers) { - if (!zones.count(worker.interf.locality.zoneId())) { - field_count[worker.interf.locality.get(pa1->attributeKey())]++; - zones.insert(worker.interf.locality.zoneId()); - } - } - // backup recruitment is not required to use degraded processes that have better fitness - // so we cannot compare degraded between the two methods - testFitness.degraded = fitness.degraded; - - int minField = 100; - - for (auto& f : field_count) { - minField = std::min(minField, f.second); - } - - if (fitness > testFitness && minField > 1) { - for (auto& w : testWorkers) { - TraceEvent("TestTLogs").detail("Interf", w.interf.address()); - } - for (auto& w : workers) { - TraceEvent("RealTLogs").detail("Interf", w.interf.address()); - } - TraceEvent("FitnessCompare") - .detail("TestF", testFitness.toString()) - .detail("RealF", fitness.toString()); - ASSERT(false); - } - } catch (Error& e) { - ASSERT(false); // Simulation only validation should not throw errors - } - } - - return workers; - } - } else if (pa1->attributeKey() == "zoneid" && embedded->name() == "One") { - ASSERT(pa1->getCount() == required); - useSimple = true; - } - } else if (policy->name() == "One") { - useSimple = true; - } - if (useSimple) { - std::map>, int> testUsed = id_used; - - auto workers = - getWorkersForTlogsSimple(conf, required, desired, id_used, checkStable, dcIds, exclusionWorkerIds); - - if (g_network->isSimulated()) { - try { - auto testWorkers = getWorkersForTlogsBackup( - conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); - RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); - RoleFitness fitness(workers, ProcessClass::TLog, id_used); - // backup recruitment is not required to use degraded processes that have better fitness - // so we cannot compare degraded between the two methods - testFitness.degraded = fitness.degraded; - - if (fitness > testFitness) { - for (auto& w : testWorkers) { - TraceEvent("TestTLogs").detail("Interf", w.interf.address()); - } - for (auto& w : workers) { - TraceEvent("RealTLogs").detail("Interf", w.interf.address()); - } - TraceEvent("FitnessCompare") - .detail("TestF", testFitness.toString()) - .detail("RealF", fitness.toString()); - ASSERT(false); - } - } catch (Error& e) { - ASSERT(false); // Simulation only validation should not throw errors - } - } - return workers; - } - TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "PolicyEngineNotOptimized"); - return getWorkersForTlogsBackup( - conf, required, desired, policy, id_used, checkStable, dcIds, exclusionWorkerIds); - } - - // FIXME: This logic will fallback unnecessarily when usable dcs > 1 because it does not check all combinations of - // potential satellite locations - std::vector getWorkersForSatelliteLogs(const DatabaseConfiguration& conf, - const RegionInfo& region, - const RegionInfo& remoteRegion, - std::map>, int>& id_used, - bool& satelliteFallback, - bool checkStable = false) { - int startDC = 0; - loop { - if (startDC > 0 && startDC >= region.satellites.size() + 1 - - (satelliteFallback ? region.satelliteTLogUsableDcsFallback - : region.satelliteTLogUsableDcs)) { - if (satelliteFallback || region.satelliteTLogUsableDcsFallback == 0) { - throw no_more_servers(); - } else { - if (!goodRecruitmentTime.isReady()) { - throw operation_failed(); - } - satelliteFallback = true; - startDC = 0; - } - } - - try { - bool remoteDCUsedAsSatellite = false; - std::set> satelliteDCs; - int32_t desiredSatelliteTLogs = 0; - for (int s = startDC; - s < std::min(startDC + (satelliteFallback ? region.satelliteTLogUsableDcsFallback - : region.satelliteTLogUsableDcs), - region.satellites.size()); - s++) { - satelliteDCs.insert(region.satellites[s].dcId); - if (region.satellites[s].satelliteDesiredTLogCount == -1 || desiredSatelliteTLogs == -1) { - desiredSatelliteTLogs = -1; - } else { - desiredSatelliteTLogs += region.satellites[s].satelliteDesiredTLogCount; - } - if (region.satellites[s].dcId == remoteRegion.dcId) { - remoteDCUsedAsSatellite = true; - } - } - std::vector exclusionWorkerIds; - // FIXME: If remote DC is used as satellite then this logic only ensures that required number of remote - // TLogs can be recruited. It does not balance the number of desired TLogs across the satellite and - // remote sides. - if (remoteDCUsedAsSatellite) { - std::map>, int> tmpIdUsed; - auto remoteLogs = getWorkersForTlogs(conf, - conf.getRemoteTLogReplicationFactor(), - conf.getRemoteTLogReplicationFactor(), - conf.getRemoteTLogPolicy(), - tmpIdUsed, - false, - { remoteRegion.dcId }, - {}); - std::transform(remoteLogs.begin(), - remoteLogs.end(), - std::back_inserter(exclusionWorkerIds), - [](const WorkerDetails& in) { return in.interf.id(); }); - } - if (satelliteFallback) { - return getWorkersForTlogs(conf, - region.satelliteTLogReplicationFactorFallback, - desiredSatelliteTLogs > 0 ? desiredSatelliteTLogs - : conf.getDesiredSatelliteLogs(region.dcId) * - region.satelliteTLogUsableDcsFallback / - region.satelliteTLogUsableDcs, - region.satelliteTLogPolicyFallback, - id_used, - checkStable, - satelliteDCs, - exclusionWorkerIds); - } else { - return getWorkersForTlogs(conf, - region.satelliteTLogReplicationFactor, - desiredSatelliteTLogs > 0 ? desiredSatelliteTLogs - : conf.getDesiredSatelliteLogs(region.dcId), - region.satelliteTLogPolicy, - id_used, - checkStable, - satelliteDCs, - exclusionWorkerIds); - } - } catch (Error& e) { - if (e.code() != error_code_no_more_servers) { - throw; - } - } - - startDC++; - } - } - - ProcessClass::Fitness getBestFitnessForRoleInDatacenter(ProcessClass::ClusterRole role) { - ProcessClass::Fitness bestFitness = ProcessClass::NeverAssign; - for (const auto& it : id_worker) { - if (it.second.priorityInfo.isExcluded || - it.second.details.interf.locality.dcId() != clusterControllerDcId) { - continue; - } - bestFitness = std::min(bestFitness, it.second.details.processClass.machineClassFitness(role)); - } - return bestFitness; - } - - WorkerFitnessInfo getWorkerForRoleInDatacenter(Optional> const& dcId, - ProcessClass::ClusterRole role, - ProcessClass::Fitness unacceptableFitness, - DatabaseConfiguration const& conf, - std::map>, int>& id_used, - std::map>, int> preferredSharing = {}, - bool checkStable = false) { - std::map, std::vector> fitness_workers; - - for (auto& it : id_worker) { - auto fitness = it.second.details.processClass.machineClassFitness(role); - if (conf.isExcludedServer(it.second.details.interf.addresses()) || - isExcludedDegradedServer(it.second.details.interf.addresses())) { - fitness = std::max(fitness, ProcessClass::ExcludeFit); - } - if (workerAvailable(it.second, checkStable) && fitness < unacceptableFitness && - it.second.details.interf.locality.dcId() == dcId) { - auto sharing = preferredSharing.find(it.first); - fitness_workers[std::make_tuple(fitness, - id_used[it.first], - isLongLivedStateless(it.first), - sharing != preferredSharing.end() ? sharing->second : 1e6)] - .push_back(it.second.details); - } - } - - if (fitness_workers.size()) { - auto worker = deterministicRandom()->randomChoice(fitness_workers.begin()->second); - id_used[worker.interf.locality.processId()]++; - return WorkerFitnessInfo(worker, - std::max(ProcessClass::GoodFit, std::get<0>(fitness_workers.begin()->first)), - std::get<1>(fitness_workers.begin()->first)); - } - - throw no_more_servers(); - } - - std::vector getWorkersForRoleInDatacenter( - Optional> const& dcId, - ProcessClass::ClusterRole role, - int amount, - DatabaseConfiguration const& conf, - std::map>, int>& id_used, - std::map>, int> preferredSharing = {}, - Optional minWorker = Optional(), - bool checkStable = false) { - std::map, std::vector> fitness_workers; - std::vector results; - if (minWorker.present()) { - results.push_back(minWorker.get().worker); - } - if (amount <= results.size()) { - return results; - } - - for (auto& it : id_worker) { - auto fitness = it.second.details.processClass.machineClassFitness(role); - if (workerAvailable(it.second, checkStable) && - !conf.isExcludedServer(it.second.details.interf.addresses()) && - !isExcludedDegradedServer(it.second.details.interf.addresses()) && - it.second.details.interf.locality.dcId() == dcId && - (!minWorker.present() || - (it.second.details.interf.id() != minWorker.get().worker.interf.id() && - (fitness < minWorker.get().fitness || - (fitness == minWorker.get().fitness && id_used[it.first] <= minWorker.get().used))))) { - auto sharing = preferredSharing.find(it.first); - fitness_workers[std::make_tuple(fitness, - id_used[it.first], - isLongLivedStateless(it.first), - sharing != preferredSharing.end() ? sharing->second : 1e6)] - .push_back(it.second.details); - } - } - - for (auto& it : fitness_workers) { - deterministicRandom()->randomShuffle(it.second); - for (int i = 0; i < it.second.size(); i++) { - results.push_back(it.second[i]); - id_used[it.second[i].interf.locality.processId()]++; - if (results.size() == amount) - return results; - } - } - - return results; - } - - // Allows the comparison of two different recruitments to determine which one is better - // Tlog recruitment is different from all the other roles, in that it avoids degraded processes - // And tried to avoid recruitment in the same DC as the cluster controller - struct RoleFitness { - ProcessClass::Fitness bestFit; - ProcessClass::Fitness worstFit; - ProcessClass::ClusterRole role; - int count; - int worstUsed = 1; - bool degraded = false; - - RoleFitness(int bestFit, int worstFit, int count, ProcessClass::ClusterRole role) - : bestFit((ProcessClass::Fitness)bestFit), worstFit((ProcessClass::Fitness)worstFit), role(role), - count(count) {} - - RoleFitness(int fitness, int count, ProcessClass::ClusterRole role) - : bestFit((ProcessClass::Fitness)fitness), worstFit((ProcessClass::Fitness)fitness), role(role), - count(count) {} - - RoleFitness() - : bestFit(ProcessClass::NeverAssign), worstFit(ProcessClass::NeverAssign), role(ProcessClass::NoRole), - count(0) {} - - RoleFitness(const std::vector& workers, - ProcessClass::ClusterRole role, - const std::map>, int>& id_used) - : role(role) { - // Every recruitment will attempt to recruit the preferred amount through GoodFit, - // So a recruitment which only has BestFit is not better than one that has a GoodFit process - worstFit = ProcessClass::GoodFit; - degraded = false; - bestFit = ProcessClass::NeverAssign; - worstUsed = 1; - for (auto& it : workers) { - auto thisFit = it.processClass.machineClassFitness(role); - auto thisUsed = id_used.find(it.interf.locality.processId()); - - if (thisUsed == id_used.end()) { - TraceEvent(SevError, "UsedNotFound").detail("ProcessId", it.interf.locality.processId().get()); - ASSERT(false); - } - if (thisUsed->second == 0) { - TraceEvent(SevError, "UsedIsZero").detail("ProcessId", it.interf.locality.processId().get()); - ASSERT(false); - } - - bestFit = std::min(bestFit, thisFit); - - if (thisFit > worstFit) { - worstFit = thisFit; - worstUsed = thisUsed->second; - } else if (thisFit == worstFit) { - worstUsed = std::max(worstUsed, thisUsed->second); - } - degraded = degraded || it.degraded; - } - - count = workers.size(); - - // degraded is only used for recruitment of tlogs - if (role != ProcessClass::TLog) { - degraded = false; - } - } - - bool operator<(RoleFitness const& r) const { - if (worstFit != r.worstFit) - return worstFit < r.worstFit; - if (worstUsed != r.worstUsed) - return worstUsed < r.worstUsed; - if (count != r.count) - return count > r.count; - if (degraded != r.degraded) - return r.degraded; - // FIXME: TLog recruitment process does not guarantee the best fit is not worsened. - if (role != ProcessClass::TLog && role != ProcessClass::LogRouter && bestFit != r.bestFit) - return bestFit < r.bestFit; - return false; - } - bool operator>(RoleFitness const& r) const { return r < *this; } - bool operator<=(RoleFitness const& r) const { return !(*this > r); } - bool operator>=(RoleFitness const& r) const { return !(*this < r); } - - bool betterCount(RoleFitness const& r) const { - if (count > r.count) - return true; - if (worstFit != r.worstFit) - return worstFit < r.worstFit; - if (worstUsed != r.worstUsed) - return worstUsed < r.worstUsed; - if (degraded != r.degraded) - return r.degraded; - return false; - } - - bool operator==(RoleFitness const& r) const { - return worstFit == r.worstFit && worstUsed == r.worstUsed && bestFit == r.bestFit && count == r.count && - degraded == r.degraded; - } - - std::string toString() const { return format("%d %d %d %d %d", worstFit, worstUsed, count, degraded, bestFit); } - }; - - std::set>> getDatacenters(DatabaseConfiguration const& conf, - bool checkStable = false) { - std::set>> result; - for (auto& it : id_worker) - if (workerAvailable(it.second, checkStable) && - !conf.isExcludedServer(it.second.details.interf.addresses()) && - !isExcludedDegradedServer(it.second.details.interf.addresses())) - result.insert(it.second.details.interf.locality.dcId()); - return result; - } - - void updateKnownIds(std::map>, int>* id_used) { - (*id_used)[masterProcessId]++; - (*id_used)[clusterControllerProcessId]++; - } - - RecruitRemoteFromConfigurationReply findRemoteWorkersForConfiguration( - RecruitRemoteFromConfigurationRequest const& req) { - RecruitRemoteFromConfigurationReply result; - std::map>, int> id_used; - - updateKnownIds(&id_used); - - std::set> remoteDC; - remoteDC.insert(req.dcId); - - auto remoteLogs = getWorkersForTlogs(req.configuration, - req.configuration.getRemoteTLogReplicationFactor(), - req.configuration.getDesiredRemoteLogs(), - req.configuration.getRemoteTLogPolicy(), - id_used, - false, - remoteDC, - req.exclusionWorkerIds); - for (int i = 0; i < remoteLogs.size(); i++) { - result.remoteTLogs.push_back(remoteLogs[i].interf); - } - - auto logRouters = getWorkersForRoleInDatacenter( - req.dcId, ProcessClass::LogRouter, req.logRouterCount, req.configuration, id_used); - for (int i = 0; i < logRouters.size(); i++) { - result.logRouters.push_back(logRouters[i].interf); - } - - if (!goodRemoteRecruitmentTime.isReady() && - ((RoleFitness( - SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredRemoteLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(remoteLogs, ProcessClass::TLog, id_used))) || - (RoleFitness(SERVER_KNOBS->EXPECTED_LOG_ROUTER_FITNESS, req.logRouterCount, ProcessClass::LogRouter) - .betterCount(RoleFitness(logRouters, ProcessClass::LogRouter, id_used))))) { - throw operation_failed(); - } - - return result; - } - - // Given datacenter ID, returns the primary and remote regions. - std::pair getPrimaryAndRemoteRegion(const std::vector& regions, Key dcId) { - RegionInfo region; - RegionInfo remoteRegion; - for (const auto& r : regions) { - if (r.dcId == dcId) { - region = r; - } else { - remoteRegion = r; - } - } - return std::make_pair(region, remoteRegion); - } - - ErrorOr findWorkersForConfigurationFromDC(RecruitFromConfigurationRequest const& req, - Optional dcId, - bool checkGoodRecruitment) { - RecruitFromConfigurationReply result; - std::map>, int> id_used; - updateKnownIds(&id_used); - - ASSERT(dcId.present()); - - std::set> primaryDC; - primaryDC.insert(dcId); - result.dcId = dcId; - - auto [region, remoteRegion] = getPrimaryAndRemoteRegion(req.configuration.regions, dcId.get()); - - if (req.recruitSeedServers) { - auto primaryStorageServers = - getWorkersForSeedServers(req.configuration, req.configuration.storagePolicy, dcId); - for (int i = 0; i < primaryStorageServers.size(); i++) { - result.storageServers.push_back(primaryStorageServers[i].interf); - } - } - - auto tlogs = getWorkersForTlogs(req.configuration, - req.configuration.tLogReplicationFactor, - req.configuration.getDesiredLogs(), - req.configuration.tLogPolicy, - id_used, - false, - primaryDC); - for (int i = 0; i < tlogs.size(); i++) { - result.tLogs.push_back(tlogs[i].interf); - } - - std::vector satelliteLogs; - if (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1) { - satelliteLogs = - getWorkersForSatelliteLogs(req.configuration, region, remoteRegion, id_used, result.satelliteFallback); - for (int i = 0; i < satelliteLogs.size(); i++) { - result.satelliteTLogs.push_back(satelliteLogs[i].interf); - } - } - - std::map>, int> preferredSharing; - auto first_commit_proxy = getWorkerForRoleInDatacenter( - dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); - preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; - auto first_grv_proxy = getWorkerForRoleInDatacenter( - dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); - preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; - auto first_resolver = getWorkerForRoleInDatacenter( - dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); - preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; - - // If one of the first process recruitments is forced to share a process, allow all of next recruitments - // to also share a process. - auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); - first_commit_proxy.used = maxUsed; - first_grv_proxy.used = maxUsed; - first_resolver.used = maxUsed; - - auto commit_proxies = getWorkersForRoleInDatacenter(dcId, - ProcessClass::CommitProxy, - req.configuration.getDesiredCommitProxies(), - req.configuration, - id_used, - preferredSharing, - first_commit_proxy); - auto grv_proxies = getWorkersForRoleInDatacenter(dcId, - ProcessClass::GrvProxy, - req.configuration.getDesiredGrvProxies(), - req.configuration, - id_used, - preferredSharing, - first_grv_proxy); - auto resolvers = getWorkersForRoleInDatacenter(dcId, - ProcessClass::Resolver, - req.configuration.getDesiredResolvers(), - req.configuration, - id_used, - preferredSharing, - first_resolver); - for (int i = 0; i < commit_proxies.size(); i++) - result.commitProxies.push_back(commit_proxies[i].interf); - for (int i = 0; i < grv_proxies.size(); i++) - result.grvProxies.push_back(grv_proxies[i].interf); - for (int i = 0; i < resolvers.size(); i++) - result.resolvers.push_back(resolvers[i].interf); - - if (req.maxOldLogRouters > 0) { - if (tlogs.size() == 1) { - result.oldLogRouters.push_back(tlogs[0].interf); - } else { - for (int i = 0; i < tlogs.size(); i++) { - if (tlogs[i].interf.locality.processId() != clusterControllerProcessId) { - result.oldLogRouters.push_back(tlogs[i].interf); - } - } - } - } - - if (req.configuration.backupWorkerEnabled) { - const int nBackup = std::max( - (req.configuration.desiredLogRouterCount > 0 ? req.configuration.desiredLogRouterCount : tlogs.size()), - req.maxOldLogRouters); - auto backupWorkers = - getWorkersForRoleInDatacenter(dcId, ProcessClass::Backup, nBackup, req.configuration, id_used); - std::transform(backupWorkers.begin(), - backupWorkers.end(), - std::back_inserter(result.backupWorkers), - [](const WorkerDetails& w) { return w.interf; }); - } - - if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && - (RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || - (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1 && - RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, - req.configuration.getDesiredSatelliteLogs(dcId), - ProcessClass::TLog) - .betterCount(RoleFitness(satelliteLogs, ProcessClass::TLog, id_used))) || - RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, - req.configuration.getDesiredCommitProxies(), - ProcessClass::CommitProxy) - .betterCount(RoleFitness(commit_proxies, ProcessClass::CommitProxy, id_used)) || - RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, - req.configuration.getDesiredGrvProxies(), - ProcessClass::GrvProxy) - .betterCount(RoleFitness(grv_proxies, ProcessClass::GrvProxy, id_used)) || - RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, - req.configuration.getDesiredResolvers(), - ProcessClass::Resolver) - .betterCount(RoleFitness(resolvers, ProcessClass::Resolver, id_used)))) { - return operation_failed(); - } - - return result; - } - - RecruitFromConfigurationReply findWorkersForConfigurationDispatch(RecruitFromConfigurationRequest const& req, - bool checkGoodRecruitment) { - if (req.configuration.regions.size() > 1) { - std::vector regions = req.configuration.regions; - if (regions[0].priority == regions[1].priority && regions[1].dcId == clusterControllerDcId.get()) { - TraceEvent("CCSwitchPrimaryDc", id) - .detail("CCDcId", clusterControllerDcId.get()) - .detail("OldPrimaryDcId", regions[0].dcId) - .detail("NewPrimaryDcId", regions[1].dcId); - std::swap(regions[0], regions[1]); - } - - if (regions[1].dcId == clusterControllerDcId.get() && - (!versionDifferenceUpdated || datacenterVersionDifference >= SERVER_KNOBS->MAX_VERSION_DIFFERENCE)) { - if (regions[1].priority >= 0) { - TraceEvent("CCSwitchPrimaryDcVersionDifference", id) - .detail("CCDcId", clusterControllerDcId.get()) - .detail("OldPrimaryDcId", regions[0].dcId) - .detail("NewPrimaryDcId", regions[1].dcId); - std::swap(regions[0], regions[1]); - } else { - TraceEvent(SevWarnAlways, "CCDcPriorityNegative") - .detail("DcId", regions[1].dcId) - .detail("Priority", regions[1].priority) - .detail("FindWorkersInDc", regions[0].dcId) - .detail("Warning", "Failover did not happen but CC is in remote DC"); - } - } - - TraceEvent("CCFindWorkersForConfiguration", id) - .detail("CCDcId", clusterControllerDcId.get()) - .detail("Region0DcId", regions[0].dcId) - .detail("Region1DcId", regions[1].dcId) - .detail("DatacenterVersionDifference", datacenterVersionDifference) - .detail("VersionDifferenceUpdated", versionDifferenceUpdated); - - bool setPrimaryDesired = false; - try { - auto reply = findWorkersForConfigurationFromDC(req, regions[0].dcId, checkGoodRecruitment); - setPrimaryDesired = true; - std::vector> dcPriority; - dcPriority.push_back(regions[0].dcId); - dcPriority.push_back(regions[1].dcId); - desiredDcIds.set(dcPriority); - if (reply.isError()) { - throw reply.getError(); - } else if (regions[0].dcId == clusterControllerDcId.get()) { - return reply.get(); - } - TraceEvent(SevWarn, "CCRecruitmentFailed", id) - .detail("Reason", "Recruited Txn system and CC are in different DCs") - .detail("CCDcId", clusterControllerDcId.get()) - .detail("RecruitedTxnSystemDcId", regions[0].dcId); - throw no_more_servers(); - } catch (Error& e) { - if (!goodRemoteRecruitmentTime.isReady() && regions[1].dcId != clusterControllerDcId.get() && - checkGoodRecruitment) { - throw operation_failed(); - } - - if (e.code() != error_code_no_more_servers || regions[1].priority < 0) { - throw; - } - TraceEvent(SevWarn, "AttemptingRecruitmentInRemoteDc", id) - .detail("SetPrimaryDesired", setPrimaryDesired) - .error(e); - auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId, checkGoodRecruitment); - if (!setPrimaryDesired) { - std::vector> dcPriority; - dcPriority.push_back(regions[1].dcId); - dcPriority.push_back(regions[0].dcId); - desiredDcIds.set(dcPriority); - } - if (reply.isError()) { - throw reply.getError(); - } else if (regions[1].dcId == clusterControllerDcId.get()) { - return reply.get(); - } - throw; - } - } else if (req.configuration.regions.size() == 1) { - std::vector> dcPriority; - dcPriority.push_back(req.configuration.regions[0].dcId); - desiredDcIds.set(dcPriority); - auto reply = - findWorkersForConfigurationFromDC(req, req.configuration.regions[0].dcId, checkGoodRecruitment); - if (reply.isError()) { - throw reply.getError(); - } else if (req.configuration.regions[0].dcId == clusterControllerDcId.get()) { - return reply.get(); - } - throw no_more_servers(); - } else { - RecruitFromConfigurationReply result; - std::map>, int> id_used; - updateKnownIds(&id_used); - auto tlogs = getWorkersForTlogs(req.configuration, - req.configuration.tLogReplicationFactor, - req.configuration.getDesiredLogs(), - req.configuration.tLogPolicy, - id_used); - for (int i = 0; i < tlogs.size(); i++) { - result.tLogs.push_back(tlogs[i].interf); - } - - if (req.maxOldLogRouters > 0) { - if (tlogs.size() == 1) { - result.oldLogRouters.push_back(tlogs[0].interf); - } else { - for (int i = 0; i < tlogs.size(); i++) { - if (tlogs[i].interf.locality.processId() != clusterControllerProcessId) { - result.oldLogRouters.push_back(tlogs[i].interf); - } - } - } - } - - if (req.recruitSeedServers) { - auto primaryStorageServers = - getWorkersForSeedServers(req.configuration, req.configuration.storagePolicy); - for (int i = 0; i < primaryStorageServers.size(); i++) - result.storageServers.push_back(primaryStorageServers[i].interf); - } - - auto datacenters = getDatacenters(req.configuration); - - std::tuple bestFitness; - int numEquivalent = 1; - Optional bestDC; - - for (auto dcId : datacenters) { - try { - // SOMEDAY: recruitment in other DCs besides the clusterControllerDcID will not account for the - // processes used by the master and cluster controller properly. - auto used = id_used; - std::map>, int> preferredSharing; - auto first_commit_proxy = getWorkerForRoleInDatacenter(dcId, - ProcessClass::CommitProxy, - ProcessClass::ExcludeFit, - req.configuration, - used, - preferredSharing); - preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; - auto first_grv_proxy = getWorkerForRoleInDatacenter(dcId, - ProcessClass::GrvProxy, - ProcessClass::ExcludeFit, - req.configuration, - used, - preferredSharing); - preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; - auto first_resolver = getWorkerForRoleInDatacenter(dcId, - ProcessClass::Resolver, - ProcessClass::ExcludeFit, - req.configuration, - used, - preferredSharing); - preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; - - // If one of the first process recruitments is forced to share a process, allow all of next - // recruitments to also share a process. - auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); - first_commit_proxy.used = maxUsed; - first_grv_proxy.used = maxUsed; - first_resolver.used = maxUsed; - - auto commit_proxies = getWorkersForRoleInDatacenter(dcId, - ProcessClass::CommitProxy, - req.configuration.getDesiredCommitProxies(), - req.configuration, - used, - preferredSharing, - first_commit_proxy); - - auto grv_proxies = getWorkersForRoleInDatacenter(dcId, - ProcessClass::GrvProxy, - req.configuration.getDesiredGrvProxies(), - req.configuration, - used, - preferredSharing, - first_grv_proxy); - - auto resolvers = getWorkersForRoleInDatacenter(dcId, - ProcessClass::Resolver, - req.configuration.getDesiredResolvers(), - req.configuration, - used, - preferredSharing, - first_resolver); - - auto fitness = std::make_tuple(RoleFitness(commit_proxies, ProcessClass::CommitProxy, used), - RoleFitness(grv_proxies, ProcessClass::GrvProxy, used), - RoleFitness(resolvers, ProcessClass::Resolver, used)); - - if (dcId == clusterControllerDcId) { - bestFitness = fitness; - bestDC = dcId; - for (int i = 0; i < resolvers.size(); i++) { - result.resolvers.push_back(resolvers[i].interf); - } - for (int i = 0; i < commit_proxies.size(); i++) { - result.commitProxies.push_back(commit_proxies[i].interf); - } - for (int i = 0; i < grv_proxies.size(); i++) { - result.grvProxies.push_back(grv_proxies[i].interf); - } - - if (req.configuration.backupWorkerEnabled) { - const int nBackup = std::max(tlogs.size(), req.maxOldLogRouters); - auto backupWorkers = getWorkersForRoleInDatacenter( - dcId, ProcessClass::Backup, nBackup, req.configuration, used); - std::transform(backupWorkers.begin(), - backupWorkers.end(), - std::back_inserter(result.backupWorkers), - [](const WorkerDetails& w) { return w.interf; }); - } - - break; - } else { - if (fitness < bestFitness) { - bestFitness = fitness; - numEquivalent = 1; - bestDC = dcId; - } else if (fitness == bestFitness && - deterministicRandom()->random01() < 1.0 / ++numEquivalent) { - bestDC = dcId; - } - } - } catch (Error& e) { - if (e.code() != error_code_no_more_servers) { - throw; - } - } - } - - if (bestDC != clusterControllerDcId) { - TraceEvent("BestDCIsNotClusterDC").log(); - std::vector> dcPriority; - dcPriority.push_back(bestDC); - desiredDcIds.set(dcPriority); - throw no_more_servers(); - } - // If this cluster controller dies, do not prioritize recruiting the next one in the same DC - desiredDcIds.set(std::vector>()); - TraceEvent("FindWorkersForConfig") - .detail("Replication", req.configuration.tLogReplicationFactor) - .detail("DesiredLogs", req.configuration.getDesiredLogs()) - .detail("ActualLogs", result.tLogs.size()) - .detail("DesiredCommitProxies", req.configuration.getDesiredCommitProxies()) - .detail("ActualCommitProxies", result.commitProxies.size()) - .detail("DesiredGrvProxies", req.configuration.getDesiredGrvProxies()) - .detail("ActualGrvProxies", result.grvProxies.size()) - .detail("DesiredResolvers", req.configuration.getDesiredResolvers()) - .detail("ActualResolvers", result.resolvers.size()); - - if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && - (RoleFitness( - SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || - RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, - req.configuration.getDesiredCommitProxies(), - ProcessClass::CommitProxy) - .betterCount(std::get<0>(bestFitness)) || - RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, - req.configuration.getDesiredGrvProxies(), - ProcessClass::GrvProxy) - .betterCount(std::get<1>(bestFitness)) || - RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, - req.configuration.getDesiredResolvers(), - ProcessClass::Resolver) - .betterCount(std::get<2>(bestFitness)))) { - throw operation_failed(); - } - - return result; - } - } - - void updateIdUsed(const std::vector& workers, - std::map>, int>& id_used) { - for (auto& it : workers) { - id_used[it.locality.processId()]++; - } - } - - void compareWorkers(const DatabaseConfiguration& conf, - const std::vector& first, - std::map>, int>& firstUsed, - const std::vector& second, - std::map>, int>& secondUsed, - ProcessClass::ClusterRole role, - std::string description) { - std::vector firstDetails; - for (auto& it : first) { - auto w = id_worker.find(it.locality.processId()); - ASSERT(w != id_worker.end()); - ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses())); - firstDetails.push_back(w->second.details); - //TraceEvent("CompareAddressesFirst").detail(description.c_str(), w->second.details.interf.address()); - } - RoleFitness firstFitness(firstDetails, role, firstUsed); - - std::vector secondDetails; - for (auto& it : second) { - auto w = id_worker.find(it.locality.processId()); - ASSERT(w != id_worker.end()); - ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses())); - secondDetails.push_back(w->second.details); - //TraceEvent("CompareAddressesSecond").detail(description.c_str(), w->second.details.interf.address()); - } - RoleFitness secondFitness(secondDetails, role, secondUsed); - - if (!(firstFitness == secondFitness)) { - TraceEvent(SevError, "NonDeterministicRecruitment") - .detail("FirstFitness", firstFitness.toString()) - .detail("SecondFitness", secondFitness.toString()) - .detail("ClusterRole", role); - } - } - - RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { - RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req, true); - if (g_network->isSimulated()) { - try { - // FIXME: The logic to pick a satellite in a remote region is not - // deterministic and can therefore break this nondeterminism check. - // Since satellites will generally be in the primary region, - // disable the determinism check for remote region satellites. - bool remoteDCUsedAsSatellite = false; - if (req.configuration.regions.size() > 1) { - auto [region, remoteRegion] = - getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId); - for (const auto& satellite : region.satellites) { - if (satellite.dcId == remoteRegion.dcId) { - remoteDCUsedAsSatellite = true; - } - } - } - if (!remoteDCUsedAsSatellite) { - RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req, false); - - std::map>, int> firstUsed; - std::map>, int> secondUsed; - updateKnownIds(&firstUsed); - updateKnownIds(&secondUsed); - - // auto mworker = id_worker.find(masterProcessId); - //TraceEvent("CompareAddressesMaster") - // .detail("Master", - // mworker != id_worker.end() ? mworker->second.details.interf.address() : - // NetworkAddress()); - - updateIdUsed(rep.tLogs, firstUsed); - updateIdUsed(compare.tLogs, secondUsed); - compareWorkers( - req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); - updateIdUsed(rep.satelliteTLogs, firstUsed); - updateIdUsed(compare.satelliteTLogs, secondUsed); - compareWorkers(req.configuration, - rep.satelliteTLogs, - firstUsed, - compare.satelliteTLogs, - secondUsed, - ProcessClass::TLog, - "Satellite"); - updateIdUsed(rep.commitProxies, firstUsed); - updateIdUsed(compare.commitProxies, secondUsed); - updateIdUsed(rep.grvProxies, firstUsed); - updateIdUsed(compare.grvProxies, secondUsed); - updateIdUsed(rep.resolvers, firstUsed); - updateIdUsed(compare.resolvers, secondUsed); - compareWorkers(req.configuration, - rep.commitProxies, - firstUsed, - compare.commitProxies, - secondUsed, - ProcessClass::CommitProxy, - "CommitProxy"); - compareWorkers(req.configuration, - rep.grvProxies, - firstUsed, - compare.grvProxies, - secondUsed, - ProcessClass::GrvProxy, - "GrvProxy"); - compareWorkers(req.configuration, - rep.resolvers, - firstUsed, - compare.resolvers, - secondUsed, - ProcessClass::Resolver, - "Resolver"); - updateIdUsed(rep.backupWorkers, firstUsed); - updateIdUsed(compare.backupWorkers, secondUsed); - compareWorkers(req.configuration, - rep.backupWorkers, - firstUsed, - compare.backupWorkers, - secondUsed, - ProcessClass::Backup, - "Backup"); - } - } catch (Error& e) { - ASSERT(false); // Simulation only validation should not throw errors - } - } - return rep; - } - - // Check if txn system is recruited successfully in each region - void checkRegions(const std::vector& regions) { - if (desiredDcIds.get().present() && desiredDcIds.get().get().size() == 2 && - desiredDcIds.get().get()[0].get() == regions[0].dcId && - desiredDcIds.get().get()[1].get() == regions[1].dcId) { - return; - } - - try { - std::map>, int> id_used; - getWorkerForRoleInDatacenter(regions[0].dcId, - ProcessClass::ClusterController, - ProcessClass::ExcludeFit, - db.config, - id_used, - {}, - true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::Master, ProcessClass::ExcludeFit, db.config, id_used, {}, true); - - std::set> primaryDC; - primaryDC.insert(regions[0].dcId); - getWorkersForTlogs(db.config, - db.config.tLogReplicationFactor, - db.config.getDesiredLogs(), - db.config.tLogPolicy, - id_used, - true, - primaryDC); - if (regions[0].satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) { - bool satelliteFallback = false; - getWorkersForSatelliteLogs(db.config, regions[0], regions[1], id_used, satelliteFallback, true); - } - - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, {}, true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, db.config, id_used, {}, true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, {}, true); - - std::vector> dcPriority; - dcPriority.push_back(regions[0].dcId); - dcPriority.push_back(regions[1].dcId); - desiredDcIds.set(dcPriority); - } catch (Error& e) { - if (e.code() != error_code_no_more_servers) { - throw; - } - } - } - - void checkRecoveryStalled() { - if ((db.serverInfo->get().recoveryState == RecoveryState::RECRUITING || - db.serverInfo->get().recoveryState == RecoveryState::ACCEPTING_COMMITS || - db.serverInfo->get().recoveryState == RecoveryState::ALL_LOGS_RECRUITED) && - db.recoveryStalled) { - if (db.config.regions.size() > 1) { - auto regions = db.config.regions; - if (clusterControllerDcId.get() == regions[0].dcId && regions[1].priority >= 0) { - std::swap(regions[0], regions[1]); - } - ASSERT(regions[1].priority < 0 || clusterControllerDcId.get() == regions[1].dcId); - checkRegions(regions); - } - } - } - - void updateIdUsed(const std::vector& workers, - std::map>, int>& id_used) { - for (auto& it : workers) { - id_used[it.interf.locality.processId()]++; - } - } - - // FIXME: determine when to fail the cluster controller when a primaryDC has not been set - - // This function returns true when the cluster controller determines it is worth forcing - // a master recovery in order to change the recruited processes in the transaction subsystem. - bool betterMasterExists() { - const ServerDBInfo dbi = db.serverInfo->get(); - - if (dbi.recoveryState < RecoveryState::ACCEPTING_COMMITS) { - return false; - } - - // Do not trigger better master exists if the cluster controller is excluded, since the master will change - // anyways once the cluster controller is moved - if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { - TraceEvent("NewRecruitmentIsWorse", id).detail("Reason", "ClusterControllerExcluded"); - return false; - } - - if (db.config.regions.size() > 1 && db.config.regions[0].priority > db.config.regions[1].priority && - db.config.regions[0].dcId != clusterControllerDcId.get() && versionDifferenceUpdated && - datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE && remoteDCIsHealthy()) { - checkRegions(db.config.regions); - } - - // Get master process - auto masterWorker = id_worker.find(dbi.master.locality.processId()); - if (masterWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindMaster") - .detail("ProcessID", dbi.master.locality.processId()); - return false; - } - - // Get tlog processes - std::vector tlogs; - std::vector remote_tlogs; - std::vector satellite_tlogs; - std::vector log_routers; - std::set logRouterAddresses; - std::vector backup_workers; - std::set backup_addresses; - - for (auto& logSet : dbi.logSystemConfig.tLogs) { - for (auto& it : logSet.tLogs) { - auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); - if (tlogWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindTLog") - .detail("ProcessID", it.interf().filteredLocality.processId()); - return false; - } - if (tlogWorker->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "TLogExcluded") - .detail("ProcessID", it.interf().filteredLocality.processId()); - return true; - } - - if (logSet.isLocal && logSet.locality == tagLocalitySatellite) { - satellite_tlogs.push_back(tlogWorker->second.details); - } else if (logSet.isLocal) { - tlogs.push_back(tlogWorker->second.details); - } else { - remote_tlogs.push_back(tlogWorker->second.details); - } - } - - for (auto& it : logSet.logRouters) { - auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); - if (tlogWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindLogRouter") - .detail("ProcessID", it.interf().filteredLocality.processId()); - return false; - } - if (tlogWorker->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "LogRouterExcluded") - .detail("ProcessID", it.interf().filteredLocality.processId()); - return true; - } - if (!logRouterAddresses.count(tlogWorker->second.details.interf.address())) { - logRouterAddresses.insert(tlogWorker->second.details.interf.address()); - log_routers.push_back(tlogWorker->second.details); - } - } - - for (const auto& worker : logSet.backupWorkers) { - auto workerIt = id_worker.find(worker.interf().locality.processId()); - if (workerIt == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindBackupWorker") - .detail("ProcessID", worker.interf().locality.processId()); - return false; - } - if (workerIt->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "BackupWorkerExcluded") - .detail("ProcessID", worker.interf().locality.processId()); - return true; - } - if (backup_addresses.count(workerIt->second.details.interf.address()) == 0) { - backup_addresses.insert(workerIt->second.details.interf.address()); - backup_workers.push_back(workerIt->second.details); - } - } - } - - // Get commit proxy classes - std::vector commitProxyClasses; - for (auto& it : dbi.client.commitProxies) { - auto commitProxyWorker = id_worker.find(it.processId); - if (commitProxyWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindCommitProxy") - .detail("ProcessID", it.processId); - return false; - } - if (commitProxyWorker->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "CommitProxyExcluded") - .detail("ProcessID", it.processId); - return true; - } - commitProxyClasses.push_back(commitProxyWorker->second.details); - } - - // Get grv proxy classes - std::vector grvProxyClasses; - for (auto& it : dbi.client.grvProxies) { - auto grvProxyWorker = id_worker.find(it.processId); - if (grvProxyWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindGrvProxy") - .detail("ProcessID", it.processId); - return false; - } - if (grvProxyWorker->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "GrvProxyExcluded") - .detail("ProcessID", it.processId); - return true; - } - grvProxyClasses.push_back(grvProxyWorker->second.details); - } - - // Get resolver classes - std::vector resolverClasses; - for (auto& it : dbi.resolvers) { - auto resolverWorker = id_worker.find(it.locality.processId()); - if (resolverWorker == id_worker.end()) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("Reason", "CannotFindResolver") - .detail("ProcessID", it.locality.processId()); - return false; - } - if (resolverWorker->second.priorityInfo.isExcluded) { - TraceEvent("BetterMasterExists", id) - .detail("Reason", "ResolverExcluded") - .detail("ProcessID", it.locality.processId()); - return true; - } - resolverClasses.push_back(resolverWorker->second.details); - } - - // Check master fitness. Don't return false if master is excluded in case all the processes are excluded, we - // still need master for recovery. - ProcessClass::Fitness oldMasterFit = - masterWorker->second.details.processClass.machineClassFitness(ProcessClass::Master); - if (db.config.isExcludedServer(dbi.master.addresses())) { - oldMasterFit = std::max(oldMasterFit, ProcessClass::ExcludeFit); - } - - std::map>, int> id_used; - std::map>, int> old_id_used; - id_used[clusterControllerProcessId]++; - old_id_used[clusterControllerProcessId]++; - WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter( - clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db.config, id_used, {}, true); - auto newMasterFit = mworker.worker.processClass.machineClassFitness(ProcessClass::Master); - if (db.config.isExcludedServer(mworker.worker.interf.addresses())) { - newMasterFit = std::max(newMasterFit, ProcessClass::ExcludeFit); - } - - old_id_used[masterWorker->first]++; - if (oldMasterFit < newMasterFit) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("OldMasterFit", oldMasterFit) - .detail("NewMasterFit", newMasterFit) - .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) - .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); - ; - return false; - } - if (oldMasterFit > newMasterFit || (dbi.master.locality.processId() == clusterControllerProcessId && - mworker.worker.interf.locality.processId() != clusterControllerProcessId)) { - TraceEvent("BetterMasterExists", id) - .detail("OldMasterFit", oldMasterFit) - .detail("NewMasterFit", newMasterFit) - .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) - .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); - return true; - } - - std::set> primaryDC; - std::set> remoteDC; - - RegionInfo region; - RegionInfo remoteRegion; - if (db.config.regions.size()) { - primaryDC.insert(clusterControllerDcId); - for (auto& r : db.config.regions) { - if (r.dcId != clusterControllerDcId.get()) { - ASSERT(remoteDC.empty()); - remoteDC.insert(r.dcId); - remoteRegion = r; - } else { - ASSERT(region.dcId == StringRef()); - region = r; - } - } - } - - // Check tLog fitness - updateIdUsed(tlogs, old_id_used); - RoleFitness oldTLogFit(tlogs, ProcessClass::TLog, old_id_used); - auto newTLogs = getWorkersForTlogs(db.config, - db.config.tLogReplicationFactor, - db.config.getDesiredLogs(), - db.config.tLogPolicy, - id_used, - true, - primaryDC); - RoleFitness newTLogFit(newTLogs, ProcessClass::TLog, id_used); - - bool oldSatelliteFallback = false; - - if (region.satelliteTLogPolicyFallback.isValid()) { - for (auto& logSet : dbi.logSystemConfig.tLogs) { - if (region.satelliteTLogPolicy.isValid() && logSet.isLocal && logSet.locality == tagLocalitySatellite) { - oldSatelliteFallback = logSet.tLogPolicy->info() != region.satelliteTLogPolicy->info(); - ASSERT(!oldSatelliteFallback || - (region.satelliteTLogPolicyFallback.isValid() && - logSet.tLogPolicy->info() == region.satelliteTLogPolicyFallback->info())); - break; - } - } - } - - updateIdUsed(satellite_tlogs, old_id_used); - RoleFitness oldSatelliteTLogFit(satellite_tlogs, ProcessClass::TLog, old_id_used); - bool newSatelliteFallback = false; - auto newSatelliteTLogs = satellite_tlogs; - RoleFitness newSatelliteTLogFit = oldSatelliteTLogFit; - if (region.satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) { - newSatelliteTLogs = - getWorkersForSatelliteLogs(db.config, region, remoteRegion, id_used, newSatelliteFallback, true); - newSatelliteTLogFit = RoleFitness(newSatelliteTLogs, ProcessClass::TLog, id_used); - } - - std::map, int32_t> satellite_priority; - for (auto& r : region.satellites) { - satellite_priority[r.dcId] = r.priority; - } - - int32_t oldSatelliteRegionFit = std::numeric_limits::max(); - for (auto& it : satellite_tlogs) { - if (satellite_priority.count(it.interf.locality.dcId())) { - oldSatelliteRegionFit = std::min(oldSatelliteRegionFit, satellite_priority[it.interf.locality.dcId()]); - } else { - oldSatelliteRegionFit = -1; - } - } - - int32_t newSatelliteRegionFit = std::numeric_limits::max(); - for (auto& it : newSatelliteTLogs) { - if (satellite_priority.count(it.interf.locality.dcId())) { - newSatelliteRegionFit = std::min(newSatelliteRegionFit, satellite_priority[it.interf.locality.dcId()]); - } else { - newSatelliteRegionFit = -1; - } - } - - if (oldSatelliteFallback && !newSatelliteFallback) { - TraceEvent("BetterMasterExists", id) - .detail("OldSatelliteFallback", oldSatelliteFallback) - .detail("NewSatelliteFallback", newSatelliteFallback); - return true; - } - if (!oldSatelliteFallback && newSatelliteFallback) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("OldSatelliteFallback", oldSatelliteFallback) - .detail("NewSatelliteFallback", newSatelliteFallback); - return false; - } - - if (oldSatelliteRegionFit < newSatelliteRegionFit) { - TraceEvent("BetterMasterExists", id) - .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) - .detail("NewSatelliteRegionFit", newSatelliteRegionFit); - return true; - } - if (oldSatelliteRegionFit > newSatelliteRegionFit) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) - .detail("NewSatelliteRegionFit", newSatelliteRegionFit); - return false; - } - - updateIdUsed(remote_tlogs, old_id_used); - RoleFitness oldRemoteTLogFit(remote_tlogs, ProcessClass::TLog, old_id_used); - std::vector exclusionWorkerIds; - auto fn = [](const WorkerDetails& in) { return in.interf.id(); }; - std::transform(newTLogs.begin(), newTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); - std::transform(newSatelliteTLogs.begin(), newSatelliteTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); - RoleFitness newRemoteTLogFit = oldRemoteTLogFit; - if (db.config.usableRegions > 1 && (dbi.recoveryState == RecoveryState::ALL_LOGS_RECRUITED || - dbi.recoveryState == RecoveryState::FULLY_RECOVERED)) { - newRemoteTLogFit = RoleFitness(getWorkersForTlogs(db.config, - db.config.getRemoteTLogReplicationFactor(), - db.config.getDesiredRemoteLogs(), - db.config.getRemoteTLogPolicy(), - id_used, - true, - remoteDC, - exclusionWorkerIds), - ProcessClass::TLog, - id_used); - } - int oldRouterCount = - oldTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, oldTLogFit.count)); - int newRouterCount = - newTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, newTLogFit.count)); - updateIdUsed(log_routers, old_id_used); - RoleFitness oldLogRoutersFit(log_routers, ProcessClass::LogRouter, old_id_used); - RoleFitness newLogRoutersFit = oldLogRoutersFit; - if (db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::FULLY_RECOVERED) { - newLogRoutersFit = RoleFitness(getWorkersForRoleInDatacenter(*remoteDC.begin(), - ProcessClass::LogRouter, - newRouterCount, - db.config, - id_used, - {}, - Optional(), - true), - ProcessClass::LogRouter, - id_used); - } - - if (oldLogRoutersFit.count < oldRouterCount) { - oldLogRoutersFit.worstFit = ProcessClass::NeverAssign; - } - if (newLogRoutersFit.count < newRouterCount) { - newLogRoutersFit.worstFit = ProcessClass::NeverAssign; - } - - // Check proxy/grvProxy/resolver fitness - updateIdUsed(commitProxyClasses, old_id_used); - updateIdUsed(grvProxyClasses, old_id_used); - updateIdUsed(resolverClasses, old_id_used); - RoleFitness oldCommitProxyFit(commitProxyClasses, ProcessClass::CommitProxy, old_id_used); - RoleFitness oldGrvProxyFit(grvProxyClasses, ProcessClass::GrvProxy, old_id_used); - RoleFitness oldResolverFit(resolverClasses, ProcessClass::Resolver, old_id_used); - - std::map>, int> preferredSharing; - auto first_commit_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, - ProcessClass::CommitProxy, - ProcessClass::ExcludeFit, - db.config, - id_used, - preferredSharing, - true); - preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; - auto first_grv_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, - ProcessClass::GrvProxy, - ProcessClass::ExcludeFit, - db.config, - id_used, - preferredSharing, - true); - preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; - auto first_resolver = getWorkerForRoleInDatacenter(clusterControllerDcId, - ProcessClass::Resolver, - ProcessClass::ExcludeFit, - db.config, - id_used, - preferredSharing, - true); - preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; - auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); - first_commit_proxy.used = maxUsed; - first_grv_proxy.used = maxUsed; - first_resolver.used = maxUsed; - auto commit_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, - ProcessClass::CommitProxy, - db.config.getDesiredCommitProxies(), - db.config, - id_used, - preferredSharing, - first_commit_proxy, - true); - auto grv_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, - ProcessClass::GrvProxy, - db.config.getDesiredGrvProxies(), - db.config, - id_used, - preferredSharing, - first_grv_proxy, - true); - auto resolvers = getWorkersForRoleInDatacenter(clusterControllerDcId, - ProcessClass::Resolver, - db.config.getDesiredResolvers(), - db.config, - id_used, - preferredSharing, - first_resolver, - true); - - RoleFitness newCommitProxyFit(commit_proxies, ProcessClass::CommitProxy, id_used); - RoleFitness newGrvProxyFit(grv_proxies, ProcessClass::GrvProxy, id_used); - RoleFitness newResolverFit(resolvers, ProcessClass::Resolver, id_used); - - // Check backup worker fitness - updateIdUsed(backup_workers, old_id_used); - RoleFitness oldBackupWorkersFit(backup_workers, ProcessClass::Backup, old_id_used); - const int nBackup = backup_addresses.size(); - RoleFitness newBackupWorkersFit(getWorkersForRoleInDatacenter(clusterControllerDcId, - ProcessClass::Backup, - nBackup, - db.config, - id_used, - {}, - Optional(), - true), - ProcessClass::Backup, - id_used); - - auto oldFit = std::make_tuple(oldTLogFit, - oldSatelliteTLogFit, - oldCommitProxyFit, - oldGrvProxyFit, - oldResolverFit, - oldBackupWorkersFit, - oldRemoteTLogFit, - oldLogRoutersFit); - auto newFit = std::make_tuple(newTLogFit, - newSatelliteTLogFit, - newCommitProxyFit, - newGrvProxyFit, - newResolverFit, - newBackupWorkersFit, - newRemoteTLogFit, - newLogRoutersFit); - - if (oldFit > newFit) { - TraceEvent("BetterMasterExists", id) - .detail("OldMasterFit", oldMasterFit) - .detail("NewMasterFit", newMasterFit) - .detail("OldTLogFit", oldTLogFit.toString()) - .detail("NewTLogFit", newTLogFit.toString()) - .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) - .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) - .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) - .detail("NewCommitProxyFit", newCommitProxyFit.toString()) - .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) - .detail("NewGrvProxyFit", newGrvProxyFit.toString()) - .detail("OldResolverFit", oldResolverFit.toString()) - .detail("NewResolverFit", newResolverFit.toString()) - .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) - .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) - .detail("OldRemoteFit", oldRemoteTLogFit.toString()) - .detail("NewRemoteFit", newRemoteTLogFit.toString()) - .detail("OldRouterFit", oldLogRoutersFit.toString()) - .detail("NewRouterFit", newLogRoutersFit.toString()) - .detail("OldSatelliteFallback", oldSatelliteFallback) - .detail("NewSatelliteFallback", newSatelliteFallback); - return true; - } - - if (oldFit < newFit) { - TraceEvent("NewRecruitmentIsWorse", id) - .detail("OldMasterFit", oldMasterFit) - .detail("NewMasterFit", newMasterFit) - .detail("OldTLogFit", oldTLogFit.toString()) - .detail("NewTLogFit", newTLogFit.toString()) - .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) - .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) - .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) - .detail("NewCommitProxyFit", newCommitProxyFit.toString()) - .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) - .detail("NewGrvProxyFit", newGrvProxyFit.toString()) - .detail("OldResolverFit", oldResolverFit.toString()) - .detail("NewResolverFit", newResolverFit.toString()) - .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) - .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) - .detail("OldRemoteFit", oldRemoteTLogFit.toString()) - .detail("NewRemoteFit", newRemoteTLogFit.toString()) - .detail("OldRouterFit", oldLogRoutersFit.toString()) - .detail("NewRouterFit", newLogRoutersFit.toString()) - .detail("OldSatelliteFallback", oldSatelliteFallback) - .detail("NewSatelliteFallback", newSatelliteFallback); - } - return false; - } - - // Returns true iff processId is currently being used - // for any non-singleton role other than master - bool isUsedNotMaster(Optional processId) const { - ASSERT(masterProcessId.present()); - if (processId == masterProcessId) - return false; - - auto& dbInfo = db.serverInfo->get(); - for (const auto& tlogset : dbInfo.logSystemConfig.tLogs) { - for (const auto& tlog : tlogset.tLogs) { - if (tlog.present() && tlog.interf().filteredLocality.processId() == processId) - return true; - } - } - for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { - if (interf.processId == processId) - return true; - } - for (const GrvProxyInterface& interf : dbInfo.client.grvProxies) { - if (interf.processId == processId) - return true; - } - for (const ResolverInterface& interf : dbInfo.resolvers) { - if (interf.locality.processId() == processId) - return true; - } - if (processId == clusterControllerProcessId) - return true; - - return false; - } - - // Returns true iff - // - role is master, or - // - role is a singleton AND worker's pid is being used for any non-singleton role - bool onMasterIsBetter(const WorkerDetails& worker, ProcessClass::ClusterRole role) const { - ASSERT(masterProcessId.present()); - const auto& pid = worker.interf.locality.processId(); - if ((role != ProcessClass::DataDistributor && role != ProcessClass::Ratekeeper && - role != ProcessClass::BlobManager) || - pid == masterProcessId.get()) { - return false; - } - return isUsedNotMaster(pid); - } - - // Returns a map of for all non-singleton roles - std::map>, int> getUsedIds() { - std::map>, int> idUsed; - updateKnownIds(&idUsed); - - auto& dbInfo = db.serverInfo->get(); - for (const auto& tlogset : dbInfo.logSystemConfig.tLogs) { - for (const auto& tlog : tlogset.tLogs) { - if (tlog.present()) { - idUsed[tlog.interf().filteredLocality.processId()]++; - } - } - } - for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { - ASSERT(interf.processId.present()); - idUsed[interf.processId]++; - } - for (const GrvProxyInterface& interf : dbInfo.client.grvProxies) { - ASSERT(interf.processId.present()); - idUsed[interf.processId]++; - } - for (const ResolverInterface& interf : dbInfo.resolvers) { - ASSERT(interf.locality.processId().present()); - idUsed[interf.locality.processId()]++; - } - return idUsed; - } - - // Updates work health signals in `workerHealth` based on `req`. - void updateWorkerHealth(const UpdateWorkerHealthRequest& req) { - std::string degradedPeersString; - for (int i = 0; i < req.degradedPeers.size(); ++i) { - degradedPeersString += (i == 0 ? "" : " ") + req.degradedPeers[i].toString(); - } - TraceEvent("ClusterControllerUpdateWorkerHealth") - .detail("WorkerAddress", req.address) - .detail("DegradedPeers", degradedPeersString); - - // `req.degradedPeers` contains the latest peer performance view from the worker. Clear the worker if the - // requested worker doesn't see any degraded peers. - if (req.degradedPeers.empty()) { - workerHealth.erase(req.address); - return; - } - - double currentTime = now(); - - // Current `workerHealth` doesn't have any information about the incoming worker. Add the worker into - // `workerHealth`. - if (workerHealth.find(req.address) == workerHealth.end()) { - workerHealth[req.address] = {}; - for (const auto& degradedPeer : req.degradedPeers) { - workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime }; - } - - return; - } - - // The incoming worker already exists in `workerHealth`. - - auto& health = workerHealth[req.address]; - - // First, remove any degraded peers recorded in the `workerHealth`, but aren't in the incoming request. These - // machines network performance should have recovered. - std::unordered_set recoveredPeers; - for (const auto& [peer, times] : health.degradedPeers) { - recoveredPeers.insert(peer); - } - for (const auto& peer : req.degradedPeers) { - if (recoveredPeers.find(peer) != recoveredPeers.end()) { - recoveredPeers.erase(peer); - } - } - for (const auto& peer : recoveredPeers) { - health.degradedPeers.erase(peer); - } - - // Update the worker's degradedPeers. - for (const auto& peer : req.degradedPeers) { - auto it = health.degradedPeers.find(peer); - if (it == health.degradedPeers.end()) { - health.degradedPeers[peer] = { currentTime, currentTime }; - continue; - } - it->second.lastRefreshTime = currentTime; - } - } - - // Checks that if any worker or their degraded peers have recovered. If so, remove them from `workerHealth`. - void updateRecoveredWorkers() { - double currentTime = now(); - for (auto& [workerAddress, health] : workerHealth) { - for (auto it = health.degradedPeers.begin(); it != health.degradedPeers.end();) { - if (currentTime - it->second.lastRefreshTime > SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL) { - TraceEvent("WorkerPeerHealthRecovered").detail("Worker", workerAddress).detail("Peer", it->first); - health.degradedPeers.erase(it++); - } else { - ++it; - } - } - } - - for (auto it = workerHealth.begin(); it != workerHealth.end();) { - if (it->second.degradedPeers.empty()) { - TraceEvent("WorkerAllPeerHealthRecovered").detail("Worker", it->first); - workerHealth.erase(it++); - } else { - ++it; - } - } - } - - // Returns a list of servers who are experiencing degraded links. These are candidates to perform exclusion. Note - // that only one endpoint of a bad link will be included in this list. - std::unordered_set getServersWithDegradedLink() { - updateRecoveredWorkers(); - - // Build a map keyed by measured degraded peer. This map gives the info that who complains a particular server. - std::unordered_map> degradedLinkDst2Src; - double currentTime = now(); - for (const auto& [server, health] : workerHealth) { - for (const auto& [degradedPeer, times] : health.degradedPeers) { - if (currentTime - times.startTime < SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL) { - // This degraded link is not long enough to be considered as degraded. - continue; - } - degradedLinkDst2Src[degradedPeer].insert(server); - } - } - - // Sort degraded peers based on the number of workers complaining about it. - std::vector> count2DegradedPeer; - for (const auto& [degradedPeer, complainers] : degradedLinkDst2Src) { - count2DegradedPeer.push_back({ complainers.size(), degradedPeer }); - } - std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), std::greater<>()); - - // Go through all reported degraded peers by decreasing order of the number of complainers. For a particular - // degraded peer, if a complainer has already be considered as degraded, we skip the current examine degraded - // peer since there has been one endpoint on the link between degradedPeer and complainer considered as - // degraded. This is to address the issue that both endpoints on a bad link may be considered as degraded - // server. - // - // For example, if server A is already considered as a degraded server, and A complains B, we won't add B as - // degraded since A is already considered as degraded. - std::unordered_set currentDegradedServers; - for (const auto& [complainerCount, badServer] : count2DegradedPeer) { - for (const auto& complainer : degradedLinkDst2Src[badServer]) { - if (currentDegradedServers.find(complainer) == currentDegradedServers.end()) { - currentDegradedServers.insert(badServer); - break; - } - } - } - - // For degraded server that are complained by more than SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, we - // don't know if it is a hot server, or the network is bad. We remove from the returned degraded server list. - std::unordered_set currentDegradedServersWithinLimit; - for (const auto& badServer : currentDegradedServers) { - if (degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) { - currentDegradedServersWithinLimit.insert(badServer); - } - } - return currentDegradedServersWithinLimit; - } - - // Whether the transaction system (in primary DC if in HA setting) contains degraded servers. - bool transactionSystemContainsDegradedServers() { - const ServerDBInfo dbi = db.serverInfo->get(); - for (const auto& excludedServer : degradedServers) { - if (dbi.master.addresses().contains(excludedServer)) { - return true; - } - - for (auto& logSet : dbi.logSystemConfig.tLogs) { - if (!logSet.isLocal || logSet.locality == tagLocalitySatellite) { - continue; - } - for (const auto& tlog : logSet.tLogs) { - if (tlog.present() && tlog.interf().addresses().contains(excludedServer)) { - return true; - } - } - } - - for (auto& proxy : dbi.client.grvProxies) { - if (proxy.addresses().contains(excludedServer)) { - return true; - } - } - - for (auto& proxy : dbi.client.commitProxies) { - if (proxy.addresses().contains(excludedServer)) { - return true; - } - } - - for (auto& resolver : dbi.resolvers) { - if (resolver.addresses().contains(excludedServer)) { - return true; - } - } - } - - return false; - } - - // Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded - // servers. - bool remoteTransactionSystemContainsDegradedServers() { - if (db.config.usableRegions <= 1) { - return false; - } - - for (const auto& excludedServer : degradedServers) { - if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) { - return true; - } - } - - return false; - } - - // Returns true if remote DC is healthy and can failover to. - bool remoteDCIsHealthy() { - // Ignore remote DC health if worker health monitor is disabled. - if (!SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) { - return true; - } - - // When we just start, we ignore any remote DC health info since the current CC may be elected at wrong DC due - // to that all the processes are still starting. - if (machineStartTime() == 0) { - return true; - } - - if (now() - machineStartTime() < SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) { - return true; - } - - // When remote DC health is not monitored, we may not know whether the remote is healthy or not. So return false - // here to prevent failover. - if (!remoteDCMonitorStarted) { - return false; - } - - return !remoteTransactionSystemContainsDegradedServers(); - } - - // Returns true when the cluster controller should trigger a recovery due to degraded servers used in the - // transaction system in the primary data center. - bool shouldTriggerRecoveryDueToDegradedServers() { - if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) { - return false; - } - - if (db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) { - return false; - } - - // Do not trigger recovery if the cluster controller is excluded, since the master will change - // anyways once the cluster controller is moved - if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { - return false; - } - - return transactionSystemContainsDegradedServers(); - } - - // Returns true when the cluster controller should trigger a failover due to degraded servers used in the - // transaction system in the primary data center, and no degradation in the remote data center. - bool shouldTriggerFailoverDueToDegradedServers() { - if (db.config.usableRegions <= 1) { - return false; - } - - if (SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION > - SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) { - TraceEvent(SevWarn, "TriggerFailoverDueToDegradedServersInvalidConfig") - .suppressFor(1.0) - .detail("Min", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION) - .detail("Max", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION); - return false; - } - - if (degradedServers.size() < SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION || - degradedServers.size() > SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) { - return false; - } - - // Do not trigger recovery if the cluster controller is excluded, since the master will change - // anyways once the cluster controller is moved - if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { - return false; - } - - return transactionSystemContainsDegradedServers() && !remoteTransactionSystemContainsDegradedServers(); - } - - int recentRecoveryCountDueToHealth() { - while (!recentHealthTriggeredRecoveryTime.empty() && - now() - recentHealthTriggeredRecoveryTime.front() > SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL) { - recentHealthTriggeredRecoveryTime.pop(); - } - return recentHealthTriggeredRecoveryTime.size(); - } - - bool isExcludedDegradedServer(const NetworkAddressList& a) { - for (const auto& server : excludedDegradedServers) { - if (a.contains(server)) - return true; - } - return false; - } - - std::map>, WorkerInfo> id_worker; - std::map>, ProcessClass> - id_class; // contains the mapping from process id to process class from the database - RangeResult lastProcessClasses; - bool gotProcessClasses; - bool gotFullyRecoveredConfig; - Optional> masterProcessId; - Optional> clusterControllerProcessId; - Optional> clusterControllerDcId; - AsyncVar>>> desiredDcIds; // desired DC priorities - AsyncVar>>>> - changingDcIds; // current DC priorities to change first, and whether that is the cluster controller - AsyncVar>>>> - changedDcIds; // current DC priorities to change second, and whether the cluster controller has been changed - UID id; - std::vector outstandingRecruitmentRequests; - std::vector outstandingRemoteRecruitmentRequests; - std::vector> outstandingStorageRequests; - std::vector> outstandingBlobWorkerRequests; - ActorCollection ac; - UpdateWorkerList updateWorkerList; - Future outstandingRequestChecker; - Future outstandingRemoteRequestChecker; - AsyncTrigger updateDBInfo; - std::set updateDBInfoEndpoints; - std::set removedDBInfoEndpoints; - - DBInfo db; - Database cx; - double startTime; - Future goodRecruitmentTime; - Future goodRemoteRecruitmentTime; - Version datacenterVersionDifference; - PromiseStream> addActor; - bool versionDifferenceUpdated; - - bool remoteDCMonitorStarted; - bool remoteTransactionSystemDegraded; - - // recruitX is used to signal when role X needs to be (re)recruited. - // recruitingXID is used to track the ID of X's interface which is being recruited. - // We use AsyncVars to kill (i.e. halt) singletons that have been replaced. - AsyncVar recruitDistributor; - Optional recruitingDistributorID; - AsyncVar recruitRatekeeper; - Optional recruitingRatekeeperID; - AsyncVar recruitBlobManager; - Optional recruitingBlobManagerID; - - // Stores the health information from a particular worker's perspective. - struct WorkerHealth { - struct DegradedTimes { - double startTime = 0; - double lastRefreshTime = 0; - }; - std::unordered_map degradedPeers; - - // TODO(zhewu): Include disk and CPU signals. - }; - std::unordered_map workerHealth; - std::unordered_set - degradedServers; // The servers that the cluster controller is considered as degraded. The servers in this list - // are not excluded unless they are added to `excludedDegradedServers`. - std::unordered_set - excludedDegradedServers; // The degraded servers to be excluded when assigning workers to roles. - std::queue recentHealthTriggeredRecoveryTime; - - CounterCollection clusterControllerMetrics; - - Counter openDatabaseRequests; - Counter registerWorkerRequests; - Counter getWorkersRequests; - Counter getClientWorkersRequests; - Counter registerMasterRequests; - Counter statusRequests; - - Reference recruitedMasterWorkerEventHolder; - - ClusterControllerData(ClusterControllerFullInterface const& ccInterface, - LocalityData const& locality, - ServerCoordinators const& coordinators) - : gotProcessClasses(false), gotFullyRecoveredConfig(false), clusterControllerProcessId(locality.processId()), - clusterControllerDcId(locality.dcId()), id(ccInterface.id()), ac(false), outstandingRequestChecker(Void()), - outstandingRemoteRequestChecker(Void()), startTime(now()), goodRecruitmentTime(Never()), - goodRemoteRecruitmentTime(Never()), datacenterVersionDifference(0), versionDifferenceUpdated(false), - remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false), recruitDistributor(false), - recruitRatekeeper(false), recruitBlobManager(false), - clusterControllerMetrics("ClusterController", id.toString()), - openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics), - registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics), - getWorkersRequests("GetWorkersRequests", clusterControllerMetrics), - getClientWorkersRequests("GetClientWorkersRequests", clusterControllerMetrics), - registerMasterRequests("RegisterMasterRequests", clusterControllerMetrics), - statusRequests("StatusRequests", clusterControllerMetrics), - recruitedMasterWorkerEventHolder(makeReference("RecruitedMasterWorker")) { - auto serverInfo = ServerDBInfo(); - serverInfo.id = deterministicRandom()->randomUniqueID(); - serverInfo.infoGeneration = ++db.dbInfoCount; - serverInfo.masterLifetime.ccID = id; - serverInfo.clusterInterface = ccInterface; - serverInfo.myLocality = locality; - db.serverInfo->set(serverInfo); - cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::True); - - specialCounter(clusterControllerMetrics, "ClientCount", [this]() { return db.clientCount; }); - } - - ~ClusterControllerData() { - ac.clear(false); - id_worker.clear(); - } -}; - // Wrapper for singleton interfaces template struct Singleton { @@ -3349,8 +147,25 @@ struct BlobManagerSingleton : Singleton { void recruit(ClusterControllerData* cc) const { cc->recruitBlobManager.set(true); } }; -ACTOR Future clusterWatchDatabase(ClusterControllerData* cluster, ClusterControllerData::DBInfo* db) { +ACTOR Future handleLeaderReplacement(Reference self, Future leaderFail) { + loop choose { + when(wait(leaderFail)) { + TraceEvent("LeaderReplaced", self->controllerData->id).log(); + // We are no longer the leader if this has changed. + self->controllerData->shouldCommitSuicide = true; + throw restart_cluster_controller(); + } + } +} + +ACTOR Future clusterWatchDatabase(ClusterControllerData* cluster, + ClusterControllerData::DBInfo* db, + ServerCoordinators coordinators, + Future leaderFail) { state MasterInterface iMaster; + state Reference recoveryData; + state PromiseStream> addActor; + state Future recoveryCore; // SOMEDAY: If there is already a non-failed master referenced by zkMasterInfo, use that one until it fails // When this someday is implemented, make sure forced failures still cause the master to be recruited again @@ -3359,102 +174,129 @@ ACTOR Future clusterWatchDatabase(ClusterControllerData* cluster, ClusterC TraceEvent("CCWDB", cluster->id).log(); try { state double recoveryStart = now(); + state MasterInterface newMaster; + state Future collection; + TraceEvent("CCWDB", cluster->id).detail("Recruiting", "Master"); + wait(recruitNewMaster(cluster, db, std::addressof(newMaster))); - // We must recruit the master in the same data center as the cluster controller. - // This should always be possible, because we can recruit the master on the same process as the cluster - // controller. - std::map>, int> id_used; - id_used[cluster->clusterControllerProcessId]++; - state WorkerFitnessInfo masterWorker = cluster->getWorkerForRoleInDatacenter( - cluster->clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db->config, id_used); - if ((masterWorker.worker.processClass.machineClassFitness(ProcessClass::Master) > - SERVER_KNOBS->EXPECTED_MASTER_FITNESS || - masterWorker.worker.interf.locality.processId() == cluster->clusterControllerProcessId) && - !cluster->goodRecruitmentTime.isReady()) { - TraceEvent("CCWDB", cluster->id) - .detail("Fitness", masterWorker.worker.processClass.machineClassFitness(ProcessClass::Master)); - wait(delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); - continue; - } - RecruitMasterRequest rmq; - rmq.lifetime = db->serverInfo->get().masterLifetime; - rmq.forceRecovery = db->forceRecovery; + iMaster = newMaster; - cluster->masterProcessId = masterWorker.worker.interf.locality.processId(); - cluster->db.unfinishedRecoveries++; - state Future> fNewMaster = masterWorker.worker.interf.master.tryGetReply(rmq); - wait(ready(fNewMaster) || db->forceMasterFailure.onTrigger()); - if (fNewMaster.isReady() && fNewMaster.get().present()) { - TraceEvent("CCWDB", cluster->id).detail("Recruited", fNewMaster.get().get().id()); + db->masterRegistrationCount = 0; + db->recoveryStalled = false; - // for status tool - TraceEvent("RecruitedMasterWorker", cluster->id) - .detail("Address", fNewMaster.get().get().address()) - .trackLatest(cluster->recruitedMasterWorkerEventHolder->trackingKey); + auto dbInfo = ServerDBInfo(); + dbInfo.master = iMaster; + dbInfo.id = deterministicRandom()->randomUniqueID(); + dbInfo.infoGeneration = ++db->dbInfoCount; + dbInfo.masterLifetime = db->serverInfo->get().masterLifetime; + ++dbInfo.masterLifetime; + dbInfo.clusterInterface = db->serverInfo->get().clusterInterface; + dbInfo.distributor = db->serverInfo->get().distributor; + dbInfo.ratekeeper = db->serverInfo->get().ratekeeper; + dbInfo.blobManager = db->serverInfo->get().blobManager; + dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig; + dbInfo.myLocality = db->serverInfo->get().myLocality; + dbInfo.client = ClientDBInfo(); - iMaster = fNewMaster.get().get(); + TraceEvent("CCWDB", cluster->id) + .detail("NewMaster", dbInfo.master.id().toString()) + .detail("Lifetime", dbInfo.masterLifetime.toString()) + .detail("ChangeID", dbInfo.id); + db->serverInfo->set(dbInfo); - db->masterRegistrationCount = 0; - db->recoveryStalled = false; + state Future spinDelay = delay( + SERVER_KNOBS + ->MASTER_SPIN_DELAY); // Don't retry master recovery more than once per second, but don't delay + // the "first" recovery after more than a second of normal operation - auto dbInfo = ServerDBInfo(); - dbInfo.master = iMaster; - dbInfo.id = deterministicRandom()->randomUniqueID(); - dbInfo.infoGeneration = ++db->dbInfoCount; - dbInfo.masterLifetime = db->serverInfo->get().masterLifetime; - ++dbInfo.masterLifetime; - dbInfo.clusterInterface = db->serverInfo->get().clusterInterface; - dbInfo.distributor = db->serverInfo->get().distributor; - dbInfo.ratekeeper = db->serverInfo->get().ratekeeper; - dbInfo.blobManager = db->serverInfo->get().blobManager; - dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig; + TraceEvent("CCWDB", cluster->id).detail("Watching", iMaster.id()); + recoveryData = makeReference(cluster, + db->serverInfo, + db->serverInfo->get().master, + db->serverInfo->get().masterLifetime, + coordinators, + db->serverInfo->get().clusterInterface, + LiteralStringRef(""), + addActor, + db->forceRecovery); - TraceEvent("CCWDB", cluster->id) - .detail("Lifetime", dbInfo.masterLifetime.toString()) - .detail("ChangeID", dbInfo.id); - db->serverInfo->set(dbInfo); + collection = actorCollection(recoveryData->addActor.getFuture()); + recoveryCore = clusterRecoveryCore(recoveryData); - state Future spinDelay = delay( - SERVER_KNOBS - ->MASTER_SPIN_DELAY); // Don't retry master recovery more than once per second, but don't delay - // the "first" recovery after more than a second of normal operation - - TraceEvent("CCWDB", cluster->id).detail("Watching", iMaster.id()); - - // Master failure detection is pretty sensitive, but if we are in the middle of a very long recovery we - // really don't want to have to start over - loop choose { - when(wait(waitFailureClient( - iMaster.waitFailure, - db->masterRegistrationCount - ? SERVER_KNOBS->MASTER_FAILURE_REACTION_TIME - : (now() - recoveryStart) * SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY, - db->masterRegistrationCount ? -SERVER_KNOBS->MASTER_FAILURE_REACTION_TIME / - SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY - : SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY) || - db->forceMasterFailure.onTrigger())) { - break; - } - when(wait(db->serverInfo->onChange())) {} + // Master failure detection is pretty sensitive, but if we are in the middle of a very long recovery we + // really don't want to have to start over + loop choose { + when(wait(recoveryCore)) {} + when(wait(waitFailureClient( + iMaster.waitFailure, + db->masterRegistrationCount + ? SERVER_KNOBS->MASTER_FAILURE_REACTION_TIME + : (now() - recoveryStart) * SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY, + db->masterRegistrationCount ? -SERVER_KNOBS->MASTER_FAILURE_REACTION_TIME / + SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY + : SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY) || + db->forceMasterFailure.onTrigger())) { + break; } - wait(spinDelay); - - TEST(true); // clusterWatchDatabase() master failed - TraceEvent(SevWarn, "DetectedFailedMaster", cluster->id).detail("OldMaster", iMaster.id()); - } else { - TEST(true); // clusterWatchDatabase() !newMaster.present() - wait(delay(SERVER_KNOBS->MASTER_SPIN_DELAY)); + when(wait(db->serverInfo->onChange())) {} + when(BackupWorkerDoneRequest req = + waitNext(db->serverInfo->get().clusterInterface.notifyBackupWorkerDone.getFuture())) { + if (recoveryData->logSystem.isValid() && recoveryData->logSystem->removeBackupWorker(req)) { + recoveryData->registrationTrigger.trigger(); + } + ++recoveryData->backupWorkerDoneRequests; + req.reply.send(Void()); + TraceEvent(SevDebug, "BackupWorkerDoneRequest", cluster->id).log(); + } + when(wait(collection)) { throw internal_error(); } + when(wait(handleLeaderReplacement(recoveryData, leaderFail))) { throw internal_error(); } + } + // failed master (better master exists) could happen while change-coordinators request processing is + // in-progress + if (cluster->shouldCommitSuicide) { + throw restart_cluster_controller(); } - } catch (Error& e) { - TraceEvent("CCWDB", cluster->id).error(e, true).detail("Master", iMaster.id()); - if (e.code() == error_code_actor_cancelled) - throw; - bool ok = e.code() == error_code_no_more_servers; - TraceEvent(ok ? SevWarn : SevError, "ClusterWatchDatabaseRetrying", cluster->id).error(e); - if (!ok) - throw e; + recoveryCore.cancel(); + wait(cleanupRecoveryActorCollection(recoveryData, /*exThrown=*/false)); + ASSERT(addActor.isEmpty()); + + wait(spinDelay); + + TEST(true); // clusterWatchDatabase() master failed + TraceEvent(SevWarn, "DetectedFailedRecovery", cluster->id).detail("OldMaster", iMaster.id()); + } catch (Error& e) { + state Error err = e; + TraceEvent("CCWDB", cluster->id).error(e, true).detail("Master", iMaster.id()); + if (e.code() != error_code_actor_cancelled) + wait(delay(0.0)); + + recoveryCore.cancel(); + wait(cleanupRecoveryActorCollection(recoveryData, true /* exThrown */)); + ASSERT(addActor.isEmpty()); + + TEST(err.code() == error_code_tlog_failed); // Terminated due to tLog failure + TEST(err.code() == error_code_commit_proxy_failed); // Terminated due to commit proxy failure + TEST(err.code() == error_code_grv_proxy_failed); // Terminated due to GRV proxy failure + TEST(err.code() == error_code_resolver_failed); // Terminated due to resolver failure + TEST(err.code() == error_code_backup_worker_failed); // Terminated due to backup worker failure + TEST(err.code() == error_code_operation_failed); // Terminated due to failed operation + TEST(err.code() == error_code_restart_cluster_controller); // Terminated due to cluster-controller restart. + + if (cluster->shouldCommitSuicide || err.code() == error_code_coordinators_changed) { + TraceEvent("ClusterControllerTerminate", cluster->id).error(err, true); + throw restart_cluster_controller(); + } + + if (isNormalClusterRecoveryError(err)) { + TraceEvent(SevWarn, "ClusterRecoveryRetrying", cluster->id).error(err); + } else { + bool ok = err.code() == error_code_no_more_servers; + TraceEvent(ok ? SevWarn : SevError, "ClusterWatchDatabaseRetrying", cluster->id).error(err); + if (!ok) + throw err; + } wait(delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); } } @@ -3494,10 +336,14 @@ ACTOR Future clusterOpenDatabase(ClusterControllerData::DBInfo* db, OpenDa void checkOutstandingRecruitmentRequests(ClusterControllerData* self) { for (int i = 0; i < self->outstandingRecruitmentRequests.size(); i++) { - RecruitFromConfigurationRequest& req = self->outstandingRecruitmentRequests[i]; + Reference info = self->outstandingRecruitmentRequests[i]; try { - RecruitFromConfigurationReply rep = self->findWorkersForConfiguration(req); - req.reply.send(rep); + info->rep = self->findWorkersForConfiguration(info->req); + if (info->dbgId.present()) { + TraceEvent("CheckOutstandingRecruitment", info->dbgId.get()) + .detail("Request", info->req.configuration.toString()); + } + info->waitForCompletion.trigger(); swapAndPop(&self->outstandingRecruitmentRequests, i--); } catch (Error& e) { if (e.code() == error_code_no_more_servers || e.code() == error_code_operation_failed) { @@ -3512,10 +358,14 @@ void checkOutstandingRecruitmentRequests(ClusterControllerData* self) { void checkOutstandingRemoteRecruitmentRequests(ClusterControllerData* self) { for (int i = 0; i < self->outstandingRemoteRecruitmentRequests.size(); i++) { - RecruitRemoteFromConfigurationRequest& req = self->outstandingRemoteRecruitmentRequests[i]; + Reference info = self->outstandingRemoteRecruitmentRequests[i]; try { - RecruitRemoteFromConfigurationReply rep = self->findRemoteWorkersForConfiguration(req); - req.reply.send(rep); + info->rep = self->findRemoteWorkersForConfiguration(info->req); + if (info->dbgId.present()) { + TraceEvent("CheckOutstandingRemoteRecruitment", info->dbgId.get()) + .detail("Request", info->req.configuration.toString()); + } + info->waitForCompletion.trigger(); swapAndPop(&self->outstandingRemoteRecruitmentRequests, i--); } catch (Error& e) { if (e.code() == error_code_no_more_servers || e.code() == error_code_operation_failed) { @@ -3992,67 +842,6 @@ void clusterRecruitBlobWorker(ClusterControllerData* self, RecruitBlobWorkerRequ } } -ACTOR Future clusterRecruitFromConfiguration(ClusterControllerData* self, RecruitFromConfigurationRequest req) { - // At the moment this doesn't really need to be an actor (it always completes immediately) - TEST(true); // ClusterController RecruitTLogsRequest - loop { - try { - auto rep = self->findWorkersForConfiguration(req); - req.reply.send(rep); - return Void(); - } catch (Error& e) { - if (e.code() == error_code_no_more_servers && self->goodRecruitmentTime.isReady()) { - self->outstandingRecruitmentRequests.push_back(req); - TraceEvent(SevWarn, "RecruitFromConfigurationNotAvailable", self->id).error(e); - return Void(); - } else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) { - // recruitment not good enough, try again - TraceEvent("RecruitFromConfigurationRetry", self->id) - .error(e) - .detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady()); - while (!self->goodRecruitmentTime.isReady()) { - wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); - } - } else { - TraceEvent(SevError, "RecruitFromConfigurationError", self->id).error(e); - throw; // goodbye, cluster controller - } - } - wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); - } -} - -ACTOR Future clusterRecruitRemoteFromConfiguration(ClusterControllerData* self, - RecruitRemoteFromConfigurationRequest req) { - // At the moment this doesn't really need to be an actor (it always completes immediately) - TEST(true); // ClusterController RecruitTLogsRequest Remote - loop { - try { - RecruitRemoteFromConfigurationReply rep = self->findRemoteWorkersForConfiguration(req); - req.reply.send(rep); - return Void(); - } catch (Error& e) { - if (e.code() == error_code_no_more_servers && self->goodRemoteRecruitmentTime.isReady()) { - self->outstandingRemoteRecruitmentRequests.push_back(req); - TraceEvent(SevWarn, "RecruitRemoteFromConfigurationNotAvailable", self->id).error(e); - return Void(); - } else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) { - // recruitment not good enough, try again - TraceEvent("RecruitRemoteFromConfigurationRetry", self->id) - .error(e) - .detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady()); - while (!self->goodRemoteRecruitmentTime.isReady()) { - wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); - } - } else { - TraceEvent(SevError, "RecruitRemoteFromConfigurationError", self->id).error(e); - throw; // goodbye, cluster controller - } - } - wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); - } -} - void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest const& req) { req.reply.send(Void()); @@ -4127,6 +916,12 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co // Construct the client information if (db->clientInfo->get().commitProxies != req.commitProxies || db->clientInfo->get().grvProxies != req.grvProxies) { + TraceEvent("PublishNewClientInfo", self->id) + .detail("Master", dbInfo.master.id()) + .detail("GrvProxies", db->clientInfo->get().grvProxies) + .detail("ReqGrvProxies", req.grvProxies) + .detail("CommitProxies", db->clientInfo->get().commitProxies) + .detail("ReqCPs", req.commitProxies); isChanged = true; // TODO why construct a new one and not just copy the old one and change proxies + id? ClientDBInfo clientInfo; @@ -5483,7 +2278,7 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, state uint64_t step = 0; state Future> error = errorOr(actorCollection(self.addActor.getFuture())); - self.addActor.send(clusterWatchDatabase(&self, &self.db)); // Start the master database + self.addActor.send(clusterWatchDatabase(&self, &self.db, coordinators, leaderFail)); // Start the master database self.addActor.send(self.updateWorkerList.init(self.db.db)); self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(), &self, @@ -5520,7 +2315,7 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, loop choose { when(ErrorOr err = wait(error)) { - if (err.isError()) { + if (err.isError() && err.getError().code() != error_code_restart_cluster_controller) { endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Stop Received Error", false, err.getError()); } else { endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Stop Received Signal", true); @@ -5534,12 +2329,6 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, ++self.openDatabaseRequests; self.addActor.send(clusterOpenDatabase(&self.db, req)); } - when(RecruitFromConfigurationRequest req = waitNext(interf.recruitFromConfiguration.getFuture())) { - self.addActor.send(clusterRecruitFromConfiguration(&self, req)); - } - when(RecruitRemoteFromConfigurationRequest req = waitNext(interf.recruitRemoteFromConfiguration.getFuture())) { - self.addActor.send(clusterRecruitRemoteFromConfiguration(&self, req)); - } when(RecruitStorageRequest req = waitNext(interf.recruitStorage.getFuture())) { clusterRecruitStorage(&self, req); } @@ -5599,12 +2388,6 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, when(GetServerDBInfoRequest req = waitNext(interf.getServerDBInfo.getFuture())) { self.addActor.send(clusterGetServerInfo(&self.db, req.knownServerInfoID, req.reply)); } - when(wait(leaderFail)) { - // We are no longer the leader if this has changed. - endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Leader Replaced", true); - TEST(true); // Lost Cluster Controller Role - return Void(); - } when(ReplyPromise ping = waitNext(interf.clientInterface.ping.getFuture())) { ping.send(Void()); } } } @@ -5929,12 +2712,17 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer NetworkAddress backup(IPAddress(0x06060606), 1); NetworkAddress proxy(IPAddress(0x07070707), 1); NetworkAddress resolver(IPAddress(0x08080808), 1); + NetworkAddress clusterController(IPAddress(0x09090909), 1); UID testUID(1, 2); // Create a ServerDBInfo using above addresses. ServerDBInfo testDbInfo; - testDbInfo.master.changeCoordinators = - RequestStream(Endpoint({ master }, testUID)); + testDbInfo.clusterInterface.changeCoordinators = + RequestStream(Endpoint({ clusterController }, UID(1, 2))); + + MasterInterface mInterface; + mInterface.getCommitVersion = RequestStream(Endpoint({ master }, UID(1, 2))); + testDbInfo.master = mInterface; TLogInterface localTLogInterf; localTLogInterf.peekMessages = RequestStream(Endpoint({ tlog }, testUID)); @@ -6035,14 +2823,15 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer NetworkAddress proxy(IPAddress(0x07070707), 1); NetworkAddress proxy2(IPAddress(0x08080808), 1); NetworkAddress resolver(IPAddress(0x09090909), 1); + NetworkAddress clusterController(IPAddress(0x10101010), 1); UID testUID(1, 2); data.db.config.usableRegions = 2; // Create a ServerDBInfo using above addresses. ServerDBInfo testDbInfo; - testDbInfo.master.changeCoordinators = - RequestStream(Endpoint({ master }, testUID)); + testDbInfo.clusterInterface.changeCoordinators = + RequestStream(Endpoint({ clusterController }, UID(1, 2))); TLogInterface localTLogInterf; localTLogInterf.peekMessages = RequestStream(Endpoint({ tlog }, testUID)); diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h new file mode 100644 index 0000000000..39618a353e --- /dev/null +++ b/fdbserver/ClusterController.actor.h @@ -0,0 +1,3286 @@ +/* + * ClusterController.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source +// version. +#include + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H) +#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H +#include "fdbserver/ClusterController.actor.g.h" +#elif !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_H) +#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_H + +#include "fdbclient/DatabaseContext.h" +#include "fdbrpc/Replication.h" +#include "fdbrpc/ReplicationUtils.h" +#include "fdbserver/Knobs.h" +#include "fdbserver/WorkerInterface.actor.h" +#include "flow/SystemMonitor.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +struct WorkerInfo : NonCopyable { + Future watcher; + ReplyPromise reply; + Generation gen; + int reboots; + ProcessClass initialClass; + ClusterControllerPriorityInfo priorityInfo; + WorkerDetails details; + Future haltRatekeeper; + Future haltDistributor; + Future haltBlobManager; + Standalone> issues; + + WorkerInfo() + : gen(-1), reboots(0), + priorityInfo(ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown) {} + WorkerInfo(Future watcher, + ReplyPromise reply, + Generation gen, + WorkerInterface interf, + ProcessClass initialClass, + ProcessClass processClass, + ClusterControllerPriorityInfo priorityInfo, + bool degraded, + Standalone> issues) + : watcher(watcher), reply(reply), gen(gen), reboots(0), initialClass(initialClass), priorityInfo(priorityInfo), + details(interf, processClass, degraded), issues(issues) {} + + WorkerInfo(WorkerInfo&& r) noexcept + : watcher(std::move(r.watcher)), reply(std::move(r.reply)), gen(r.gen), reboots(r.reboots), + initialClass(r.initialClass), priorityInfo(r.priorityInfo), details(std::move(r.details)), + haltRatekeeper(r.haltRatekeeper), haltDistributor(r.haltDistributor), haltBlobManager(r.haltBlobManager), + issues(r.issues) {} + void operator=(WorkerInfo&& r) noexcept { + watcher = std::move(r.watcher); + reply = std::move(r.reply); + gen = r.gen; + reboots = r.reboots; + initialClass = r.initialClass; + priorityInfo = r.priorityInfo; + details = std::move(r.details); + haltRatekeeper = r.haltRatekeeper; + haltDistributor = r.haltDistributor; + haltBlobManager = r.haltBlobManager; + issues = r.issues; + } +}; + +struct WorkerFitnessInfo { + WorkerDetails worker; + ProcessClass::Fitness fitness; + int used; + + WorkerFitnessInfo() : fitness(ProcessClass::NeverAssign), used(0) {} + WorkerFitnessInfo(WorkerDetails worker, ProcessClass::Fitness fitness, int used) + : worker(worker), fitness(fitness), used(used) {} +}; + +struct RecruitWorkersInfo : ReferenceCounted { + RecruitFromConfigurationRequest req; + RecruitFromConfigurationReply rep; + AsyncTrigger waitForCompletion; + Optional dbgId; + + RecruitWorkersInfo(RecruitFromConfigurationRequest const& req) : req(req) {} +}; + +struct RecruitRemoteWorkersInfo : ReferenceCounted { + RecruitRemoteFromConfigurationRequest req; + RecruitRemoteFromConfigurationReply rep; + AsyncTrigger waitForCompletion; + Optional dbgId; + + RecruitRemoteWorkersInfo(RecruitRemoteFromConfigurationRequest const& req) : req(req) {} +}; + +class ClusterControllerData { +public: + struct DBInfo { + Reference> clientInfo; + Reference> serverInfo; + std::map incompatibleConnections; + AsyncTrigger forceMasterFailure; + int64_t masterRegistrationCount; + int64_t dbInfoCount; + bool recoveryStalled; + bool forceRecovery; + DatabaseConfiguration config; // Asynchronously updated via master registration + DatabaseConfiguration fullyRecoveredConfig; + Database db; + int unfinishedRecoveries; + int logGenerations; + bool cachePopulated; + std::map> clientStatus; + Future clientCounter; + int clientCount; + + DBInfo() + : clientInfo(new AsyncVar()), serverInfo(new AsyncVar()), + masterRegistrationCount(0), dbInfoCount(0), recoveryStalled(false), forceRecovery(false), + db(DatabaseContext::create(clientInfo, + Future(), + LocalityData(), + EnableLocalityLoadBalance::True, + TaskPriority::DefaultEndpoint, + LockAware::True)), // SOMEDAY: Locality! + unfinishedRecoveries(0), logGenerations(0), cachePopulated(false), clientCount(0) { + clientCounter = countClients(this); + } + + void setDistributor(const DataDistributorInterface& interf) { + auto newInfo = serverInfo->get(); + newInfo.id = deterministicRandom()->randomUniqueID(); + newInfo.infoGeneration = ++dbInfoCount; + newInfo.distributor = interf; + serverInfo->set(newInfo); + } + + void setRatekeeper(const RatekeeperInterface& interf) { + auto newInfo = serverInfo->get(); + newInfo.id = deterministicRandom()->randomUniqueID(); + newInfo.infoGeneration = ++dbInfoCount; + newInfo.ratekeeper = interf; + serverInfo->set(newInfo); + } + + void setBlobManager(const BlobManagerInterface& interf) { + auto newInfo = serverInfo->get(); + newInfo.id = deterministicRandom()->randomUniqueID(); + newInfo.infoGeneration = ++dbInfoCount; + newInfo.blobManager = interf; + serverInfo->set(newInfo); + } + + void clearInterf(ProcessClass::ClassType t) { + auto newInfo = serverInfo->get(); + newInfo.id = deterministicRandom()->randomUniqueID(); + newInfo.infoGeneration = ++dbInfoCount; + if (t == ProcessClass::DataDistributorClass) { + newInfo.distributor = Optional(); + } else if (t == ProcessClass::RatekeeperClass) { + newInfo.ratekeeper = Optional(); + } else if (t == ProcessClass::BlobManagerClass) { + newInfo.blobManager = Optional(); + } + serverInfo->set(newInfo); + } + + ACTOR static Future countClients(DBInfo* self) { + loop { + wait(delay(SERVER_KNOBS->CC_PRUNE_CLIENTS_INTERVAL)); + + self->clientCount = 0; + for (auto itr = self->clientStatus.begin(); itr != self->clientStatus.end();) { + if (now() - itr->second.first < 2 * SERVER_KNOBS->COORDINATOR_REGISTER_INTERVAL) { + self->clientCount += itr->second.second.clientCount; + ++itr; + } else { + itr = self->clientStatus.erase(itr); + } + } + } + } + }; + + struct UpdateWorkerList { + Future init(Database const& db) { return update(this, db); } + + void set(Optional> processID, Optional data) { + delta[processID] = data; + anyDelta.set(true); + } + + private: + std::map>, Optional> delta; + AsyncVar anyDelta; + + ACTOR static Future update(UpdateWorkerList* self, Database db) { + // The Database we are using is based on worker registrations to this cluster controller, which come only + // from master servers that we started, so it shouldn't be possible for multiple cluster controllers to + // fight. + state Transaction tr(db); + loop { + try { + tr.clear(workerListKeys); + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + loop { + tr.reset(); + + // Wait for some changes + while (!self->anyDelta.get()) + wait(self->anyDelta.onChange()); + self->anyDelta.set(false); + + state std::map>, Optional> delta; + delta.swap(self->delta); + + TraceEvent("UpdateWorkerList").detail("DeltaCount", delta.size()); + + // Do a transaction to write the changes + loop { + try { + for (auto w = delta.begin(); w != delta.end(); ++w) { + if (w->second.present()) { + tr.set(workerListKeyFor(w->first.get()), workerListValue(w->second.get())); + } else + tr.clear(workerListKeyFor(w->first.get())); + } + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + } + }; + + bool workerAvailable(WorkerInfo const& worker, bool checkStable) { + return (now() - startTime < 2 * FLOW_KNOBS->SERVER_REQUEST_INTERVAL) || + (IFailureMonitor::failureMonitor().getState(worker.details.interf.storage.getEndpoint()).isAvailable() && + (!checkStable || worker.reboots < 2)); + } + + bool isLongLivedStateless(Optional const& processId) { + return (db.serverInfo->get().distributor.present() && + db.serverInfo->get().distributor.get().locality.processId() == processId) || + (db.serverInfo->get().ratekeeper.present() && + db.serverInfo->get().ratekeeper.get().locality.processId() == processId) || + (db.serverInfo->get().blobManager.present() && + db.serverInfo->get().blobManager.get().locality.processId() == processId); + } + + WorkerDetails getStorageWorker(RecruitStorageRequest const& req) { + std::set>> excludedMachines(req.excludeMachines.begin(), + req.excludeMachines.end()); + std::set>> includeDCs(req.includeDCs.begin(), req.includeDCs.end()); + std::set excludedAddresses(req.excludeAddresses.begin(), req.excludeAddresses.end()); + + for (auto& it : id_worker) + if (workerAvailable(it.second, false) && + !excludedMachines.count(it.second.details.interf.locality.zoneId()) && + (includeDCs.size() == 0 || includeDCs.count(it.second.details.interf.locality.dcId())) && + !addressExcluded(excludedAddresses, it.second.details.interf.address()) && + (!it.second.details.interf.secondaryAddress().present() || + !addressExcluded(excludedAddresses, it.second.details.interf.secondaryAddress().get())) && + it.second.details.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::UnsetFit) { + return it.second.details; + } + + if (req.criticalRecruitment) { + ProcessClass::Fitness bestFit = ProcessClass::NeverAssign; + Optional bestInfo; + for (auto& it : id_worker) { + ProcessClass::Fitness fit = it.second.details.processClass.machineClassFitness(ProcessClass::Storage); + if (workerAvailable(it.second, false) && + !excludedMachines.count(it.second.details.interf.locality.zoneId()) && + (includeDCs.size() == 0 || includeDCs.count(it.second.details.interf.locality.dcId())) && + !addressExcluded(excludedAddresses, it.second.details.interf.address()) && fit < bestFit) { + bestFit = fit; + bestInfo = it.second.details; + } + } + + if (bestInfo.present()) { + return bestInfo.get(); + } + } + + throw no_more_servers(); + } + + // Returns a worker that can be used by a blob worker + // Note: we restrict the set of possible workers to those in the same DC as the BM/CC + WorkerDetails getBlobWorker(RecruitBlobWorkerRequest const& req) { + std::set excludedAddresses(req.excludeAddresses.begin(), req.excludeAddresses.end()); + for (auto& it : id_worker) { + // the worker must be available, have the same dcID as CC, + // not be one of the excluded addrs from req and have the approriate fitness + if (workerAvailable(it.second, false) && + clusterControllerDcId == it.second.details.interf.locality.dcId() && + !addressExcluded(excludedAddresses, it.second.details.interf.address()) && + (!it.second.details.interf.secondaryAddress().present() || + !addressExcluded(excludedAddresses, it.second.details.interf.secondaryAddress().get())) && + it.second.details.processClass.machineClassFitness(ProcessClass::BlobWorker) == ProcessClass::BestFit) { + return it.second.details; + } + } + + throw no_more_servers(); + } + + std::vector getWorkersForSeedServers( + DatabaseConfiguration const& conf, + Reference const& policy, + Optional>> const& dcId = Optional>>()) { + std::map> fitness_workers; + std::vector results; + Reference logServerSet = Reference(new LocalityMap()); + LocalityMap* logServerMap = (LocalityMap*)logServerSet.getPtr(); + bool bCompleted = false; + + for (auto& it : id_worker) { + auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage); + if (workerAvailable(it.second, false) && !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses()) && + fitness != ProcessClass::NeverAssign && + (!dcId.present() || it.second.details.interf.locality.dcId() == dcId.get())) { + fitness_workers[fitness].push_back(it.second.details); + } + } + + for (auto& it : fitness_workers) { + for (auto& worker : it.second) { + logServerMap->add(worker.interf.locality, &worker); + } + + std::vector bestSet; + if (logServerSet->selectReplicas(policy, bestSet)) { + results.reserve(bestSet.size()); + for (auto& entry : bestSet) { + auto object = logServerMap->getObject(entry); + results.push_back(*object); + } + bCompleted = true; + break; + } + } + + logServerSet->clear(); + logServerSet.clear(); + + if (!bCompleted) { + throw no_more_servers(); + } + + return results; + } + + // Adds workers to the result such that each field is used in the result set as evenly as possible, + // with a secondary criteria of minimizing the reuse of zoneIds + // only add workers which have a field which is already in the result set + void addWorkersByLowestField(StringRef field, + int desired, + const std::vector& workers, + std::set& resultSet) { + typedef Optional> Field; + typedef Optional> Zone; + typedef std::tuple FieldCount; + typedef std::pair ZoneCount; + + std::priority_queue, std::greater> fieldQueue; + std::map, std::greater>> + field_zoneQueue; + + std::map> field_count; + std::map> zone_count; + std::map> zone_workers; + + // Count the amount of fields and zones already in the result set + for (auto& worker : resultSet) { + auto thisField = worker.interf.locality.get(field); + auto thisZone = worker.interf.locality.zoneId(); + auto thisDc = worker.interf.locality.dcId(); + + auto& fitness = field_count[thisField]; + fitness.first++; + fitness.second = thisDc == clusterControllerDcId; + + auto& zc = zone_count[thisZone]; + zc.first++; + zc.second = thisField; + } + + for (auto& worker : workers) { + auto thisField = worker.interf.locality.get(field); + auto thisZone = worker.interf.locality.zoneId(); + + if (field_count.count(thisField)) { + zone_workers[thisZone].push_back(worker); + zone_count[thisZone].second = thisField; + } + } + + // try to avoid fields in the cluster controller datacenter if everything else is equal + for (auto& it : field_count) { + fieldQueue.emplace(it.second.first, it.second.second, it.first); + } + + for (auto& it : zone_count) { + field_zoneQueue[it.second.second].emplace(it.second.first, it.first); + } + + // start with the least used field, and try to find a worker with that field + while (fieldQueue.size()) { + auto lowestField = fieldQueue.top(); + auto& lowestZoneQueue = field_zoneQueue[std::get<2>(lowestField)]; + bool added = false; + // start with the least used zoneId, and try and find a worker with that zone + while (lowestZoneQueue.size() && !added) { + auto lowestZone = lowestZoneQueue.top(); + auto& zoneWorkers = zone_workers[lowestZone.second]; + + while (zoneWorkers.size() && !added) { + if (!resultSet.count(zoneWorkers.back())) { + resultSet.insert(zoneWorkers.back()); + if (resultSet.size() == desired) { + return; + } + added = true; + } + zoneWorkers.pop_back(); + } + lowestZoneQueue.pop(); + if (added && zoneWorkers.size()) { + ++lowestZone.first; + lowestZoneQueue.push(lowestZone); + } + } + fieldQueue.pop(); + if (added) { + ++std::get<0>(lowestField); + fieldQueue.push(lowestField); + } + } + } + + // Adds workers to the result which minimize the reuse of zoneIds + void addWorkersByLowestZone(int desired, + const std::vector& workers, + std::set& resultSet) { + typedef Optional> Zone; + typedef std::pair ZoneCount; + + std::map zone_count; + std::map> zone_workers; + std::priority_queue, std::greater> zoneQueue; + + for (const auto& worker : workers) { + auto thisZone = worker.interf.locality.zoneId(); + zone_count[thisZone] = 0; + zone_workers[thisZone].push_back(worker); + } + + for (auto& worker : resultSet) { + auto thisZone = worker.interf.locality.zoneId(); + zone_count[thisZone]++; + } + + for (auto& it : zone_count) { + zoneQueue.emplace(it.second, it.first); + } + + while (zoneQueue.size()) { + auto lowestZone = zoneQueue.top(); + auto& zoneWorkers = zone_workers[lowestZone.second]; + + bool added = false; + while (zoneWorkers.size() && !added) { + if (!resultSet.count(zoneWorkers.back())) { + resultSet.insert(zoneWorkers.back()); + if (resultSet.size() == desired) { + return; + } + added = true; + } + zoneWorkers.pop_back(); + } + zoneQueue.pop(); + if (added && zoneWorkers.size()) { + ++lowestZone.first; + zoneQueue.push(lowestZone); + } + } + } + + // Log the reason why the worker is considered as unavailable. + void logWorkerUnavailable(const Severity severity, + const UID& id, + const std::string& method, + const std::string& reason, + const WorkerDetails& details, + const ProcessClass::Fitness& fitness, + const std::set>& dcIds) { + // Construct the list of DCs where the TLog recruitment is happening. This is mainly for logging purpose. + std::string dcList; + for (const auto& dc : dcIds) { + if (!dcList.empty()) { + dcList += ','; + } + dcList += printable(dc); + } + // Logging every possible options is a lot for every recruitment; logging all of the options with GoodFit or + // BestFit may work because there should only be like 30 tlog class processes. Plus, the recruitment happens + // only during initial database creation and recovery. So these trace events should be sparse. + if (fitness == ProcessClass::GoodFit || fitness == ProcessClass::BestFit || + fitness == ProcessClass::NeverAssign) { + TraceEvent(severity, "GetTLogTeamWorkerUnavailable", id) + .detail("TLogRecruitMethod", method) + .detail("Reason", reason) + .detail("WorkerID", details.interf.id()) + .detail("WorkerDC", details.interf.locality.dcId()) + .detail("Address", details.interf.addresses().toString()) + .detail("Fitness", fitness) + .detail("RecruitmentDcIds", dcList); + } + } + + // A TLog recruitment method specialized for three_data_hall and three_datacenter configurations + // It attempts to evenly recruit processes from across data_halls or datacenters + std::vector getWorkersForTlogsComplex(DatabaseConfiguration const& conf, + int32_t desired, + std::map>, int>& id_used, + StringRef field, + int minFields, + int minPerField, + bool allowDegraded, + bool checkStable, + const std::set>& dcIds, + const std::vector& exclusionWorkerIds) { + std::map, std::vector> fitness_workers; + + // Go through all the workers to list all the workers that can be recruited. + for (const auto& [worker_process_id, worker_info] : id_worker) { + const auto& worker_details = worker_info.details; + auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); + + if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != + exclusionWorkerIds.end()) { + logWorkerUnavailable(SevInfo, id, "complex", "Worker is excluded", worker_details, fitness, dcIds); + continue; + } + if (!workerAvailable(worker_info, checkStable)) { + logWorkerUnavailable(SevInfo, id, "complex", "Worker is not available", worker_details, fitness, dcIds); + continue; + } + if (conf.isExcludedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "complex", + "Worker server is excluded from the cluster", + worker_details, + fitness, + dcIds); + continue; + } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "complex", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } + if (fitness == ProcessClass::NeverAssign) { + logWorkerUnavailable( + SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); + continue; + } + if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { + logWorkerUnavailable( + SevDebug, id, "complex", "Worker is not in the target DC", worker_details, fitness, dcIds); + continue; + } + if (!allowDegraded && worker_details.degraded) { + logWorkerUnavailable( + SevInfo, id, "complex", "Worker is degraded and not allowed", worker_details, fitness, dcIds); + continue; + } + + fitness_workers[std::make_tuple( + fitness, id_used[worker_process_id], isLongLivedStateless(worker_process_id))] + .push_back(worker_details); + } + + auto requiredFitness = ProcessClass::NeverAssign; + int requiredUsed = 1e6; + + typedef Optional> Field; + typedef Optional> Zone; + std::map, std::vector>> field_zones; + std::set fieldsWithMin; + std::map field_count; + std::map> field_fitness; + + // Determine the best required workers by finding the workers with enough unique zoneIds per field + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + deterministicRandom()->randomShuffle(workerIter->second); + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + + for (auto& worker : workerIter->second) { + auto thisField = worker.interf.locality.get(field); + auto& zones = field_zones[thisField]; + if (zones.first.insert(worker.interf.locality.zoneId()).second) { + zones.second.push_back(worker); + if (zones.first.size() == minPerField) { + fieldsWithMin.insert(thisField); + } + } + field_count[thisField]++; + field_fitness.insert( + { thisField, + std::make_tuple(fitness, used, worker.interf.locality.dcId() == clusterControllerDcId) }); + } + if (fieldsWithMin.size() >= minFields) { + requiredFitness = fitness; + requiredUsed = used; + } + } + + if (fieldsWithMin.size() < minFields) { + throw no_more_servers(); + } + + std::set chosenFields; + // If we cannot use all of the fields, use the fields which allow the best workers to be chosen + if (fieldsWithMin.size() * minPerField > desired) { + std::vector> orderedFields; + for (auto& it : fieldsWithMin) { + auto& fitness = field_fitness[it]; + orderedFields.emplace_back( + std::get<0>(fitness), std::get<1>(fitness), std::get<2>(fitness), field_count[it], it); + } + std::sort(orderedFields.begin(), orderedFields.end()); + int totalFields = desired / minPerField; + int maxCount = 0; + for (int i = 0; i < orderedFields.size() && chosenFields.size() < totalFields; i++) { + if (chosenFields.size() == totalFields - 1 && maxCount + std::get<3>(orderedFields[i]) < desired) { + for (int j = i + 1; j < orderedFields.size(); j++) { + if (maxCount + std::get<3>(orderedFields[j]) >= desired) { + chosenFields.insert(std::get<4>(orderedFields[j])); + break; + } + } + } + if (chosenFields.size() < totalFields) { + maxCount += std::get<3>(orderedFields[i]); + chosenFields.insert(std::get<4>(orderedFields[i])); + } + } + } else { + chosenFields = fieldsWithMin; + } + + // Create a result set with fulfills the minField and minPerField requirements before adding more workers + std::set resultSet; + for (auto& it : chosenFields) { + auto& w = field_zones[it].second; + for (int i = 0; i < minPerField; i++) { + resultSet.insert(w[i]); + } + } + + // Continue adding workers to the result set until we reach the desired number of workers + for (auto workerIter = fitness_workers.begin(); + workerIter != fitness_workers.end() && resultSet.size() < desired; + ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + if (workerIter->second.size() + resultSet.size() <= desired) { + for (auto& worker : workerIter->second) { + if (chosenFields.count(worker.interf.locality.get(field))) { + resultSet.insert(worker); + } + } + } else { + addWorkersByLowestField(field, desired, workerIter->second, resultSet); + } + } + + for (auto& result : resultSet) { + id_used[result.interf.locality.processId()]++; + } + + return std::vector(resultSet.begin(), resultSet.end()); + } + + // Attempt to recruit TLogs without degraded processes and see if it improves the configuration + std::vector getWorkersForTlogsComplex(DatabaseConfiguration const& conf, + int32_t desired, + std::map>, int>& id_used, + StringRef field, + int minFields, + int minPerField, + bool checkStable, + const std::set>& dcIds, + const std::vector& exclusionWorkerIds) { + desired = std::max(desired, minFields * minPerField); + std::map>, int> withDegradedUsed = id_used; + auto withDegraded = getWorkersForTlogsComplex(conf, + desired, + withDegradedUsed, + field, + minFields, + minPerField, + true, + checkStable, + dcIds, + exclusionWorkerIds); + RoleFitness withDegradedFitness(withDegraded, ProcessClass::TLog, withDegradedUsed); + ASSERT(withDegraded.size() <= desired); + + bool usedDegraded = false; + for (auto& it : withDegraded) { + if (it.degraded) { + usedDegraded = true; + break; + } + } + + if (!usedDegraded) { + id_used = withDegradedUsed; + return withDegraded; + } + + try { + std::map>, int> withoutDegradedUsed = id_used; + auto withoutDegraded = getWorkersForTlogsComplex(conf, + desired, + withoutDegradedUsed, + field, + minFields, + minPerField, + false, + checkStable, + dcIds, + exclusionWorkerIds); + RoleFitness withoutDegradedFitness(withoutDegraded, ProcessClass::TLog, withoutDegradedUsed); + ASSERT(withoutDegraded.size() <= desired); + + if (withDegradedFitness < withoutDegradedFitness) { + id_used = withDegradedUsed; + return withDegraded; + } + id_used = withoutDegradedUsed; + return withoutDegraded; + } catch (Error& e) { + if (e.code() != error_code_no_more_servers) { + throw; + } + id_used = withDegradedUsed; + return withDegraded; + } + } + + // A TLog recruitment method specialized for single, double, and triple configurations + // It recruits processes from with unique zoneIds until it reaches the desired amount + std::vector getWorkersForTlogsSimple(DatabaseConfiguration const& conf, + int32_t required, + int32_t desired, + std::map>, int>& id_used, + bool checkStable, + const std::set>& dcIds, + const std::vector& exclusionWorkerIds) { + std::map, std::vector> fitness_workers; + + // Go through all the workers to list all the workers that can be recruited. + for (const auto& [worker_process_id, worker_info] : id_worker) { + const auto& worker_details = worker_info.details; + auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); + + if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != + exclusionWorkerIds.end()) { + logWorkerUnavailable(SevInfo, id, "simple", "Worker is excluded", worker_details, fitness, dcIds); + continue; + } + if (!workerAvailable(worker_info, checkStable)) { + logWorkerUnavailable(SevInfo, id, "simple", "Worker is not available", worker_details, fitness, dcIds); + continue; + } + if (conf.isExcludedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "simple", + "Worker server is excluded from the cluster", + worker_details, + fitness, + dcIds); + continue; + } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "simple", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } + if (fitness == ProcessClass::NeverAssign) { + logWorkerUnavailable( + SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); + continue; + } + if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { + logWorkerUnavailable( + SevDebug, id, "simple", "Worker is not in the target DC", worker_details, fitness, dcIds); + continue; + } + + // This worker is a candidate for TLog recruitment. + bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId; + // Prefer recruiting a TransactionClass non-degraded process over a LogClass degraded process + if (worker_details.degraded) { + fitness = std::max(fitness, ProcessClass::GoodFit); + } + + fitness_workers[std::make_tuple(fitness, + id_used[worker_process_id], + worker_details.degraded, + isLongLivedStateless(worker_process_id), + inCCDC)] + .push_back(worker_details); + } + + auto requiredFitness = ProcessClass::BestFit; + int requiredUsed = 0; + + std::set>> zones; + std::set resultSet; + + // Determine the best required workers by finding the workers with enough unique zoneIds + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + deterministicRandom()->randomShuffle(workerIter->second); + for (auto& worker : workerIter->second) { + if (!zones.count(worker.interf.locality.zoneId())) { + zones.insert(worker.interf.locality.zoneId()); + resultSet.insert(worker); + if (resultSet.size() == required) { + break; + } + } + } + if (resultSet.size() == required) { + requiredFitness = fitness; + requiredUsed = used; + break; + } + } + + if (resultSet.size() < required) { + throw no_more_servers(); + } + + // Continue adding workers to the result set until we reach the desired number of workers + for (auto workerIter = fitness_workers.begin(); + workerIter != fitness_workers.end() && resultSet.size() < desired; + ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + if (workerIter->second.size() + resultSet.size() <= desired) { + for (auto& worker : workerIter->second) { + resultSet.insert(worker); + } + } else { + addWorkersByLowestZone(desired, workerIter->second, resultSet); + } + } + + ASSERT(resultSet.size() >= required && resultSet.size() <= desired); + + for (auto& result : resultSet) { + id_used[result.interf.locality.processId()]++; + } + + return std::vector(resultSet.begin(), resultSet.end()); + } + + // A backup method for TLog recruitment that is used for custom policies, but does a worse job + // selecting the best workers. + // conf: the database configuration. + // required: the required number of TLog workers to select. + // desired: the desired number of TLog workers to select. + // policy: the TLog replication policy the selection needs to satisfy. + // id_used: keep track of process IDs of selected workers. + // checkStable: when true, only select from workers that are considered as stable worker (not rebooted more than + // twice recently). + // dcIds: the target data centers the workers are in. The selected workers must all be from these + // data centers: + // exclusionWorkerIds: the workers to be excluded from the selection. + std::vector getWorkersForTlogsBackup( + DatabaseConfiguration const& conf, + int32_t required, + int32_t desired, + Reference const& policy, + std::map>, int>& id_used, + bool checkStable = false, + const std::set>& dcIds = std::set>(), + const std::vector& exclusionWorkerIds = {}) { + std::map, std::vector> fitness_workers; + std::vector results; + Reference logServerSet = Reference(new LocalityMap()); + LocalityMap* logServerMap = (LocalityMap*)logServerSet.getPtr(); + bool bCompleted = false; + desired = std::max(required, desired); + + // Go through all the workers to list all the workers that can be recruited. + for (const auto& [worker_process_id, worker_info] : id_worker) { + const auto& worker_details = worker_info.details; + auto fitness = worker_details.processClass.machineClassFitness(ProcessClass::TLog); + + if (std::find(exclusionWorkerIds.begin(), exclusionWorkerIds.end(), worker_details.interf.id()) != + exclusionWorkerIds.end()) { + logWorkerUnavailable(SevInfo, id, "deprecated", "Worker is excluded", worker_details, fitness, dcIds); + continue; + } + if (!workerAvailable(worker_info, checkStable)) { + logWorkerUnavailable( + SevInfo, id, "deprecated", "Worker is not available", worker_details, fitness, dcIds); + continue; + } + if (conf.isExcludedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "deprecated", + "Worker server is excluded from the cluster", + worker_details, + fitness, + dcIds); + continue; + } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "deprecated", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } + if (fitness == ProcessClass::NeverAssign) { + logWorkerUnavailable( + SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); + continue; + } + if (!dcIds.empty() && dcIds.count(worker_details.interf.locality.dcId()) == 0) { + logWorkerUnavailable( + SevDebug, id, "deprecated", "Worker is not in the target DC", worker_details, fitness, dcIds); + continue; + } + + // This worker is a candidate for TLog recruitment. + bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId; + // Prefer recruiting a TransactionClass non-degraded process over a LogClass degraded process + if (worker_details.degraded) { + fitness = std::max(fitness, ProcessClass::GoodFit); + } + + fitness_workers[std::make_tuple(fitness, id_used[worker_process_id], worker_details.degraded, inCCDC)] + .push_back(worker_details); + } + + auto requiredFitness = ProcessClass::BestFit; + int requiredUsed = 0; + bool requiredDegraded = false; + bool requiredInCCDC = false; + + // Determine the minimum fitness and used necessary to fulfill the policy + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || used > requiredUsed) { + if (logServerSet->size() >= required && logServerSet->validate(policy)) { + bCompleted = true; + break; + } + requiredFitness = fitness; + requiredUsed = used; + } + + if (std::get<2>(workerIter->first)) { + requiredDegraded = true; + } + if (std::get<3>(workerIter->first)) { + requiredInCCDC = true; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + + if (!bCompleted && !(logServerSet->size() >= required && logServerSet->validate(policy))) { + std::vector tLocalities; + for (auto& object : logServerMap->getObjects()) { + tLocalities.push_back(object->interf.locality); + } + + logServerSet->clear(); + logServerSet.clear(); + throw no_more_servers(); + } + + // If we have less than the desired amount, return all of the processes we have + if (logServerSet->size() <= desired) { + for (auto& object : logServerMap->getObjects()) { + results.push_back(*object); + } + for (auto& result : results) { + id_used[result.interf.locality.processId()]++; + } + return results; + } + + // If we have added any degraded processes, try and remove them to see if we can still + // have the desired amount of processes + if (requiredDegraded) { + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + auto addingDegraded = std::get<2>(workerIter->first); + if (addingDegraded) { + continue; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + if (logServerSet->size() >= desired && logServerSet->validate(policy)) { + requiredDegraded = false; + } + } + + // If we have added any processes in the CC DC, try and remove them to see if we can still + // have the desired amount of processes + if (requiredInCCDC) { + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + auto addingDegraded = std::get<2>(workerIter->first); + auto inCCDC = std::get<3>(workerIter->first); + if (inCCDC || (!requiredDegraded && addingDegraded)) { + continue; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + if (logServerSet->size() >= desired && logServerSet->validate(policy)) { + requiredInCCDC = false; + } + } + + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + auto addingDegraded = std::get<2>(workerIter->first); + auto inCCDC = std::get<3>(workerIter->first); + if ((!requiredInCCDC && inCCDC) || (!requiredDegraded && addingDegraded)) { + continue; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + + if (logServerSet->size() == desired) { + for (auto& object : logServerMap->getObjects()) { + results.push_back(*object); + } + for (auto& result : results) { + id_used[result.interf.locality.processId()]++; + } + return results; + } + + std::vector bestSet; + std::vector tLocalities; + + // We have more than the desired number of processes, so use the policy engine to + // pick a diverse subset of them + bCompleted = findBestPolicySet(bestSet, + logServerSet, + policy, + desired, + SERVER_KNOBS->POLICY_RATING_TESTS, + SERVER_KNOBS->POLICY_GENERATIONS); + ASSERT(bCompleted); + results.reserve(results.size() + bestSet.size()); + for (auto& entry : bestSet) { + auto object = logServerMap->getObject(entry); + ASSERT(object); + results.push_back(*object); + tLocalities.push_back(object->interf.locality); + } + for (auto& result : results) { + id_used[result.interf.locality.processId()]++; + } + TraceEvent("GetTLogTeamDone") + .detail("Policy", policy->info()) + .detail("Results", results.size()) + .detail("Processes", logServerSet->size()) + .detail("Workers", id_worker.size()) + .detail("Required", required) + .detail("Desired", desired) + .detail("Fitness", requiredFitness) + .detail("Used", requiredUsed) + .detail("AddingDegraded", requiredDegraded) + .detail("InCCDC", requiredInCCDC) + .detail("BestCount", bestSet.size()) + .detail("BestZones", ::describeZones(tLocalities)) + .detail("BestDataHalls", ::describeDataHalls(tLocalities)); + return results; + } + + // Selects the best method for TLog recruitment based on the specified policy + std::vector getWorkersForTlogs(DatabaseConfiguration const& conf, + int32_t required, + int32_t desired, + Reference const& policy, + std::map>, int>& id_used, + bool checkStable = false, + const std::set>& dcIds = std::set>(), + const std::vector& exclusionWorkerIds = {}) { + desired = std::max(required, desired); + bool useSimple = false; + if (policy->name() == "Across") { + PolicyAcross* pa1 = (PolicyAcross*)policy.getPtr(); + Reference embedded = pa1->embeddedPolicy(); + if (embedded->name() == "Across") { + PolicyAcross* pa2 = (PolicyAcross*)embedded.getPtr(); + if (pa2->attributeKey() == "zoneid" && pa2->embeddedPolicyName() == "One") { + std::map>, int> testUsed = id_used; + + auto workers = getWorkersForTlogsComplex(conf, + desired, + id_used, + pa1->attributeKey(), + pa1->getCount(), + pa2->getCount(), + checkStable, + dcIds, + exclusionWorkerIds); + + if (g_network->isSimulated()) { + try { + auto testWorkers = getWorkersForTlogsBackup( + conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); + RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); + RoleFitness fitness(workers, ProcessClass::TLog, id_used); + + std::map>, int> field_count; + std::set>> zones; + for (auto& worker : testWorkers) { + if (!zones.count(worker.interf.locality.zoneId())) { + field_count[worker.interf.locality.get(pa1->attributeKey())]++; + zones.insert(worker.interf.locality.zoneId()); + } + } + // backup recruitment is not required to use degraded processes that have better fitness + // so we cannot compare degraded between the two methods + testFitness.degraded = fitness.degraded; + + int minField = 100; + + for (auto& f : field_count) { + minField = std::min(minField, f.second); + } + + if (fitness > testFitness && minField > 1) { + for (auto& w : testWorkers) { + TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + } + for (auto& w : workers) { + TraceEvent("RealTLogs").detail("Interf", w.interf.address()); + } + TraceEvent("FitnessCompare") + .detail("TestF", testFitness.toString()) + .detail("RealF", fitness.toString()); + ASSERT(false); + } + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors + } + } + + return workers; + } + } else if (pa1->attributeKey() == "zoneid" && embedded->name() == "One") { + ASSERT(pa1->getCount() == required); + useSimple = true; + } + } else if (policy->name() == "One") { + useSimple = true; + } + if (useSimple) { + std::map>, int> testUsed = id_used; + + auto workers = + getWorkersForTlogsSimple(conf, required, desired, id_used, checkStable, dcIds, exclusionWorkerIds); + + if (g_network->isSimulated()) { + try { + auto testWorkers = getWorkersForTlogsBackup( + conf, required, desired, policy, testUsed, checkStable, dcIds, exclusionWorkerIds); + RoleFitness testFitness(testWorkers, ProcessClass::TLog, testUsed); + RoleFitness fitness(workers, ProcessClass::TLog, id_used); + // backup recruitment is not required to use degraded processes that have better fitness + // so we cannot compare degraded between the two methods + testFitness.degraded = fitness.degraded; + + if (fitness > testFitness) { + for (auto& w : testWorkers) { + TraceEvent("TestTLogs").detail("Interf", w.interf.address()); + } + for (auto& w : workers) { + TraceEvent("RealTLogs").detail("Interf", w.interf.address()); + } + TraceEvent("FitnessCompare") + .detail("TestF", testFitness.toString()) + .detail("RealF", fitness.toString()); + ASSERT(false); + } + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors + } + } + return workers; + } + TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "PolicyEngineNotOptimized"); + return getWorkersForTlogsBackup( + conf, required, desired, policy, id_used, checkStable, dcIds, exclusionWorkerIds); + } + + // FIXME: This logic will fallback unnecessarily when usable dcs > 1 because it does not check all combinations of + // potential satellite locations + std::vector getWorkersForSatelliteLogs(const DatabaseConfiguration& conf, + const RegionInfo& region, + const RegionInfo& remoteRegion, + std::map>, int>& id_used, + bool& satelliteFallback, + bool checkStable = false) { + int startDC = 0; + loop { + if (startDC > 0 && startDC >= region.satellites.size() + 1 - + (satelliteFallback ? region.satelliteTLogUsableDcsFallback + : region.satelliteTLogUsableDcs)) { + if (satelliteFallback || region.satelliteTLogUsableDcsFallback == 0) { + throw no_more_servers(); + } else { + if (!goodRecruitmentTime.isReady()) { + throw operation_failed(); + } + satelliteFallback = true; + startDC = 0; + } + } + + try { + bool remoteDCUsedAsSatellite = false; + std::set> satelliteDCs; + int32_t desiredSatelliteTLogs = 0; + for (int s = startDC; + s < std::min(startDC + (satelliteFallback ? region.satelliteTLogUsableDcsFallback + : region.satelliteTLogUsableDcs), + region.satellites.size()); + s++) { + satelliteDCs.insert(region.satellites[s].dcId); + if (region.satellites[s].satelliteDesiredTLogCount == -1 || desiredSatelliteTLogs == -1) { + desiredSatelliteTLogs = -1; + } else { + desiredSatelliteTLogs += region.satellites[s].satelliteDesiredTLogCount; + } + if (region.satellites[s].dcId == remoteRegion.dcId) { + remoteDCUsedAsSatellite = true; + } + } + std::vector exclusionWorkerIds; + // FIXME: If remote DC is used as satellite then this logic only ensures that required number of remote + // TLogs can be recruited. It does not balance the number of desired TLogs across the satellite and + // remote sides. + if (remoteDCUsedAsSatellite) { + std::map>, int> tmpIdUsed; + auto remoteLogs = getWorkersForTlogs(conf, + conf.getRemoteTLogReplicationFactor(), + conf.getRemoteTLogReplicationFactor(), + conf.getRemoteTLogPolicy(), + tmpIdUsed, + false, + { remoteRegion.dcId }, + {}); + std::transform(remoteLogs.begin(), + remoteLogs.end(), + std::back_inserter(exclusionWorkerIds), + [](const WorkerDetails& in) { return in.interf.id(); }); + } + if (satelliteFallback) { + return getWorkersForTlogs(conf, + region.satelliteTLogReplicationFactorFallback, + desiredSatelliteTLogs > 0 ? desiredSatelliteTLogs + : conf.getDesiredSatelliteLogs(region.dcId) * + region.satelliteTLogUsableDcsFallback / + region.satelliteTLogUsableDcs, + region.satelliteTLogPolicyFallback, + id_used, + checkStable, + satelliteDCs, + exclusionWorkerIds); + } else { + return getWorkersForTlogs(conf, + region.satelliteTLogReplicationFactor, + desiredSatelliteTLogs > 0 ? desiredSatelliteTLogs + : conf.getDesiredSatelliteLogs(region.dcId), + region.satelliteTLogPolicy, + id_used, + checkStable, + satelliteDCs, + exclusionWorkerIds); + } + } catch (Error& e) { + if (e.code() != error_code_no_more_servers) { + throw; + } + } + + startDC++; + } + } + + ProcessClass::Fitness getBestFitnessForRoleInDatacenter(ProcessClass::ClusterRole role) { + ProcessClass::Fitness bestFitness = ProcessClass::NeverAssign; + for (const auto& it : id_worker) { + if (it.second.priorityInfo.isExcluded || + it.second.details.interf.locality.dcId() != clusterControllerDcId) { + continue; + } + bestFitness = std::min(bestFitness, it.second.details.processClass.machineClassFitness(role)); + } + return bestFitness; + } + + WorkerFitnessInfo getWorkerForRoleInDatacenter(Optional> const& dcId, + ProcessClass::ClusterRole role, + ProcessClass::Fitness unacceptableFitness, + DatabaseConfiguration const& conf, + std::map>, int>& id_used, + std::map>, int> preferredSharing = {}, + bool checkStable = false) { + std::map, std::vector> fitness_workers; + + for (auto& it : id_worker) { + auto fitness = it.second.details.processClass.machineClassFitness(role); + if (conf.isExcludedServer(it.second.details.interf.addresses()) || + isExcludedDegradedServer(it.second.details.interf.addresses())) { + fitness = std::max(fitness, ProcessClass::ExcludeFit); + } + if (workerAvailable(it.second, checkStable) && fitness < unacceptableFitness && + it.second.details.interf.locality.dcId() == dcId) { + auto sharing = preferredSharing.find(it.first); + fitness_workers[std::make_tuple(fitness, + id_used[it.first], + isLongLivedStateless(it.first), + sharing != preferredSharing.end() ? sharing->second : 1e6)] + .push_back(it.second.details); + } + } + + if (fitness_workers.size()) { + auto worker = deterministicRandom()->randomChoice(fitness_workers.begin()->second); + id_used[worker.interf.locality.processId()]++; + return WorkerFitnessInfo(worker, + std::max(ProcessClass::GoodFit, std::get<0>(fitness_workers.begin()->first)), + std::get<1>(fitness_workers.begin()->first)); + } + + throw no_more_servers(); + } + + std::vector getWorkersForRoleInDatacenter( + Optional> const& dcId, + ProcessClass::ClusterRole role, + int amount, + DatabaseConfiguration const& conf, + std::map>, int>& id_used, + std::map>, int> preferredSharing = {}, + Optional minWorker = Optional(), + bool checkStable = false) { + std::map, std::vector> fitness_workers; + std::vector results; + if (minWorker.present()) { + results.push_back(minWorker.get().worker); + } + if (amount <= results.size()) { + return results; + } + + for (auto& it : id_worker) { + auto fitness = it.second.details.processClass.machineClassFitness(role); + if (workerAvailable(it.second, checkStable) && + !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses()) && + it.second.details.interf.locality.dcId() == dcId && + (!minWorker.present() || + (it.second.details.interf.id() != minWorker.get().worker.interf.id() && + (fitness < minWorker.get().fitness || + (fitness == minWorker.get().fitness && id_used[it.first] <= minWorker.get().used))))) { + auto sharing = preferredSharing.find(it.first); + fitness_workers[std::make_tuple(fitness, + id_used[it.first], + isLongLivedStateless(it.first), + sharing != preferredSharing.end() ? sharing->second : 1e6)] + .push_back(it.second.details); + } + } + + for (auto& it : fitness_workers) { + deterministicRandom()->randomShuffle(it.second); + for (int i = 0; i < it.second.size(); i++) { + results.push_back(it.second[i]); + id_used[it.second[i].interf.locality.processId()]++; + if (results.size() == amount) + return results; + } + } + + return results; + } + + // Allows the comparison of two different recruitments to determine which one is better + // Tlog recruitment is different from all the other roles, in that it avoids degraded processes + // And tried to avoid recruitment in the same DC as the cluster controller + struct RoleFitness { + ProcessClass::Fitness bestFit; + ProcessClass::Fitness worstFit; + ProcessClass::ClusterRole role; + int count; + int worstUsed = 1; + bool degraded = false; + + RoleFitness(int bestFit, int worstFit, int count, ProcessClass::ClusterRole role) + : bestFit((ProcessClass::Fitness)bestFit), worstFit((ProcessClass::Fitness)worstFit), role(role), + count(count) {} + + RoleFitness(int fitness, int count, ProcessClass::ClusterRole role) + : bestFit((ProcessClass::Fitness)fitness), worstFit((ProcessClass::Fitness)fitness), role(role), + count(count) {} + + RoleFitness() + : bestFit(ProcessClass::NeverAssign), worstFit(ProcessClass::NeverAssign), role(ProcessClass::NoRole), + count(0) {} + + RoleFitness(const std::vector& workers, + ProcessClass::ClusterRole role, + const std::map>, int>& id_used) + : role(role) { + // Every recruitment will attempt to recruit the preferred amount through GoodFit, + // So a recruitment which only has BestFit is not better than one that has a GoodFit process + worstFit = ProcessClass::GoodFit; + degraded = false; + bestFit = ProcessClass::NeverAssign; + worstUsed = 1; + for (auto& it : workers) { + auto thisFit = it.processClass.machineClassFitness(role); + auto thisUsed = id_used.find(it.interf.locality.processId()); + + if (thisUsed == id_used.end()) { + TraceEvent(SevError, "UsedNotFound").detail("ProcessId", it.interf.locality.processId().get()); + ASSERT(false); + } + if (thisUsed->second == 0) { + TraceEvent(SevError, "UsedIsZero").detail("ProcessId", it.interf.locality.processId().get()); + ASSERT(false); + } + + bestFit = std::min(bestFit, thisFit); + + if (thisFit > worstFit) { + worstFit = thisFit; + worstUsed = thisUsed->second; + } else if (thisFit == worstFit) { + worstUsed = std::max(worstUsed, thisUsed->second); + } + degraded = degraded || it.degraded; + } + + count = workers.size(); + + // degraded is only used for recruitment of tlogs + if (role != ProcessClass::TLog) { + degraded = false; + } + } + + bool operator<(RoleFitness const& r) const { + if (worstFit != r.worstFit) + return worstFit < r.worstFit; + if (worstUsed != r.worstUsed) + return worstUsed < r.worstUsed; + if (count != r.count) + return count > r.count; + if (degraded != r.degraded) + return r.degraded; + // FIXME: TLog recruitment process does not guarantee the best fit is not worsened. + if (role != ProcessClass::TLog && role != ProcessClass::LogRouter && bestFit != r.bestFit) + return bestFit < r.bestFit; + return false; + } + bool operator>(RoleFitness const& r) const { return r < *this; } + bool operator<=(RoleFitness const& r) const { return !(*this > r); } + bool operator>=(RoleFitness const& r) const { return !(*this < r); } + + bool betterCount(RoleFitness const& r) const { + if (count > r.count) + return true; + if (worstFit != r.worstFit) + return worstFit < r.worstFit; + if (worstUsed != r.worstUsed) + return worstUsed < r.worstUsed; + if (degraded != r.degraded) + return r.degraded; + return false; + } + + bool operator==(RoleFitness const& r) const { + return worstFit == r.worstFit && worstUsed == r.worstUsed && bestFit == r.bestFit && count == r.count && + degraded == r.degraded; + } + + std::string toString() const { return format("%d %d %d %d %d", worstFit, worstUsed, count, degraded, bestFit); } + }; + + std::set>> getDatacenters(DatabaseConfiguration const& conf, + bool checkStable = false) { + std::set>> result; + for (auto& it : id_worker) + if (workerAvailable(it.second, checkStable) && + !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses())) + result.insert(it.second.details.interf.locality.dcId()); + return result; + } + + void updateKnownIds(std::map>, int>* id_used) { + (*id_used)[masterProcessId]++; + (*id_used)[clusterControllerProcessId]++; + } + + RecruitRemoteFromConfigurationReply findRemoteWorkersForConfiguration( + RecruitRemoteFromConfigurationRequest const& req) { + RecruitRemoteFromConfigurationReply result; + std::map>, int> id_used; + + updateKnownIds(&id_used); + + if (req.dbgId.present()) { + TraceEvent(SevDebug, "FindRemoteWorkersForConf", req.dbgId.get()) + .detail("RemoteDcId", req.dcId) + .detail("Configuration", req.configuration.toString()) + .detail("Policy", req.configuration.getRemoteTLogPolicy()->name()); + } + + std::set> remoteDC; + remoteDC.insert(req.dcId); + + auto remoteLogs = getWorkersForTlogs(req.configuration, + req.configuration.getRemoteTLogReplicationFactor(), + req.configuration.getDesiredRemoteLogs(), + req.configuration.getRemoteTLogPolicy(), + id_used, + false, + remoteDC, + req.exclusionWorkerIds); + for (int i = 0; i < remoteLogs.size(); i++) { + result.remoteTLogs.push_back(remoteLogs[i].interf); + } + + auto logRouters = getWorkersForRoleInDatacenter( + req.dcId, ProcessClass::LogRouter, req.logRouterCount, req.configuration, id_used); + for (int i = 0; i < logRouters.size(); i++) { + result.logRouters.push_back(logRouters[i].interf); + } + + if (!goodRemoteRecruitmentTime.isReady() && + ((RoleFitness( + SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredRemoteLogs(), ProcessClass::TLog) + .betterCount(RoleFitness(remoteLogs, ProcessClass::TLog, id_used))) || + (RoleFitness(SERVER_KNOBS->EXPECTED_LOG_ROUTER_FITNESS, req.logRouterCount, ProcessClass::LogRouter) + .betterCount(RoleFitness(logRouters, ProcessClass::LogRouter, id_used))))) { + throw operation_failed(); + } + + if (req.dbgId.present()) { + TraceEvent(SevDebug, "FindRemoteWorkersForConf_ReturnResult", req.dbgId.get()) + .detail("RemoteDcId", req.dcId) + .detail("ResultRemoteLogs", result.remoteTLogs.size()); + result.dbgId = req.dbgId; + } + + return result; + } + + // Given datacenter ID, returns the primary and remote regions. + std::pair getPrimaryAndRemoteRegion(const std::vector& regions, Key dcId) { + RegionInfo region; + RegionInfo remoteRegion; + for (const auto& r : regions) { + if (r.dcId == dcId) { + region = r; + } else { + remoteRegion = r; + } + } + return std::make_pair(region, remoteRegion); + } + + ErrorOr findWorkersForConfigurationFromDC(RecruitFromConfigurationRequest const& req, + Optional dcId, + bool checkGoodRecruitment) { + RecruitFromConfigurationReply result; + std::map>, int> id_used; + updateKnownIds(&id_used); + + ASSERT(dcId.present()); + + std::set> primaryDC; + primaryDC.insert(dcId); + result.dcId = dcId; + + auto [region, remoteRegion] = getPrimaryAndRemoteRegion(req.configuration.regions, dcId.get()); + + if (req.recruitSeedServers) { + auto primaryStorageServers = + getWorkersForSeedServers(req.configuration, req.configuration.storagePolicy, dcId); + for (int i = 0; i < primaryStorageServers.size(); i++) { + result.storageServers.push_back(primaryStorageServers[i].interf); + } + } + + auto tlogs = getWorkersForTlogs(req.configuration, + req.configuration.tLogReplicationFactor, + req.configuration.getDesiredLogs(), + req.configuration.tLogPolicy, + id_used, + false, + primaryDC); + for (int i = 0; i < tlogs.size(); i++) { + result.tLogs.push_back(tlogs[i].interf); + } + + std::vector satelliteLogs; + if (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1) { + satelliteLogs = + getWorkersForSatelliteLogs(req.configuration, region, remoteRegion, id_used, result.satelliteFallback); + for (int i = 0; i < satelliteLogs.size(); i++) { + result.satelliteTLogs.push_back(satelliteLogs[i].interf); + } + } + + std::map>, int> preferredSharing; + auto first_commit_proxy = getWorkerForRoleInDatacenter( + dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); + preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; + auto first_grv_proxy = getWorkerForRoleInDatacenter( + dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); + preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; + auto first_resolver = getWorkerForRoleInDatacenter( + dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, id_used, preferredSharing); + preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; + + // If one of the first process recruitments is forced to share a process, allow all of next recruitments + // to also share a process. + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; + + auto commit_proxies = getWorkersForRoleInDatacenter(dcId, + ProcessClass::CommitProxy, + req.configuration.getDesiredCommitProxies(), + req.configuration, + id_used, + preferredSharing, + first_commit_proxy); + auto grv_proxies = getWorkersForRoleInDatacenter(dcId, + ProcessClass::GrvProxy, + req.configuration.getDesiredGrvProxies(), + req.configuration, + id_used, + preferredSharing, + first_grv_proxy); + auto resolvers = getWorkersForRoleInDatacenter(dcId, + ProcessClass::Resolver, + req.configuration.getDesiredResolvers(), + req.configuration, + id_used, + preferredSharing, + first_resolver); + for (int i = 0; i < commit_proxies.size(); i++) + result.commitProxies.push_back(commit_proxies[i].interf); + for (int i = 0; i < grv_proxies.size(); i++) + result.grvProxies.push_back(grv_proxies[i].interf); + for (int i = 0; i < resolvers.size(); i++) + result.resolvers.push_back(resolvers[i].interf); + + if (req.maxOldLogRouters > 0) { + if (tlogs.size() == 1) { + result.oldLogRouters.push_back(tlogs[0].interf); + } else { + for (int i = 0; i < tlogs.size(); i++) { + if (tlogs[i].interf.locality.processId() != clusterControllerProcessId) { + result.oldLogRouters.push_back(tlogs[i].interf); + } + } + } + } + + if (req.configuration.backupWorkerEnabled) { + const int nBackup = std::max( + (req.configuration.desiredLogRouterCount > 0 ? req.configuration.desiredLogRouterCount : tlogs.size()), + req.maxOldLogRouters); + auto backupWorkers = + getWorkersForRoleInDatacenter(dcId, ProcessClass::Backup, nBackup, req.configuration, id_used); + std::transform(backupWorkers.begin(), + backupWorkers.end(), + std::back_inserter(result.backupWorkers), + [](const WorkerDetails& w) { return w.interf; }); + } + + if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && + (RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) + .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || + (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1 && + RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, + req.configuration.getDesiredSatelliteLogs(dcId), + ProcessClass::TLog) + .betterCount(RoleFitness(satelliteLogs, ProcessClass::TLog, id_used))) || + RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, + req.configuration.getDesiredCommitProxies(), + ProcessClass::CommitProxy) + .betterCount(RoleFitness(commit_proxies, ProcessClass::CommitProxy, id_used)) || + RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, + req.configuration.getDesiredGrvProxies(), + ProcessClass::GrvProxy) + .betterCount(RoleFitness(grv_proxies, ProcessClass::GrvProxy, id_used)) || + RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, + req.configuration.getDesiredResolvers(), + ProcessClass::Resolver) + .betterCount(RoleFitness(resolvers, ProcessClass::Resolver, id_used)))) { + return operation_failed(); + } + + return result; + } + + RecruitFromConfigurationReply findWorkersForConfigurationDispatch(RecruitFromConfigurationRequest const& req, + bool checkGoodRecruitment) { + if (req.configuration.regions.size() > 1) { + std::vector regions = req.configuration.regions; + if (regions[0].priority == regions[1].priority && regions[1].dcId == clusterControllerDcId.get()) { + TraceEvent("CCSwitchPrimaryDc", id) + .detail("CCDcId", clusterControllerDcId.get()) + .detail("OldPrimaryDcId", regions[0].dcId) + .detail("NewPrimaryDcId", regions[1].dcId); + std::swap(regions[0], regions[1]); + } + + if (regions[1].dcId == clusterControllerDcId.get() && + (!versionDifferenceUpdated || datacenterVersionDifference >= SERVER_KNOBS->MAX_VERSION_DIFFERENCE)) { + if (regions[1].priority >= 0) { + TraceEvent("CCSwitchPrimaryDcVersionDifference", id) + .detail("CCDcId", clusterControllerDcId.get()) + .detail("OldPrimaryDcId", regions[0].dcId) + .detail("NewPrimaryDcId", regions[1].dcId); + std::swap(regions[0], regions[1]); + } else { + TraceEvent(SevWarnAlways, "CCDcPriorityNegative") + .detail("DcId", regions[1].dcId) + .detail("Priority", regions[1].priority) + .detail("FindWorkersInDc", regions[0].dcId) + .detail("Warning", "Failover did not happen but CC is in remote DC"); + } + } + + TraceEvent("CCFindWorkersForConfiguration", id) + .detail("CCDcId", clusterControllerDcId.get()) + .detail("Region0DcId", regions[0].dcId) + .detail("Region1DcId", regions[1].dcId) + .detail("DatacenterVersionDifference", datacenterVersionDifference) + .detail("VersionDifferenceUpdated", versionDifferenceUpdated); + + bool setPrimaryDesired = false; + try { + auto reply = findWorkersForConfigurationFromDC(req, regions[0].dcId, checkGoodRecruitment); + setPrimaryDesired = true; + std::vector> dcPriority; + dcPriority.push_back(regions[0].dcId); + dcPriority.push_back(regions[1].dcId); + desiredDcIds.set(dcPriority); + if (reply.isError()) { + throw reply.getError(); + } else if (regions[0].dcId == clusterControllerDcId.get()) { + return reply.get(); + } + TraceEvent(SevWarn, "CCRecruitmentFailed", id) + .detail("Reason", "Recruited Txn system and CC are in different DCs") + .detail("CCDcId", clusterControllerDcId.get()) + .detail("RecruitedTxnSystemDcId", regions[0].dcId); + throw no_more_servers(); + } catch (Error& e) { + if (!goodRemoteRecruitmentTime.isReady() && regions[1].dcId != clusterControllerDcId.get() && + checkGoodRecruitment) { + throw operation_failed(); + } + + if (e.code() != error_code_no_more_servers || regions[1].priority < 0) { + throw; + } + TraceEvent(SevWarn, "AttemptingRecruitmentInRemoteDc", id) + .detail("SetPrimaryDesired", setPrimaryDesired) + .error(e); + auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId, checkGoodRecruitment); + if (!setPrimaryDesired) { + std::vector> dcPriority; + dcPriority.push_back(regions[1].dcId); + dcPriority.push_back(regions[0].dcId); + desiredDcIds.set(dcPriority); + } + if (reply.isError()) { + throw reply.getError(); + } else if (regions[1].dcId == clusterControllerDcId.get()) { + return reply.get(); + } + throw; + } + } else if (req.configuration.regions.size() == 1) { + std::vector> dcPriority; + dcPriority.push_back(req.configuration.regions[0].dcId); + desiredDcIds.set(dcPriority); + auto reply = + findWorkersForConfigurationFromDC(req, req.configuration.regions[0].dcId, checkGoodRecruitment); + if (reply.isError()) { + throw reply.getError(); + } else if (req.configuration.regions[0].dcId == clusterControllerDcId.get()) { + return reply.get(); + } + throw no_more_servers(); + } else { + RecruitFromConfigurationReply result; + std::map>, int> id_used; + updateKnownIds(&id_used); + auto tlogs = getWorkersForTlogs(req.configuration, + req.configuration.tLogReplicationFactor, + req.configuration.getDesiredLogs(), + req.configuration.tLogPolicy, + id_used); + for (int i = 0; i < tlogs.size(); i++) { + result.tLogs.push_back(tlogs[i].interf); + } + + if (req.maxOldLogRouters > 0) { + if (tlogs.size() == 1) { + result.oldLogRouters.push_back(tlogs[0].interf); + } else { + for (int i = 0; i < tlogs.size(); i++) { + if (tlogs[i].interf.locality.processId() != clusterControllerProcessId) { + result.oldLogRouters.push_back(tlogs[i].interf); + } + } + } + } + + if (req.recruitSeedServers) { + auto primaryStorageServers = + getWorkersForSeedServers(req.configuration, req.configuration.storagePolicy); + for (int i = 0; i < primaryStorageServers.size(); i++) + result.storageServers.push_back(primaryStorageServers[i].interf); + } + + auto datacenters = getDatacenters(req.configuration); + + std::tuple bestFitness; + int numEquivalent = 1; + Optional bestDC; + + for (auto dcId : datacenters) { + try { + // SOMEDAY: recruitment in other DCs besides the clusterControllerDcID will not account for the + // processes used by the master and cluster controller properly. + auto used = id_used; + std::map>, int> preferredSharing; + auto first_commit_proxy = getWorkerForRoleInDatacenter(dcId, + ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, + req.configuration, + used, + preferredSharing); + preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; + auto first_grv_proxy = getWorkerForRoleInDatacenter(dcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + req.configuration, + used, + preferredSharing); + preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; + auto first_resolver = getWorkerForRoleInDatacenter(dcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + req.configuration, + used, + preferredSharing); + preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; + + // If one of the first process recruitments is forced to share a process, allow all of next + // recruitments to also share a process. + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; + + auto commit_proxies = getWorkersForRoleInDatacenter(dcId, + ProcessClass::CommitProxy, + req.configuration.getDesiredCommitProxies(), + req.configuration, + used, + preferredSharing, + first_commit_proxy); + + auto grv_proxies = getWorkersForRoleInDatacenter(dcId, + ProcessClass::GrvProxy, + req.configuration.getDesiredGrvProxies(), + req.configuration, + used, + preferredSharing, + first_grv_proxy); + + auto resolvers = getWorkersForRoleInDatacenter(dcId, + ProcessClass::Resolver, + req.configuration.getDesiredResolvers(), + req.configuration, + used, + preferredSharing, + first_resolver); + + auto fitness = std::make_tuple(RoleFitness(commit_proxies, ProcessClass::CommitProxy, used), + RoleFitness(grv_proxies, ProcessClass::GrvProxy, used), + RoleFitness(resolvers, ProcessClass::Resolver, used)); + + if (dcId == clusterControllerDcId) { + bestFitness = fitness; + bestDC = dcId; + for (int i = 0; i < resolvers.size(); i++) { + result.resolvers.push_back(resolvers[i].interf); + } + for (int i = 0; i < commit_proxies.size(); i++) { + result.commitProxies.push_back(commit_proxies[i].interf); + } + for (int i = 0; i < grv_proxies.size(); i++) { + result.grvProxies.push_back(grv_proxies[i].interf); + } + + if (req.configuration.backupWorkerEnabled) { + const int nBackup = std::max(tlogs.size(), req.maxOldLogRouters); + auto backupWorkers = getWorkersForRoleInDatacenter( + dcId, ProcessClass::Backup, nBackup, req.configuration, used); + std::transform(backupWorkers.begin(), + backupWorkers.end(), + std::back_inserter(result.backupWorkers), + [](const WorkerDetails& w) { return w.interf; }); + } + + break; + } else { + if (fitness < bestFitness) { + bestFitness = fitness; + numEquivalent = 1; + bestDC = dcId; + } else if (fitness == bestFitness && + deterministicRandom()->random01() < 1.0 / ++numEquivalent) { + bestDC = dcId; + } + } + } catch (Error& e) { + if (e.code() != error_code_no_more_servers) { + throw; + } + } + } + + if (bestDC != clusterControllerDcId) { + TraceEvent("BestDCIsNotClusterDC").log(); + std::vector> dcPriority; + dcPriority.push_back(bestDC); + desiredDcIds.set(dcPriority); + throw no_more_servers(); + } + // If this cluster controller dies, do not prioritize recruiting the next one in the same DC + desiredDcIds.set(std::vector>()); + TraceEvent("FindWorkersForConfig") + .detail("Replication", req.configuration.tLogReplicationFactor) + .detail("DesiredLogs", req.configuration.getDesiredLogs()) + .detail("ActualLogs", result.tLogs.size()) + .detail("DesiredCommitProxies", req.configuration.getDesiredCommitProxies()) + .detail("ActualCommitProxies", result.commitProxies.size()) + .detail("DesiredGrvProxies", req.configuration.getDesiredGrvProxies()) + .detail("ActualGrvProxies", result.grvProxies.size()) + .detail("DesiredResolvers", req.configuration.getDesiredResolvers()) + .detail("ActualResolvers", result.resolvers.size()); + + if (!goodRecruitmentTime.isReady() && checkGoodRecruitment && + (RoleFitness( + SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) + .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used)) || + RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, + req.configuration.getDesiredCommitProxies(), + ProcessClass::CommitProxy) + .betterCount(std::get<0>(bestFitness)) || + RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, + req.configuration.getDesiredGrvProxies(), + ProcessClass::GrvProxy) + .betterCount(std::get<1>(bestFitness)) || + RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, + req.configuration.getDesiredResolvers(), + ProcessClass::Resolver) + .betterCount(std::get<2>(bestFitness)))) { + throw operation_failed(); + } + + return result; + } + } + + void updateIdUsed(const std::vector& workers, + std::map>, int>& id_used) { + for (auto& it : workers) { + id_used[it.locality.processId()]++; + } + } + + void compareWorkers(const DatabaseConfiguration& conf, + const std::vector& first, + std::map>, int>& firstUsed, + const std::vector& second, + std::map>, int>& secondUsed, + ProcessClass::ClusterRole role, + std::string description) { + std::vector firstDetails; + for (auto& it : first) { + auto w = id_worker.find(it.locality.processId()); + ASSERT(w != id_worker.end()); + ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses())); + firstDetails.push_back(w->second.details); + //TraceEvent("CompareAddressesFirst").detail(description.c_str(), w->second.details.interf.address()); + } + RoleFitness firstFitness(firstDetails, role, firstUsed); + + std::vector secondDetails; + for (auto& it : second) { + auto w = id_worker.find(it.locality.processId()); + ASSERT(w != id_worker.end()); + ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses())); + secondDetails.push_back(w->second.details); + //TraceEvent("CompareAddressesSecond").detail(description.c_str(), w->second.details.interf.address()); + } + RoleFitness secondFitness(secondDetails, role, secondUsed); + + if (!(firstFitness == secondFitness)) { + TraceEvent(SevError, "NonDeterministicRecruitment") + .detail("FirstFitness", firstFitness.toString()) + .detail("SecondFitness", secondFitness.toString()) + .detail("ClusterRole", role); + } + } + + RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { + RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req, true); + if (g_network->isSimulated()) { + try { + // FIXME: The logic to pick a satellite in a remote region is not + // deterministic and can therefore break this nondeterminism check. + // Since satellites will generally be in the primary region, + // disable the determinism check for remote region satellites. + bool remoteDCUsedAsSatellite = false; + if (req.configuration.regions.size() > 1) { + auto [region, remoteRegion] = + getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId); + for (const auto& satellite : region.satellites) { + if (satellite.dcId == remoteRegion.dcId) { + remoteDCUsedAsSatellite = true; + } + } + } + if (!remoteDCUsedAsSatellite) { + RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req, false); + + std::map>, int> firstUsed; + std::map>, int> secondUsed; + updateKnownIds(&firstUsed); + updateKnownIds(&secondUsed); + + // auto mworker = id_worker.find(masterProcessId); + //TraceEvent("CompareAddressesMaster") + // .detail("Master", + // mworker != id_worker.end() ? mworker->second.details.interf.address() : + // NetworkAddress()); + + updateIdUsed(rep.tLogs, firstUsed); + updateIdUsed(compare.tLogs, secondUsed); + compareWorkers( + req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); + updateIdUsed(rep.satelliteTLogs, firstUsed); + updateIdUsed(compare.satelliteTLogs, secondUsed); + compareWorkers(req.configuration, + rep.satelliteTLogs, + firstUsed, + compare.satelliteTLogs, + secondUsed, + ProcessClass::TLog, + "Satellite"); + updateIdUsed(rep.commitProxies, firstUsed); + updateIdUsed(compare.commitProxies, secondUsed); + updateIdUsed(rep.grvProxies, firstUsed); + updateIdUsed(compare.grvProxies, secondUsed); + updateIdUsed(rep.resolvers, firstUsed); + updateIdUsed(compare.resolvers, secondUsed); + compareWorkers(req.configuration, + rep.commitProxies, + firstUsed, + compare.commitProxies, + secondUsed, + ProcessClass::CommitProxy, + "CommitProxy"); + compareWorkers(req.configuration, + rep.grvProxies, + firstUsed, + compare.grvProxies, + secondUsed, + ProcessClass::GrvProxy, + "GrvProxy"); + compareWorkers(req.configuration, + rep.resolvers, + firstUsed, + compare.resolvers, + secondUsed, + ProcessClass::Resolver, + "Resolver"); + updateIdUsed(rep.backupWorkers, firstUsed); + updateIdUsed(compare.backupWorkers, secondUsed); + compareWorkers(req.configuration, + rep.backupWorkers, + firstUsed, + compare.backupWorkers, + secondUsed, + ProcessClass::Backup, + "Backup"); + } + } catch (Error& e) { + ASSERT(false); // Simulation only validation should not throw errors + } + } + return rep; + } + + // Check if txn system is recruited successfully in each region + void checkRegions(const std::vector& regions) { + if (desiredDcIds.get().present() && desiredDcIds.get().get().size() == 2 && + desiredDcIds.get().get()[0].get() == regions[0].dcId && + desiredDcIds.get().get()[1].get() == regions[1].dcId) { + return; + } + + try { + std::map>, int> id_used; + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::ClusterController, + ProcessClass::ExcludeFit, + db.config, + id_used, + {}, + true); + getWorkerForRoleInDatacenter( + regions[0].dcId, ProcessClass::Master, ProcessClass::ExcludeFit, db.config, id_used, {}, true); + + std::set> primaryDC; + primaryDC.insert(regions[0].dcId); + getWorkersForTlogs(db.config, + db.config.tLogReplicationFactor, + db.config.getDesiredLogs(), + db.config.tLogPolicy, + id_used, + true, + primaryDC); + if (regions[0].satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) { + bool satelliteFallback = false; + getWorkersForSatelliteLogs(db.config, regions[0], regions[1], id_used, satelliteFallback, true); + } + + getWorkerForRoleInDatacenter( + regions[0].dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, {}, true); + getWorkerForRoleInDatacenter( + regions[0].dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, db.config, id_used, {}, true); + getWorkerForRoleInDatacenter( + regions[0].dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, {}, true); + + std::vector> dcPriority; + dcPriority.push_back(regions[0].dcId); + dcPriority.push_back(regions[1].dcId); + desiredDcIds.set(dcPriority); + } catch (Error& e) { + if (e.code() != error_code_no_more_servers) { + throw; + } + } + } + + void checkRecoveryStalled() { + if ((db.serverInfo->get().recoveryState == RecoveryState::RECRUITING || + db.serverInfo->get().recoveryState == RecoveryState::ACCEPTING_COMMITS || + db.serverInfo->get().recoveryState == RecoveryState::ALL_LOGS_RECRUITED) && + db.recoveryStalled) { + if (db.config.regions.size() > 1) { + auto regions = db.config.regions; + if (clusterControllerDcId.get() == regions[0].dcId && regions[1].priority >= 0) { + std::swap(regions[0], regions[1]); + } + ASSERT(regions[1].priority < 0 || clusterControllerDcId.get() == regions[1].dcId); + checkRegions(regions); + } + } + } + + void updateIdUsed(const std::vector& workers, + std::map>, int>& id_used) { + for (auto& it : workers) { + id_used[it.interf.locality.processId()]++; + } + } + + // FIXME: determine when to fail the cluster controller when a primaryDC has not been set + + // This function returns true when the cluster controller determines it is worth forcing + // a master recovery in order to change the recruited processes in the transaction subsystem. + bool betterMasterExists() { + const ServerDBInfo dbi = db.serverInfo->get(); + + if (dbi.recoveryState < RecoveryState::ACCEPTING_COMMITS) { + return false; + } + + // Do not trigger better master exists if the cluster controller is excluded, since the master will change + // anyways once the cluster controller is moved + if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { + TraceEvent("NewRecruitmentIsWorse", id).detail("Reason", "ClusterControllerExcluded"); + return false; + } + + if (db.config.regions.size() > 1 && db.config.regions[0].priority > db.config.regions[1].priority && + db.config.regions[0].dcId != clusterControllerDcId.get() && versionDifferenceUpdated && + datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE && remoteDCIsHealthy()) { + checkRegions(db.config.regions); + } + + // Get master process + auto masterWorker = id_worker.find(dbi.master.locality.processId()); + if (masterWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindMaster") + .detail("ProcessID", dbi.master.locality.processId()); + return false; + } + + // Get tlog processes + std::vector tlogs; + std::vector remote_tlogs; + std::vector satellite_tlogs; + std::vector log_routers; + std::set logRouterAddresses; + std::vector backup_workers; + std::set backup_addresses; + + for (auto& logSet : dbi.logSystemConfig.tLogs) { + for (auto& it : logSet.tLogs) { + auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); + if (tlogWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindTLog") + .detail("ProcessID", it.interf().filteredLocality.processId()); + return false; + } + if (tlogWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "TLogExcluded") + .detail("ProcessID", it.interf().filteredLocality.processId()); + return true; + } + + if (logSet.isLocal && logSet.locality == tagLocalitySatellite) { + satellite_tlogs.push_back(tlogWorker->second.details); + } else if (logSet.isLocal) { + tlogs.push_back(tlogWorker->second.details); + } else { + remote_tlogs.push_back(tlogWorker->second.details); + } + } + + for (auto& it : logSet.logRouters) { + auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); + if (tlogWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindLogRouter") + .detail("ProcessID", it.interf().filteredLocality.processId()); + return false; + } + if (tlogWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "LogRouterExcluded") + .detail("ProcessID", it.interf().filteredLocality.processId()); + return true; + } + if (!logRouterAddresses.count(tlogWorker->second.details.interf.address())) { + logRouterAddresses.insert(tlogWorker->second.details.interf.address()); + log_routers.push_back(tlogWorker->second.details); + } + } + + for (const auto& worker : logSet.backupWorkers) { + auto workerIt = id_worker.find(worker.interf().locality.processId()); + if (workerIt == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindBackupWorker") + .detail("ProcessID", worker.interf().locality.processId()); + return false; + } + if (workerIt->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "BackupWorkerExcluded") + .detail("ProcessID", worker.interf().locality.processId()); + return true; + } + if (backup_addresses.count(workerIt->second.details.interf.address()) == 0) { + backup_addresses.insert(workerIt->second.details.interf.address()); + backup_workers.push_back(workerIt->second.details); + } + } + } + + // Get commit proxy classes + std::vector commitProxyClasses; + for (auto& it : dbi.client.commitProxies) { + auto commitProxyWorker = id_worker.find(it.processId); + if (commitProxyWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindCommitProxy") + .detail("ProcessID", it.processId); + return false; + } + if (commitProxyWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "CommitProxyExcluded") + .detail("ProcessID", it.processId); + return true; + } + commitProxyClasses.push_back(commitProxyWorker->second.details); + } + + // Get grv proxy classes + std::vector grvProxyClasses; + for (auto& it : dbi.client.grvProxies) { + auto grvProxyWorker = id_worker.find(it.processId); + if (grvProxyWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindGrvProxy") + .detail("ProcessID", it.processId); + return false; + } + if (grvProxyWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "GrvProxyExcluded") + .detail("ProcessID", it.processId); + return true; + } + grvProxyClasses.push_back(grvProxyWorker->second.details); + } + + // Get resolver classes + std::vector resolverClasses; + for (auto& it : dbi.resolvers) { + auto resolverWorker = id_worker.find(it.locality.processId()); + if (resolverWorker == id_worker.end()) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("Reason", "CannotFindResolver") + .detail("ProcessID", it.locality.processId()); + return false; + } + if (resolverWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "ResolverExcluded") + .detail("ProcessID", it.locality.processId()); + return true; + } + resolverClasses.push_back(resolverWorker->second.details); + } + + // Check master fitness. Don't return false if master is excluded in case all the processes are excluded, we + // still need master for recovery. + ProcessClass::Fitness oldMasterFit = + masterWorker->second.details.processClass.machineClassFitness(ProcessClass::Master); + if (db.config.isExcludedServer(dbi.master.addresses())) { + oldMasterFit = std::max(oldMasterFit, ProcessClass::ExcludeFit); + } + + std::map>, int> id_used; + std::map>, int> old_id_used; + id_used[clusterControllerProcessId]++; + old_id_used[clusterControllerProcessId]++; + WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter( + clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db.config, id_used, {}, true); + auto newMasterFit = mworker.worker.processClass.machineClassFitness(ProcessClass::Master); + if (db.config.isExcludedServer(mworker.worker.interf.addresses())) { + newMasterFit = std::max(newMasterFit, ProcessClass::ExcludeFit); + } + + old_id_used[masterWorker->first]++; + if (oldMasterFit < newMasterFit) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) + .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); + ; + return false; + } + if (oldMasterFit > newMasterFit || (dbi.master.locality.processId() == clusterControllerProcessId && + mworker.worker.interf.locality.processId() != clusterControllerProcessId)) { + TraceEvent("BetterMasterExists", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) + .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); + return true; + } + + std::set> primaryDC; + std::set> remoteDC; + + RegionInfo region; + RegionInfo remoteRegion; + if (db.config.regions.size()) { + primaryDC.insert(clusterControllerDcId); + for (auto& r : db.config.regions) { + if (r.dcId != clusterControllerDcId.get()) { + ASSERT(remoteDC.empty()); + remoteDC.insert(r.dcId); + remoteRegion = r; + } else { + ASSERT(region.dcId == StringRef()); + region = r; + } + } + } + + // Check tLog fitness + updateIdUsed(tlogs, old_id_used); + RoleFitness oldTLogFit(tlogs, ProcessClass::TLog, old_id_used); + auto newTLogs = getWorkersForTlogs(db.config, + db.config.tLogReplicationFactor, + db.config.getDesiredLogs(), + db.config.tLogPolicy, + id_used, + true, + primaryDC); + RoleFitness newTLogFit(newTLogs, ProcessClass::TLog, id_used); + + bool oldSatelliteFallback = false; + + if (region.satelliteTLogPolicyFallback.isValid()) { + for (auto& logSet : dbi.logSystemConfig.tLogs) { + if (region.satelliteTLogPolicy.isValid() && logSet.isLocal && logSet.locality == tagLocalitySatellite) { + oldSatelliteFallback = logSet.tLogPolicy->info() != region.satelliteTLogPolicy->info(); + ASSERT(!oldSatelliteFallback || + (region.satelliteTLogPolicyFallback.isValid() && + logSet.tLogPolicy->info() == region.satelliteTLogPolicyFallback->info())); + break; + } + } + } + + updateIdUsed(satellite_tlogs, old_id_used); + RoleFitness oldSatelliteTLogFit(satellite_tlogs, ProcessClass::TLog, old_id_used); + bool newSatelliteFallback = false; + auto newSatelliteTLogs = satellite_tlogs; + RoleFitness newSatelliteTLogFit = oldSatelliteTLogFit; + if (region.satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) { + newSatelliteTLogs = + getWorkersForSatelliteLogs(db.config, region, remoteRegion, id_used, newSatelliteFallback, true); + newSatelliteTLogFit = RoleFitness(newSatelliteTLogs, ProcessClass::TLog, id_used); + } + + std::map, int32_t> satellite_priority; + for (auto& r : region.satellites) { + satellite_priority[r.dcId] = r.priority; + } + + int32_t oldSatelliteRegionFit = std::numeric_limits::max(); + for (auto& it : satellite_tlogs) { + if (satellite_priority.count(it.interf.locality.dcId())) { + oldSatelliteRegionFit = std::min(oldSatelliteRegionFit, satellite_priority[it.interf.locality.dcId()]); + } else { + oldSatelliteRegionFit = -1; + } + } + + int32_t newSatelliteRegionFit = std::numeric_limits::max(); + for (auto& it : newSatelliteTLogs) { + if (satellite_priority.count(it.interf.locality.dcId())) { + newSatelliteRegionFit = std::min(newSatelliteRegionFit, satellite_priority[it.interf.locality.dcId()]); + } else { + newSatelliteRegionFit = -1; + } + } + + if (oldSatelliteFallback && !newSatelliteFallback) { + TraceEvent("BetterMasterExists", id) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); + return true; + } + if (!oldSatelliteFallback && newSatelliteFallback) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); + return false; + } + + if (oldSatelliteRegionFit < newSatelliteRegionFit) { + TraceEvent("BetterMasterExists", id) + .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) + .detail("NewSatelliteRegionFit", newSatelliteRegionFit); + return true; + } + if (oldSatelliteRegionFit > newSatelliteRegionFit) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) + .detail("NewSatelliteRegionFit", newSatelliteRegionFit); + return false; + } + + updateIdUsed(remote_tlogs, old_id_used); + RoleFitness oldRemoteTLogFit(remote_tlogs, ProcessClass::TLog, old_id_used); + std::vector exclusionWorkerIds; + auto fn = [](const WorkerDetails& in) { return in.interf.id(); }; + std::transform(newTLogs.begin(), newTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); + std::transform(newSatelliteTLogs.begin(), newSatelliteTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); + RoleFitness newRemoteTLogFit = oldRemoteTLogFit; + if (db.config.usableRegions > 1 && (dbi.recoveryState == RecoveryState::ALL_LOGS_RECRUITED || + dbi.recoveryState == RecoveryState::FULLY_RECOVERED)) { + newRemoteTLogFit = RoleFitness(getWorkersForTlogs(db.config, + db.config.getRemoteTLogReplicationFactor(), + db.config.getDesiredRemoteLogs(), + db.config.getRemoteTLogPolicy(), + id_used, + true, + remoteDC, + exclusionWorkerIds), + ProcessClass::TLog, + id_used); + } + int oldRouterCount = + oldTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, oldTLogFit.count)); + int newRouterCount = + newTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, newTLogFit.count)); + updateIdUsed(log_routers, old_id_used); + RoleFitness oldLogRoutersFit(log_routers, ProcessClass::LogRouter, old_id_used); + RoleFitness newLogRoutersFit = oldLogRoutersFit; + if (db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::FULLY_RECOVERED) { + newLogRoutersFit = RoleFitness(getWorkersForRoleInDatacenter(*remoteDC.begin(), + ProcessClass::LogRouter, + newRouterCount, + db.config, + id_used, + {}, + Optional(), + true), + ProcessClass::LogRouter, + id_used); + } + + if (oldLogRoutersFit.count < oldRouterCount) { + oldLogRoutersFit.worstFit = ProcessClass::NeverAssign; + } + if (newLogRoutersFit.count < newRouterCount) { + newLogRoutersFit.worstFit = ProcessClass::NeverAssign; + } + + // Check proxy/grvProxy/resolver fitness + updateIdUsed(commitProxyClasses, old_id_used); + updateIdUsed(grvProxyClasses, old_id_used); + updateIdUsed(resolverClasses, old_id_used); + RoleFitness oldCommitProxyFit(commitProxyClasses, ProcessClass::CommitProxy, old_id_used); + RoleFitness oldGrvProxyFit(grvProxyClasses, ProcessClass::GrvProxy, old_id_used); + RoleFitness oldResolverFit(resolverClasses, ProcessClass::Resolver, old_id_used); + + std::map>, int> preferredSharing; + auto first_commit_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + preferredSharing, + true); + preferredSharing[first_commit_proxy.worker.interf.locality.processId()] = 0; + auto first_grv_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + preferredSharing, + true); + preferredSharing[first_grv_proxy.worker.interf.locality.processId()] = 1; + auto first_resolver = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + db.config, + id_used, + preferredSharing, + true); + preferredSharing[first_resolver.worker.interf.locality.processId()] = 2; + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; + auto commit_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, + ProcessClass::CommitProxy, + db.config.getDesiredCommitProxies(), + db.config, + id_used, + preferredSharing, + first_commit_proxy, + true); + auto grv_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, + ProcessClass::GrvProxy, + db.config.getDesiredGrvProxies(), + db.config, + id_used, + preferredSharing, + first_grv_proxy, + true); + auto resolvers = getWorkersForRoleInDatacenter(clusterControllerDcId, + ProcessClass::Resolver, + db.config.getDesiredResolvers(), + db.config, + id_used, + preferredSharing, + first_resolver, + true); + + RoleFitness newCommitProxyFit(commit_proxies, ProcessClass::CommitProxy, id_used); + RoleFitness newGrvProxyFit(grv_proxies, ProcessClass::GrvProxy, id_used); + RoleFitness newResolverFit(resolvers, ProcessClass::Resolver, id_used); + + // Check backup worker fitness + updateIdUsed(backup_workers, old_id_used); + RoleFitness oldBackupWorkersFit(backup_workers, ProcessClass::Backup, old_id_used); + const int nBackup = backup_addresses.size(); + RoleFitness newBackupWorkersFit(getWorkersForRoleInDatacenter(clusterControllerDcId, + ProcessClass::Backup, + nBackup, + db.config, + id_used, + {}, + Optional(), + true), + ProcessClass::Backup, + id_used); + + auto oldFit = std::make_tuple(oldTLogFit, + oldSatelliteTLogFit, + oldCommitProxyFit, + oldGrvProxyFit, + oldResolverFit, + oldBackupWorkersFit, + oldRemoteTLogFit, + oldLogRoutersFit); + auto newFit = std::make_tuple(newTLogFit, + newSatelliteTLogFit, + newCommitProxyFit, + newGrvProxyFit, + newResolverFit, + newBackupWorkersFit, + newRemoteTLogFit, + newLogRoutersFit); + + if (oldFit > newFit) { + TraceEvent("BetterMasterExists", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldTLogFit", oldTLogFit.toString()) + .detail("NewTLogFit", newTLogFit.toString()) + .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) + .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) + .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) + .detail("NewCommitProxyFit", newCommitProxyFit.toString()) + .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) + .detail("NewGrvProxyFit", newGrvProxyFit.toString()) + .detail("OldResolverFit", oldResolverFit.toString()) + .detail("NewResolverFit", newResolverFit.toString()) + .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) + .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) + .detail("OldRemoteFit", oldRemoteTLogFit.toString()) + .detail("NewRemoteFit", newRemoteTLogFit.toString()) + .detail("OldRouterFit", oldLogRoutersFit.toString()) + .detail("NewRouterFit", newLogRoutersFit.toString()) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); + return true; + } + + if (oldFit < newFit) { + TraceEvent("NewRecruitmentIsWorse", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldTLogFit", oldTLogFit.toString()) + .detail("NewTLogFit", newTLogFit.toString()) + .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) + .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) + .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) + .detail("NewCommitProxyFit", newCommitProxyFit.toString()) + .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) + .detail("NewGrvProxyFit", newGrvProxyFit.toString()) + .detail("OldResolverFit", oldResolverFit.toString()) + .detail("NewResolverFit", newResolverFit.toString()) + .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) + .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) + .detail("OldRemoteFit", oldRemoteTLogFit.toString()) + .detail("NewRemoteFit", newRemoteTLogFit.toString()) + .detail("OldRouterFit", oldLogRoutersFit.toString()) + .detail("NewRouterFit", newLogRoutersFit.toString()) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); + } + return false; + } + + // Returns true iff processId is currently being used + // for any non-singleton role other than master + bool isUsedNotMaster(Optional processId) const { + ASSERT(masterProcessId.present()); + if (processId == masterProcessId) + return false; + + auto& dbInfo = db.serverInfo->get(); + for (const auto& tlogset : dbInfo.logSystemConfig.tLogs) { + for (const auto& tlog : tlogset.tLogs) { + if (tlog.present() && tlog.interf().filteredLocality.processId() == processId) + return true; + } + } + for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { + if (interf.processId == processId) + return true; + } + for (const GrvProxyInterface& interf : dbInfo.client.grvProxies) { + if (interf.processId == processId) + return true; + } + for (const ResolverInterface& interf : dbInfo.resolvers) { + if (interf.locality.processId() == processId) + return true; + } + if (processId == clusterControllerProcessId) + return true; + + return false; + } + + // Returns true iff + // - role is master, or + // - role is a singleton AND worker's pid is being used for any non-singleton role + bool onMasterIsBetter(const WorkerDetails& worker, ProcessClass::ClusterRole role) const { + ASSERT(masterProcessId.present()); + const auto& pid = worker.interf.locality.processId(); + if ((role != ProcessClass::DataDistributor && role != ProcessClass::Ratekeeper && + role != ProcessClass::BlobManager) || + pid == masterProcessId.get()) { + return false; + } + return isUsedNotMaster(pid); + } + + // Returns a map of for all non-singleton roles + std::map>, int> getUsedIds() { + std::map>, int> idUsed; + updateKnownIds(&idUsed); + + auto& dbInfo = db.serverInfo->get(); + for (const auto& tlogset : dbInfo.logSystemConfig.tLogs) { + for (const auto& tlog : tlogset.tLogs) { + if (tlog.present()) { + idUsed[tlog.interf().filteredLocality.processId()]++; + } + } + } + for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { + ASSERT(interf.processId.present()); + idUsed[interf.processId]++; + } + for (const GrvProxyInterface& interf : dbInfo.client.grvProxies) { + ASSERT(interf.processId.present()); + idUsed[interf.processId]++; + } + for (const ResolverInterface& interf : dbInfo.resolvers) { + ASSERT(interf.locality.processId().present()); + idUsed[interf.locality.processId()]++; + } + return idUsed; + } + + // Updates work health signals in `workerHealth` based on `req`. + void updateWorkerHealth(const UpdateWorkerHealthRequest& req) { + std::string degradedPeersString; + for (int i = 0; i < req.degradedPeers.size(); ++i) { + degradedPeersString += (i == 0 ? "" : " ") + req.degradedPeers[i].toString(); + } + TraceEvent("ClusterControllerUpdateWorkerHealth") + .detail("WorkerAddress", req.address) + .detail("DegradedPeers", degradedPeersString); + + // `req.degradedPeers` contains the latest peer performance view from the worker. Clear the worker if the + // requested worker doesn't see any degraded peers. + if (req.degradedPeers.empty()) { + workerHealth.erase(req.address); + return; + } + + double currentTime = now(); + + // Current `workerHealth` doesn't have any information about the incoming worker. Add the worker into + // `workerHealth`. + if (workerHealth.find(req.address) == workerHealth.end()) { + workerHealth[req.address] = {}; + for (const auto& degradedPeer : req.degradedPeers) { + workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime }; + } + + return; + } + + // The incoming worker already exists in `workerHealth`. + + auto& health = workerHealth[req.address]; + + // First, remove any degraded peers recorded in the `workerHealth`, but aren't in the incoming request. These + // machines network performance should have recovered. + std::unordered_set recoveredPeers; + for (const auto& [peer, times] : health.degradedPeers) { + recoveredPeers.insert(peer); + } + for (const auto& peer : req.degradedPeers) { + if (recoveredPeers.find(peer) != recoveredPeers.end()) { + recoveredPeers.erase(peer); + } + } + for (const auto& peer : recoveredPeers) { + health.degradedPeers.erase(peer); + } + + // Update the worker's degradedPeers. + for (const auto& peer : req.degradedPeers) { + auto it = health.degradedPeers.find(peer); + if (it == health.degradedPeers.end()) { + health.degradedPeers[peer] = { currentTime, currentTime }; + continue; + } + it->second.lastRefreshTime = currentTime; + } + } + + // Checks that if any worker or their degraded peers have recovered. If so, remove them from `workerHealth`. + void updateRecoveredWorkers() { + double currentTime = now(); + for (auto& [workerAddress, health] : workerHealth) { + for (auto it = health.degradedPeers.begin(); it != health.degradedPeers.end();) { + if (currentTime - it->second.lastRefreshTime > SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL) { + TraceEvent("WorkerPeerHealthRecovered").detail("Worker", workerAddress).detail("Peer", it->first); + health.degradedPeers.erase(it++); + } else { + ++it; + } + } + } + + for (auto it = workerHealth.begin(); it != workerHealth.end();) { + if (it->second.degradedPeers.empty()) { + TraceEvent("WorkerAllPeerHealthRecovered").detail("Worker", it->first); + workerHealth.erase(it++); + } else { + ++it; + } + } + } + + // Returns a list of servers who are experiencing degraded links. These are candidates to perform exclusion. Note + // that only one endpoint of a bad link will be included in this list. + std::unordered_set getServersWithDegradedLink() { + updateRecoveredWorkers(); + + // Build a map keyed by measured degraded peer. This map gives the info that who complains a particular server. + std::unordered_map> degradedLinkDst2Src; + double currentTime = now(); + for (const auto& [server, health] : workerHealth) { + for (const auto& [degradedPeer, times] : health.degradedPeers) { + if (currentTime - times.startTime < SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL) { + // This degraded link is not long enough to be considered as degraded. + continue; + } + degradedLinkDst2Src[degradedPeer].insert(server); + } + } + + // Sort degraded peers based on the number of workers complaining about it. + std::vector> count2DegradedPeer; + for (const auto& [degradedPeer, complainers] : degradedLinkDst2Src) { + count2DegradedPeer.push_back({ complainers.size(), degradedPeer }); + } + std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), std::greater<>()); + + // Go through all reported degraded peers by decreasing order of the number of complainers. For a particular + // degraded peer, if a complainer has already be considered as degraded, we skip the current examine degraded + // peer since there has been one endpoint on the link between degradedPeer and complainer considered as + // degraded. This is to address the issue that both endpoints on a bad link may be considered as degraded + // server. + // + // For example, if server A is already considered as a degraded server, and A complains B, we won't add B as + // degraded since A is already considered as degraded. + std::unordered_set currentDegradedServers; + for (const auto& [complainerCount, badServer] : count2DegradedPeer) { + for (const auto& complainer : degradedLinkDst2Src[badServer]) { + if (currentDegradedServers.find(complainer) == currentDegradedServers.end()) { + currentDegradedServers.insert(badServer); + break; + } + } + } + + // For degraded server that are complained by more than SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, we + // don't know if it is a hot server, or the network is bad. We remove from the returned degraded server list. + std::unordered_set currentDegradedServersWithinLimit; + for (const auto& badServer : currentDegradedServers) { + if (degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) { + currentDegradedServersWithinLimit.insert(badServer); + } + } + return currentDegradedServersWithinLimit; + } + + // Whether the transaction system (in primary DC if in HA setting) contains degraded servers. + bool transactionSystemContainsDegradedServers() { + const ServerDBInfo dbi = db.serverInfo->get(); + for (const auto& excludedServer : degradedServers) { + if (dbi.master.addresses().contains(excludedServer)) { + return true; + } + + for (auto& logSet : dbi.logSystemConfig.tLogs) { + if (!logSet.isLocal || logSet.locality == tagLocalitySatellite) { + continue; + } + for (const auto& tlog : logSet.tLogs) { + if (tlog.present() && tlog.interf().addresses().contains(excludedServer)) { + return true; + } + } + } + + for (auto& proxy : dbi.client.grvProxies) { + if (proxy.addresses().contains(excludedServer)) { + return true; + } + } + + for (auto& proxy : dbi.client.commitProxies) { + if (proxy.addresses().contains(excludedServer)) { + return true; + } + } + + for (auto& resolver : dbi.resolvers) { + if (resolver.addresses().contains(excludedServer)) { + return true; + } + } + } + + return false; + } + + // Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded + // servers. + bool remoteTransactionSystemContainsDegradedServers() { + if (db.config.usableRegions <= 1) { + return false; + } + + for (const auto& excludedServer : degradedServers) { + if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) { + return true; + } + } + + return false; + } + + // Returns true if remote DC is healthy and can failover to. + bool remoteDCIsHealthy() { + // Ignore remote DC health if worker health monitor is disabled. + if (!SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) { + return true; + } + + // When we just start, we ignore any remote DC health info since the current CC may be elected at wrong DC due + // to that all the processes are still starting. + if (machineStartTime() == 0) { + return true; + } + + if (now() - machineStartTime() < SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) { + return true; + } + + // When remote DC health is not monitored, we may not know whether the remote is healthy or not. So return false + // here to prevent failover. + if (!remoteDCMonitorStarted) { + return false; + } + + return !remoteTransactionSystemContainsDegradedServers(); + } + + // Returns true when the cluster controller should trigger a recovery due to degraded servers used in the + // transaction system in the primary data center. + bool shouldTriggerRecoveryDueToDegradedServers() { + if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) { + return false; + } + + if (db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) { + return false; + } + + // Do not trigger recovery if the cluster controller is excluded, since the master will change + // anyways once the cluster controller is moved + if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { + return false; + } + + return transactionSystemContainsDegradedServers(); + } + + // Returns true when the cluster controller should trigger a failover due to degraded servers used in the + // transaction system in the primary data center, and no degradation in the remote data center. + bool shouldTriggerFailoverDueToDegradedServers() { + if (db.config.usableRegions <= 1) { + return false; + } + + if (SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION > + SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) { + TraceEvent(SevWarn, "TriggerFailoverDueToDegradedServersInvalidConfig") + .suppressFor(1.0) + .detail("Min", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION) + .detail("Max", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION); + return false; + } + + if (degradedServers.size() < SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION || + degradedServers.size() > SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) { + return false; + } + + // Do not trigger recovery if the cluster controller is excluded, since the master will change + // anyways once the cluster controller is moved + if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { + return false; + } + + return transactionSystemContainsDegradedServers() && !remoteTransactionSystemContainsDegradedServers(); + } + + int recentRecoveryCountDueToHealth() { + while (!recentHealthTriggeredRecoveryTime.empty() && + now() - recentHealthTriggeredRecoveryTime.front() > SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL) { + recentHealthTriggeredRecoveryTime.pop(); + } + return recentHealthTriggeredRecoveryTime.size(); + } + + bool isExcludedDegradedServer(const NetworkAddressList& a) { + for (const auto& server : excludedDegradedServers) { + if (a.contains(server)) + return true; + } + return false; + } + + std::map>, WorkerInfo> id_worker; + std::map>, ProcessClass> + id_class; // contains the mapping from process id to process class from the database + RangeResult lastProcessClasses; + bool gotProcessClasses; + bool gotFullyRecoveredConfig; + bool shouldCommitSuicide; + Optional> masterProcessId; + Optional> clusterControllerProcessId; + Optional> clusterControllerDcId; + AsyncVar>>> desiredDcIds; // desired DC priorities + AsyncVar>>>> + changingDcIds; // current DC priorities to change first, and whether that is the cluster controller + AsyncVar>>>> + changedDcIds; // current DC priorities to change second, and whether the cluster controller has been changed + UID id; + std::vector> outstandingRecruitmentRequests; + std::vector> outstandingRemoteRecruitmentRequests; + std::vector> outstandingStorageRequests; + std::vector> outstandingBlobWorkerRequests; + ActorCollection ac; + UpdateWorkerList updateWorkerList; + Future outstandingRequestChecker; + Future outstandingRemoteRequestChecker; + AsyncTrigger updateDBInfo; + std::set updateDBInfoEndpoints; + std::set removedDBInfoEndpoints; + + DBInfo db; + Database cx; + double startTime; + Future goodRecruitmentTime; + Future goodRemoteRecruitmentTime; + Version datacenterVersionDifference; + PromiseStream> addActor; + bool versionDifferenceUpdated; + + bool remoteDCMonitorStarted; + bool remoteTransactionSystemDegraded; + + // recruitX is used to signal when role X needs to be (re)recruited. + // recruitingXID is used to track the ID of X's interface which is being recruited. + // We use AsyncVars to kill (i.e. halt) singletons that have been replaced. + AsyncVar recruitDistributor; + Optional recruitingDistributorID; + AsyncVar recruitRatekeeper; + Optional recruitingRatekeeperID; + AsyncVar recruitBlobManager; + Optional recruitingBlobManagerID; + + // Stores the health information from a particular worker's perspective. + struct WorkerHealth { + struct DegradedTimes { + double startTime = 0; + double lastRefreshTime = 0; + }; + std::unordered_map degradedPeers; + + // TODO(zhewu): Include disk and CPU signals. + }; + std::unordered_map workerHealth; + std::unordered_set + degradedServers; // The servers that the cluster controller is considered as degraded. The servers in this list + // are not excluded unless they are added to `excludedDegradedServers`. + std::unordered_set + excludedDegradedServers; // The degraded servers to be excluded when assigning workers to roles. + std::queue recentHealthTriggeredRecoveryTime; + + CounterCollection clusterControllerMetrics; + + Counter openDatabaseRequests; + Counter registerWorkerRequests; + Counter getWorkersRequests; + Counter getClientWorkersRequests; + Counter registerMasterRequests; + Counter statusRequests; + + Reference recruitedMasterWorkerEventHolder; + + ClusterControllerData(ClusterControllerFullInterface const& ccInterface, + LocalityData const& locality, + ServerCoordinators const& coordinators) + : gotProcessClasses(false), gotFullyRecoveredConfig(false), shouldCommitSuicide(false), + clusterControllerProcessId(locality.processId()), clusterControllerDcId(locality.dcId()), id(ccInterface.id()), + ac(false), outstandingRequestChecker(Void()), outstandingRemoteRequestChecker(Void()), startTime(now()), + goodRecruitmentTime(Never()), goodRemoteRecruitmentTime(Never()), datacenterVersionDifference(0), + versionDifferenceUpdated(false), remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false), + recruitDistributor(false), recruitRatekeeper(false), recruitBlobManager(false), + clusterControllerMetrics("ClusterController", id.toString()), + openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics), + registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics), + getWorkersRequests("GetWorkersRequests", clusterControllerMetrics), + getClientWorkersRequests("GetClientWorkersRequests", clusterControllerMetrics), + registerMasterRequests("RegisterMasterRequests", clusterControllerMetrics), + statusRequests("StatusRequests", clusterControllerMetrics), + recruitedMasterWorkerEventHolder(makeReference("RecruitedMasterWorker")) { + auto serverInfo = ServerDBInfo(); + serverInfo.id = deterministicRandom()->randomUniqueID(); + serverInfo.infoGeneration = ++db.dbInfoCount; + serverInfo.masterLifetime.ccID = id; + serverInfo.clusterInterface = ccInterface; + serverInfo.myLocality = locality; + db.serverInfo->set(serverInfo); + cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::True); + + specialCounter(clusterControllerMetrics, "ClientCount", [this]() { return db.clientCount; }); + } + + ~ClusterControllerData() { + ac.clear(false); + id_worker.clear(); + } +}; + +#include "flow/unactorcompiler.h" + +#endif diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp new file mode 100644 index 0000000000..8fbac1fc9f --- /dev/null +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -0,0 +1,1888 @@ +/* + * ClusterRecovery.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbrpc/sim_validation.h" +#include "fdbserver/ApplyMetadataMutation.h" +#include "fdbserver/BackupProgress.actor.h" +#include "fdbserver/ClusterRecovery.actor.h" +#include "fdbserver/MasterInterface.h" +#include "fdbserver/WaitFailure.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +static std::set const& normalClusterRecoveryErrors() { + static std::set s; + if (s.empty()) { + s.insert(error_code_operation_failed); + s.insert(error_code_tlog_stopped); + s.insert(error_code_tlog_failed); + s.insert(error_code_commit_proxy_failed); + s.insert(error_code_grv_proxy_failed); + s.insert(error_code_resolver_failed); + s.insert(error_code_backup_worker_failed); + s.insert(error_code_recruitment_failed); + s.insert(error_code_no_more_servers); + s.insert(error_code_cluster_recovery_failed); + s.insert(error_code_coordinated_state_conflict); + s.insert(error_code_master_max_versions_in_flight); + s.insert(error_code_worker_removed); + s.insert(error_code_new_coordinators_timed_out); + s.insert(error_code_broken_promise); + } + return s; +} + +ACTOR Future recoveryTerminateOnConflict(UID dbgid, + Promise fullyRecovered, + Future onConflict, + Future switchedState) { + choose { + when(wait(onConflict)) { + if (!fullyRecovered.isSet()) { + TraceEvent("RecoveryTerminated", dbgid).detail("Reason", "Conflict"); + TEST(true); // Coordinated state conflict, recovery terminating + throw worker_removed(); + } + return Void(); + } + when(wait(switchedState)) { return Void(); } + } +} + +ACTOR Future recruitNewMaster(ClusterControllerData* cluster, + ClusterControllerData::DBInfo* db, + MasterInterface* newMaster) { + state Future> fNewMaster; + state WorkerFitnessInfo masterWorker; + + loop { + // We must recruit the master in the same data center as the cluster controller. + // This should always be possible, because we can recruit the master on the same process as the cluster + // controller. + std::map>, int> id_used; + id_used[cluster->clusterControllerProcessId]++; + masterWorker = cluster->getWorkerForRoleInDatacenter( + cluster->clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db->config, id_used); + if ((masterWorker.worker.processClass.machineClassFitness(ProcessClass::Master) > + SERVER_KNOBS->EXPECTED_MASTER_FITNESS || + masterWorker.worker.interf.locality.processId() == cluster->clusterControllerProcessId) && + !cluster->goodRecruitmentTime.isReady()) { + TraceEvent("RecruitNewMaster", cluster->id) + .detail("Fitness", masterWorker.worker.processClass.machineClassFitness(ProcessClass::Master)); + wait(delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); + continue; + } + RecruitMasterRequest rmq; + rmq.lifetime = db->serverInfo->get().masterLifetime; + rmq.forceRecovery = db->forceRecovery; + + cluster->masterProcessId = masterWorker.worker.interf.locality.processId(); + cluster->db.unfinishedRecoveries++; + fNewMaster = masterWorker.worker.interf.master.tryGetReply(rmq); + wait(ready(fNewMaster) || db->forceMasterFailure.onTrigger()); + if (fNewMaster.isReady() && fNewMaster.get().present()) { + TraceEvent("RecruitNewMaster", cluster->id).detail("Recruited", fNewMaster.get().get().id()); + + // for status tool + TraceEvent("RecruitedMasterWorker", cluster->id) + .detail("Address", fNewMaster.get().get().address()) + .trackLatest(cluster->recruitedMasterWorkerEventHolder->trackingKey); + + *newMaster = fNewMaster.get().get(); + + return Void(); + } else { + TEST(true); // clusterWatchDatabase() !newMaster.present() + wait(delay(SERVER_KNOBS->MASTER_SPIN_DELAY)); + } + } +} + +ACTOR Future clusterRecruitFromConfiguration(ClusterControllerData* self, Reference req) { + // At the moment this doesn't really need to be an actor (it always completes immediately) + TEST(true); // ClusterController RecruitTLogsRequest + loop { + try { + req->rep = self->findWorkersForConfiguration(req->req); + return Void(); + } catch (Error& e) { + if (e.code() == error_code_no_more_servers && self->goodRecruitmentTime.isReady()) { + self->outstandingRecruitmentRequests.push_back(req); + TraceEvent(SevWarn, "RecruitFromConfigurationNotAvailable", self->id).error(e); + wait(req->waitForCompletion.onTrigger()); + return Void(); + } else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) { + // recruitment not good enough, try again + TraceEvent("RecruitFromConfigurationRetry", self->id) + .error(e) + .detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady()); + while (!self->goodRecruitmentTime.isReady()) { + wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); + } + } else { + TraceEvent(SevError, "RecruitFromConfigurationError", self->id).error(e); + throw; + } + } + wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); + } +} + +ACTOR Future clusterRecruitRemoteFromConfiguration( + ClusterControllerData* self, + Reference req) { + // At the moment this doesn't really need to be an actor (it always completes immediately) + TEST(true); // ClusterController RecruitTLogsRequest Remote + loop { + try { + auto rep = self->findRemoteWorkersForConfiguration(req->req); + return rep; + } catch (Error& e) { + if (e.code() == error_code_no_more_servers && self->goodRemoteRecruitmentTime.isReady()) { + self->outstandingRemoteRecruitmentRequests.push_back(req); + TraceEvent(SevWarn, "RecruitRemoteFromConfigurationNotAvailable", self->id).error(e); + wait(req->waitForCompletion.onTrigger()); + return req->rep; + } else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) { + // recruitment not good enough, try again + TraceEvent("RecruitRemoteFromConfigurationRetry", self->id) + .error(e) + .detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady()); + while (!self->goodRemoteRecruitmentTime.isReady()) { + wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); + } + } else { + TraceEvent(SevError, "RecruitRemoteFromConfigurationError", self->id).error(e); + throw; + } + } + wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY)); + } +} + +ACTOR Future newCommitProxies(Reference self, RecruitFromConfigurationReply recr) { + std::vector> initializationReplies; + for (int i = 0; i < recr.commitProxies.size(); i++) { + InitializeCommitProxyRequest req; + req.master = self->masterInterface; + req.masterLifetime = self->masterLifetime; + req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; + req.recoveryTransactionVersion = self->recoveryTransactionVersion; + req.firstProxy = i == 0; + TraceEvent("CommitProxyReplies", self->dbgid) + .detail("WorkerID", recr.commitProxies[i].id()) + .detail("ReocoveryTxnVersion", self->recoveryTransactionVersion) + .detail("FirstProxy", req.firstProxy ? "True" : "False"); + initializationReplies.push_back( + transformErrors(throwErrorOr(recr.commitProxies[i].commitProxy.getReplyUnlessFailedFor( + req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + commit_proxy_failed())); + } + + std::vector newRecruits = wait(getAll(initializationReplies)); + // It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->commitProxies[0] is the firstCommitProxy. + self->commitProxies = newRecruits; + + return Void(); +} + +ACTOR Future newGrvProxies(Reference self, RecruitFromConfigurationReply recr) { + std::vector> initializationReplies; + for (int i = 0; i < recr.grvProxies.size(); i++) { + InitializeGrvProxyRequest req; + req.master = self->masterInterface; + req.masterLifetime = self->masterLifetime; + req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; + TraceEvent("GrvProxyReplies", self->dbgid).detail("WorkerID", recr.grvProxies[i].id()); + initializationReplies.push_back( + transformErrors(throwErrorOr(recr.grvProxies[i].grvProxy.getReplyUnlessFailedFor( + req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + grv_proxy_failed())); + } + + std::vector newRecruits = wait(getAll(initializationReplies)); + self->grvProxies = newRecruits; + return Void(); +} + +ACTOR Future newResolvers(Reference self, RecruitFromConfigurationReply recr) { + std::vector> initializationReplies; + for (int i = 0; i < recr.resolvers.size(); i++) { + InitializeResolverRequest req; + req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; + req.commitProxyCount = recr.commitProxies.size(); + req.resolverCount = recr.resolvers.size(); + TraceEvent("ResolverReplies", self->dbgid).detail("WorkerID", recr.resolvers[i].id()); + initializationReplies.push_back( + transformErrors(throwErrorOr(recr.resolvers[i].resolver.getReplyUnlessFailedFor( + req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + resolver_failed())); + } + + std::vector newRecruits = wait(getAll(initializationReplies)); + self->resolvers = newRecruits; + + return Void(); +} + +ACTOR Future newTLogServers(Reference self, + RecruitFromConfigurationReply recr, + Reference oldLogSystem, + std::vector>* initialConfChanges) { + if (self->configuration.usableRegions > 1) { + state Optional remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional(); + if (!self->dcId_locality.count(recr.dcId)) { + int8_t loc = self->getNextLocality(); + Standalone tr; + tr.set(tr.arena(), tagLocalityListKeyFor(recr.dcId), tagLocalityListValue(loc)); + initialConfChanges->push_back(tr); + self->dcId_locality[recr.dcId] = loc; + TraceEvent(SevWarn, "UnknownPrimaryDCID", self->dbgid).detail("PrimaryId", recr.dcId).detail("Loc", loc); + } + + if (!self->dcId_locality.count(remoteDcId)) { + int8_t loc = self->getNextLocality(); + Standalone tr; + tr.set(tr.arena(), tagLocalityListKeyFor(remoteDcId), tagLocalityListValue(loc)); + initialConfChanges->push_back(tr); + self->dcId_locality[remoteDcId] = loc; + TraceEvent(SevWarn, "UnknownRemoteDCID", self->dbgid).detail("RemoteId", remoteDcId).detail("Loc", loc); + } + + std::vector exclusionWorkerIds; + std::transform(recr.tLogs.begin(), + recr.tLogs.end(), + std::back_inserter(exclusionWorkerIds), + [](const WorkerInterface& in) { return in.id(); }); + std::transform(recr.satelliteTLogs.begin(), + recr.satelliteTLogs.end(), + std::back_inserter(exclusionWorkerIds), + [](const WorkerInterface& in) { return in.id(); }); + + RecruitRemoteFromConfigurationRequest remoteRecruitReq( + self->configuration, + remoteDcId, + recr.tLogs.size() * + std::max(1, self->configuration.desiredLogRouterCount / std::max(1, recr.tLogs.size())), + exclusionWorkerIds); + remoteRecruitReq.dbgId = self->dbgid; + state Reference recruitWorkersInfo = + makeReference(remoteRecruitReq); + recruitWorkersInfo->dbgId = self->dbgid; + Future fRemoteWorkers = + clusterRecruitRemoteFromConfiguration(self->controllerData, recruitWorkersInfo); + + self->primaryLocality = self->dcId_locality[recr.dcId]; + self->logSystem = Reference(); // Cancels the actors in the previous log system. + Reference newLogSystem = wait(oldLogSystem->newEpoch(recr, + fRemoteWorkers, + self->clusterId, + self->configuration, + self->cstate.myDBState.recoveryCount + 1, + self->primaryLocality, + self->dcId_locality[remoteDcId], + self->allTags, + self->recruitmentStalled)); + self->logSystem = newLogSystem; + } else { + self->primaryLocality = tagLocalitySpecial; + self->logSystem = Reference(); // Cancels the actors in the previous log system. + Reference newLogSystem = wait(oldLogSystem->newEpoch(recr, + Never(), + self->clusterId, + self->configuration, + self->cstate.myDBState.recoveryCount + 1, + self->primaryLocality, + tagLocalitySpecial, + self->allTags, + self->recruitmentStalled)); + self->logSystem = newLogSystem; + } + return Void(); +} + +ACTOR Future newSeedServers(Reference self, + RecruitFromConfigurationReply recruits, + std::vector* servers) { + // This is only necessary if the database is at version 0 + servers->clear(); + if (self->lastEpochEnd) + return Void(); + + state int idx = 0; + state std::map, Tag> dcId_tags; + state int8_t nextLocality = 0; + while (idx < recruits.storageServers.size()) { + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_SS_RECRUITMENT_EVENT_NAME).c_str(), + self->dbgid) + .detail("CandidateWorker", recruits.storageServers[idx].locality.toString()); + + InitializeStorageRequest isr; + isr.seedTag = dcId_tags.count(recruits.storageServers[idx].locality.dcId()) + ? dcId_tags[recruits.storageServers[idx].locality.dcId()] + : Tag(nextLocality, 0); + isr.storeType = self->configuration.storageServerStoreType; + isr.reqId = deterministicRandom()->randomUniqueID(); + isr.interfaceId = deterministicRandom()->randomUniqueID(); + isr.clusterId = self->clusterId; + + ErrorOr newServer = wait(recruits.storageServers[idx].storage.tryGetReply(isr)); + + if (newServer.isError()) { + if (!newServer.isError(error_code_recruitment_failed) && + !newServer.isError(error_code_request_maybe_delivered)) + throw newServer.getError(); + + TEST(true); // initial storage recuitment loop failed to get new server + wait(delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY)); + } else { + if (!dcId_tags.count(recruits.storageServers[idx].locality.dcId())) { + dcId_tags[recruits.storageServers[idx].locality.dcId()] = Tag(nextLocality, 0); + nextLocality++; + } + + Tag& tag = dcId_tags[recruits.storageServers[idx].locality.dcId()]; + tag.id++; + idx++; + + servers->push_back(newServer.get().interf); + } + } + + self->dcId_locality.clear(); + for (auto& it : dcId_tags) { + self->dcId_locality[it.first] = it.second.locality; + } + + TraceEvent("ClusterRecoveryRecruitedInitialStorageServers", self->dbgid) + .detail("TargetCount", self->configuration.storageTeamSize) + .detail("Servers", describe(*servers)); + + return Void(); +} + +Future waitCommitProxyFailure(std::vector const& commitProxies) { + std::vector> failed; + failed.reserve(commitProxies.size()); + for (auto commitProxy : commitProxies) { + failed.push_back(waitFailureClient(commitProxy.waitFailure, + SERVER_KNOBS->TLOG_TIMEOUT, + -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, + /*trace=*/true)); + } + ASSERT(failed.size() >= 1); + return tagError(quorum(failed, 1), commit_proxy_failed()); +} + +Future waitGrvProxyFailure(std::vector const& grvProxies) { + std::vector> failed; + failed.reserve(grvProxies.size()); + for (int i = 0; i < grvProxies.size(); i++) + failed.push_back(waitFailureClient(grvProxies[i].waitFailure, + SERVER_KNOBS->TLOG_TIMEOUT, + -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, + /*trace=*/true)); + ASSERT(failed.size() >= 1); + return tagError(quorum(failed, 1), grv_proxy_failed()); +} + +Future waitResolverFailure(std::vector const& resolvers) { + std::vector> failed; + failed.reserve(resolvers.size()); + for (auto resolver : resolvers) { + failed.push_back(waitFailureClient(resolver.waitFailure, + SERVER_KNOBS->TLOG_TIMEOUT, + -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, + /*trace=*/true)); + } + ASSERT(failed.size() >= 1); + return tagError(quorum(failed, 1), resolver_failed()); +} + +ACTOR Future rejoinRequestHandler(Reference self) { + loop { + TLogRejoinRequest req = waitNext(self->clusterController.tlogRejoin.getFuture()); + TraceEvent(SevDebug, "TLogRejoinRequestHandler") + .detail("MasterLifeTime", self->dbInfo->get().masterLifetime.toString()); + req.reply.send(true); + } +} + +// Keeps the coordinated state (cstate) updated as the set of recruited tlogs change through recovery. +ACTOR Future trackTlogRecovery(Reference self, + Reference>> oldLogSystems, + Future minRecoveryDuration) { + state Future rejoinRequests = Never(); + state DBRecoveryCount recoverCount = self->cstate.myDBState.recoveryCount + 1; + state DatabaseConfiguration configuration = + self->configuration; // self-configuration can be changed by configurationMonitor so we need a copy + loop { + state DBCoreState newState; + self->logSystem->toCoreState(newState); + newState.recoveryCount = recoverCount; + state Future changed = self->logSystem->onCoreStateChanged(); + + ASSERT(newState.tLogs[0].tLogWriteAntiQuorum == configuration.tLogWriteAntiQuorum && + newState.tLogs[0].tLogReplicationFactor == configuration.tLogReplicationFactor); + + state bool allLogs = + newState.tLogs.size() == + configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional()); + state bool finalUpdate = !newState.oldTLogData.size() && allLogs; + TraceEvent("TrackTlogRecovery") + .detail("FinalUpdate", finalUpdate) + .detail("NewState.tlogs", newState.tLogs.size()) + .detail("NewState.OldTLogs", newState.oldTLogData.size()) + .detail("Expected.tlogs", + configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional())); + wait(self->cstate.write(newState, finalUpdate)); + if (self->cstateUpdated.canBeSet()) { + self->cstateUpdated.send(Void()); + } + + wait(minRecoveryDuration); + self->logSystem->coreStateWritten(newState); + + if (self->recoveryReadyForCommits.canBeSet()) { + self->recoveryReadyForCommits.send(Void()); + } + + if (finalUpdate) { + self->recoveryState = RecoveryState::FULLY_RECOVERED; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->dbgid) + .detail("StatusCode", RecoveryStatus::fully_recovered) + .detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered]) + .detail("FullyRecoveredAtVersion", self->version) + .detail("ClusterId", self->clusterId) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME).c_str(), + self->dbgid) + .detail("ActiveGenerations", 1) + .trackLatest(self->clusterRecoveryGenerationsEventHolder->trackingKey); + } else if (!newState.oldTLogData.size() && self->recoveryState < RecoveryState::STORAGE_RECOVERED) { + self->recoveryState = RecoveryState::STORAGE_RECOVERED; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->dbgid) + .detail("StatusCode", RecoveryStatus::storage_recovered) + .detail("Status", RecoveryStatus::names[RecoveryStatus::storage_recovered]) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + } else if (allLogs && self->recoveryState < RecoveryState::ALL_LOGS_RECRUITED) { + self->recoveryState = RecoveryState::ALL_LOGS_RECRUITED; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->dbgid) + .detail("StatusCode", RecoveryStatus::all_logs_recruited) + .detail("Status", RecoveryStatus::names[RecoveryStatus::all_logs_recruited]) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + } + + if (newState.oldTLogData.size() && configuration.repopulateRegionAntiQuorum > 0 && + self->logSystem->remoteStorageRecovered()) { + TraceEvent(SevWarnAlways, "RecruitmentStalled_RemoteStorageRecovered", self->dbgid).log(); + self->recruitmentStalled->set(true); + } + self->registrationTrigger.trigger(); + + if (finalUpdate) { + oldLogSystems->get()->stopRejoins(); + rejoinRequests = rejoinRequestHandler(self); + return Void(); + } + + wait(changed); + } +} + +std::pair findRange(CoalescedKeyRangeMap& key_resolver, + Standalone>& movedRanges, + int src, + int dest) { + auto ranges = key_resolver.ranges(); + auto prev = ranges.begin(); + auto it = ranges.begin(); + ++it; + if (it == ranges.end()) { + if (ranges.begin().value() != src || + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(ranges.begin()->range(), dest)) != + movedRanges.end()) + throw operation_failed(); + return std::make_pair(ranges.begin().range(), true); + } + + std::set borders; + // If possible expand an existing boundary between the two resolvers + for (; it != ranges.end(); ++it) { + if (it->value() == src && prev->value() == dest && + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == + movedRanges.end()) { + return std::make_pair(it->range(), true); + } + if (it->value() == dest && prev->value() == src && + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(prev->range(), dest)) == + movedRanges.end()) { + return std::make_pair(prev->range(), false); + } + if (it->value() == dest) + borders.insert(prev->value()); + if (prev->value() == dest) + borders.insert(it->value()); + ++prev; + } + + prev = ranges.begin(); + it = ranges.begin(); + ++it; + // If possible create a new boundry which doesn't exist yet + for (; it != ranges.end(); ++it) { + if (it->value() == src && !borders.count(prev->value()) && + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == + movedRanges.end()) { + return std::make_pair(it->range(), true); + } + if (prev->value() == src && !borders.count(it->value()) && + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(prev->range(), dest)) == + movedRanges.end()) { + return std::make_pair(prev->range(), false); + } + ++prev; + } + + it = ranges.begin(); + for (; it != ranges.end(); ++it) { + if (it->value() == src && + std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == + movedRanges.end()) { + return std::make_pair(it->range(), true); + } + } + throw operation_failed(); // we are already attempting to move all of the data one resolver is assigned, so do not + // move anything +} + +ACTOR Future resolutionBalancing(Reference self) { + state CoalescedKeyRangeMap key_resolver; + key_resolver.insert(allKeys, 0); + loop { + wait(delay(SERVER_KNOBS->MIN_BALANCE_TIME, TaskPriority::ResolutionMetrics)); + while (self->resolverChanges.get().size()) + wait(self->resolverChanges.onChange()); + state std::vector> futures; + for (auto& p : self->resolvers) + futures.push_back( + brokenPromiseToNever(p.metrics.getReply(ResolutionMetricsRequest(), TaskPriority::ResolutionMetrics))); + wait(waitForAll(futures)); + state IndexedSet, NoMetric> metrics; + + int64_t total = 0; + for (int i = 0; i < futures.size(); i++) { + total += futures[i].get().value; + metrics.insert(std::make_pair(futures[i].get().value, i), NoMetric()); + //TraceEvent("ResolverMetric").detail("I", i).detail("Metric", futures[i].get()); + } + if (metrics.lastItem()->first - metrics.begin()->first > SERVER_KNOBS->MIN_BALANCE_DIFFERENCE) { + try { + state int src = metrics.lastItem()->second; + state int dest = metrics.begin()->second; + state int64_t amount = std::min(metrics.lastItem()->first - total / self->resolvers.size(), + total / self->resolvers.size() - metrics.begin()->first) / + 2; + state Standalone> movedRanges; + + loop { + state std::pair range = findRange(key_resolver, movedRanges, src, dest); + + ResolutionSplitRequest req; + req.front = range.second; + req.offset = amount; + req.range = range.first; + + ResolutionSplitReply split = + wait(brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply( + req, TaskPriority::ResolutionMetrics))); + KeyRangeRef moveRange = range.second ? KeyRangeRef(range.first.begin, split.key) + : KeyRangeRef(split.key, range.first.end); + movedRanges.push_back_deep(movedRanges.arena(), ResolverMoveRef(moveRange, dest)); + TraceEvent("MovingResolutionRange") + .detail("Src", src) + .detail("Dest", dest) + .detail("Amount", amount) + .detail("StartRange", range.first) + .detail("MoveRange", moveRange) + .detail("Used", split.used) + .detail("KeyResolverRanges", key_resolver.size()); + amount -= split.used; + if (moveRange != range.first || amount <= 0) + break; + } + for (auto& it : movedRanges) + key_resolver.insert(it.range, it.dest); + // for(auto& it : key_resolver.ranges()) + // TraceEvent("KeyResolver").detail("Range", it.range()).detail("Value", it.value()); + + self->resolverChangesVersion = self->version + 1; + for (auto& p : self->commitProxies) + self->resolverNeedingChanges.insert(p.id()); + self->resolverChanges.set(movedRanges); + } catch (Error& e) { + if (e.code() != error_code_operation_failed) + throw; + } + } + } +} + +ACTOR Future changeCoordinators(Reference self) { + loop { + ChangeCoordinatorsRequest req = waitNext(self->clusterController.changeCoordinators.getFuture()); + TraceEvent("ChangeCoordinators", self->dbgid).log(); + ++self->changeCoordinatorsRequests; + state ChangeCoordinatorsRequest changeCoordinatorsRequest = req; + + // Kill cluster controller to facilitate coordinator registration update + if (self->controllerData->shouldCommitSuicide) { + throw restart_cluster_controller(); + } + self->controllerData->shouldCommitSuicide = true; + + while (!self->cstate.previousWrite.isReady()) { + wait(self->cstate.previousWrite); + wait(delay( + 0)); // if a new core state is ready to be written, have that take priority over our finalizing write; + } + + if (!self->cstate.fullyRecovered.isSet()) { + wait(self->cstate.write(self->cstate.myDBState, true)); + } + + try { + wait(self->cstate.move(ClusterConnectionString(changeCoordinatorsRequest.newConnectionString.toString()))); + } catch (Error& e) { + if (e.code() != error_code_actor_cancelled) + changeCoordinatorsRequest.reply.sendError(e); + + throw; + } + + throw internal_error(); + } +} + +ACTOR Future configurationMonitor(Reference self, Database cx) { + loop { + state ReadYourWritesTransaction tr(cx); + + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY)); + ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY); + + DatabaseConfiguration conf; + conf.fromKeyValues((VectorRef)results); + TraceEvent("ConfigurationMonitor", self->dbgid) + .detail(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->recoveryState); + if (conf != self->configuration) { + if (self->recoveryState != RecoveryState::ALL_LOGS_RECRUITED && + self->recoveryState != RecoveryState::FULLY_RECOVERED) { + self->controllerData->shouldCommitSuicide = true; + throw restart_cluster_controller(); + } + + self->configuration = conf; + self->registrationTrigger.trigger(); + } + + state Future watchFuture = + tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) || + tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) || + tr.watch(failedLocalityVersionKey); + wait(tr.commit()); + wait(watchFuture); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } +} + +ACTOR static Future> getMinBackupVersion(Reference self, Database cx) { + loop { + state ReadYourWritesTransaction tr(cx); + + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + Optional value = wait(tr.get(backupStartedKey)); + Optional minVersion; + if (value.present()) { + auto uidVersions = decodeBackupStartedValue(value.get()); + TraceEvent e("GotBackupStartKey", self->dbgid); + int i = 1; + for (auto [uid, version] : uidVersions) { + e.detail(format("BackupID%d", i), uid).detail(format("Version%d", i), version); + i++; + minVersion = minVersion.present() ? std::min(version, minVersion.get()) : version; + } + } else { + TraceEvent("EmptyBackupStartKey", self->dbgid).log(); + } + return minVersion; + + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + +ACTOR static Future recruitBackupWorkers(Reference self, Database cx) { + ASSERT(self->backupWorkers.size() > 0); + + // Avoid race between a backup worker's save progress and the reads below. + wait(delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER)); + + state LogEpoch epoch = self->cstate.myDBState.recoveryCount; + state Reference backupProgress( + new BackupProgress(self->dbgid, self->logSystem->getOldEpochTagsVersionsInfo())); + state Future gotProgress = getBackupProgress(cx, self->dbgid, backupProgress, /*logging=*/true); + state std::vector> initializationReplies; + + state std::vector> idsTags; // worker IDs and tags for current epoch + state int logRouterTags = self->logSystem->getLogRouterTags(); + idsTags.reserve(logRouterTags); + for (int i = 0; i < logRouterTags; i++) { + idsTags.emplace_back(deterministicRandom()->randomUniqueID(), Tag(tagLocalityLogRouter, i)); + } + + const Version startVersion = self->logSystem->getBackupStartVersion(); + state int i = 0; + for (; i < logRouterTags; i++) { + const auto& worker = self->backupWorkers[i % self->backupWorkers.size()]; + InitializeBackupRequest req(idsTags[i].first); + req.recruitedEpoch = epoch; + req.backupEpoch = epoch; + req.routerTag = idsTags[i].second; + req.totalTags = logRouterTags; + req.startVersion = startVersion; + TraceEvent("BackupRecruitment", self->dbgid) + .detail("RequestID", req.reqId) + .detail("Tag", req.routerTag.toString()) + .detail("Epoch", epoch) + .detail("BackupEpoch", epoch) + .detail("StartVersion", req.startVersion); + initializationReplies.push_back( + transformErrors(throwErrorOr(worker.backup.getReplyUnlessFailedFor( + req, SERVER_KNOBS->BACKUP_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + backup_worker_failed())); + } + + state Future> fMinVersion = getMinBackupVersion(self, cx); + wait(gotProgress && success(fMinVersion)); + TraceEvent("MinBackupVersion", self->dbgid).detail("Version", fMinVersion.get().present() ? fMinVersion.get() : -1); + + std::map, std::map> toRecruit = + backupProgress->getUnfinishedBackup(); + for (const auto& [epochVersionTags, tagVersions] : toRecruit) { + const Version oldEpochEnd = std::get<1>(epochVersionTags); + if (!fMinVersion.get().present() || fMinVersion.get().get() + 1 >= oldEpochEnd) { + TraceEvent("SkipBackupRecruitment", self->dbgid) + .detail("MinVersion", fMinVersion.get().present() ? fMinVersion.get() : -1) + .detail("Epoch", epoch) + .detail("OldEpoch", std::get<0>(epochVersionTags)) + .detail("OldEpochEnd", oldEpochEnd); + continue; + } + for (const auto& [tag, version] : tagVersions) { + const auto& worker = self->backupWorkers[i % self->backupWorkers.size()]; + i++; + InitializeBackupRequest req(deterministicRandom()->randomUniqueID()); + req.recruitedEpoch = epoch; + req.backupEpoch = std::get<0>(epochVersionTags); + req.routerTag = tag; + req.totalTags = std::get<2>(epochVersionTags); + req.startVersion = version; // savedVersion + 1 + req.endVersion = std::get<1>(epochVersionTags) - 1; + TraceEvent("BackupRecruitment", self->dbgid) + .detail("RequestID", req.reqId) + .detail("Tag", req.routerTag.toString()) + .detail("Epoch", epoch) + .detail("BackupEpoch", req.backupEpoch) + .detail("StartVersion", req.startVersion) + .detail("EndVersion", req.endVersion.get()); + initializationReplies.push_back(transformErrors( + throwErrorOr(worker.backup.getReplyUnlessFailedFor( + req, SERVER_KNOBS->BACKUP_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + backup_worker_failed())); + } + } + + std::vector newRecruits = wait(getAll(initializationReplies)); + self->logSystem->setBackupWorkers(newRecruits); + TraceEvent("BackupRecruitmentDone", self->dbgid).log(); + self->registrationTrigger.trigger(); + return Void(); +} + +ACTOR Future updateLogsValue(Reference self, Database cx) { + state Transaction tr(cx); + loop { + try { + Optional> value = wait(tr.get(logsKey)); + ASSERT(value.present()); + auto logs = decodeLogsValue(value.get()); + + std::set logIds; + for (auto& log : logs.first) { + logIds.insert(log.first); + } + + bool found = false; + for (auto& logSet : self->logSystem->getLogSystemConfig().tLogs) { + for (auto& log : logSet.tLogs) { + if (logIds.count(log.id())) { + found = true; + break; + } + } + if (found) { + break; + } + } + + if (!found) { + TEST(true); // old master attempted to change logsKey + return Void(); + } + + tr.set(logsKey, self->logSystem->getLogsValue()); + wait(tr.commit()); + return Void(); + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + +// TODO(ahusain): ClusterController orchestrating recovery, self message can be avoided. +Future sendMasterRegistration(ClusterRecoveryData* self, + LogSystemConfig const& logSystemConfig, + std::vector commitProxies, + std::vector grvProxies, + std::vector resolvers, + DBRecoveryCount recoveryCount, + std::vector priorCommittedLogServers) { + RegisterMasterRequest masterReq; + masterReq.id = self->masterInterface.id(); + masterReq.mi = self->masterInterface.locality; + masterReq.logSystemConfig = logSystemConfig; + masterReq.commitProxies = commitProxies; + masterReq.grvProxies = grvProxies; + masterReq.resolvers = resolvers; + masterReq.recoveryCount = recoveryCount; + if (self->hasConfiguration) + masterReq.configuration = self->configuration; + masterReq.registrationCount = ++self->registrationCount; + masterReq.priorCommittedLogServers = priorCommittedLogServers; + masterReq.recoveryState = self->recoveryState; + masterReq.recoveryStalled = self->recruitmentStalled->get(); + masterReq.clusterId = self->clusterId; + return brokenPromiseToNever(self->clusterController.registerMaster.getReply(masterReq)); +} + +ACTOR Future updateRegistration(Reference self, Reference logSystem) { + state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True); + state Future trigger = self->registrationTrigger.onTrigger(); + state Future updateLogsKey; + + loop { + wait(trigger); + wait(delay(.001)); // Coalesce multiple changes + + trigger = self->registrationTrigger.onTrigger(); + + auto logSystemConfig = logSystem->getLogSystemConfig(); + TraceEvent("UpdateRegistration", self->dbgid) + .detail("RecoveryCount", self->cstate.myDBState.recoveryCount) + .detail("OldestBackupEpoch", logSystemConfig.oldestBackupEpoch) + .detail("Logs", describe(logSystemConfig.tLogs)) + .detail("CStateUpdated", self->cstateUpdated.isSet()) + .detail("RecoveryTxnVersion", self->recoveryTransactionVersion) + .detail("LastEpochEnd", self->lastEpochEnd); + + if (!self->cstateUpdated.isSet()) { + wait(sendMasterRegistration(self.getPtr(), + logSystemConfig, + self->provisionalCommitProxies, + self->provisionalGrvProxies, + self->resolvers, + self->cstate.myDBState.recoveryCount, + self->cstate.prevDBState.getPriorCommittedLogServers())); + + } else if (self->recoveryState >= RecoveryState::ACCEPTING_COMMITS) { + updateLogsKey = updateLogsValue(self, cx); + wait(sendMasterRegistration(self.getPtr(), + logSystemConfig, + self->commitProxies, + self->grvProxies, + self->resolvers, + self->cstate.myDBState.recoveryCount, + std::vector())); + } else { + // The cluster should enter the accepting commits phase soon, and then we will register again + TEST(true); // cstate is updated but we aren't accepting commits yet + } + } +} + +ACTOR Future> provisionalMaster(Reference parent, + Future activate) { + wait(activate); + + // Register a fake commit proxy (to be provided right here) to make ourselves available to clients + parent->provisionalCommitProxies = std::vector(1); + parent->provisionalCommitProxies[0].provisional = true; + parent->provisionalCommitProxies[0].initEndpoints(); + parent->provisionalGrvProxies = std::vector(1); + parent->provisionalGrvProxies[0].provisional = true; + parent->provisionalGrvProxies[0].initEndpoints(); + state Future waitCommitProxyFailure = + waitFailureServer(parent->provisionalCommitProxies[0].waitFailure.getFuture()); + state Future waitGrvProxyFailure = + waitFailureServer(parent->provisionalGrvProxies[0].waitFailure.getFuture()); + parent->registrationTrigger.trigger(); + + auto lockedKey = parent->txnStateStore->readValue(databaseLockedKey).get(); + state bool locked = lockedKey.present() && lockedKey.get().size(); + + state Optional metadataVersion = parent->txnStateStore->readValue(metadataVersionKey).get(); + + // We respond to a minimal subset of the commit proxy protocol. Our sole purpose is to receive a single write-only + // transaction which might repair our configuration, and return it. + loop choose { + when(GetReadVersionRequest req = + waitNext(parent->provisionalGrvProxies[0].getConsistentReadVersion.getFuture())) { + if ((req.flags & GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY) && + (req.flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES) && parent->lastEpochEnd) { + GetReadVersionReply rep; + rep.version = parent->lastEpochEnd; + rep.locked = locked; + rep.metadataVersion = metadataVersion; + req.reply.send(rep); + } else + req.reply.send(Never()); // We can't perform causally consistent reads without recovering + } + when(CommitTransactionRequest req = waitNext(parent->provisionalCommitProxies[0].commit.getFuture())) { + req.reply.send(Never()); // don't reply (clients always get commit_unknown_result) + auto t = &req.transaction; + if (t->read_snapshot == parent->lastEpochEnd && //< So no transactions can fall between the read snapshot + // and the recovery transaction this (might) be merged with + // vvv and also the changes we will make in the recovery + // transaction (most notably to lastEpochEndKey) BEFORE we + // merge initialConfChanges won't conflict + !std::any_of(t->read_conflict_ranges.begin(), t->read_conflict_ranges.end(), [](KeyRangeRef const& r) { + return r.contains(lastEpochEndKey); + })) { + for (auto m = t->mutations.begin(); m != t->mutations.end(); ++m) { + TraceEvent("PM_CTM", parent->dbgid) + .detail("MType", m->type) + .detail("Param1", m->param1) + .detail("Param2", m->param2); + if (isMetadataMutation(*m)) { + // We keep the mutations and write conflict ranges from this transaction, but not its read + // conflict ranges + Standalone out; + out.read_snapshot = invalidVersion; + out.mutations.append_deep(out.arena(), t->mutations.begin(), t->mutations.size()); + out.write_conflict_ranges.append_deep( + out.arena(), t->write_conflict_ranges.begin(), t->write_conflict_ranges.size()); + return out; + } + } + } + } + when(GetKeyServerLocationsRequest req = + waitNext(parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture())) { + req.reply.send(Never()); + } + when(wait(waitCommitProxyFailure)) { throw worker_removed(); } + when(wait(waitGrvProxyFailure)) { throw worker_removed(); } + } +} + +ACTOR Future>> recruitEverything( + Reference self, + std::vector* seedServers, + Reference oldLogSystem) { + if (!self->configuration.isValid()) { + RecoveryStatus::RecoveryStatus status; + if (self->configuration.initialized) { + TraceEvent( + SevWarn, + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_INVALID_CONFIG_EVENT_NAME).c_str(), + self->dbgid) + .setMaxEventLength(11000) + .setMaxFieldLength(10000) + .detail("Conf", self->configuration.toString()); + status = RecoveryStatus::configuration_invalid; + } else if (!self->cstate.prevDBState.tLogs.size()) { + status = RecoveryStatus::configuration_never_created; + self->neverCreated = true; + } else { + status = RecoveryStatus::configuration_missing; + } + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->dbgid) + .detail("StatusCode", status) + .detail("Status", RecoveryStatus::names[status]) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + return Never(); + } else + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), + self->dbgid) + .detail("StatusCode", RecoveryStatus::recruiting_transaction_servers) + .detail("Status", RecoveryStatus::names[RecoveryStatus::recruiting_transaction_servers]) + .detail("Conf", self->configuration.toString()) + .detail("RequiredCommitProxies", 1) + .detail("RequiredGrvProxies", 1) + .detail("RequiredResolvers", 1) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + // FIXME: we only need log routers for the same locality as the master + int maxLogRouters = self->cstate.prevDBState.logRouterTags; + for (auto& old : self->cstate.prevDBState.oldTLogData) { + maxLogRouters = std::max(maxLogRouters, old.logRouterTags); + } + + RecruitFromConfigurationRequest recruitReq(self->configuration, self->lastEpochEnd == 0, maxLogRouters); + state Reference recruitWorkersInfo = makeReference(recruitReq); + recruitWorkersInfo->dbgId = self->dbgid; + wait(clusterRecruitFromConfiguration(self->controllerData, recruitWorkersInfo)); + state RecruitFromConfigurationReply recruits = recruitWorkersInfo->rep; + + std::string primaryDcIds, remoteDcIds; + + self->primaryDcId.clear(); + self->remoteDcIds.clear(); + if (recruits.dcId.present()) { + self->primaryDcId.push_back(recruits.dcId); + if (!primaryDcIds.empty()) { + primaryDcIds += ','; + } + primaryDcIds += printable(recruits.dcId); + if (self->configuration.regions.size() > 1) { + Key remoteDcId = recruits.dcId.get() == self->configuration.regions[0].dcId + ? self->configuration.regions[1].dcId + : self->configuration.regions[0].dcId; + self->remoteDcIds.push_back(remoteDcId); + if (!remoteDcIds.empty()) { + remoteDcIds += ','; + } + remoteDcIds += printable(remoteDcId); + } + } + self->backupWorkers.swap(recruits.backupWorkers); + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::initializing_transaction_servers) + .detail("Status", RecoveryStatus::names[RecoveryStatus::initializing_transaction_servers]) + .detail("CommitProxies", recruits.commitProxies.size()) + .detail("GrvProxies", recruits.grvProxies.size()) + .detail("TLogs", recruits.tLogs.size()) + .detail("Resolvers", recruits.resolvers.size()) + .detail("SatelliteTLogs", recruits.satelliteTLogs.size()) + .detail("OldLogRouters", recruits.oldLogRouters.size()) + .detail("StorageServers", recruits.storageServers.size()) + .detail("BackupWorkers", self->backupWorkers.size()) + .detail("PrimaryDcIds", primaryDcIds) + .detail("RemoteDcIds", remoteDcIds) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + // Actually, newSeedServers does both the recruiting and initialization of the seed servers; so if this is a brand + // new database we are sort of lying that we are past the recruitment phase. In a perfect world we would split that + // up so that the recruitment part happens above (in parallel with recruiting the transaction servers?). + wait(newSeedServers(self, recruits, seedServers)); + state std::vector> confChanges; + wait(newCommitProxies(self, recruits) && newGrvProxies(self, recruits) && newResolvers(self, recruits) && + newTLogServers(self, recruits, oldLogSystem, &confChanges)); + + // Update recovery related information to the newly elected sequencer (master) process. + wait(brokenPromiseToNever(self->masterInterface.updateRecoveryData.getReply( + UpdateRecoveryDataRequest(self->recoveryTransactionVersion, self->lastEpochEnd, self->commitProxies)))); + + return confChanges; +} + +ACTOR Future updateLocalityForDcId(Optional dcId, + Reference oldLogSystem, + Reference> locality) { + loop { + std::pair loc = oldLogSystem->getLogSystemConfig().getLocalityForDcId(dcId); + Version ver = locality->get().knownCommittedVersion; + if (ver == invalidVersion) { + ver = oldLogSystem->getKnownCommittedVersion(); + } + locality->set(PeekTxsInfo(loc.first, loc.second, ver)); + TraceEvent("UpdatedLocalityForDcId") + .detail("DcId", dcId) + .detail("Locality0", loc.first) + .detail("Locality1", loc.second) + .detail("Version", ver); + wait(oldLogSystem->onLogSystemConfigChange() || oldLogSystem->onKnownCommittedVersionChange()); + } +} + +ACTOR Future readTransactionSystemState(Reference self, + Reference oldLogSystem, + Version txsPoppedVersion) { + state Reference> myLocality = Reference>( + new AsyncVar(PeekTxsInfo(tagLocalityInvalid, tagLocalityInvalid, invalidVersion))); + state Future localityUpdater = + updateLocalityForDcId(self->masterInterface.locality.dcId(), oldLogSystem, myLocality); + // Peek the txnStateTag in oldLogSystem and recover self->txnStateStore + + // For now, we also obtain the recovery metadata that the log system obtained during the end_epoch process for + // comparison + + // Sets self->lastEpochEnd and self->recoveryTransactionVersion + // Sets self->configuration to the configuration (FF/conf/ keys) at self->lastEpochEnd + + // Recover transaction state store + if (self->txnStateStore) + self->txnStateStore->close(); + self->txnStateLogAdapter = openDiskQueueAdapter(oldLogSystem, myLocality, txsPoppedVersion); + self->txnStateStore = + keyValueStoreLogSystem(self->txnStateLogAdapter, self->dbgid, self->memoryLimit, false, false, true); + + // Versionstamped operations (particularly those applied from DR) define a minimum commit version + // that we may recover to, as they embed the version in user-readable data and require that no + // transactions will be committed at a lower version. + Optional> requiredCommitVersion = + wait(self->txnStateStore->readValue(minRequiredCommitVersionKey)); + + Version minRequiredCommitVersion = -1; + if (requiredCommitVersion.present()) { + minRequiredCommitVersion = BinaryReader::fromStringRef(requiredCommitVersion.get(), Unversioned()); + } + + // Recover version info + self->lastEpochEnd = oldLogSystem->getEnd() - 1; + if (self->lastEpochEnd == 0) { + self->recoveryTransactionVersion = 1; + } else { + if (self->forceRecovery) { + self->recoveryTransactionVersion = self->lastEpochEnd + SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT_FORCED; + } else { + self->recoveryTransactionVersion = self->lastEpochEnd + SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT; + } + + if (BUGGIFY) { + self->recoveryTransactionVersion += + deterministicRandom()->randomInt64(0, SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT); + } + if (self->recoveryTransactionVersion < minRequiredCommitVersion) + self->recoveryTransactionVersion = minRequiredCommitVersion; + } + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERED_EVENT_NAME).c_str(), + self->dbgid) + .detail("LastEpochEnd", self->lastEpochEnd) + .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); + + RangeResult rawConf = wait(self->txnStateStore->readRange(configKeys)); + self->configuration.fromKeyValues(rawConf.castTo>()); + self->originalConfiguration = self->configuration; + self->hasConfiguration = true; + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERED_EVENT_NAME).c_str(), + self->dbgid) + .setMaxEventLength(11000) + .setMaxFieldLength(10000) + .detail("Conf", self->configuration.toString()) + .trackLatest(self->recoveredConfigEventHolder->trackingKey); + + RangeResult rawLocalities = wait(self->txnStateStore->readRange(tagLocalityListKeys)); + self->dcId_locality.clear(); + for (auto& kv : rawLocalities) { + self->dcId_locality[decodeTagLocalityListKey(kv.key)] = decodeTagLocalityListValue(kv.value); + } + + RangeResult rawTags = wait(self->txnStateStore->readRange(serverTagKeys)); + self->allTags.clear(); + if (self->lastEpochEnd > 0) { + self->allTags.push_back(cacheTag); + } + + if (self->forceRecovery) { + self->safeLocality = oldLogSystem->getLogSystemConfig().tLogs[0].locality; + for (auto& kv : rawTags) { + Tag tag = decodeServerTagValue(kv.value); + if (tag.locality == self->safeLocality) { + self->allTags.push_back(tag); + } + } + } else { + for (auto& kv : rawTags) { + self->allTags.push_back(decodeServerTagValue(kv.value)); + } + } + + RangeResult rawHistoryTags = wait(self->txnStateStore->readRange(serverTagHistoryKeys)); + for (auto& kv : rawHistoryTags) { + self->allTags.push_back(decodeServerTagValue(kv.value)); + } + + uniquify(self->allTags); + + // auto kvs = self->txnStateStore->readRange( systemKeys ); + // for( auto & kv : kvs.get() ) + // TraceEvent("ClusterRecoveredTXS", self->dbgid).detail("K", kv.key).detail("V", kv.value); + + self->txnStateLogAdapter->setNextVersion( + oldLogSystem->getEnd()); //< FIXME: (1) the log adapter should do this automatically after recovery; (2) if we + // make KeyValueStoreMemory guarantee immediate reads, we should be able to get rid of + // the discardCommit() below and not need a writable log adapter + + TraceEvent("RTSSComplete", self->dbgid).log(); + + return Void(); +} + +ACTOR Future sendInitialCommitToResolvers(Reference self) { + state KeyRange txnKeys = allKeys; + state Sequence txnSequence = 0; + ASSERT(self->recoveryTransactionVersion); + + state RangeResult data = + self->txnStateStore + ->readRange(txnKeys, BUGGIFY ? 3 : SERVER_KNOBS->DESIRED_TOTAL_BYTES, SERVER_KNOBS->DESIRED_TOTAL_BYTES) + .get(); + state std::vector> txnReplies; + state int64_t dataOutstanding = 0; + + state std::vector endpoints; + for (auto& it : self->commitProxies) { + endpoints.push_back(it.txnState.getEndpoint()); + } + + loop { + if (!data.size()) + break; + ((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end); + RangeResult nextData = + self->txnStateStore + ->readRange(txnKeys, BUGGIFY ? 3 : SERVER_KNOBS->DESIRED_TOTAL_BYTES, SERVER_KNOBS->DESIRED_TOTAL_BYTES) + .get(); + + TxnStateRequest req; + req.arena = data.arena(); + req.data = data; + req.sequence = txnSequence; + req.last = !nextData.size(); + req.broadcastInfo = endpoints; + txnReplies.push_back(broadcastTxnRequest(req, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, false)); + dataOutstanding += SERVER_KNOBS->TXN_STATE_SEND_AMOUNT * data.arena().getSize(); + data = nextData; + txnSequence++; + + if (dataOutstanding > SERVER_KNOBS->MAX_TXS_SEND_MEMORY) { + wait(waitForAll(txnReplies)); + txnReplies = std::vector>(); + dataOutstanding = 0; + } + + wait(yield()); + } + wait(waitForAll(txnReplies)); + TraceEvent("RecoveryInternal", self->dbgid) + .detail("StatusCode", RecoveryStatus::recovery_transaction) + .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) + .detail("RecoveryTxnVersion", self->recoveryTransactionVersion) + .detail("LastEpochEnd", self->lastEpochEnd) + .detail("Step", "SentTxnStateStoreToCommitProxies"); + + std::vector> replies; + for (auto& r : self->resolvers) { + ResolveTransactionBatchRequest req; + req.prevVersion = -1; + req.version = self->lastEpochEnd; + req.lastReceivedVersion = -1; + + replies.push_back(brokenPromiseToNever(r.resolve.getReply(req))); + } + + wait(waitForAll(replies)); + TraceEvent("RecoveryInternal", self->dbgid) + .detail("StatusCode", RecoveryStatus::recovery_transaction) + .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) + .detail("RecoveryTxnVersion", self->recoveryTransactionVersion) + .detail("LastEpochEnd", self->lastEpochEnd) + .detail("Step", "InitializedAllResolvers"); + return Void(); +} + +ACTOR Future triggerUpdates(Reference self, Reference oldLogSystem) { + loop { + wait(oldLogSystem->onLogSystemConfigChange() || self->cstate.fullyRecovered.getFuture() || + self->recruitmentStalled->onChange()); + if (self->cstate.fullyRecovered.isSet()) + return Void(); + + self->registrationTrigger.trigger(); + } +} + +ACTOR Future discardCommit(IKeyValueStore* store, LogSystemDiskQueueAdapter* adapter) { + state Future fcm = adapter->getCommitMessage(); + state Future committed = store->commit(); + LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm); + ASSERT(!committed.isReady()); + cm.acknowledge.send(Void()); + ASSERT(committed.isReady()); + return Void(); +} + +void updateConfigForForcedRecovery(Reference self, + std::vector>* initialConfChanges) { + bool regionsChanged = false; + for (auto& it : self->configuration.regions) { + if (it.dcId == self->controllerData->clusterControllerDcId.get() && it.priority < 0) { + it.priority = 1; + regionsChanged = true; + } else if (it.dcId != self->controllerData->clusterControllerDcId.get() && it.priority >= 0) { + it.priority = -1; + regionsChanged = true; + } + } + Standalone regionCommit; + regionCommit.mutations.push_back_deep( + regionCommit.arena(), + MutationRef(MutationRef::SetValue, configKeysPrefix.toString() + "usable_regions", LiteralStringRef("1"))); + self->configuration.applyMutation(regionCommit.mutations.back()); + if (regionsChanged) { + std::sort( + self->configuration.regions.begin(), self->configuration.regions.end(), RegionInfo::sort_by_priority()); + StatusObject regionJSON; + regionJSON["regions"] = self->configuration.getRegionJSON(); + regionCommit.mutations.push_back_deep( + regionCommit.arena(), + MutationRef(MutationRef::SetValue, + configKeysPrefix.toString() + "regions", + BinaryWriter::toValue(regionJSON, IncludeVersion(ProtocolVersion::withRegionConfiguration())) + .toString())); + self->configuration.applyMutation( + regionCommit.mutations.back()); // modifying the configuration directly does not change the configuration + // when it is re-serialized unless we call applyMutation + TraceEvent("ForcedRecoveryConfigChange", self->dbgid) + .setMaxEventLength(11000) + .setMaxFieldLength(10000) + .detail("Conf", self->configuration.toString()); + } + initialConfChanges->push_back(regionCommit); +} + +ACTOR Future recoverFrom(Reference self, + Reference oldLogSystem, + std::vector* seedServers, + std::vector>* initialConfChanges, + Future poppedTxsVersion, + bool* clusterIdExists) { + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::reading_transaction_system_state) + .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_transaction_system_state]) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + self->hasConfiguration = false; + + if (BUGGIFY) + wait(delay(10.0)); + + Version txsPoppedVersion = wait(poppedTxsVersion); + wait(readTransactionSystemState(self, oldLogSystem, txsPoppedVersion)); + for (auto& itr : *initialConfChanges) { + for (auto& m : itr.mutations) { + self->configuration.applyMutation(m); + } + } + + if (self->forceRecovery) { + updateConfigForForcedRecovery(self, initialConfChanges); + } + + debug_checkMaxRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery"); + + // Generate a cluster ID to uniquely identify the cluster if it doesn't + // already exist in the txnStateStore. + Optional clusterId = self->txnStateStore->readValue(clusterIdKey).get(); + *clusterIdExists = clusterId.present(); + if (!clusterId.present()) { + self->clusterId = deterministicRandom()->randomUniqueID(); + } else { + self->clusterId = BinaryReader::fromStringRef(clusterId.get(), Unversioned()); + } + + // Ordinarily we pass through this loop once and recover. We go around the loop if recovery stalls for more than a + // second, a provisional master is initialized, and an "emergency transaction" is submitted that might change the + // configuration so that we can finish recovery. + + state std::map, int8_t> originalLocalityMap = self->dcId_locality; + state Future>> recruitments = + recruitEverything(self, seedServers, oldLogSystem); + state double provisionalDelay = SERVER_KNOBS->PROVISIONAL_START_DELAY; + loop { + state Future> provisional = provisionalMaster(self, delay(provisionalDelay)); + provisionalDelay = + std::min(SERVER_KNOBS->PROVISIONAL_MAX_DELAY, provisionalDelay * SERVER_KNOBS->PROVISIONAL_DELAY_GROWTH); + choose { + when(std::vector> confChanges = wait(recruitments)) { + initialConfChanges->insert(initialConfChanges->end(), confChanges.begin(), confChanges.end()); + provisional.cancel(); + break; + } + when(Standalone _req = wait(provisional)) { + state Standalone req = _req; // mutable + TEST(true); // Emergency transaction processing during recovery + TraceEvent("EmergencyTransaction", self->dbgid).log(); + for (auto m = req.mutations.begin(); m != req.mutations.end(); ++m) + TraceEvent("EmergencyTransactionMutation", self->dbgid) + .detail("MType", m->type) + .detail("P1", m->param1) + .detail("P2", m->param2); + + DatabaseConfiguration oldConf = self->configuration; + self->configuration = self->originalConfiguration; + for (auto& m : req.mutations) + self->configuration.applyMutation(m); + + initialConfChanges->clear(); + if (self->originalConfiguration.isValid() && + self->configuration.usableRegions != self->originalConfiguration.usableRegions) { + TraceEvent(SevWarnAlways, "CannotChangeUsableRegions", self->dbgid).log(); + self->configuration = self->originalConfiguration; + } else { + initialConfChanges->push_back(req); + } + if (self->forceRecovery) { + updateConfigForForcedRecovery(self, initialConfChanges); + } + + if (self->configuration != oldConf) { // confChange does not trigger when including servers + self->dcId_locality = originalLocalityMap; + recruitments = recruitEverything(self, seedServers, oldLogSystem); + } + } + } + + provisional.cancel(); + } + + return Void(); +} + +ACTOR Future clusterRecoveryCore(Reference self) { + state TraceInterval recoveryInterval("ClusterRecovery"); + state double recoverStartTime = now(); + + self->addActor.send(waitFailureServer(self->masterInterface.waitFailure.getFuture())); + + TraceEvent(recoveryInterval.begin(), self->dbgid).log(); + + self->recoveryState = RecoveryState::READING_CSTATE; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::reading_coordinated_state) + .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_coordinated_state]) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + wait(self->cstate.read()); + + self->recoveryState = RecoveryState::LOCKING_CSTATE; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::locking_coordinated_state) + .detail("Status", RecoveryStatus::names[RecoveryStatus::locking_coordinated_state]) + .detail("TLogs", self->cstate.prevDBState.tLogs.size()) + .detail("ActiveGenerations", self->cstate.myDBState.oldTLogData.size() + 1) + .detail("MyRecoveryCount", self->cstate.prevDBState.recoveryCount + 2) + .detail("ForceRecovery", self->forceRecovery) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + // for (const auto& old : self->cstate.prevDBState.oldTLogData) { + // TraceEvent("BWReadCoreState", self->dbgid).detail("Epoch", old.epoch).detail("Version", old.epochEnd); + //} + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME).c_str(), + self->dbgid) + .detail("ActiveGenerations", self->cstate.myDBState.oldTLogData.size() + 1) + .trackLatest(self->clusterRecoveryGenerationsEventHolder->trackingKey); + + if (self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_OVERRIDE) { + if (self->cstate.myDBState.oldTLogData.size() >= CLIENT_KNOBS->MAX_GENERATIONS) { + TraceEvent(SevError, "RecoveryStoppedTooManyOldGenerations") + .detail("OldGenerations", self->cstate.myDBState.oldTLogData.size()) + .detail("Reason", + "Recovery stopped because too many recoveries have happened since the last time the cluster " + "was fully_recovered. Set --knob-max-generations-override on your server processes to a value " + "larger than OldGenerations to resume recovery once the underlying problem has been fixed."); + wait(Future(Never())); + } else if (self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION) { + TraceEvent(SevError, "RecoveryDelayedTooManyOldGenerations") + .detail("OldGenerations", self->cstate.myDBState.oldTLogData.size()) + .detail("Reason", + "Recovery is delayed because too many recoveries have happened since the last time the cluster " + "was fully_recovered. Set --knob-max-generations-override on your server processes to a value " + "larger than OldGenerations to resume recovery once the underlying problem has been fixed."); + wait(delay(CLIENT_KNOBS->RECOVERY_DELAY_SECONDS_PER_GENERATION * + (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION))); + } + if (g_network->isSimulated() && self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_SIM) { + g_simulator.connectionFailuresDisableDuration = 1e6; + g_simulator.speedUpSimulation = true; + TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyGenerations").log(); + } + } + + state Reference>> oldLogSystems(new AsyncVar>); + state Future recoverAndEndEpoch = + ILogSystem::recoverAndEndEpoch(oldLogSystems, + self->dbgid, + self->cstate.prevDBState, + self->clusterController.tlogRejoin.getFuture(), + self->controllerData->db.serverInfo->get().myLocality, + std::addressof(self->forceRecovery)); + + DBCoreState newState = self->cstate.myDBState; + newState.recoveryCount++; + wait(self->cstate.write(newState) || recoverAndEndEpoch); + + self->recoveryState = RecoveryState::RECRUITING; + + state std::vector seedServers; + state std::vector> initialConfChanges; + state Future logChanges; + state Future minRecoveryDuration; + state Future poppedTxsVersion; + state bool clusterIdExists = false; + + loop { + Reference oldLogSystem = oldLogSystems->get(); + if (oldLogSystem) { + logChanges = triggerUpdates(self, oldLogSystem); + if (!minRecoveryDuration.isValid()) { + minRecoveryDuration = delay(SERVER_KNOBS->ENFORCED_MIN_RECOVERY_DURATION); + poppedTxsVersion = oldLogSystem->getTxsPoppedVersion(); + } + } + + state Future reg = oldLogSystem ? updateRegistration(self, oldLogSystem) : Never(); + self->registrationTrigger.trigger(); + + choose { + when(wait(oldLogSystem ? recoverFrom(self, + oldLogSystem, + &seedServers, + &initialConfChanges, + poppedTxsVersion, + std::addressof(clusterIdExists)) + : Never())) { + reg.cancel(); + break; + } + when(wait(oldLogSystems->onChange())) {} + when(wait(reg)) { throw internal_error(); } + when(wait(recoverAndEndEpoch)) { throw internal_error(); } + } + } + + if (self->neverCreated) { + recoverStartTime = now(); + } + + recoverAndEndEpoch.cancel(); + + ASSERT(self->commitProxies.size() <= self->configuration.getDesiredCommitProxies()); + ASSERT(self->commitProxies.size() >= 1); + ASSERT(self->grvProxies.size() <= self->configuration.getDesiredGrvProxies()); + ASSERT(self->grvProxies.size() >= 1); + ASSERT(self->resolvers.size() <= self->configuration.getDesiredResolvers()); + ASSERT(self->resolvers.size() >= 1); + + self->recoveryState = RecoveryState::RECOVERY_TRANSACTION; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::recovery_transaction) + .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) + .detail("PrimaryLocality", self->primaryLocality) + .detail("DcId", self->masterInterface.locality.dcId()) + .detail("ClusterId", self->clusterId) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + // Recovery transaction + state bool debugResult = debug_checkMinRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery", SevWarn); + + CommitTransactionRequest recoveryCommitRequest; + recoveryCommitRequest.flags = recoveryCommitRequest.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE; + CommitTransactionRef& tr = recoveryCommitRequest.transaction; + int mmApplied = 0; // The number of mutations in tr.mutations that have been applied to the txnStateStore so far + if (self->lastEpochEnd != 0) { + Optional snapRecoveryFlag = self->txnStateStore->readValue(writeRecoveryKey).get(); + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_SNAPSHOT_CHECK_EVENT_NAME).c_str()) + .detail("SnapRecoveryFlag", snapRecoveryFlag.present() ? snapRecoveryFlag.get().toString() : "N/A") + .detail("LastEpochEnd", self->lastEpochEnd); + if (snapRecoveryFlag.present()) { + TEST(true); // Recovering from snapshot, writing to snapShotEndVersionKey + BinaryWriter bw(Unversioned()); + tr.set(recoveryCommitRequest.arena, snapshotEndVersionKey, (bw << self->lastEpochEnd).toValue()); + // Pause the backups that got restored in this snapshot to avoid data corruption + // Requires further operational work to abort the backup + TraceEvent( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_PAUSE_AGENT_BACKUP_EVENT_NAME).c_str()) + .log(); + Key backupPauseKey = FileBackupAgent::getPauseKey(); + tr.set(recoveryCommitRequest.arena, backupPauseKey, StringRef()); + // Clear the key so multiple recoveries will not overwrite the first version recorded + tr.clear(recoveryCommitRequest.arena, singleKeyRange(writeRecoveryKey)); + } + if (self->forceRecovery) { + BinaryWriter bw(Unversioned()); + tr.set(recoveryCommitRequest.arena, killStorageKey, (bw << self->safeLocality).toValue()); + } + + // This transaction sets \xff/lastEpochEnd, which the shard servers can use to roll back speculatively + // processed semi-committed transactions from the previous epoch. + // It also guarantees the shard servers and tlog servers eventually get versions in the new epoch, which + // clients might rely on. + // This transaction is by itself in a batch (has its own version number), which simplifies storage servers + // slightly (they assume there are no modifications to serverKeys in the same batch) The proxy also expects the + // lastEpochEndKey mutation to be first in the transaction + BinaryWriter bw(Unversioned()); + tr.set(recoveryCommitRequest.arena, lastEpochEndKey, (bw << self->lastEpochEnd).toValue()); + + if (self->forceRecovery) { + tr.set(recoveryCommitRequest.arena, rebootWhenDurableKey, StringRef()); + tr.set(recoveryCommitRequest.arena, + moveKeysLockOwnerKey, + BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned())); + } + } else { + // Recruit and seed initial shard servers + // This transaction must be the very first one in the database (version 1) + seedShardServers(recoveryCommitRequest.arena, tr, seedServers); + } + // initialConfChanges have not been conflict checked against any earlier writes in the recovery transaction, so do + // this as early as possible in the recovery transaction but see above comments as to why it can't be absolutely + // first. Theoretically emergency transactions should conflict check against the lastEpochEndKey. + for (auto& itr : initialConfChanges) { + tr.mutations.append_deep(recoveryCommitRequest.arena, itr.mutations.begin(), itr.mutations.size()); + tr.write_conflict_ranges.append_deep( + recoveryCommitRequest.arena, itr.write_conflict_ranges.begin(), itr.write_conflict_ranges.size()); + } + + tr.set( + recoveryCommitRequest.arena, primaryLocalityKey, BinaryWriter::toValue(self->primaryLocality, Unversioned())); + tr.set(recoveryCommitRequest.arena, backupVersionKey, backupVersionValue); + tr.set(recoveryCommitRequest.arena, coordinatorsKey, self->coordinators.ccr->getConnectionString().toString()); + tr.set(recoveryCommitRequest.arena, logsKey, self->logSystem->getLogsValue()); + tr.set(recoveryCommitRequest.arena, + primaryDatacenterKey, + self->controllerData->clusterControllerDcId.present() ? self->controllerData->clusterControllerDcId.get() + : StringRef()); + + tr.clear(recoveryCommitRequest.arena, tLogDatacentersKeys); + for (auto& dc : self->primaryDcId) { + tr.set(recoveryCommitRequest.arena, tLogDatacentersKeyFor(dc), StringRef()); + } + if (self->configuration.usableRegions > 1) { + for (auto& dc : self->remoteDcIds) { + tr.set(recoveryCommitRequest.arena, tLogDatacentersKeyFor(dc), StringRef()); + } + } + + // Write cluster ID into txnStateStore if it is missing. + if (!clusterIdExists) { + tr.set(recoveryCommitRequest.arena, clusterIdKey, BinaryWriter::toValue(self->clusterId, Unversioned())); + } + + applyMetadataMutations(SpanID(), + self->dbgid, + recoveryCommitRequest.arena, + tr.mutations.slice(mmApplied, tr.mutations.size()), + self->txnStateStore); + mmApplied = tr.mutations.size(); + + tr.read_snapshot = self->recoveryTransactionVersion; // lastEpochEnd would make more sense, but isn't in the initial + // window of the resolver(s) + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_EVENT_NAME).c_str(), self->dbgid) + .log(); + state Future> recoveryCommit = self->commitProxies[0].commit.tryGetReply(recoveryCommitRequest); + self->addActor.send(self->logSystem->onError()); + self->addActor.send(waitResolverFailure(self->resolvers)); + self->addActor.send(waitCommitProxyFailure(self->commitProxies)); + self->addActor.send(waitGrvProxyFailure(self->grvProxies)); + self->addActor.send(reportErrors(updateRegistration(self, self->logSystem), "UpdateRegistration", self->dbgid)); + self->registrationTrigger.trigger(); + + wait(discardCommit(self->txnStateStore, self->txnStateLogAdapter)); + + // Wait for the recovery transaction to complete. + // SOMEDAY: For faster recovery, do this and setDBState asynchronously and don't wait for them + // unless we want to change TLogs + wait((success(recoveryCommit) && sendInitialCommitToResolvers(self))); + if (recoveryCommit.isReady() && recoveryCommit.get().isError()) { + TEST(true); // Cluster recovery failed because of the initial commit failed + throw cluster_recovery_failed(); + } + + ASSERT(self->recoveryTransactionVersion != 0); + + self->recoveryState = RecoveryState::WRITING_CSTATE; + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::writing_coordinated_state) + .detail("Status", RecoveryStatus::names[RecoveryStatus::writing_coordinated_state]) + .detail("TLogList", self->logSystem->describe()) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + // Multiple masters prevent conflicts between themselves via CoordinatedState (self->cstate) + // 1. If SetMaster succeeds, then by CS's contract, these "new" Tlogs are the immediate + // successors of the "old" ones we are replacing + // 2. logSystem->recoverAndEndEpoch ensured that a co-quorum of the "old" tLogs were stopped at + // versions <= self->lastEpochEnd, so no versions > self->lastEpochEnd could be (fully) committed to them. + // 3. No other master will attempt to commit anything to our "new" Tlogs + // because it didn't recruit them + // 4. Therefore, no full commit can come between self->lastEpochEnd and the first commit + // we made to the new Tlogs (self->recoveryTransactionVersion), and only our own semi-commits can come between + // our first commit and the next new TLogs + + self->addActor.send(trackTlogRecovery(self, oldLogSystems, minRecoveryDuration)); + debug_advanceMaxCommittedVersion(UID(), self->recoveryTransactionVersion); + wait(self->cstateUpdated.getFuture()); + debug_advanceMinCommittedVersion(UID(), self->recoveryTransactionVersion); + + if (debugResult) { + TraceEvent(self->forceRecovery ? SevWarn : SevError, "DBRecoveryDurabilityError").log(); + } + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_EVENT_NAME).c_str(), self->dbgid) + .detail("TLogs", self->logSystem->describe()) + .detail("RecoveryCount", self->cstate.myDBState.recoveryCount) + .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); + + TraceEvent(recoveryInterval.end(), self->dbgid) + .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); + + self->recoveryState = RecoveryState::ACCEPTING_COMMITS; + double recoveryDuration = now() - recoverStartTime; + + TraceEvent((recoveryDuration > 4 && !g_network->isSimulated()) ? SevWarnAlways : SevInfo, + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_DURATION_EVENT_NAME).c_str(), + self->dbgid) + .detail("RecoveryDuration", recoveryDuration) + .trackLatest(self->clusterRecoveryDurationEventHolder->trackingKey); + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) + .detail("StatusCode", RecoveryStatus::accepting_commits) + .detail("Status", RecoveryStatus::names[RecoveryStatus::accepting_commits]) + .detail("StoreType", self->configuration.storageServerStoreType) + .detail("RecoveryDuration", recoveryDuration) + .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey); + + TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME).c_str(), + self->dbgid) + .detail("AvailableAtVersion", self->version) + .trackLatest(self->clusterRecoveryAvailableEventHolder->trackingKey); + + if (self->resolvers.size() > 1) + self->addActor.send(resolutionBalancing(self)); + + self->addActor.send(changeCoordinators(self)); + Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True); + self->addActor.send(configurationMonitor(self, cx)); + if (self->configuration.backupWorkerEnabled) { + self->addActor.send(recruitBackupWorkers(self, cx)); + } else { + self->logSystem->setOldestBackupEpoch(self->cstate.myDBState.recoveryCount); + } + + wait(Future(Never())); + throw internal_error(); +} + +ACTOR Future cleanupRecoveryActorCollection(Reference self, bool exThrown) { + if (self.isValid()) { + wait(delay(0.0)); + + while (!self->addActor.isEmpty()) { + self->addActor.getFuture().pop(); + } + } + + return Void(); +} + +bool isNormalClusterRecoveryError(const Error& error) { + return normalClusterRecoveryErrors().count(error.code()); +} + +std::string& getRecoveryEventName(ClusterRecoveryEventType type) { + ASSERT(type >= ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME && + type < ClusterRecoveryEventType::CLUSTER_RECOVERY_LAST); + + // Cluster recovery state machine used to be driven from master/sequencer process, recovery state + // tracking is used by test environment and tooling scripts. To ease the process of migration, prefix + // is controlled by a ServerKnob for now. + // + // TODO: ServerKnob should be removed ones all tooling scripts and usage is identified and updated accordingly + + static std::map recoveryEventNameMap; + if (recoveryEventNameMap.empty()) { + // initialize the map + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryState" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_TLOG_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryCommittedTLogs" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_DURATION_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryDuration" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryGenerations" }); + recoveryEventNameMap.insert( + { ClusterRecoveryEventType::CLUSTER_RECOVERY_SS_RECRUITMENT_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryRecruitingInitialStorageServer" }); + recoveryEventNameMap.insert( + { ClusterRecoveryEventType::CLUSTER_RECOVERY_INVALID_CONFIG_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryInvalidConfiguration" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERING_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "Recovering" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERED_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveredConfig" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_SNAPSHOT_CHECK_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoverySnapshotCheck" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_SS_RECRUITMENT_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoverySnapshotCheck" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_PAUSE_AGENT_BACKUP_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryPauseBackupAgents" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryCommit" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryAvailable" }); + recoveryEventNameMap.insert({ ClusterRecoveryEventType::CLUSTER_RECOVERY_METRICS_EVENT_NAME, + SERVER_KNOBS->CLUSTER_RECOVERY_EVENT_NAME_PREFIX + "RecoveryMetrics" }); + } + + auto iter = recoveryEventNameMap.find(type); + ASSERT(iter != recoveryEventNameMap.end()); + return iter->second; +} \ No newline at end of file diff --git a/fdbserver/ClusterRecovery.actor.h b/fdbserver/ClusterRecovery.actor.h new file mode 100644 index 0000000000..d53e136484 --- /dev/null +++ b/fdbserver/ClusterRecovery.actor.h @@ -0,0 +1,314 @@ +/* + * ClusterRecovery.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source +// version. +#include + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H) +#define FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H +#include "fdbserver/ClusterRecovery.actor.g.h" +#elif !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_H) +#define FDBSERVER_CLUSTERRECOVERY_ACTOR_H + +#include "fdbclient/DatabaseContext.h" +#include "fdbrpc/Replication.h" +#include "fdbrpc/ReplicationUtils.h" +#include "fdbserver/CoordinatedState.h" +#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class +#include "fdbserver/ClusterController.actor.h" +#include "fdbserver/DBCoreState.h" +#include "fdbserver/Knobs.h" +#include "fdbserver/LogSystem.h" +#include "fdbserver/LogSystemConfig.h" +#include "fdbserver/LogSystemDiskQueueAdapter.h" +#include "fdbserver/MoveKeys.actor.h" +#include "fdbserver/WorkerInterface.actor.h" +#include "flow/Error.h" +#include "flow/SystemMonitor.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +typedef enum { + CLUSTER_RECOVERY_STATE_EVENT_NAME, + CLUSTER_RECOVERY_COMMIT_TLOG_EVENT_NAME, + CLUSTER_RECOVERY_DURATION_EVENT_NAME, + CLUSTER_RECOVERY_GENERATION_EVENT_NAME, + CLUSTER_RECOVERY_SS_RECRUITMENT_EVENT_NAME, + CLUSTER_RECOVERY_INVALID_CONFIG_EVENT_NAME, + CLUSTER_RECOVERY_RECOVERING_EVENT_NAME, + CLUSTER_RECOVERY_RECOVERED_EVENT_NAME, + CLUSTER_RECOVERY_SNAPSHOT_CHECK_EVENT_NAME, + CLUSTER_RECOVERY_PAUSE_AGENT_BACKUP_EVENT_NAME, + CLUSTER_RECOVERY_COMMIT_EVENT_NAME, + CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME, + CLUSTER_RECOVERY_METRICS_EVENT_NAME, + CLUSTER_RECOVERY_LAST // Always the last entry +} ClusterRecoveryEventType; + +ACTOR Future recoveryTerminateOnConflict(UID dbgid, + Promise fullyRecovered, + Future onConflict, + Future switchedState); +std::string& getRecoveryEventName(ClusterRecoveryEventType type); + +class ReusableCoordinatedState : NonCopyable { +public: + Promise fullyRecovered; + DBCoreState prevDBState; + DBCoreState myDBState; + bool finalWriteStarted; + Future previousWrite; + + ReusableCoordinatedState(ServerCoordinators const& coordinators, + PromiseStream> const& addActor, + UID const& dbgid) + : finalWriteStarted(false), previousWrite(Void()), cstate(coordinators), coordinators(coordinators), + addActor(addActor), dbgid(dbgid) {} + + Future read() { return _read(this); } + + Future write(DBCoreState newState, bool finalWrite = false) { + previousWrite = _write(this, newState, finalWrite); + return previousWrite; + } + + Future move(ClusterConnectionString const& nc) { return cstate.move(nc); } + +private: + MovableCoordinatedState cstate; + ServerCoordinators coordinators; + PromiseStream> addActor; + Promise switchedState; + UID dbgid; + + ACTOR Future _read(ReusableCoordinatedState* self) { + Value prevDBStateRaw = wait(self->cstate.read()); + Future onConflict = recoveryTerminateOnConflict( + self->dbgid, self->fullyRecovered, self->cstate.onConflict(), self->switchedState.getFuture()); + if (onConflict.isReady() && onConflict.isError()) { + throw onConflict.getError(); + } + self->addActor.send(onConflict); + + if (prevDBStateRaw.size()) { + self->prevDBState = BinaryReader::fromStringRef(prevDBStateRaw, IncludeVersion()); + self->myDBState = self->prevDBState; + } + + return Void(); + } + + ACTOR Future _write(ReusableCoordinatedState* self, DBCoreState newState, bool finalWrite) { + if (self->finalWriteStarted) { + wait(Future(Never())); + } + + if (finalWrite) { + self->finalWriteStarted = true; + } + + try { + wait(self->cstate.setExclusive( + BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withDBCoreState())))); + } catch (Error& e) { + TEST(true); // Master displaced during writeMasterState + throw; + } + + self->myDBState = newState; + + if (!finalWrite) { + self->switchedState.send(Void()); + self->cstate = MovableCoordinatedState(self->coordinators); + Value rereadDBStateRaw = wait(self->cstate.read()); + DBCoreState readState; + if (rereadDBStateRaw.size()) + readState = BinaryReader::fromStringRef(rereadDBStateRaw, IncludeVersion()); + + if (readState != newState) { + TraceEvent("RecoveryTerminated", self->dbgid).detail("Reason", "CStateChanged"); + TEST(true); // Coordinated state changed between writing and reading, recovery restarting + throw worker_removed(); + } + self->switchedState = Promise(); + self->addActor.send(recoveryTerminateOnConflict( + self->dbgid, self->fullyRecovered, self->cstate.onConflict(), self->switchedState.getFuture())); + } else { + self->fullyRecovered.send(Void()); + } + + return Void(); + } +}; + +struct ClusterRecoveryData : NonCopyable, ReferenceCounted { + ClusterControllerData* controllerData; + + UID dbgid; + + AsyncTrigger registrationTrigger; + Version lastEpochEnd, // The last version in the old epoch not (to be) rolled back in this recovery + recoveryTransactionVersion; // The first version in this epoch + double lastCommitTime; + + Version liveCommittedVersion; // The largest live committed version reported by commit proxies. + bool databaseLocked; + Optional proxyMetadataVersion; + Version minKnownCommittedVersion; + + DatabaseConfiguration originalConfiguration; + DatabaseConfiguration configuration; + std::vector> primaryDcId; + std::vector> remoteDcIds; + bool hasConfiguration; + + ServerCoordinators coordinators; + + Reference logSystem; + Version version; // The last version assigned to a proxy by getVersion() + double lastVersionTime; + LogSystemDiskQueueAdapter* txnStateLogAdapter; + IKeyValueStore* txnStateStore; + int64_t memoryLimit; + std::map, int8_t> dcId_locality; + std::vector allTags; + + int8_t getNextLocality() { + int8_t maxLocality = -1; + for (auto it : dcId_locality) { + maxLocality = std::max(maxLocality, it.second); + } + return maxLocality + 1; + } + + std::vector commitProxies; + std::vector provisionalCommitProxies; + std::vector grvProxies; + std::vector provisionalGrvProxies; + std::vector resolvers; + + std::map lastCommitProxyVersionReplies; + + UID clusterId; + Standalone dbId; + + MasterInterface masterInterface; + LifetimeToken masterLifetime; + const ClusterControllerFullInterface + clusterController; // If the cluster controller changes, this master will die, so this is immutable. + + ReusableCoordinatedState cstate; + Promise recoveryReadyForCommits; + Promise cstateUpdated; + Reference const> dbInfo; + int64_t registrationCount; // Number of different MasterRegistrationRequests sent to clusterController + + RecoveryState recoveryState; + + AsyncVar>> resolverChanges; + Version resolverChangesVersion; + std::set resolverNeedingChanges; + + PromiseStream> addActor; + Reference> recruitmentStalled; + bool forceRecovery; + bool neverCreated; + int8_t safeLocality; + int8_t primaryLocality; + + std::vector backupWorkers; // Recruited backup workers from cluster controller. + + CounterCollection cc; + Counter changeCoordinatorsRequests; + Counter getCommitVersionRequests; + Counter backupWorkerDoneRequests; + Counter getLiveCommittedVersionRequests; + Counter reportLiveCommittedVersionRequests; + + Future logger; + + Reference recoveredConfigEventHolder; + Reference clusterRecoveryStateEventHolder; + Reference clusterRecoveryGenerationsEventHolder; + Reference clusterRecoveryDurationEventHolder; + Reference clusterRecoveryAvailableEventHolder; + + ClusterRecoveryData(ClusterControllerData* controllerData, + Reference const> const& dbInfo, + MasterInterface const& masterInterface, + LifetimeToken const& masterLifetimeToken, + ServerCoordinators const& coordinators, + ClusterControllerFullInterface const& clusterController, + Standalone const& dbId, + PromiseStream> const& addActor, + bool forceRecovery) + + : controllerData(controllerData), dbgid(masterInterface.id()), lastEpochEnd(invalidVersion), + recoveryTransactionVersion(invalidVersion), lastCommitTime(0), liveCommittedVersion(invalidVersion), + databaseLocked(false), minKnownCommittedVersion(invalidVersion), hasConfiguration(false), + coordinators(coordinators), version(invalidVersion), lastVersionTime(0), txnStateStore(nullptr), + memoryLimit(2e9), dbId(dbId), masterInterface(masterInterface), masterLifetime(masterLifetimeToken), + clusterController(clusterController), cstate(coordinators, addActor, dbgid), dbInfo(dbInfo), + registrationCount(0), addActor(addActor), recruitmentStalled(makeReference>(false)), + forceRecovery(forceRecovery), neverCreated(false), safeLocality(tagLocalityInvalid), + primaryLocality(tagLocalityInvalid), cc("Master", dbgid.toString()), + changeCoordinatorsRequests("ChangeCoordinatorsRequests", cc), + getCommitVersionRequests("GetCommitVersionRequests", cc), + backupWorkerDoneRequests("BackupWorkerDoneRequests", cc), + getLiveCommittedVersionRequests("GetLiveCommittedVersionRequests", cc), + reportLiveCommittedVersionRequests("ReportLiveCommittedVersionRequests", cc), + recoveredConfigEventHolder(makeReference("RecoveredConfig")) { + clusterRecoveryStateEventHolder = makeReference( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME)); + clusterRecoveryGenerationsEventHolder = makeReference( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME)); + clusterRecoveryDurationEventHolder = makeReference( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_DURATION_EVENT_NAME)); + clusterRecoveryAvailableEventHolder = makeReference( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME)); + logger = traceCounters(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_METRICS_EVENT_NAME), + dbgid, + SERVER_KNOBS->WORKER_LOGGING_INTERVAL, + &cc, + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_METRICS_EVENT_NAME)); + if (forceRecovery && !controllerData->clusterControllerDcId.present()) { + TraceEvent(SevError, "ForcedRecoveryRequiresDcID").log(); + forceRecovery = false; + } + } + ~ClusterRecoveryData() { + if (txnStateStore) + txnStateStore->close(); + } +}; + +ACTOR Future recruitNewMaster(ClusterControllerData* cluster, + ClusterControllerData::DBInfo* db, + MasterInterface* newMaster); +ACTOR Future cleanupRecoveryActorCollection(Reference self, bool exThrown); +ACTOR Future clusterRecoveryCore(Reference self); +bool isNormalClusterRecoveryError(const Error&); + +#include "flow/unactorcompiler.h" + +#endif diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index d2c5d7aa28..9de55eea45 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -642,6 +642,8 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { self->commitVersion = versionReply.version; self->prevVersion = versionReply.prevVersion; + //TraceEvent("CPGetVersion", pProxyCommitData->dbgid).detail("Master", pProxyCommitData->master.id().toString()).detail("CommitVersion", self->commitVersion).detail("PrvVersion", self->prevVersion); + for (auto it : versionReply.resolverChanges) { auto rs = pProxyCommitData->keyResolvers.modify(it.range); for (auto r = rs.begin(); r != rs.end(); ++r) @@ -878,7 +880,7 @@ ACTOR Future applyMetadataToCommittedTransactions(CommitBatchContext* self if (!self->isMyFirstBatch && pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get() != self->oldCoordinators.get()) { - wait(brokenPromiseToNever(pProxyCommitData->master.changeCoordinators.getReply( + wait(brokenPromiseToNever(pProxyCommitData->db->get().clusterInterface.changeCoordinators.getReply( ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get())))); ASSERT(false); // ChangeCoordinatorsRequest should always throw } @@ -1093,8 +1095,16 @@ ACTOR Future postResolution(CommitBatchContext* self) { applyMetadataEffect(self); + if (debugID.present()) { + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.ApplyMetadaEffect"); + } + determineCommittedTransactions(self); + if (debugID.present()) { + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.ApplyMetadaEffect"); + } + if (self->forceRecovery) { wait(Future(Never())); } @@ -1102,9 +1112,18 @@ ACTOR Future postResolution(CommitBatchContext* self) { // First pass wait(applyMetadataToCommittedTransactions(self)); + if (debugID.present()) { + g_traceBatch.addEvent( + "CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.ApplyMetadaToCommittedTxn"); + } + // Second pass wait(assignMutationsToStorageServers(self)); + if (debugID.present()) { + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.AssignMutationToSS"); + } + // Serialize and backup the mutations as a single mutation if ((pProxyCommitData->vecBackupKeys.size() > 1) && self->logRangeMutations.size()) { wait(addBackupMutations(pProxyCommitData, @@ -1241,7 +1260,7 @@ ACTOR Future transactionLogging(CommitBatchContext* self) { } } catch (Error& e) { if (e.code() == error_code_broken_promise) { - throw master_tlog_failed(); + throw tlog_failed(); } throw; } @@ -1273,8 +1292,10 @@ ACTOR Future reply(CommitBatchContext* self) { const Optional& debugID = self->debugID; - if (self->prevVersion && self->commitVersion - self->prevVersion < SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT / 2) + if (self->prevVersion && self->commitVersion - self->prevVersion < SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT / 2) { + //TraceEvent("CPAdvanceMinVersion", self->pProxyCommitData->dbgid).detail("PrvVersion", self->prevVersion).detail("CommitVersion", self->commitVersion).detail("Master", self->pProxyCommitData->master.id().toString()).detail("TxSize", self->trs.size()); debug_advanceMinCommittedVersion(UID(), self->commitVersion); + } //TraceEvent("ProxyPushed", pProxyCommitData->dbgid).detail("PrevVersion", prevVersion).detail("Version", commitVersion); if (debugID.present()) @@ -1999,6 +2020,7 @@ ACTOR Future processTransactionStateRequestPart(TransactionStateResolveCon ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, MasterInterface master, + LifetimeToken masterLifetime, Reference const> db, LogEpoch epoch, Version recoveryTransactionVersion, @@ -2013,8 +2035,7 @@ ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, state Future lastCommitComplete = Void(); state PromiseStream> addActor; - state Future onError = - transformError(actorCollection(addActor.getFuture()), broken_promise(), master_tlog_failed()); + state Future onError = transformError(actorCollection(addActor.getFuture()), broken_promise(), tlog_failed()); state double lastCommit = 0; state GetHealthMetricsReply healthMetricsReply; @@ -2026,7 +2047,7 @@ ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, //TraceEvent("CommitProxyInit1", proxy.id()); // Wait until we can load the "real" logsystem, since we don't support switching them currently - while (!(commitData.db->get().master.id() == master.id() && + while (!(masterLifetime.isEqual(commitData.db->get().masterLifetime) && commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) { //TraceEvent("ProxyInit2", proxy.id()).detail("LSEpoch", db->get().logSystemConfig.epoch).detail("Need", epoch); wait(commitData.db->onChange()); @@ -2088,7 +2109,7 @@ ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, loop choose { when(wait(dbInfoChange)) { dbInfoChange = commitData.db->onChange(); - if (commitData.db->get().master.id() == master.id() && + if (masterLifetime.isEqual(commitData.db->get().masterLifetime) && commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION) { commitData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor); for (auto it : commitData.tag_popped) { @@ -2106,7 +2127,9 @@ ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, const std::vector& trs = batchedRequests.first; int batchBytes = batchedRequests.second; //TraceEvent("CommitProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount); + //TraceEvent("CommitProxyCore", commitData.dbgid).detail("TxSize", trs.size()).detail("MasterLifetime", masterLifetime.toString()).detail("DbMasterLifetime", commitData.db->get().masterLifetime.toString()).detail("RecoveryState", commitData.db->get().recoveryState).detail("CCInf", commitData.db->get().clusterInterface.id().toString()); if (trs.size() || (commitData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && + masterLifetime.isEqual(commitData.db->get().masterLifetime) && now() - lastCommit >= SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL)) { lastCommit = now(); @@ -2155,6 +2178,7 @@ ACTOR Future commitProxyServer(CommitProxyInterface proxy, try { state Future core = commitProxyServerCore(proxy, req.master, + req.masterLifetime, db, req.recoveryCount, req.recoveryTransactionVersion, @@ -2165,7 +2189,7 @@ ACTOR Future commitProxyServer(CommitProxyInterface proxy, TraceEvent("CommitProxyTerminated", proxy.id()).error(e, true); if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped && - e.code() != error_code_master_tlog_failed && e.code() != error_code_coordinators_changed && + e.code() != error_code_tlog_failed && e.code() != error_code_coordinators_changed && e.code() != error_code_coordinated_state_conflict && e.code() != error_code_new_coordinators_timed_out && e.code() != error_code_failed_to_progress) { throw; diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp index d400666a43..5f427dd0b0 100644 --- a/fdbserver/GrvProxyServer.actor.cpp +++ b/fdbserver/GrvProxyServer.actor.cpp @@ -920,12 +920,12 @@ ACTOR static Future transactionStarter(GrvProxyInterface proxy, ACTOR Future grvProxyServerCore(GrvProxyInterface proxy, MasterInterface master, + LifetimeToken masterLifetime, Reference const> db) { state GrvProxyData grvProxyData(proxy.id(), master, proxy.getConsistentReadVersion, db); state PromiseStream> addActor; - state Future onError = - transformError(actorCollection(addActor.getFuture()), broken_promise(), master_tlog_failed()); + state Future onError = transformError(actorCollection(addActor.getFuture()), broken_promise(), tlog_failed()); state GetHealthMetricsReply healthMetricsReply; state GetHealthMetricsReply detailedHealthMetricsReply; @@ -933,9 +933,14 @@ ACTOR Future grvProxyServerCore(GrvProxyInterface proxy, addActor.send(waitFailureServer(proxy.waitFailure.getFuture())); addActor.send(traceRole(Role::GRV_PROXY, proxy.id())); + TraceEvent("GrvProxyServerCore", proxy.id()) + .detail("MasterId", master.id()) + .detail("MasterLifetime", masterLifetime.toString()) + .detail("RecoveryCount", db->get().recoveryCount); + // Wait until we can load the "real" logsystem, since we don't support switching them currently - while (!(grvProxyData.db->get().master.id() == master.id() && - grvProxyData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) { + while (!(masterLifetime.isEqual(grvProxyData.db->get().masterLifetime) && + grvProxyData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS)) { wait(grvProxyData.db->onChange()); } // Do we need to wait for any db info change? Yes. To update latency band. @@ -956,7 +961,7 @@ ACTOR Future grvProxyServerCore(GrvProxyInterface proxy, when(wait(dbInfoChange)) { dbInfoChange = grvProxyData.db->onChange(); - if (grvProxyData.db->get().master.id() == master.id() && + if (masterLifetime.isEqual(grvProxyData.db->get().masterLifetime) && grvProxyData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION) { grvProxyData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), grvProxyData.db->get(), false, addActor); @@ -983,13 +988,13 @@ ACTOR Future grvProxyServer(GrvProxyInterface proxy, InitializeGrvProxyRequest req, Reference const> db) { try { - state Future core = grvProxyServerCore(proxy, req.master, db); + state Future core = grvProxyServerCore(proxy, req.master, req.masterLifetime, db); wait(core || checkRemoved(db, req.recoveryCount, proxy)); } catch (Error& e) { TraceEvent("GrvProxyTerminated", proxy.id()).error(e, true); if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped && - e.code() != error_code_master_tlog_failed && e.code() != error_code_coordinators_changed && + e.code() != error_code_tlog_failed && e.code() != error_code_coordinators_changed && e.code() != error_code_coordinated_state_conflict && e.code() != error_code_new_coordinators_timed_out) { throw; } diff --git a/fdbserver/MasterInterface.h b/fdbserver/MasterInterface.h index a9fb8122aa..a092226f75 100644 --- a/fdbserver/MasterInterface.h +++ b/fdbserver/MasterInterface.h @@ -22,11 +22,13 @@ #define FDBSERVER_MASTERINTERFACE_H #pragma once +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/StorageServerInterface.h" #include "fdbclient/CommitTransaction.h" #include "fdbclient/DatabaseConfiguration.h" #include "fdbserver/TLogInterface.h" +#include "fdbclient/Notified.h" typedef uint64_t DBRecoveryCount; @@ -34,20 +36,17 @@ struct MasterInterface { constexpr static FileIdentifier file_identifier = 5979145; LocalityData locality; RequestStream> waitFailure; - RequestStream - tlogRejoin; // sent by tlog (whether or not rebooted) to communicate with a new master - RequestStream changeCoordinators; RequestStream getCommitVersion; - RequestStream notifyBackupWorkerDone; // Get the centralized live committed version reported by commit proxies. RequestStream getLiveCommittedVersion; // Report a proxy's committed version. RequestStream reportLiveCommittedVersion; + RequestStream updateRecoveryData; - NetworkAddress address() const { return changeCoordinators.getEndpoint().getPrimaryAddress(); } - NetworkAddressList addresses() const { return changeCoordinators.getEndpoint().addresses; } + NetworkAddress address() const { return getCommitVersion.getEndpoint().getPrimaryAddress(); } + NetworkAddressList addresses() const { return getCommitVersion.getEndpoint().addresses; } - UID id() const { return changeCoordinators.getEndpoint().token; } + UID id() const { return getCommitVersion.getEndpoint().token; } template void serialize(Archive& ar) { if constexpr (!is_fb_function) { @@ -55,61 +54,28 @@ struct MasterInterface { } serializer(ar, locality, waitFailure); if (Archive::isDeserializing) { - tlogRejoin = RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(1)); - changeCoordinators = - RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(2)); getCommitVersion = - RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(3)); - notifyBackupWorkerDone = - RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(4)); + RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(1)); getLiveCommittedVersion = - RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(5)); + RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(2)); reportLiveCommittedVersion = RequestStream( - waitFailure.getEndpoint().getAdjustedEndpoint(6)); + waitFailure.getEndpoint().getAdjustedEndpoint(3)); + updateRecoveryData = + RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(4)); } } void initEndpoints() { std::vector> streams; streams.push_back(waitFailure.getReceiver()); - streams.push_back(tlogRejoin.getReceiver(TaskPriority::MasterTLogRejoin)); - streams.push_back(changeCoordinators.getReceiver()); streams.push_back(getCommitVersion.getReceiver(TaskPriority::GetConsistentReadVersion)); - streams.push_back(notifyBackupWorkerDone.getReceiver()); streams.push_back(getLiveCommittedVersion.getReceiver(TaskPriority::GetLiveCommittedVersion)); streams.push_back(reportLiveCommittedVersion.getReceiver(TaskPriority::ReportLiveCommittedVersion)); + streams.push_back(updateRecoveryData.getReceiver(TaskPriority::UpdateRecoveryTransactionVersion)); FlowTransport::transport().addEndpoints(streams); } }; -struct TLogRejoinReply { - constexpr static FileIdentifier file_identifier = 11; - - // false means someone else registered, so we should re-register. true means this master is recovered, so don't - // send again to the same master. - bool masterIsRecovered; - TLogRejoinReply() = default; - explicit TLogRejoinReply(bool masterIsRecovered) : masterIsRecovered(masterIsRecovered) {} - - template - void serialize(Ar& ar) { - serializer(ar, masterIsRecovered); - } -}; - -struct TLogRejoinRequest { - constexpr static FileIdentifier file_identifier = 15692200; - TLogInterface myInterface; - ReplyPromise reply; - - TLogRejoinRequest() {} - explicit TLogRejoinRequest(const TLogInterface& interf) : myInterface(interf) {} - template - void serialize(Ar& ar) { - serializer(ar, myInterface, reply); - } -}; - struct ChangeCoordinatorsRequest { constexpr static FileIdentifier file_identifier = 13605416; Standalone newConnectionString; @@ -184,6 +150,26 @@ struct GetCommitVersionRequest { } }; +struct UpdateRecoveryDataRequest { + constexpr static FileIdentifier file_identifier = 13605417; + Version recoveryTransactionVersion; + Version lastEpochEnd; + std::vector commitProxies; + ReplyPromise reply; + + UpdateRecoveryDataRequest() {} + UpdateRecoveryDataRequest(Version recoveryTransactionVersion, + Version lastEpochEnd, + std::vector commitProxies) + : recoveryTransactionVersion(recoveryTransactionVersion), lastEpochEnd(lastEpochEnd), + commitProxies(commitProxies) {} + + template + void serialize(Ar& ar) { + serializer(ar, recoveryTransactionVersion, lastEpochEnd, commitProxies, reply); + } +}; + struct ReportRawCommittedVersionRequest { constexpr static FileIdentifier file_identifier = 1853148; Version version; @@ -207,21 +193,6 @@ struct ReportRawCommittedVersionRequest { } }; -struct BackupWorkerDoneRequest { - constexpr static FileIdentifier file_identifier = 8736351; - UID workerUID; - LogEpoch backupEpoch; - ReplyPromise reply; - - BackupWorkerDoneRequest() : workerUID(), backupEpoch(-1) {} - BackupWorkerDoneRequest(UID id, LogEpoch epoch) : workerUID(id), backupEpoch(epoch) {} - - template - void serialize(Ar& ar) { - serializer(ar, workerUID, backupEpoch, reply); - } -}; - struct LifetimeToken { UID ccID; int64_t count; @@ -231,6 +202,9 @@ struct LifetimeToken { bool isStillValid(LifetimeToken const& latestToken, bool isLatestID) const { return ccID == latestToken.ccID && (count >= latestToken.count || isLatestID); } + bool isEqual(LifetimeToken const& toCompare) { + return ccID.compare(toCompare.ccID) == 0 && count == toCompare.count; + } std::string toString() const { return ccID.shortString() + format("#%lld", count); } void operator++() { ++count; } @@ -240,4 +214,18 @@ struct LifetimeToken { } }; +struct CommitProxyVersionReplies { + std::map replies; + NotifiedVersion latestRequestNum; + + CommitProxyVersionReplies(CommitProxyVersionReplies&& r) noexcept + : replies(std::move(r.replies)), latestRequestNum(std::move(r.latestRequestNum)) {} + void operator=(CommitProxyVersionReplies&& r) noexcept { + replies = std::move(r.replies); + latestRequestNum = std::move(r.latestRequestNum); + } + + CommitProxyVersionReplies() : latestRequestNum(0) {} +}; + #endif diff --git a/fdbserver/OldTLogServer_4_6.actor.cpp b/fdbserver/OldTLogServer_4_6.actor.cpp index 9c4131e500..eca9c17390 100644 --- a/fdbserver/OldTLogServer_4_6.actor.cpp +++ b/fdbserver/OldTLogServer_4_6.actor.cpp @@ -1249,11 +1249,11 @@ ACTOR Future commitQueue(TLogData* self) { } } -ACTOR Future rejoinMasters(TLogData* self, - TLogInterface tli, - DBRecoveryCount recoveryCount, - Future registerWithMaster) { - state UID lastMasterID(0, 0); +ACTOR Future rejoinClusterController(TLogData* self, + TLogInterface tli, + DBRecoveryCount recoveryCount, + Future registerWithCC) { + state LifetimeToken lastMasterLifetime; loop { auto const& inf = self->dbInfo->get(); bool isDisplaced = @@ -1274,18 +1274,21 @@ ACTOR Future rejoinMasters(TLogData* self, throw worker_removed(); } - if (registerWithMaster.isReady()) { - if (self->dbInfo->get().master.id() != lastMasterID) { + if (registerWithCC.isReady()) { + if (!lastMasterLifetime.isEqual(self->dbInfo->get().masterLifetime)) { // The TLogRejoinRequest is needed to establish communications with a new master, which doesn't have our // TLogInterface TLogRejoinRequest req; req.myInterface = tli; - TraceEvent("TLogRejoining", tli.id()).detail("Master", self->dbInfo->get().master.id()); + TraceEvent("TLogRejoining", tli.id()) + .detail("ClusterController", self->dbInfo->get().clusterInterface.id()) + .detail("DbInfoMasterLifeTime", self->dbInfo->get().masterLifetime.toString()) + .detail("LastMasterLifeTime", lastMasterLifetime.toString()); choose { - when(TLogRejoinReply rep = - wait(brokenPromiseToNever(self->dbInfo->get().master.tlogRejoin.getReply(req)))) { + when(TLogRejoinReply rep = wait( + brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)))) { if (rep.masterIsRecovered) - lastMasterID = self->dbInfo->get().master.id(); + lastMasterLifetime = self->dbInfo->get().masterLifetime; } when(wait(self->dbInfo->onChange())) {} } @@ -1293,7 +1296,7 @@ ACTOR Future rejoinMasters(TLogData* self, wait(self->dbInfo->onChange()); } } else { - wait(registerWithMaster || self->dbInfo->onChange()); + wait(registerWithCC || self->dbInfo->onChange()); } } } @@ -1479,7 +1482,7 @@ ACTOR Future restorePersistentState(TLogData* self, LocalityData locality) ASSERT(fVers.get().size() == fRecoverCounts.get().size()); state int idx = 0; - state Promise registerWithMaster; + state Promise registerWithCC; for (idx = 0; idx < fVers.get().size(); idx++) { state KeyRef rawId = fVers.get()[idx].key.removePrefix(persistCurrentVersionKeys.begin); UID id1 = BinaryReader::fromStringRef(rawId, Unversioned()); @@ -1513,7 +1516,7 @@ ACTOR Future restorePersistentState(TLogData* self, LocalityData locality) logData->version.set(ver); logData->recoveryCount = BinaryReader::fromStringRef(fRecoverCounts.get()[idx].value, Unversioned()); - logData->removed = rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture()); + logData->removed = rejoinClusterController(self, recruited, logData->recoveryCount, registerWithCC.getFuture()); removed.push_back(errorOr(logData->removed)); TraceEvent("TLogRestorePersistentStateVer", id1).detail("Ver", ver); @@ -1617,8 +1620,8 @@ ACTOR Future restorePersistentState(TLogData* self, LocalityData locality) self->sharedActors.send(tLogCore(self, it.second)); } - if (registerWithMaster.canBeSet()) - registerWithMaster.send(Void()); + if (registerWithCC.canBeSet()) + registerWithCC.send(Void()); return Void(); } diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp index 6f3e5da3a9..98e8d74672 100644 --- a/fdbserver/OldTLogServer_6_0.actor.cpp +++ b/fdbserver/OldTLogServer_6_0.actor.cpp @@ -1729,12 +1729,12 @@ ACTOR Future initPersistentState(TLogData* self, Reference logDat return Void(); } -ACTOR Future rejoinMasters(TLogData* self, - TLogInterface tli, - DBRecoveryCount recoveryCount, - Future registerWithMaster, - bool isPrimary) { - state UID lastMasterID(0, 0); +ACTOR Future rejoinClusterController(TLogData* self, + TLogInterface tli, + DBRecoveryCount recoveryCount, + Future registerWithCC, + bool isPrimary) { + state LifetimeToken lastMasterLifetime; loop { auto const& inf = self->dbInfo->get(); bool isDisplaced = @@ -1762,17 +1762,20 @@ ACTOR Future rejoinMasters(TLogData* self, throw worker_removed(); } - if (registerWithMaster.isReady()) { - if (self->dbInfo->get().master.id() != lastMasterID) { + if (registerWithCC.isReady()) { + if (!lastMasterLifetime.isEqual(self->dbInfo->get().masterLifetime)) { // The TLogRejoinRequest is needed to establish communications with a new master, which doesn't have our // TLogInterface TLogRejoinRequest req(tli); - TraceEvent("TLogRejoining", tli.id()).detail("Master", self->dbInfo->get().master.id()); + TraceEvent("TLogRejoining", tli.id()) + .detail("ClusterController", self->dbInfo->get().clusterInterface.id()) + .detail("DbInfoMasterLifeTime", self->dbInfo->get().masterLifetime.toString()) + .detail("LastMasterLifeTime", lastMasterLifetime.toString()); choose { - when(TLogRejoinReply rep = - wait(brokenPromiseToNever(self->dbInfo->get().master.tlogRejoin.getReply(req)))) { + when(TLogRejoinReply rep = wait( + brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)))) { if (rep.masterIsRecovered) - lastMasterID = self->dbInfo->get().master.id(); + lastMasterLifetime = self->dbInfo->get().masterLifetime; } when(wait(self->dbInfo->onChange())) {} } @@ -1780,7 +1783,7 @@ ACTOR Future rejoinMasters(TLogData* self, wait(self->dbInfo->onChange()); } } else { - wait(registerWithMaster || self->dbInfo->onChange()); + wait(registerWithCC || self->dbInfo->onChange()); } } } @@ -2384,7 +2387,7 @@ ACTOR Future restorePersistentState(TLogData* self, } state int idx = 0; - state Promise registerWithMaster; + state Promise registerWithCC; state std::map id_interf; for (idx = 0; idx < fVers.get().size(); idx++) { state KeyRef rawId = fVers.get()[idx].key.removePrefix(persistCurrentVersionKeys.begin); @@ -2432,7 +2435,7 @@ ACTOR Future restorePersistentState(TLogData* self, logData->recoveryCount = BinaryReader::fromStringRef(fRecoverCounts.get()[idx].value, Unversioned()); logData->removed = - rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false); + rejoinClusterController(self, recruited, logData->recoveryCount, registerWithCC.getFuture(), false); removed.push_back(errorOr(logData->removed)); TraceEvent("TLogRestorePersistentStateVer", id1).detail("Ver", ver); @@ -2534,8 +2537,8 @@ ACTOR Future restorePersistentState(TLogData* self, self->sharedActors.send(tLogCore(self, it.second, id_interf[it.first], false)); } - if (registerWithMaster.canBeSet()) - registerWithMaster.send(Void()); + if (registerWithCC.canBeSet()) + registerWithCC.send(Void()); return Void(); } @@ -2653,7 +2656,7 @@ ACTOR Future tLogStart(TLogData* self, InitializeTLogRequest req, Locality self->id_data[recruited.id()] = logData; logData->locality = req.locality; logData->recoveryCount = req.epoch; - logData->removed = rejoinMasters(self, recruited, req.epoch, Future(Void()), req.isPrimary); + logData->removed = rejoinClusterController(self, recruited, req.epoch, Future(Void()), req.isPrimary); self->queueOrder.push_back(recruited.id()); TraceEvent("TLogStart", logData->logId).log(); diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp index 2d2ed59703..53be793c7c 100644 --- a/fdbserver/OldTLogServer_6_2.actor.cpp +++ b/fdbserver/OldTLogServer_6_2.actor.cpp @@ -2174,12 +2174,12 @@ ACTOR Future initPersistentState(TLogData* self, Reference logDat return Void(); } -ACTOR Future rejoinMasters(TLogData* self, - TLogInterface tli, - DBRecoveryCount recoveryCount, - Future registerWithMaster, - bool isPrimary) { - state UID lastMasterID(0, 0); +ACTOR Future rejoinClusterController(TLogData* self, + TLogInterface tli, + DBRecoveryCount recoveryCount, + Future registerWithCC, + bool isPrimary) { + state LifetimeToken lastMasterLifetime; loop { auto const& inf = self->dbInfo->get(); bool isDisplaced = @@ -2207,17 +2207,20 @@ ACTOR Future rejoinMasters(TLogData* self, throw worker_removed(); } - if (registerWithMaster.isReady()) { - if (self->dbInfo->get().master.id() != lastMasterID) { + if (registerWithCC.isReady()) { + if (!lastMasterLifetime.isEqual(self->dbInfo->get().masterLifetime)) { // The TLogRejoinRequest is needed to establish communications with a new master, which doesn't have our // TLogInterface TLogRejoinRequest req(tli); - TraceEvent("TLogRejoining", tli.id()).detail("Master", self->dbInfo->get().master.id()); + TraceEvent("TLogRejoining", tli.id()) + .detail("ClusterController", self->dbInfo->get().clusterInterface.id()) + .detail("DbInfoMasterLifeTime", self->dbInfo->get().masterLifetime.toString()) + .detail("LastMasterLifeTime", lastMasterLifetime.toString()); choose { - when(TLogRejoinReply rep = - wait(brokenPromiseToNever(self->dbInfo->get().master.tlogRejoin.getReply(req)))) { + when(TLogRejoinReply rep = wait( + brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)))) { if (rep.masterIsRecovered) - lastMasterID = self->dbInfo->get().master.id(); + lastMasterLifetime = self->dbInfo->get().masterLifetime; } when(wait(self->dbInfo->onChange())) {} } @@ -2225,7 +2228,7 @@ ACTOR Future rejoinMasters(TLogData* self, wait(self->dbInfo->onChange()); } } else { - wait(registerWithMaster || self->dbInfo->onChange()); + wait(registerWithCC || self->dbInfo->onChange()); } } } @@ -2846,7 +2849,7 @@ ACTOR Future restorePersistentState(TLogData* self, } state int idx = 0; - state Promise registerWithMaster; + state Promise registerWithCC; state std::map id_interf; state std::vector> logsByVersion; for (idx = 0; idx < fVers.get().size(); idx++) { @@ -2894,7 +2897,7 @@ ACTOR Future restorePersistentState(TLogData* self, logData->recoveryCount = BinaryReader::fromStringRef(fRecoverCounts.get()[idx].value, Unversioned()); logData->removed = - rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false); + rejoinClusterController(self, recruited, logData->recoveryCount, registerWithCC.getFuture(), false); removed.push_back(errorOr(logData->removed)); logsByVersion.emplace_back(ver, id1); @@ -3017,8 +3020,8 @@ ACTOR Future restorePersistentState(TLogData* self, self->sharedActors.send(tLogCore(self, it.second, id_interf[it.first], false)); } - if (registerWithMaster.canBeSet()) - registerWithMaster.send(Void()); + if (registerWithCC.canBeSet()) + registerWithCC.send(Void()); return Void(); } @@ -3135,7 +3138,7 @@ ACTOR Future tLogStart(TLogData* self, InitializeTLogRequest req, Locality self->id_data[recruited.id()] = logData; logData->locality = req.locality; logData->recoveryCount = req.epoch; - logData->removed = rejoinMasters(self, recruited, req.epoch, Future(Void()), req.isPrimary); + logData->removed = rejoinClusterController(self, recruited, req.epoch, Future(Void()), req.isPrimary); self->popOrder.push_back(recruited.id()); self->spillOrder.push_back(recruited.id()); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 85cdd71866..a6045ea8f0 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -22,12 +22,14 @@ #include "contrib/fmt-8.0.1/include/fmt/format.h" #include "fdbclient/BlobWorkerInterface.h" #include "fdbserver/Status.h" +#include "flow/ITrace.h" #include "flow/Trace.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" #include "fdbclient/ReadYourWrites.h" #include "fdbserver/WorkerInterface.actor.h" #include +#include "fdbserver/ClusterRecovery.actor.h" #include "fdbserver/CoordinationInterface.h" #include "fdbserver/DataDistribution.actor.h" #include "flow/UnitTest.h" @@ -1149,6 +1151,7 @@ static JsonBuilderObject clientStatusFetcher( } ACTOR static Future recoveryStateStatusFetcher(Database cx, + WorkerDetails ccWorker, WorkerDetails mWorker, int workerCount, std::set* incomplete_reasons, @@ -1156,13 +1159,18 @@ ACTOR static Future recoveryStateStatusFetcher(Database cx, state JsonBuilderObject message; state Transaction tr(cx); try { - state Future mdActiveGensF = timeoutError( - mWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MasterRecoveryGenerations"))), - 1.0); - state Future mdF = timeoutError( - mWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MasterRecoveryState"))), 1.0); - state Future mDBAvailableF = timeoutError( - mWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MasterRecoveryAvailable"))), 1.0); + state Future mdActiveGensF = + timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME)))), + 1.0); + state Future mdF = + timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME)))), + 1.0); + state Future mDBAvailableF = + timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef( + getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME)))), + 1.0); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); state Future> rvF = errorOr(timeoutError(tr.getReadVersion(), 1.0)); @@ -2709,6 +2717,7 @@ ACTOR Future clusterGetStatus( state JsonBuilderArray messages; state std::set status_incomplete_reasons; state WorkerDetails mWorker; // Master worker + state WorkerDetails ccWorker; // Cluster-Controller worker state WorkerDetails ddWorker; // DataDistributor worker state WorkerDetails rkWorker; // Ratekeeper worker @@ -2722,6 +2731,15 @@ ACTOR Future clusterGetStatus( JsonString::makeMessage("unreachable_master_worker", "Unable to locate the master worker.")); } + // Get the cluster-controller Worker interface + Optional _ccWorker = getWorker(workers, db->get().clusterInterface.address()); + if (_ccWorker.present()) { + ccWorker = _ccWorker.get(); + } else { + messages.push_back(JsonString::makeMessage("unreachable_cluster_controller_worker", + "Unable to locate the cluster-controller worker.")); + } + // Get the DataDistributor worker interface Optional _ddWorker; if (db->get().distributor.present()) { @@ -2789,8 +2807,8 @@ ACTOR Future clusterGetStatus( // construct status information for cluster subsections state int statusCode = (int)RecoveryStatus::END; - state JsonBuilderObject recoveryStateStatus = - wait(recoveryStateStatusFetcher(cx, mWorker, workers.size(), &status_incomplete_reasons, &statusCode)); + state JsonBuilderObject recoveryStateStatus = wait( + recoveryStateStatusFetcher(cx, ccWorker, mWorker, workers.size(), &status_incomplete_reasons, &statusCode)); // machine metrics state WorkerEvents mMetrics = workerEventsVec[0].present() ? workerEventsVec[0].get().first : WorkerEvents(); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 681f7fde2d..313fdbe508 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -319,9 +319,9 @@ struct TLogData : NonCopyable { // data is written from. This value is restored from disk when the tlog // restarts. UID durableClusterId; - // The master cluster ID stores the cluster ID read from the txnStateStore. + // The cluster-controller cluster ID stores the cluster ID read from the txnStateStore. // It is cached in this variable. - UID masterClusterId; + UID ccClusterId; UID dbgid; UID workerID; @@ -783,7 +783,7 @@ void TLogQueue::updateVersionSizes(const TLogQueueEntry& result, ACTOR Future tLogLock(TLogData* self, ReplyPromise reply, Reference logData) { state Version stopVersion = logData->version.get(); - TEST(true); // TLog stopped by recovering master + TEST(true); // TLog stopped by recovering cluster-controller TEST(logData->stopped); // logData already stopped TEST(!logData->stopped); // logData not yet stopped @@ -2251,12 +2251,12 @@ ACTOR Future getClusterId(TLogData* self) { } } -ACTOR Future rejoinMasters(TLogData* self, - TLogInterface tli, - DBRecoveryCount recoveryCount, - Future registerWithMaster, - bool isPrimary) { - state UID lastMasterID(0, 0); +ACTOR Future rejoinClusterController(TLogData* self, + TLogInterface tli, + DBRecoveryCount recoveryCount, + Future registerWithCC, + bool isPrimary) { + state LifetimeToken lastMasterLifetime; loop { auto const& inf = self->dbInfo->get(); bool isDisplaced = @@ -2284,24 +2284,27 @@ ACTOR Future rejoinMasters(TLogData* self, // with a different cluster ID. state UID clusterId = wait(getClusterId(self)); ASSERT(clusterId.isValid()); - self->masterClusterId = clusterId; + self->ccClusterId = clusterId; ev.detail("ClusterId", clusterId).detail("SelfClusterId", self->durableClusterId); if (BUGGIFY) wait(delay(SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * deterministicRandom()->random01())); throw worker_removed(); } - if (registerWithMaster.isReady()) { - if (self->dbInfo->get().master.id() != lastMasterID) { + if (registerWithCC.isReady()) { + if (!lastMasterLifetime.isEqual(self->dbInfo->get().masterLifetime)) { // The TLogRejoinRequest is needed to establish communications with a new master, which doesn't have our // TLogInterface TLogRejoinRequest req(tli); - TraceEvent("TLogRejoining", tli.id()).detail("Master", self->dbInfo->get().master.id()); + TraceEvent("TLogRejoining", tli.id()) + .detail("ClusterController", self->dbInfo->get().clusterInterface.id()) + .detail("DbInfoMasterLifeTime", self->dbInfo->get().masterLifetime.toString()) + .detail("LastMasterLifeTime", lastMasterLifetime.toString()); choose { - when(TLogRejoinReply rep = - wait(brokenPromiseToNever(self->dbInfo->get().master.tlogRejoin.getReply(req)))) { + when(TLogRejoinReply rep = wait( + brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)))) { if (rep.masterIsRecovered) - lastMasterID = self->dbInfo->get().master.id(); + lastMasterLifetime = self->dbInfo->get().masterLifetime; } when(wait(self->dbInfo->onChange())) {} } @@ -2309,7 +2312,7 @@ ACTOR Future rejoinMasters(TLogData* self, wait(self->dbInfo->onChange()); } } else { - wait(registerWithMaster || self->dbInfo->onChange()); + wait(registerWithCC || self->dbInfo->onChange()); } } } @@ -2506,13 +2509,13 @@ ACTOR Future serveTLogInterface(TLogData* self, } // Persist cluster ID once cluster has recovered. - auto masterClusterId = self->dbInfo->get().clusterId; + auto ccClusterId = self->dbInfo->get().clusterId; if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED && !self->durableClusterId.isValid()) { - ASSERT(masterClusterId.isValid()); - self->durableClusterId = masterClusterId; + ASSERT(ccClusterId.isValid()); + self->durableClusterId = ccClusterId; self->persistentData->set( - KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(masterClusterId, Unversioned()))); + KeyValueRef(persistClusterIdKey, BinaryWriter::toValue(ccClusterId, Unversioned()))); wait(self->persistentData->commit()); } } @@ -2958,7 +2961,7 @@ ACTOR Future restorePersistentState(TLogData* self, } state int idx = 0; - state Promise registerWithMaster; + state Promise registerWithCC; state std::map id_interf; state std::vector> logsByVersion; for (idx = 0; idx < fVers.get().size(); idx++) { @@ -3014,7 +3017,7 @@ ACTOR Future restorePersistentState(TLogData* self, logData->recoveryCount = BinaryReader::fromStringRef(fRecoverCounts.get()[idx].value, Unversioned()); logData->removed = - rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false); + rejoinClusterController(self, recruited, logData->recoveryCount, registerWithCC.getFuture(), false); removed.push_back(errorOr(logData->removed)); logsByVersion.emplace_back(ver, id1); @@ -3137,8 +3140,8 @@ ACTOR Future restorePersistentState(TLogData* self, self->sharedActors.send(tLogCore(self, it.second, id_interf[it.first], false)); } - if (registerWithMaster.canBeSet()) - registerWithMaster.send(Void()); + if (registerWithCC.canBeSet()) + registerWithCC.send(Void()); return Void(); } @@ -3263,7 +3266,7 @@ ACTOR Future tLogStart(TLogData* self, InitializeTLogRequest req, Locality self->id_data[recruited.id()] = logData; logData->locality = req.locality; logData->recoveryCount = req.epoch; - logData->removed = rejoinMasters(self, recruited, req.epoch, Future(Void()), req.isPrimary); + logData->removed = rejoinClusterController(self, recruited, req.epoch, Future(Void()), req.isPrimary); self->popOrder.push_back(recruited.id()); self->spillOrder.push_back(recruited.id()); @@ -3491,13 +3494,13 @@ ACTOR Future tLog(IKeyValueStore* persistentData, // it should automatically exclude itself to avoid being used in // the new cluster. auto recoveryState = self.dbInfo->get().recoveryState; - if (recoveryState == RecoveryState::FULLY_RECOVERED && self.masterClusterId.isValid() && - self.durableClusterId.isValid() && self.masterClusterId != self.durableClusterId) { + if (recoveryState == RecoveryState::FULLY_RECOVERED && self.ccClusterId.isValid() && + self.durableClusterId.isValid() && self.ccClusterId != self.durableClusterId) { state NetworkAddress address = g_network->getLocalAddress(); wait(excludeServers(self.cx, { AddressExclusion{ address.ip, address.port } })); TraceEvent(SevWarnAlways, "TLogBelongsToExistingCluster") .detail("ClusterId", self.durableClusterId) - .detail("NewClusterId", self.masterClusterId); + .detail("NewClusterId", self.ccClusterId); } // If the tlog has a valid durable cluster ID, we don't want it to // wipe its data! Throw this error to signal to `tlogTerminated` to diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 96cc285272..c6e6d568b7 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -461,8 +461,8 @@ ACTOR Future TagPartitionedLogSystem::onError_internal(TagPartitionedLogSy changes.push_back(self->backupWorkerChanged.onTrigger()); ASSERT(failed.size() >= 1); - wait(quorum(changes, 1) || tagError(quorum(failed, 1), master_tlog_failed()) || - tagError(quorum(backupFailed, 1), master_backup_worker_failed())); + wait(quorum(changes, 1) || tagError(quorum(failed, 1), tlog_failed()) || + tagError(quorum(backupFailed, 1), backup_worker_failed())); } } @@ -2300,7 +2300,7 @@ ACTOR Future TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL auto reply = transformErrors( throwErrorOr(workers[nextRouter].logRouter.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed()); + cluster_recovery_failed()); logRouterInitializationReplies.back().push_back(reply); allReplies.push_back(reply); nextRouter = (nextRouter + 1) % workers.size(); @@ -2349,7 +2349,7 @@ ACTOR Future TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL auto reply = transformErrors( throwErrorOr(workers[nextRouter].logRouter.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed()); + cluster_recovery_failed()); logRouterInitializationReplies.back().push_back(reply); allReplies.push_back(reply); nextRouter = (nextRouter + 1) % workers.size(); @@ -2410,7 +2410,7 @@ ACTOR Future TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL if (!forRemote) { self->logSystemConfigChanged.trigger(); - wait(failed.size() ? tagError(quorum(failed, 1), master_tlog_failed()) : Future(Never())); + wait(failed.size() ? tagError(quorum(failed, 1), tlog_failed()) : Future(Never())); throw internal_error(); } return Void(); @@ -2509,7 +2509,7 @@ ACTOR Future TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst throwErrorOr( remoteWorkers.logRouters[i % remoteWorkers.logRouters.size()].logRouter.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); } std::vector localTags = TagPartitionedLogSystem::getLocalTags(remoteLocality, allTags); @@ -2587,7 +2587,7 @@ ACTOR Future TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst remoteTLogInitializationReplies.push_back(transformErrors( throwErrorOr(remoteWorkers.remoteTLogs[i].tLog.getReplyUnlessFailedFor( remoteTLogReqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); TraceEvent("RemoteLogRecruitment_InitializingRemoteLogs") .detail("StartVersion", logSet->startVersion) @@ -2616,7 +2616,7 @@ ACTOR Future TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst TLogRecoveryFinishedRequest(), SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); self->remoteRecoveryComplete = waitForAll(recoveryComplete); self->tLogs.push_back(logSet); @@ -2857,7 +2857,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( initializationReplies.push_back(transformErrors( throwErrorOr(recr.tLogs[i].tLog.getReplyUnlessFailedFor( reqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); state std::vector> recoveryComplete; @@ -2924,7 +2924,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( satelliteInitializationReplies.push_back(transformErrors( throwErrorOr(recr.satelliteTLogs[i].tLog.getReplyUnlessFailedFor( sreqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); wait(waitForAll(satelliteInitializationReplies) || oldRouterRecruitment); @@ -2940,7 +2940,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( TLogRecoveryFinishedRequest(), SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); } wait(waitForAll(initializationReplies) || oldRouterRecruitment); @@ -2955,7 +2955,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( // Don't force failure of recovery if it took us a long time to recover. This avoids multiple long running // recoveries causing tests to timeout if (BUGGIFY && now() - startTime < 300 && g_network->isSimulated() && g_simulator.speedUpSimulation) - throw master_recovery_failed(); + throw cluster_recovery_failed(); for (int i = 0; i < logSystem->tLogs[0]->logServers.size(); i++) recoveryComplete.push_back(transformErrors( @@ -2963,7 +2963,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( TLogRecoveryFinishedRequest(), SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); + cluster_recovery_failed())); logSystem->recoveryComplete = waitForAll(recoveryComplete); if (configuration.usableRegions > 1) { @@ -3057,7 +3057,7 @@ ACTOR Future TagPartitionedLogSystem::lockTLog( UID myID, Reference>> tlog) { - TraceEvent("TLogLockStarted", myID).detail("TLog", tlog->get().id()); + TraceEvent("TLogLockStarted", myID).detail("TLog", tlog->get().id()).detail("InfPresent", tlog->get().present()); loop { choose { when(TLogLockResult data = wait( diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index 1da9de249d..baf4cece98 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -163,17 +163,25 @@ struct ClusterControllerFullInterface { RequestStream getServerDBInfo; // only used by testers; the cluster controller will send the serverDBInfo to workers RequestStream updateWorkerHealth; + RequestStream + tlogRejoin; // sent by tlog (whether or not rebooted) to communicate with a new controller + RequestStream notifyBackupWorkerDone; + RequestStream changeCoordinators; UID id() const { return clientInterface.id(); } bool operator==(ClusterControllerFullInterface const& r) const { return id() == r.id(); } bool operator!=(ClusterControllerFullInterface const& r) const { return id() != r.id(); } + NetworkAddress address() const { return clientInterface.address(); } + bool hasMessage() const { return clientInterface.hasMessage() || recruitFromConfiguration.getFuture().isReady() || recruitRemoteFromConfiguration.getFuture().isReady() || recruitStorage.getFuture().isReady() || recruitBlobWorker.getFuture().isReady() || registerWorker.getFuture().isReady() || getWorkers.getFuture().isReady() || registerMaster.getFuture().isReady() || - getServerDBInfo.getFuture().isReady() || updateWorkerHealth.getFuture().isReady(); + getServerDBInfo.getFuture().isReady() || updateWorkerHealth.getFuture().isReady() || + tlogRejoin.getFuture().isReady() || notifyBackupWorkerDone.getFuture().isReady() || + changeCoordinators.getFuture().isReady(); } void initEndpoints() { @@ -187,6 +195,9 @@ struct ClusterControllerFullInterface { registerMaster.getEndpoint(TaskPriority::ClusterControllerRegister); getServerDBInfo.getEndpoint(TaskPriority::ClusterController); updateWorkerHealth.getEndpoint(TaskPriority::ClusterController); + tlogRejoin.getEndpoint(TaskPriority::MasterTLogRejoin); + notifyBackupWorkerDone.getEndpoint(TaskPriority::ClusterController); + changeCoordinators.getEndpoint(TaskPriority::DefaultEndpoint); } template @@ -204,7 +215,10 @@ struct ClusterControllerFullInterface { getWorkers, registerMaster, getServerDBInfo, - updateWorkerHealth); + updateWorkerHealth, + tlogRejoin, + notifyBackupWorkerDone, + changeCoordinators); } }; @@ -323,10 +337,11 @@ struct RecruitRemoteFromConfigurationReply { constexpr static FileIdentifier file_identifier = 9091392; std::vector remoteTLogs; std::vector logRouters; + Optional dbgId; template void serialize(Ar& ar) { - serializer(ar, remoteTLogs, logRouters); + serializer(ar, remoteTLogs, logRouters, dbgId); } }; @@ -336,6 +351,7 @@ struct RecruitRemoteFromConfigurationRequest { Optional dcId; int logRouterCount; std::vector exclusionWorkerIds; + Optional dbgId; ReplyPromise reply; RecruitRemoteFromConfigurationRequest() {} @@ -348,7 +364,7 @@ struct RecruitRemoteFromConfigurationRequest { template void serialize(Ar& ar) { - serializer(ar, configuration, dcId, logRouterCount, exclusionWorkerIds, reply); + serializer(ar, configuration, dcId, logRouterCount, exclusionWorkerIds, dbgId, reply); } }; @@ -486,6 +502,49 @@ struct UpdateWorkerHealthRequest { } }; +struct TLogRejoinReply { + constexpr static FileIdentifier file_identifier = 11; + + // false means someone else registered, so we should re-register. true means this master is recovered, so don't + // send again to the same master. + bool masterIsRecovered; + TLogRejoinReply() = default; + explicit TLogRejoinReply(bool masterIsRecovered) : masterIsRecovered(masterIsRecovered) {} + + template + void serialize(Ar& ar) { + serializer(ar, masterIsRecovered); + } +}; + +struct TLogRejoinRequest { + constexpr static FileIdentifier file_identifier = 15692200; + TLogInterface myInterface; + ReplyPromise reply; + + TLogRejoinRequest() {} + explicit TLogRejoinRequest(const TLogInterface& interf) : myInterface(interf) {} + template + void serialize(Ar& ar) { + serializer(ar, myInterface, reply); + } +}; + +struct BackupWorkerDoneRequest { + constexpr static FileIdentifier file_identifier = 8736351; + UID workerUID; + LogEpoch backupEpoch; + ReplyPromise reply; + + BackupWorkerDoneRequest() : workerUID(), backupEpoch(-1) {} + BackupWorkerDoneRequest(UID id, LogEpoch epoch) : workerUID(id), backupEpoch(epoch) {} + + template + void serialize(Ar& ar) { + serializer(ar, workerUID, backupEpoch, reply); + } +}; + struct InitializeTLogRequest { constexpr static FileIdentifier file_identifier = 15604392; UID recruitmentID; @@ -605,6 +664,7 @@ struct RecruitMasterRequest { struct InitializeCommitProxyRequest { constexpr static FileIdentifier file_identifier = 10344153; MasterInterface master; + LifetimeToken masterLifetime; uint64_t recoveryCount; Version recoveryTransactionVersion; bool firstProxy; @@ -612,19 +672,20 @@ struct InitializeCommitProxyRequest { template void serialize(Ar& ar) { - serializer(ar, master, recoveryCount, recoveryTransactionVersion, firstProxy, reply); + serializer(ar, master, masterLifetime, recoveryCount, recoveryTransactionVersion, firstProxy, reply); } }; struct InitializeGrvProxyRequest { constexpr static FileIdentifier file_identifier = 8265613; MasterInterface master; + LifetimeToken masterLifetime; uint64_t recoveryCount; ReplyPromise reply; template void serialize(Ar& ar) { - serializer(ar, master, recoveryCount, reply); + serializer(ar, master, masterLifetime, recoveryCount, reply); } }; diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index 426e9ebc49..4abae4d5e8 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -36,7 +36,6 @@ #include "fdbserver/DataDistribution.actor.h" #include "fdbserver/IKeyValueStore.h" #include "fdbserver/Knobs.h" -#include "fdbserver/LogSystem.h" #include "fdbserver/LogSystemDiskQueueAdapter.h" #include "fdbserver/MasterInterface.h" #include "fdbserver/ProxyCommitData.actor.h" @@ -49,216 +48,41 @@ #include "flow/actorcompiler.h" // This must be the last #include. -struct CommitProxyVersionReplies { - std::map replies; - NotifiedVersion latestRequestNum; - - CommitProxyVersionReplies(CommitProxyVersionReplies&& r) noexcept - : replies(std::move(r.replies)), latestRequestNum(std::move(r.latestRequestNum)) {} - void operator=(CommitProxyVersionReplies&& r) noexcept { - replies = std::move(r.replies); - latestRequestNum = std::move(r.latestRequestNum); - } - - CommitProxyVersionReplies() : latestRequestNum(0) {} -}; - -ACTOR Future masterTerminateOnConflict(UID dbgid, - Promise fullyRecovered, - Future onConflict, - Future switchedState) { - choose { - when(wait(onConflict)) { - if (!fullyRecovered.isSet()) { - TraceEvent("MasterTerminated", dbgid).detail("Reason", "Conflict"); - TEST(true); // Coordinated state conflict, master dying - throw worker_removed(); - } - return Void(); - } - when(wait(switchedState)) { return Void(); } - } -} - -class ReusableCoordinatedState : NonCopyable { -public: - Promise fullyRecovered; - DBCoreState prevDBState; - DBCoreState myDBState; - bool finalWriteStarted; - Future previousWrite; - - ReusableCoordinatedState(ServerCoordinators const& coordinators, - PromiseStream> const& addActor, - UID const& dbgid) - : finalWriteStarted(false), previousWrite(Void()), cstate(coordinators), coordinators(coordinators), - addActor(addActor), dbgid(dbgid) {} - - Future read() { return _read(this); } - - Future write(DBCoreState newState, bool finalWrite = false) { - previousWrite = _write(this, newState, finalWrite); - return previousWrite; - } - - Future move(ClusterConnectionString const& nc) { return cstate.move(nc); } - -private: - MovableCoordinatedState cstate; - ServerCoordinators coordinators; - PromiseStream> addActor; - Promise switchedState; - UID dbgid; - - ACTOR Future _read(ReusableCoordinatedState* self) { - Value prevDBStateRaw = wait(self->cstate.read()); - Future onConflict = masterTerminateOnConflict( - self->dbgid, self->fullyRecovered, self->cstate.onConflict(), self->switchedState.getFuture()); - if (onConflict.isReady() && onConflict.isError()) { - throw onConflict.getError(); - } - self->addActor.send(onConflict); - - if (prevDBStateRaw.size()) { - self->prevDBState = BinaryReader::fromStringRef(prevDBStateRaw, IncludeVersion()); - self->myDBState = self->prevDBState; - } - - return Void(); - } - - ACTOR Future _write(ReusableCoordinatedState* self, DBCoreState newState, bool finalWrite) { - if (self->finalWriteStarted) { - wait(Future(Never())); - } - - if (finalWrite) { - self->finalWriteStarted = true; - } - - try { - wait(self->cstate.setExclusive( - BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withDBCoreState())))); - } catch (Error& e) { - TEST(true); // Master displaced during writeMasterState - throw; - } - - self->myDBState = newState; - - if (!finalWrite) { - self->switchedState.send(Void()); - self->cstate = MovableCoordinatedState(self->coordinators); - Value rereadDBStateRaw = wait(self->cstate.read()); - DBCoreState readState; - if (rereadDBStateRaw.size()) - readState = BinaryReader::fromStringRef(rereadDBStateRaw, IncludeVersion()); - - if (readState != newState) { - TraceEvent("MasterTerminated", self->dbgid).detail("Reason", "CStateChanged"); - TEST(true); // Coordinated state changed between writing and reading, master dying - throw worker_removed(); - } - self->switchedState = Promise(); - self->addActor.send(masterTerminateOnConflict( - self->dbgid, self->fullyRecovered, self->cstate.onConflict(), self->switchedState.getFuture())); - } else { - self->fullyRecovered.send(Void()); - } - - return Void(); - } -}; - struct MasterData : NonCopyable, ReferenceCounted { UID dbgid; - AsyncTrigger registrationTrigger; Version lastEpochEnd, // The last version in the old epoch not (to be) rolled back in this recovery recoveryTransactionVersion; // The first version in this epoch - double lastCommitTime; Version liveCommittedVersion; // The largest live committed version reported by commit proxies. bool databaseLocked; Optional proxyMetadataVersion; Version minKnownCommittedVersion; - DatabaseConfiguration originalConfiguration; - DatabaseConfiguration configuration; - std::vector> primaryDcId; - std::vector> remoteDcIds; - bool hasConfiguration; - ServerCoordinators coordinators; - Reference logSystem; Version version; // The last version assigned to a proxy by getVersion() double lastVersionTime; - LogSystemDiskQueueAdapter* txnStateLogAdapter; IKeyValueStore* txnStateStore; - int64_t memoryLimit; - std::map, int8_t> dcId_locality; - std::vector allTags; - - int8_t getNextLocality() { - int8_t maxLocality = -1; - for (auto it : dcId_locality) { - maxLocality = std::max(maxLocality, it.second); - } - return maxLocality + 1; - } std::vector commitProxies; - std::vector provisionalCommitProxies; - std::vector grvProxies; - std::vector provisionalGrvProxies; - std::vector resolvers; - std::map lastCommitProxyVersionReplies; - UID clusterId; - Standalone dbId; - MasterInterface myInterface; - const ClusterControllerFullInterface - clusterController; // If the cluster controller changes, this master will die, so this is immutable. - - ReusableCoordinatedState cstate; - Promise recoveryReadyForCommits; - Promise cstateUpdated; - Reference const> dbInfo; - int64_t registrationCount; // Number of different MasterRegistrationRequests sent to clusterController - - RecoveryState recoveryState; AsyncVar>> resolverChanges; Version resolverChangesVersion; std::set resolverNeedingChanges; - PromiseStream> addActor; - Reference> recruitmentStalled; bool forceRecovery; - bool neverCreated; - int8_t safeLocality; - int8_t primaryLocality; - - std::vector backupWorkers; // Recruited backup workers from cluster controller. CounterCollection cc; - Counter changeCoordinatorsRequests; Counter getCommitVersionRequests; - Counter backupWorkerDoneRequests; Counter getLiveCommittedVersionRequests; Counter reportLiveCommittedVersionRequests; Future logger; - Reference masterRecoveryStateEventHolder; - Reference masterRecoveryGenerationsEventHolder; - Reference masterRecoveryDurationEventHolder; - Reference masterRecoveryAvailableEventHolder; - Reference recoveredConfigEventHolder; - MasterData(Reference const> const& dbInfo, MasterInterface const& myInterface, ServerCoordinators const& coordinators, @@ -268,23 +92,12 @@ struct MasterData : NonCopyable, ReferenceCounted { bool forceRecovery) : dbgid(myInterface.id()), lastEpochEnd(invalidVersion), recoveryTransactionVersion(invalidVersion), - lastCommitTime(0), liveCommittedVersion(invalidVersion), databaseLocked(false), - minKnownCommittedVersion(invalidVersion), hasConfiguration(false), coordinators(coordinators), - version(invalidVersion), lastVersionTime(0), txnStateStore(nullptr), memoryLimit(2e9), dbId(dbId), - myInterface(myInterface), clusterController(clusterController), cstate(coordinators, addActor, dbgid), - dbInfo(dbInfo), registrationCount(0), addActor(addActor), - recruitmentStalled(makeReference>(false)), forceRecovery(forceRecovery), neverCreated(false), - safeLocality(tagLocalityInvalid), primaryLocality(tagLocalityInvalid), cc("Master", dbgid.toString()), - changeCoordinatorsRequests("ChangeCoordinatorsRequests", cc), + liveCommittedVersion(invalidVersion), databaseLocked(false), minKnownCommittedVersion(invalidVersion), + coordinators(coordinators), version(invalidVersion), lastVersionTime(0), txnStateStore(nullptr), + myInterface(myInterface), forceRecovery(forceRecovery), cc("Master", dbgid.toString()), getCommitVersionRequests("GetCommitVersionRequests", cc), - backupWorkerDoneRequests("BackupWorkerDoneRequests", cc), getLiveCommittedVersionRequests("GetLiveCommittedVersionRequests", cc), - reportLiveCommittedVersionRequests("ReportLiveCommittedVersionRequests", cc), - masterRecoveryStateEventHolder(makeReference("MasterRecoveryState")), - masterRecoveryGenerationsEventHolder(makeReference("MasterRecoveryGenerations")), - masterRecoveryDurationEventHolder(makeReference("MasterRecoveryDuration")), - masterRecoveryAvailableEventHolder(makeReference("MasterRecoveryAvailable")), - recoveredConfigEventHolder(makeReference("RecoveredConfig")) { + reportLiveCommittedVersionRequests("ReportLiveCommittedVersionRequests", cc) { logger = traceCounters("MasterMetrics", dbgid, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "MasterMetrics"); if (forceRecovery && !myInterface.locality.dcId().present()) { TraceEvent(SevError, "ForcedRecoveryRequiresDcID").log(); @@ -297,866 +110,6 @@ struct MasterData : NonCopyable, ReferenceCounted { } }; -ACTOR Future newCommitProxies(Reference self, RecruitFromConfigurationReply recr) { - std::vector> initializationReplies; - for (int i = 0; i < recr.commitProxies.size(); i++) { - InitializeCommitProxyRequest req; - req.master = self->myInterface; - req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; - req.recoveryTransactionVersion = self->recoveryTransactionVersion; - req.firstProxy = i == 0; - TraceEvent("CommitProxyReplies", self->dbgid) - .detail("WorkerID", recr.commitProxies[i].id()) - .detail("FirstProxy", req.firstProxy ? "True" : "False"); - initializationReplies.push_back( - transformErrors(throwErrorOr(recr.commitProxies[i].commitProxy.getReplyUnlessFailedFor( - req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); - } - - std::vector newRecruits = wait(getAll(initializationReplies)); - // It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->commitProxies[0] is the firstCommitProxy. - self->commitProxies = newRecruits; - - return Void(); -} - -ACTOR Future newGrvProxies(Reference self, RecruitFromConfigurationReply recr) { - std::vector> initializationReplies; - for (int i = 0; i < recr.grvProxies.size(); i++) { - InitializeGrvProxyRequest req; - req.master = self->myInterface; - req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; - TraceEvent("GrvProxyReplies", self->dbgid).detail("WorkerID", recr.grvProxies[i].id()); - initializationReplies.push_back( - transformErrors(throwErrorOr(recr.grvProxies[i].grvProxy.getReplyUnlessFailedFor( - req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); - } - - std::vector newRecruits = wait(getAll(initializationReplies)); - self->grvProxies = newRecruits; - return Void(); -} - -ACTOR Future newResolvers(Reference self, RecruitFromConfigurationReply recr) { - std::vector> initializationReplies; - for (int i = 0; i < recr.resolvers.size(); i++) { - InitializeResolverRequest req; - req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; - req.commitProxyCount = recr.commitProxies.size(); - req.resolverCount = recr.resolvers.size(); - TraceEvent("ResolverReplies", self->dbgid).detail("WorkerID", recr.resolvers[i].id()); - initializationReplies.push_back( - transformErrors(throwErrorOr(recr.resolvers[i].resolver.getReplyUnlessFailedFor( - req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_recovery_failed())); - } - - std::vector newRecruits = wait(getAll(initializationReplies)); - self->resolvers = newRecruits; - - return Void(); -} - -ACTOR Future newTLogServers(Reference self, - RecruitFromConfigurationReply recr, - Reference oldLogSystem, - std::vector>* initialConfChanges) { - if (self->configuration.usableRegions > 1) { - state Optional remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional(); - if (!self->dcId_locality.count(recr.dcId)) { - int8_t loc = self->getNextLocality(); - Standalone tr; - tr.set(tr.arena(), tagLocalityListKeyFor(recr.dcId), tagLocalityListValue(loc)); - initialConfChanges->push_back(tr); - self->dcId_locality[recr.dcId] = loc; - TraceEvent(SevWarn, "UnknownPrimaryDCID", self->dbgid).detail("PrimaryId", recr.dcId).detail("Loc", loc); - } - - if (!self->dcId_locality.count(remoteDcId)) { - int8_t loc = self->getNextLocality(); - Standalone tr; - tr.set(tr.arena(), tagLocalityListKeyFor(remoteDcId), tagLocalityListValue(loc)); - initialConfChanges->push_back(tr); - self->dcId_locality[remoteDcId] = loc; - TraceEvent(SevWarn, "UnknownRemoteDCID", self->dbgid).detail("RemoteId", remoteDcId).detail("Loc", loc); - } - - std::vector exclusionWorkerIds; - std::transform(recr.tLogs.begin(), - recr.tLogs.end(), - std::back_inserter(exclusionWorkerIds), - [](const WorkerInterface& in) { return in.id(); }); - std::transform(recr.satelliteTLogs.begin(), - recr.satelliteTLogs.end(), - std::back_inserter(exclusionWorkerIds), - [](const WorkerInterface& in) { return in.id(); }); - Future fRemoteWorkers = brokenPromiseToNever( - self->clusterController.recruitRemoteFromConfiguration.getReply(RecruitRemoteFromConfigurationRequest( - self->configuration, - remoteDcId, - recr.tLogs.size() * - std::max(1, self->configuration.desiredLogRouterCount / std::max(1, recr.tLogs.size())), - exclusionWorkerIds))); - - self->primaryLocality = self->dcId_locality[recr.dcId]; - self->logSystem = Reference(); // Cancels the actors in the previous log system. - Reference newLogSystem = wait(oldLogSystem->newEpoch(recr, - fRemoteWorkers, - self->clusterId, - self->configuration, - self->cstate.myDBState.recoveryCount + 1, - self->primaryLocality, - self->dcId_locality[remoteDcId], - self->allTags, - self->recruitmentStalled)); - self->logSystem = newLogSystem; - } else { - self->primaryLocality = tagLocalitySpecial; - self->logSystem = Reference(); // Cancels the actors in the previous log system. - Reference newLogSystem = wait(oldLogSystem->newEpoch(recr, - Never(), - self->clusterId, - self->configuration, - self->cstate.myDBState.recoveryCount + 1, - self->primaryLocality, - tagLocalitySpecial, - self->allTags, - self->recruitmentStalled)); - self->logSystem = newLogSystem; - } - return Void(); -} - -ACTOR Future newSeedServers(Reference self, - RecruitFromConfigurationReply recruits, - std::vector* servers) { - // This is only necessary if the database is at version 0 - servers->clear(); - if (self->lastEpochEnd) - return Void(); - - state int idx = 0; - state std::map, Tag> dcId_tags; - state int8_t nextLocality = 0; - while (idx < recruits.storageServers.size()) { - TraceEvent("MasterRecruitingInitialStorageServer", self->dbgid) - .detail("CandidateWorker", recruits.storageServers[idx].locality.toString()); - - InitializeStorageRequest isr; - isr.seedTag = dcId_tags.count(recruits.storageServers[idx].locality.dcId()) - ? dcId_tags[recruits.storageServers[idx].locality.dcId()] - : Tag(nextLocality, 0); - isr.storeType = self->configuration.storageServerStoreType; - isr.reqId = deterministicRandom()->randomUniqueID(); - isr.interfaceId = deterministicRandom()->randomUniqueID(); - isr.clusterId = self->clusterId; - - ErrorOr newServer = wait(recruits.storageServers[idx].storage.tryGetReply(isr)); - - if (newServer.isError()) { - if (!newServer.isError(error_code_recruitment_failed) && - !newServer.isError(error_code_request_maybe_delivered)) - throw newServer.getError(); - - TEST(true); // masterserver initial storage recuitment loop failed to get new server - wait(delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY)); - } else { - if (!dcId_tags.count(recruits.storageServers[idx].locality.dcId())) { - dcId_tags[recruits.storageServers[idx].locality.dcId()] = Tag(nextLocality, 0); - nextLocality++; - } - - Tag& tag = dcId_tags[recruits.storageServers[idx].locality.dcId()]; - tag.id++; - idx++; - - servers->push_back(newServer.get().interf); - } - } - - self->dcId_locality.clear(); - for (auto& it : dcId_tags) { - self->dcId_locality[it.first] = it.second.locality; - } - - TraceEvent("MasterRecruitedInitialStorageServers", self->dbgid) - .detail("TargetCount", self->configuration.storageTeamSize) - .detail("Servers", describe(*servers)); - - return Void(); -} - -Future waitCommitProxyFailure(std::vector const& commitProxies) { - std::vector> failed; - failed.reserve(commitProxies.size()); - for (auto commitProxy : commitProxies) { - failed.push_back(waitFailureClient(commitProxy.waitFailure, - SERVER_KNOBS->TLOG_TIMEOUT, - -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, - /*trace=*/true)); - } - ASSERT(failed.size() >= 1); - return tagError(quorum(failed, 1), commit_proxy_failed()); -} - -Future waitGrvProxyFailure(std::vector const& grvProxies) { - std::vector> failed; - failed.reserve(grvProxies.size()); - for (int i = 0; i < grvProxies.size(); i++) - failed.push_back(waitFailureClient(grvProxies[i].waitFailure, - SERVER_KNOBS->TLOG_TIMEOUT, - -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, - /*trace=*/true)); - ASSERT(failed.size() >= 1); - return tagError(quorum(failed, 1), grv_proxy_failed()); -} - -Future waitResolverFailure(std::vector const& resolvers) { - std::vector> failed; - failed.reserve(resolvers.size()); - for (auto resolver : resolvers) { - failed.push_back(waitFailureClient(resolver.waitFailure, - SERVER_KNOBS->TLOG_TIMEOUT, - -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, - /*trace=*/true)); - } - ASSERT(failed.size() >= 1); - return tagError(quorum(failed, 1), master_resolver_failed()); -} - -ACTOR Future updateLogsValue(Reference self, Database cx) { - state Transaction tr(cx); - loop { - try { - Optional> value = wait(tr.get(logsKey)); - ASSERT(value.present()); - auto logs = decodeLogsValue(value.get()); - - std::set logIds; - for (auto& log : logs.first) { - logIds.insert(log.first); - } - - bool found = false; - for (auto& logSet : self->logSystem->getLogSystemConfig().tLogs) { - for (auto& log : logSet.tLogs) { - if (logIds.count(log.id())) { - found = true; - break; - } - } - if (found) { - break; - } - } - - if (!found) { - TEST(true); // old master attempted to change logsKey - return Void(); - } - - tr.set(logsKey, self->logSystem->getLogsValue()); - wait(tr.commit()); - return Void(); - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -Future sendMasterRegistration(MasterData* self, - LogSystemConfig const& logSystemConfig, - std::vector commitProxies, - std::vector grvProxies, - std::vector resolvers, - DBRecoveryCount recoveryCount, - std::vector priorCommittedLogServers) { - RegisterMasterRequest masterReq; - masterReq.id = self->myInterface.id(); - masterReq.mi = self->myInterface.locality; - masterReq.logSystemConfig = logSystemConfig; - masterReq.commitProxies = commitProxies; - masterReq.grvProxies = grvProxies; - masterReq.resolvers = resolvers; - masterReq.recoveryCount = recoveryCount; - if (self->hasConfiguration) - masterReq.configuration = self->configuration; - masterReq.registrationCount = ++self->registrationCount; - masterReq.priorCommittedLogServers = priorCommittedLogServers; - masterReq.recoveryState = self->recoveryState; - masterReq.recoveryStalled = self->recruitmentStalled->get(); - masterReq.clusterId = self->clusterId; - return brokenPromiseToNever(self->clusterController.registerMaster.getReply(masterReq)); -} - -ACTOR Future updateRegistration(Reference self, Reference logSystem) { - state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True); - state Future trigger = self->registrationTrigger.onTrigger(); - state Future updateLogsKey; - - loop { - wait(trigger); - wait(delay(.001)); // Coalesce multiple changes - - trigger = self->registrationTrigger.onTrigger(); - - auto logSystemConfig = logSystem->getLogSystemConfig(); - TraceEvent("MasterUpdateRegistration", self->dbgid) - .detail("RecoveryCount", self->cstate.myDBState.recoveryCount) - .detail("OldestBackupEpoch", logSystemConfig.oldestBackupEpoch) - .detail("Logs", describe(logSystemConfig.tLogs)) - .detail("CStateUpdated", self->cstateUpdated.isSet()) - .detail("RecoveryState", self->recoveryState); - - if (!self->cstateUpdated.isSet()) { - wait(sendMasterRegistration(self.getPtr(), - logSystemConfig, - self->provisionalCommitProxies, - self->provisionalGrvProxies, - self->resolvers, - self->cstate.myDBState.recoveryCount, - self->cstate.prevDBState.getPriorCommittedLogServers())); - } else if (self->recoveryState >= RecoveryState::ACCEPTING_COMMITS) { - updateLogsKey = updateLogsValue(self, cx); - wait(sendMasterRegistration(self.getPtr(), - logSystemConfig, - self->commitProxies, - self->grvProxies, - self->resolvers, - self->cstate.myDBState.recoveryCount, - std::vector())); - } else { - // The master should enter the accepting commits phase soon, and then we will register again - TEST(true); // cstate is updated but we aren't accepting commits yet - } - } -} - -ACTOR Future> provisionalMaster(Reference parent, Future activate) { - wait(activate); - - // Register a fake commit proxy (to be provided right here) to make ourselves available to clients - parent->provisionalCommitProxies = std::vector(1); - parent->provisionalCommitProxies[0].provisional = true; - parent->provisionalCommitProxies[0].initEndpoints(); - parent->provisionalGrvProxies = std::vector(1); - parent->provisionalGrvProxies[0].provisional = true; - parent->provisionalGrvProxies[0].initEndpoints(); - state Future waitCommitProxyFailure = - waitFailureServer(parent->provisionalCommitProxies[0].waitFailure.getFuture()); - state Future waitGrvProxyFailure = - waitFailureServer(parent->provisionalGrvProxies[0].waitFailure.getFuture()); - parent->registrationTrigger.trigger(); - - auto lockedKey = parent->txnStateStore->readValue(databaseLockedKey).get(); - state bool locked = lockedKey.present() && lockedKey.get().size(); - - state Optional metadataVersion = parent->txnStateStore->readValue(metadataVersionKey).get(); - - // We respond to a minimal subset of the commit proxy protocol. Our sole purpose is to receive a single write-only - // transaction which might repair our configuration, and return it. - loop choose { - when(GetReadVersionRequest req = - waitNext(parent->provisionalGrvProxies[0].getConsistentReadVersion.getFuture())) { - if ((req.flags & GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY) && - (req.flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES) && parent->lastEpochEnd) { - GetReadVersionReply rep; - rep.version = parent->lastEpochEnd; - rep.locked = locked; - rep.metadataVersion = metadataVersion; - req.reply.send(rep); - } else - req.reply.send(Never()); // We can't perform causally consistent reads without recovering - } - when(CommitTransactionRequest req = waitNext(parent->provisionalCommitProxies[0].commit.getFuture())) { - req.reply.send(Never()); // don't reply (clients always get commit_unknown_result) - auto t = &req.transaction; - if (t->read_snapshot == parent->lastEpochEnd && //< So no transactions can fall between the read snapshot - // and the recovery transaction this (might) be merged with - // vvv and also the changes we will make in the recovery - // transaction (most notably to lastEpochEndKey) BEFORE we - // merge initialConfChanges won't conflict - !std::any_of(t->read_conflict_ranges.begin(), t->read_conflict_ranges.end(), [](KeyRangeRef const& r) { - return r.contains(lastEpochEndKey); - })) { - for (auto m = t->mutations.begin(); m != t->mutations.end(); ++m) { - TraceEvent("PM_CTM", parent->dbgid) - .detail("MType", m->type) - .detail("Param1", m->param1) - .detail("Param2", m->param2); - if (isMetadataMutation(*m)) { - // We keep the mutations and write conflict ranges from this transaction, but not its read - // conflict ranges - Standalone out; - out.read_snapshot = invalidVersion; - out.mutations.append_deep(out.arena(), t->mutations.begin(), t->mutations.size()); - out.write_conflict_ranges.append_deep( - out.arena(), t->write_conflict_ranges.begin(), t->write_conflict_ranges.size()); - return out; - } - } - } - } - when(GetKeyServerLocationsRequest req = - waitNext(parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture())) { - req.reply.send(Never()); - } - when(wait(waitCommitProxyFailure)) { throw worker_removed(); } - when(wait(waitGrvProxyFailure)) { throw worker_removed(); } - } -} - -ACTOR Future>> recruitEverything( - Reference self, - std::vector* seedServers, - Reference oldLogSystem) { - if (!self->configuration.isValid()) { - RecoveryStatus::RecoveryStatus status; - if (self->configuration.initialized) { - TraceEvent(SevWarn, "MasterRecoveryInvalidConfiguration", self->dbgid) - .setMaxEventLength(11000) - .setMaxFieldLength(10000) - .detail("Conf", self->configuration.toString()); - status = RecoveryStatus::configuration_invalid; - } else if (!self->cstate.prevDBState.tLogs.size()) { - status = RecoveryStatus::configuration_never_created; - self->neverCreated = true; - } else { - status = RecoveryStatus::configuration_missing; - } - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", status) - .detail("Status", RecoveryStatus::names[status]) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - return Never(); - } else - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::recruiting_transaction_servers) - .detail("Status", RecoveryStatus::names[RecoveryStatus::recruiting_transaction_servers]) - .detail("Conf", self->configuration.toString()) - .detail("RequiredCommitProxies", 1) - .detail("RequiredGrvProxies", 1) - .detail("RequiredResolvers", 1) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - // FIXME: we only need log routers for the same locality as the master - int maxLogRouters = self->cstate.prevDBState.logRouterTags; - for (auto& old : self->cstate.prevDBState.oldTLogData) { - maxLogRouters = std::max(maxLogRouters, old.logRouterTags); - } - - state RecruitFromConfigurationReply recruits = - wait(brokenPromiseToNever(self->clusterController.recruitFromConfiguration.getReply( - RecruitFromConfigurationRequest(self->configuration, self->lastEpochEnd == 0, maxLogRouters)))); - - std::string primaryDcIds, remoteDcIds; - - self->primaryDcId.clear(); - self->remoteDcIds.clear(); - if (recruits.dcId.present()) { - self->primaryDcId.push_back(recruits.dcId); - if (!primaryDcIds.empty()) { - primaryDcIds += ','; - } - primaryDcIds += printable(recruits.dcId); - if (self->configuration.regions.size() > 1) { - Key remoteDcId = recruits.dcId.get() == self->configuration.regions[0].dcId - ? self->configuration.regions[1].dcId - : self->configuration.regions[0].dcId; - self->remoteDcIds.push_back(remoteDcId); - if (!remoteDcIds.empty()) { - remoteDcIds += ','; - } - remoteDcIds += printable(remoteDcId); - } - } - self->backupWorkers.swap(recruits.backupWorkers); - - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::initializing_transaction_servers) - .detail("Status", RecoveryStatus::names[RecoveryStatus::initializing_transaction_servers]) - .detail("CommitProxies", recruits.commitProxies.size()) - .detail("GrvProxies", recruits.grvProxies.size()) - .detail("TLogs", recruits.tLogs.size()) - .detail("Resolvers", recruits.resolvers.size()) - .detail("SatelliteTLogs", recruits.satelliteTLogs.size()) - .detail("OldLogRouters", recruits.oldLogRouters.size()) - .detail("StorageServers", recruits.storageServers.size()) - .detail("BackupWorkers", self->backupWorkers.size()) - .detail("PrimaryDcIds", primaryDcIds) - .detail("RemoteDcIds", remoteDcIds) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - // Actually, newSeedServers does both the recruiting and initialization of the seed servers; so if this is a brand - // new database we are sort of lying that we are past the recruitment phase. In a perfect world we would split that - // up so that the recruitment part happens above (in parallel with recruiting the transaction servers?). - wait(newSeedServers(self, recruits, seedServers)); - state std::vector> confChanges; - wait(newCommitProxies(self, recruits) && newGrvProxies(self, recruits) && newResolvers(self, recruits) && - newTLogServers(self, recruits, oldLogSystem, &confChanges)); - return confChanges; -} - -ACTOR Future updateLocalityForDcId(Optional dcId, - Reference oldLogSystem, - Reference> locality) { - loop { - std::pair loc = oldLogSystem->getLogSystemConfig().getLocalityForDcId(dcId); - Version ver = locality->get().knownCommittedVersion; - if (ver == invalidVersion) { - ver = oldLogSystem->getKnownCommittedVersion(); - } - locality->set(PeekTxsInfo(loc.first, loc.second, ver)); - TraceEvent("UpdatedLocalityForDcId") - .detail("DcId", dcId) - .detail("Locality0", loc.first) - .detail("Locality1", loc.second) - .detail("Version", ver); - wait(oldLogSystem->onLogSystemConfigChange() || oldLogSystem->onKnownCommittedVersionChange()); - } -} - -ACTOR Future readTransactionSystemState(Reference self, - Reference oldLogSystem, - Version txsPoppedVersion) { - state Reference> myLocality = Reference>( - new AsyncVar(PeekTxsInfo(tagLocalityInvalid, tagLocalityInvalid, invalidVersion))); - state Future localityUpdater = - updateLocalityForDcId(self->myInterface.locality.dcId(), oldLogSystem, myLocality); - // Peek the txnStateTag in oldLogSystem and recover self->txnStateStore - - // For now, we also obtain the recovery metadata that the log system obtained during the end_epoch process for - // comparison - - // Sets self->lastEpochEnd and self->recoveryTransactionVersion - // Sets self->configuration to the configuration (FF/conf/ keys) at self->lastEpochEnd - - // Recover transaction state store - if (self->txnStateStore) - self->txnStateStore->close(); - self->txnStateLogAdapter = openDiskQueueAdapter(oldLogSystem, myLocality, txsPoppedVersion); - self->txnStateStore = - keyValueStoreLogSystem(self->txnStateLogAdapter, self->dbgid, self->memoryLimit, false, false, true); - - // Versionstamped operations (particularly those applied from DR) define a minimum commit version - // that we may recover to, as they embed the version in user-readable data and require that no - // transactions will be committed at a lower version. - Optional> requiredCommitVersion = - wait(self->txnStateStore->readValue(minRequiredCommitVersionKey)); - Version minRequiredCommitVersion = -1; - if (requiredCommitVersion.present()) { - minRequiredCommitVersion = BinaryReader::fromStringRef(requiredCommitVersion.get(), Unversioned()); - } - - // Recover version info - self->lastEpochEnd = oldLogSystem->getEnd() - 1; - if (self->lastEpochEnd == 0) { - self->recoveryTransactionVersion = 1; - } else { - if (self->forceRecovery) { - self->recoveryTransactionVersion = self->lastEpochEnd + SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT_FORCED; - } else { - self->recoveryTransactionVersion = self->lastEpochEnd + SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT; - } - - if (BUGGIFY) { - self->recoveryTransactionVersion += - deterministicRandom()->randomInt64(0, SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT); - } - if (self->recoveryTransactionVersion < minRequiredCommitVersion) - self->recoveryTransactionVersion = minRequiredCommitVersion; - } - - TraceEvent("MasterRecovering", self->dbgid) - .detail("LastEpochEnd", self->lastEpochEnd) - .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); - - RangeResult rawConf = wait(self->txnStateStore->readRange(configKeys)); - self->configuration.fromKeyValues(rawConf.castTo>()); - self->originalConfiguration = self->configuration; - self->hasConfiguration = true; - - TraceEvent("MasterRecoveredConfig", self->dbgid) - .setMaxEventLength(11000) - .setMaxFieldLength(10000) - .detail("Conf", self->configuration.toString()) - .trackLatest(self->recoveredConfigEventHolder->trackingKey); - - RangeResult rawLocalities = wait(self->txnStateStore->readRange(tagLocalityListKeys)); - self->dcId_locality.clear(); - for (auto& kv : rawLocalities) { - self->dcId_locality[decodeTagLocalityListKey(kv.key)] = decodeTagLocalityListValue(kv.value); - } - - RangeResult rawTags = wait(self->txnStateStore->readRange(serverTagKeys)); - self->allTags.clear(); - if (self->lastEpochEnd > 0) { - self->allTags.push_back(cacheTag); - } - - if (self->forceRecovery) { - self->safeLocality = oldLogSystem->getLogSystemConfig().tLogs[0].locality; - for (auto& kv : rawTags) { - Tag tag = decodeServerTagValue(kv.value); - if (tag.locality == self->safeLocality) { - self->allTags.push_back(tag); - } - } - } else { - for (auto& kv : rawTags) { - self->allTags.push_back(decodeServerTagValue(kv.value)); - } - } - - RangeResult rawHistoryTags = wait(self->txnStateStore->readRange(serverTagHistoryKeys)); - for (auto& kv : rawHistoryTags) { - self->allTags.push_back(decodeServerTagValue(kv.value)); - } - - uniquify(self->allTags); - - // auto kvs = self->txnStateStore->readRange( systemKeys ); - // for( auto & kv : kvs.get() ) - // TraceEvent("MasterRecoveredTXS", self->dbgid).detail("K", kv.key).detail("V", kv.value); - - self->txnStateLogAdapter->setNextVersion( - oldLogSystem->getEnd()); //< FIXME: (1) the log adapter should do this automatically after recovery; (2) if we - // make KeyValueStoreMemory guarantee immediate reads, we should be able to get rid of - // the discardCommit() below and not need a writable log adapter - - TraceEvent("RTSSComplete", self->dbgid).log(); - - return Void(); -} - -ACTOR Future sendInitialCommitToResolvers(Reference self) { - state KeyRange txnKeys = allKeys; - state Sequence txnSequence = 0; - ASSERT(self->recoveryTransactionVersion); - - state RangeResult data = - self->txnStateStore - ->readRange(txnKeys, BUGGIFY ? 3 : SERVER_KNOBS->DESIRED_TOTAL_BYTES, SERVER_KNOBS->DESIRED_TOTAL_BYTES) - .get(); - state std::vector> txnReplies; - state int64_t dataOutstanding = 0; - - state std::vector endpoints; - for (auto& it : self->commitProxies) { - endpoints.push_back(it.txnState.getEndpoint()); - } - - loop { - if (!data.size()) - break; - ((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end); - RangeResult nextData = - self->txnStateStore - ->readRange(txnKeys, BUGGIFY ? 3 : SERVER_KNOBS->DESIRED_TOTAL_BYTES, SERVER_KNOBS->DESIRED_TOTAL_BYTES) - .get(); - - TxnStateRequest req; - req.arena = data.arena(); - req.data = data; - req.sequence = txnSequence; - req.last = !nextData.size(); - req.broadcastInfo = endpoints; - txnReplies.push_back(broadcastTxnRequest(req, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, false)); - dataOutstanding += SERVER_KNOBS->TXN_STATE_SEND_AMOUNT * data.arena().getSize(); - data = nextData; - txnSequence++; - - if (dataOutstanding > SERVER_KNOBS->MAX_TXS_SEND_MEMORY) { - wait(waitForAll(txnReplies)); - txnReplies = std::vector>(); - dataOutstanding = 0; - } - - wait(yield()); - } - wait(waitForAll(txnReplies)); - TraceEvent("RecoveryInternal", self->dbgid) - .detail("StatusCode", RecoveryStatus::recovery_transaction) - .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) - .detail("Step", "SentTxnStateStoreToCommitProxies"); - - std::vector> replies; - for (auto& r : self->resolvers) { - ResolveTransactionBatchRequest req; - req.prevVersion = -1; - req.version = self->lastEpochEnd; - req.lastReceivedVersion = -1; - - replies.push_back(brokenPromiseToNever(r.resolve.getReply(req))); - } - - wait(waitForAll(replies)); - TraceEvent("RecoveryInternal", self->dbgid) - .detail("StatusCode", RecoveryStatus::recovery_transaction) - .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) - .detail("Step", "InitializedAllResolvers"); - return Void(); -} - -ACTOR Future triggerUpdates(Reference self, Reference oldLogSystem) { - loop { - wait(oldLogSystem->onLogSystemConfigChange() || self->cstate.fullyRecovered.getFuture() || - self->recruitmentStalled->onChange()); - if (self->cstate.fullyRecovered.isSet()) - return Void(); - - self->registrationTrigger.trigger(); - } -} - -ACTOR Future discardCommit(IKeyValueStore* store, LogSystemDiskQueueAdapter* adapter) { - state Future fcm = adapter->getCommitMessage(); - state Future committed = store->commit(); - LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm); - ASSERT(!committed.isReady()); - cm.acknowledge.send(Void()); - ASSERT(committed.isReady()); - return Void(); -} - -void updateConfigForForcedRecovery(Reference self, - std::vector>* initialConfChanges) { - bool regionsChanged = false; - for (auto& it : self->configuration.regions) { - if (it.dcId == self->myInterface.locality.dcId().get() && it.priority < 0) { - it.priority = 1; - regionsChanged = true; - } else if (it.dcId != self->myInterface.locality.dcId().get() && it.priority >= 0) { - it.priority = -1; - regionsChanged = true; - } - } - Standalone regionCommit; - regionCommit.mutations.push_back_deep( - regionCommit.arena(), - MutationRef(MutationRef::SetValue, configKeysPrefix.toString() + "usable_regions", LiteralStringRef("1"))); - self->configuration.applyMutation(regionCommit.mutations.back()); - if (regionsChanged) { - std::sort( - self->configuration.regions.begin(), self->configuration.regions.end(), RegionInfo::sort_by_priority()); - StatusObject regionJSON; - regionJSON["regions"] = self->configuration.getRegionJSON(); - regionCommit.mutations.push_back_deep( - regionCommit.arena(), - MutationRef(MutationRef::SetValue, - configKeysPrefix.toString() + "regions", - BinaryWriter::toValue(regionJSON, IncludeVersion(ProtocolVersion::withRegionConfiguration())) - .toString())); - self->configuration.applyMutation( - regionCommit.mutations.back()); // modifying the configuration directly does not change the configuration - // when it is re-serialized unless we call applyMutation - TraceEvent("ForcedRecoveryConfigChange", self->dbgid) - .setMaxEventLength(11000) - .setMaxFieldLength(10000) - .detail("Conf", self->configuration.toString()); - } - initialConfChanges->push_back(regionCommit); -} - -ACTOR Future recoverFrom(Reference self, - Reference oldLogSystem, - std::vector* seedServers, - std::vector>* initialConfChanges, - Future poppedTxsVersion, - bool* clusterIdExists) { - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::reading_transaction_system_state) - .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_transaction_system_state]) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - self->hasConfiguration = false; - - if (BUGGIFY) - wait(delay(10.0)); - - Version txsPoppedVersion = wait(poppedTxsVersion); - wait(readTransactionSystemState(self, oldLogSystem, txsPoppedVersion)); - for (auto& itr : *initialConfChanges) { - for (auto& m : itr.mutations) { - self->configuration.applyMutation(m); - } - } - - if (self->forceRecovery) { - updateConfigForForcedRecovery(self, initialConfChanges); - } - - debug_checkMaxRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery"); - - // Generate a cluster ID to uniquely identify the cluster if it doesn't - // already exist in the txnStateStore. - Optional clusterId = self->txnStateStore->readValue(clusterIdKey).get(); - *clusterIdExists = clusterId.present(); - if (!clusterId.present()) { - self->clusterId = deterministicRandom()->randomUniqueID(); - } else { - self->clusterId = BinaryReader::fromStringRef(clusterId.get(), Unversioned()); - } - - // Ordinarily we pass through this loop once and recover. We go around the loop if recovery stalls for more than a - // second, a provisional master is initialized, and an "emergency transaction" is submitted that might change the - // configuration so that we can finish recovery. - - state std::map, int8_t> originalLocalityMap = self->dcId_locality; - state Future>> recruitments = - recruitEverything(self, seedServers, oldLogSystem); - state double provisionalDelay = SERVER_KNOBS->PROVISIONAL_START_DELAY; - loop { - state Future> provisional = provisionalMaster(self, delay(provisionalDelay)); - provisionalDelay = - std::min(SERVER_KNOBS->PROVISIONAL_MAX_DELAY, provisionalDelay * SERVER_KNOBS->PROVISIONAL_DELAY_GROWTH); - choose { - when(std::vector> confChanges = wait(recruitments)) { - initialConfChanges->insert(initialConfChanges->end(), confChanges.begin(), confChanges.end()); - provisional.cancel(); - break; - } - when(Standalone _req = wait(provisional)) { - state Standalone req = _req; // mutable - TEST(true); // Emergency transaction processing during recovery - TraceEvent("EmergencyTransaction", self->dbgid).log(); - for (auto m = req.mutations.begin(); m != req.mutations.end(); ++m) - TraceEvent("EmergencyTransactionMutation", self->dbgid) - .detail("MType", m->type) - .detail("P1", m->param1) - .detail("P2", m->param2); - - DatabaseConfiguration oldConf = self->configuration; - self->configuration = self->originalConfiguration; - for (auto& m : req.mutations) - self->configuration.applyMutation(m); - - initialConfChanges->clear(); - if (self->originalConfiguration.isValid() && - self->configuration.usableRegions != self->originalConfiguration.usableRegions) { - TraceEvent(SevWarnAlways, "CannotChangeUsableRegions", self->dbgid).log(); - self->configuration = self->originalConfiguration; - } else { - initialConfChanges->push_back(req); - } - if (self->forceRecovery) { - updateConfigForForcedRecovery(self, initialConfChanges); - } - - if (self->configuration != oldConf) { // confChange does not trigger when including servers - self->dcId_locality = originalLocalityMap; - recruitments = recruitEverything(self, seedServers, oldLogSystem); - } - } - } - - provisional.cancel(); - } - - return Void(); -} - ACTOR Future getVersion(Reference self, GetCommitVersionRequest req) { state Span span("M:getVersion"_loc, { req.spanContext }); state std::map::iterator proxyItr = @@ -1283,139 +236,31 @@ ACTOR Future serveLiveCommittedVersion(Reference self) { } } -std::pair findRange(CoalescedKeyRangeMap& key_resolver, - Standalone>& movedRanges, - int src, - int dest) { - auto ranges = key_resolver.ranges(); - auto prev = ranges.begin(); - auto it = ranges.begin(); - ++it; - if (it == ranges.end()) { - if (ranges.begin().value() != src || - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(ranges.begin()->range(), dest)) != - movedRanges.end()) - throw operation_failed(); - return std::make_pair(ranges.begin().range(), true); - } - - std::set borders; - // If possible expand an existing boundary between the two resolvers - for (; it != ranges.end(); ++it) { - if (it->value() == src && prev->value() == dest && - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == - movedRanges.end()) { - return std::make_pair(it->range(), true); - } - if (it->value() == dest && prev->value() == src && - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(prev->range(), dest)) == - movedRanges.end()) { - return std::make_pair(prev->range(), false); - } - if (it->value() == dest) - borders.insert(prev->value()); - if (prev->value() == dest) - borders.insert(it->value()); - ++prev; - } - - prev = ranges.begin(); - it = ranges.begin(); - ++it; - // If possible create a new boundry which doesn't exist yet - for (; it != ranges.end(); ++it) { - if (it->value() == src && !borders.count(prev->value()) && - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == - movedRanges.end()) { - return std::make_pair(it->range(), true); - } - if (prev->value() == src && !borders.count(it->value()) && - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(prev->range(), dest)) == - movedRanges.end()) { - return std::make_pair(prev->range(), false); - } - ++prev; - } - - it = ranges.begin(); - for (; it != ranges.end(); ++it) { - if (it->value() == src && - std::find(movedRanges.begin(), movedRanges.end(), ResolverMoveRef(it->range(), dest)) == - movedRanges.end()) { - return std::make_pair(it->range(), true); - } - } - throw operation_failed(); // we are already attempting to move all of the data one resolver is assigned, so do not - // move anything -} - -ACTOR Future resolutionBalancing(Reference self) { - state CoalescedKeyRangeMap key_resolver; - key_resolver.insert(allKeys, 0); +ACTOR Future updateRecoveryData(Reference self) { loop { - wait(delay(SERVER_KNOBS->MIN_BALANCE_TIME, TaskPriority::ResolutionMetrics)); - while (self->resolverChanges.get().size()) - wait(self->resolverChanges.onChange()); - state std::vector> futures; - for (auto& p : self->resolvers) - futures.push_back( - brokenPromiseToNever(p.metrics.getReply(ResolutionMetricsRequest(), TaskPriority::ResolutionMetrics))); - wait(waitForAll(futures)); - state IndexedSet, NoMetric> metrics; + choose { + when(UpdateRecoveryDataRequest req = waitNext(self->myInterface.updateRecoveryData.getFuture())) { + TraceEvent("UpdateRecoveryData", self->dbgid) + .detail("RecoveryTxnVersion", req.recoveryTransactionVersion) + .detail("LastEpochEnd", req.lastEpochEnd) + .detail("NumCommitProxies", req.commitProxies.size()); - int64_t total = 0; - for (int i = 0; i < futures.size(); i++) { - total += futures[i].get().value; - metrics.insert(std::make_pair(futures[i].get().value, i), NoMetric()); - //TraceEvent("ResolverMetric").detail("I", i).detail("Metric", futures[i].get()); - } - if (metrics.lastItem()->first - metrics.begin()->first > SERVER_KNOBS->MIN_BALANCE_DIFFERENCE) { - try { - state int src = metrics.lastItem()->second; - state int dest = metrics.begin()->second; - state int64_t amount = std::min(metrics.lastItem()->first - total / self->resolvers.size(), - total / self->resolvers.size() - metrics.begin()->first) / - 2; - state Standalone> movedRanges; - - loop { - state std::pair range = findRange(key_resolver, movedRanges, src, dest); - - ResolutionSplitRequest req; - req.front = range.second; - req.offset = amount; - req.range = range.first; - - ResolutionSplitReply split = - wait(brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply( - req, TaskPriority::ResolutionMetrics))); - KeyRangeRef moveRange = range.second ? KeyRangeRef(range.first.begin, split.key) - : KeyRangeRef(split.key, range.first.end); - movedRanges.push_back_deep(movedRanges.arena(), ResolverMoveRef(moveRange, dest)); - TraceEvent("MovingResolutionRange") - .detail("Src", src) - .detail("Dest", dest) - .detail("Amount", amount) - .detail("StartRange", range.first) - .detail("MoveRange", moveRange) - .detail("Used", split.used) - .detail("KeyResolverRanges", key_resolver.size()); - amount -= split.used; - if (moveRange != range.first || amount <= 0) - break; + if (self->recoveryTransactionVersion == invalidVersion || + req.recoveryTransactionVersion > self->recoveryTransactionVersion) { + self->recoveryTransactionVersion = req.recoveryTransactionVersion; } - for (auto& it : movedRanges) - key_resolver.insert(it.range, it.dest); - // for(auto& it : key_resolver.ranges()) - // TraceEvent("KeyResolver").detail("Range", it.range()).detail("Value", it.value()); + if (self->lastEpochEnd == invalidVersion || req.lastEpochEnd > self->lastEpochEnd) { + self->lastEpochEnd = req.lastEpochEnd; + } + if (req.commitProxies.size() > 0) { + self->commitProxies = req.commitProxies; + self->lastCommitProxyVersionReplies.clear(); - self->resolverChangesVersion = self->version + 1; - for (auto& p : self->commitProxies) - self->resolverNeedingChanges.insert(p.id()); - self->resolverChanges.set(movedRanges); - } catch (Error& e) { - if (e.code() != error_code_operation_failed) - throw; + for (auto& p : self->commitProxies) { + self->lastCommitProxyVersionReplies[p.id()] = CommitProxyVersionReplies(); + } + } + req.reply.send(Void()); } } } @@ -1425,14 +270,14 @@ static std::set const& normalMasterErrors() { static std::set s; if (s.empty()) { s.insert(error_code_tlog_stopped); - s.insert(error_code_master_tlog_failed); + s.insert(error_code_tlog_failed); s.insert(error_code_commit_proxy_failed); s.insert(error_code_grv_proxy_failed); - s.insert(error_code_master_resolver_failed); - s.insert(error_code_master_backup_worker_failed); + s.insert(error_code_resolver_failed); + s.insert(error_code_backup_worker_failed); s.insert(error_code_recruitment_failed); s.insert(error_code_no_more_servers); - s.insert(error_code_master_recovery_failed); + s.insert(error_code_cluster_recovery_failed); s.insert(error_code_coordinated_state_conflict); s.insert(error_code_master_max_versions_in_flight); s.insert(error_code_worker_removed); @@ -1442,591 +287,6 @@ static std::set const& normalMasterErrors() { return s; } -ACTOR Future changeCoordinators(Reference self) { - loop { - ChangeCoordinatorsRequest req = waitNext(self->myInterface.changeCoordinators.getFuture()); - ++self->changeCoordinatorsRequests; - state ChangeCoordinatorsRequest changeCoordinatorsRequest = req; - - while (!self->cstate.previousWrite.isReady()) { - wait(self->cstate.previousWrite); - wait(delay( - 0)); // if a new core state is ready to be written, have that take priority over our finalizing write; - } - - if (!self->cstate.fullyRecovered.isSet()) { - wait(self->cstate.write(self->cstate.myDBState, true)); - } - - try { - wait(self->cstate.move(ClusterConnectionString(changeCoordinatorsRequest.newConnectionString.toString()))); - } catch (Error& e) { - if (e.code() != error_code_actor_cancelled) - changeCoordinatorsRequest.reply.sendError(e); - - throw; - } - - throw internal_error(); - } -} - -ACTOR Future rejoinRequestHandler(Reference self) { - loop { - TLogRejoinRequest req = waitNext(self->myInterface.tlogRejoin.getFuture()); - req.reply.send(true); - } -} - -// Keeps the coordinated state (cstate) updated as the set of recruited tlogs change through recovery. -ACTOR Future trackTlogRecovery(Reference self, - Reference>> oldLogSystems, - Future minRecoveryDuration) { - state Future rejoinRequests = Never(); - state DBRecoveryCount recoverCount = self->cstate.myDBState.recoveryCount + 1; - state DatabaseConfiguration configuration = - self->configuration; // self-configuration can be changed by configurationMonitor so we need a copy - loop { - state DBCoreState newState; - self->logSystem->toCoreState(newState); - newState.recoveryCount = recoverCount; - state Future changed = self->logSystem->onCoreStateChanged(); - - ASSERT(newState.tLogs[0].tLogWriteAntiQuorum == configuration.tLogWriteAntiQuorum && - newState.tLogs[0].tLogReplicationFactor == configuration.tLogReplicationFactor); - - state bool allLogs = - newState.tLogs.size() == - configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional()); - state bool finalUpdate = !newState.oldTLogData.size() && allLogs; - wait(self->cstate.write(newState, finalUpdate)); - if (self->cstateUpdated.canBeSet()) { - self->cstateUpdated.send(Void()); - } - - wait(minRecoveryDuration); - self->logSystem->coreStateWritten(newState); - - if (self->recoveryReadyForCommits.canBeSet()) { - self->recoveryReadyForCommits.send(Void()); - } - - if (finalUpdate) { - self->recoveryState = RecoveryState::FULLY_RECOVERED; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::fully_recovered) - .detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered]) - .detail("FullyRecoveredAtVersion", self->version) - .detail("ClusterId", self->clusterId) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - TraceEvent("MasterRecoveryGenerations", self->dbgid) - .detail("ActiveGenerations", 1) - .trackLatest("MasterRecoveryGenerations"); - } else if (!newState.oldTLogData.size() && self->recoveryState < RecoveryState::STORAGE_RECOVERED) { - self->recoveryState = RecoveryState::STORAGE_RECOVERED; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::storage_recovered) - .detail("Status", RecoveryStatus::names[RecoveryStatus::storage_recovered]) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - } else if (allLogs && self->recoveryState < RecoveryState::ALL_LOGS_RECRUITED) { - self->recoveryState = RecoveryState::ALL_LOGS_RECRUITED; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::all_logs_recruited) - .detail("Status", RecoveryStatus::names[RecoveryStatus::all_logs_recruited]) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - } - - if (newState.oldTLogData.size() && configuration.repopulateRegionAntiQuorum > 0 && - self->logSystem->remoteStorageRecovered()) { - TraceEvent(SevWarnAlways, "RecruitmentStalled_RemoteStorageRecovered", self->dbgid).log(); - self->recruitmentStalled->set(true); - } - self->registrationTrigger.trigger(); - - if (finalUpdate) { - oldLogSystems->get()->stopRejoins(); - rejoinRequests = rejoinRequestHandler(self); - return Void(); - } - - wait(changed); - } -} - -ACTOR Future configurationMonitor(Reference self, Database cx) { - loop { - state ReadYourWritesTransaction tr(cx); - - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY)); - ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY); - - DatabaseConfiguration conf; - conf.fromKeyValues((VectorRef)results); - if (conf != self->configuration) { - if (self->recoveryState != RecoveryState::ALL_LOGS_RECRUITED && - self->recoveryState != RecoveryState::FULLY_RECOVERED) { - throw master_recovery_failed(); - } - - self->configuration = conf; - self->registrationTrigger.trigger(); - } - - state Future watchFuture = - tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) || - tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) || - tr.watch(failedLocalityVersionKey); - wait(tr.commit()); - wait(watchFuture); - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - } -} - -ACTOR static Future> getMinBackupVersion(Reference self, Database cx) { - loop { - state ReadYourWritesTransaction tr(cx); - - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - Optional value = wait(tr.get(backupStartedKey)); - Optional minVersion; - if (value.present()) { - auto uidVersions = decodeBackupStartedValue(value.get()); - TraceEvent e("GotBackupStartKey", self->dbgid); - int i = 1; - for (auto [uid, version] : uidVersions) { - e.detail(format("BackupID%d", i), uid).detail(format("Version%d", i), version); - i++; - minVersion = minVersion.present() ? std::min(version, minVersion.get()) : version; - } - } else { - TraceEvent("EmptyBackupStartKey", self->dbgid).log(); - } - return minVersion; - - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -ACTOR static Future recruitBackupWorkers(Reference self, Database cx) { - ASSERT(self->backupWorkers.size() > 0); - - // Avoid race between a backup worker's save progress and the reads below. - wait(delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER)); - - state LogEpoch epoch = self->cstate.myDBState.recoveryCount; - state Reference backupProgress( - new BackupProgress(self->dbgid, self->logSystem->getOldEpochTagsVersionsInfo())); - state Future gotProgress = getBackupProgress(cx, self->dbgid, backupProgress, /*logging=*/true); - state std::vector> initializationReplies; - - state std::vector> idsTags; // worker IDs and tags for current epoch - state int logRouterTags = self->logSystem->getLogRouterTags(); - idsTags.reserve(logRouterTags); - for (int i = 0; i < logRouterTags; i++) { - idsTags.emplace_back(deterministicRandom()->randomUniqueID(), Tag(tagLocalityLogRouter, i)); - } - - const Version startVersion = self->logSystem->getBackupStartVersion(); - state int i = 0; - for (; i < logRouterTags; i++) { - const auto& worker = self->backupWorkers[i % self->backupWorkers.size()]; - InitializeBackupRequest req(idsTags[i].first); - req.recruitedEpoch = epoch; - req.backupEpoch = epoch; - req.routerTag = idsTags[i].second; - req.totalTags = logRouterTags; - req.startVersion = startVersion; - TraceEvent("BackupRecruitment", self->dbgid) - .detail("RequestID", req.reqId) - .detail("Tag", req.routerTag.toString()) - .detail("Epoch", epoch) - .detail("BackupEpoch", epoch) - .detail("StartVersion", req.startVersion); - initializationReplies.push_back( - transformErrors(throwErrorOr(worker.backup.getReplyUnlessFailedFor( - req, SERVER_KNOBS->BACKUP_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_backup_worker_failed())); - } - - state Future> fMinVersion = getMinBackupVersion(self, cx); - wait(gotProgress && success(fMinVersion)); - TraceEvent("MinBackupVersion", self->dbgid).detail("Version", fMinVersion.get().present() ? fMinVersion.get() : -1); - - std::map, std::map> toRecruit = - backupProgress->getUnfinishedBackup(); - for (const auto& [epochVersionTags, tagVersions] : toRecruit) { - const Version oldEpochEnd = std::get<1>(epochVersionTags); - if (!fMinVersion.get().present() || fMinVersion.get().get() + 1 >= oldEpochEnd) { - TraceEvent("SkipBackupRecruitment", self->dbgid) - .detail("MinVersion", fMinVersion.get().present() ? fMinVersion.get() : -1) - .detail("Epoch", epoch) - .detail("OldEpoch", std::get<0>(epochVersionTags)) - .detail("OldEpochEnd", oldEpochEnd); - continue; - } - for (const auto& [tag, version] : tagVersions) { - const auto& worker = self->backupWorkers[i % self->backupWorkers.size()]; - i++; - InitializeBackupRequest req(deterministicRandom()->randomUniqueID()); - req.recruitedEpoch = epoch; - req.backupEpoch = std::get<0>(epochVersionTags); - req.routerTag = tag; - req.totalTags = std::get<2>(epochVersionTags); - req.startVersion = version; // savedVersion + 1 - req.endVersion = std::get<1>(epochVersionTags) - 1; - TraceEvent("BackupRecruitment", self->dbgid) - .detail("RequestID", req.reqId) - .detail("Tag", req.routerTag.toString()) - .detail("Epoch", epoch) - .detail("BackupEpoch", req.backupEpoch) - .detail("StartVersion", req.startVersion) - .detail("EndVersion", req.endVersion.get()); - initializationReplies.push_back(transformErrors( - throwErrorOr(worker.backup.getReplyUnlessFailedFor( - req, SERVER_KNOBS->BACKUP_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), - master_backup_worker_failed())); - } - } - - std::vector newRecruits = wait(getAll(initializationReplies)); - self->logSystem->setBackupWorkers(newRecruits); - TraceEvent("BackupRecruitmentDone", self->dbgid).log(); - self->registrationTrigger.trigger(); - return Void(); -} - -ACTOR Future masterCore(Reference self) { - state TraceInterval recoveryInterval("MasterRecovery"); - state double recoverStartTime = now(); - - self->addActor.send(waitFailureServer(self->myInterface.waitFailure.getFuture())); - - TraceEvent(recoveryInterval.begin(), self->dbgid); - - self->recoveryState = RecoveryState::READING_CSTATE; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::reading_coordinated_state) - .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_coordinated_state]) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - wait(self->cstate.read()); - - self->recoveryState = RecoveryState::LOCKING_CSTATE; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::locking_coordinated_state) - .detail("Status", RecoveryStatus::names[RecoveryStatus::locking_coordinated_state]) - .detail("TLogs", self->cstate.prevDBState.tLogs.size()) - .detail("ActiveGenerations", self->cstate.myDBState.oldTLogData.size() + 1) - .detail("MyRecoveryCount", self->cstate.prevDBState.recoveryCount + 2) - .detail("ForceRecovery", self->forceRecovery) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - // for (const auto& old : self->cstate.prevDBState.oldTLogData) { - // TraceEvent("BWReadCoreState", self->dbgid).detail("Epoch", old.epoch).detail("Version", old.epochEnd); - //} - - TraceEvent("MasterRecoveryGenerations", self->dbgid) - .detail("ActiveGenerations", self->cstate.myDBState.oldTLogData.size() + 1) - .trackLatest(self->masterRecoveryGenerationsEventHolder->trackingKey); - - if (self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_OVERRIDE) { - if (self->cstate.myDBState.oldTLogData.size() >= CLIENT_KNOBS->MAX_GENERATIONS) { - TraceEvent(SevError, "RecoveryStoppedTooManyOldGenerations") - .detail("OldGenerations", self->cstate.myDBState.oldTLogData.size()) - .detail("Reason", - "Recovery stopped because too many recoveries have happened since the last time the cluster " - "was fully_recovered. Set --knob-max-generations-override on your server processes to a value " - "larger than OldGenerations to resume recovery once the underlying problem has been fixed."); - wait(Future(Never())); - } else if (self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION) { - TraceEvent(SevError, "RecoveryDelayedTooManyOldGenerations") - .detail("OldGenerations", self->cstate.myDBState.oldTLogData.size()) - .detail("Reason", - "Recovery is delayed because too many recoveries have happened since the last time the cluster " - "was fully_recovered. Set --knob-max-generations-override on your server processes to a value " - "larger than OldGenerations to resume recovery once the underlying problem has been fixed."); - wait(delay(CLIENT_KNOBS->RECOVERY_DELAY_SECONDS_PER_GENERATION * - (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION))); - } - if (g_network->isSimulated() && self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_SIM) { - g_simulator.connectionFailuresDisableDuration = 1e6; - g_simulator.speedUpSimulation = true; - TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyGenerations").log(); - } - } - - state Reference>> oldLogSystems(new AsyncVar>); - state Future recoverAndEndEpoch = ILogSystem::recoverAndEndEpoch(oldLogSystems, - self->dbgid, - self->cstate.prevDBState, - self->myInterface.tlogRejoin.getFuture(), - self->myInterface.locality, - &self->forceRecovery); - - DBCoreState newState = self->cstate.myDBState; - newState.recoveryCount++; - wait(self->cstate.write(newState) || recoverAndEndEpoch); - - self->recoveryState = RecoveryState::RECRUITING; - - state std::vector seedServers; - state std::vector> initialConfChanges; - state Future logChanges; - state Future minRecoveryDuration; - state Future poppedTxsVersion; - state bool clusterIdExists = false; - - loop { - Reference oldLogSystem = oldLogSystems->get(); - if (oldLogSystem) { - logChanges = triggerUpdates(self, oldLogSystem); - if (!minRecoveryDuration.isValid()) { - minRecoveryDuration = delay(SERVER_KNOBS->ENFORCED_MIN_RECOVERY_DURATION); - poppedTxsVersion = oldLogSystem->getTxsPoppedVersion(); - } - } - - state Future reg = oldLogSystem ? updateRegistration(self, oldLogSystem) : Never(); - self->registrationTrigger.trigger(); - - choose { - when(wait(oldLogSystem ? recoverFrom(self, - oldLogSystem, - &seedServers, - &initialConfChanges, - poppedTxsVersion, - std::addressof(clusterIdExists)) - : Never())) { - reg.cancel(); - break; - } - when(wait(oldLogSystems->onChange())) {} - when(wait(reg)) { throw internal_error(); } - when(wait(recoverAndEndEpoch)) { throw internal_error(); } - } - } - - if (self->neverCreated) { - recoverStartTime = now(); - } - - recoverAndEndEpoch.cancel(); - - ASSERT(self->commitProxies.size() <= self->configuration.getDesiredCommitProxies()); - ASSERT(self->commitProxies.size() >= 1); - ASSERT(self->grvProxies.size() <= self->configuration.getDesiredGrvProxies()); - ASSERT(self->grvProxies.size() >= 1); - ASSERT(self->resolvers.size() <= self->configuration.getDesiredResolvers()); - ASSERT(self->resolvers.size() >= 1); - - self->recoveryState = RecoveryState::RECOVERY_TRANSACTION; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::recovery_transaction) - .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction]) - .detail("PrimaryLocality", self->primaryLocality) - .detail("DcId", self->myInterface.locality.dcId()) - .detail("ClusterId", self->clusterId) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - // Recovery transaction - state bool debugResult = debug_checkMinRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery", SevWarn); - - CommitTransactionRequest recoveryCommitRequest; - recoveryCommitRequest.flags = recoveryCommitRequest.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE; - CommitTransactionRef& tr = recoveryCommitRequest.transaction; - int mmApplied = 0; // The number of mutations in tr.mutations that have been applied to the txnStateStore so far - if (self->lastEpochEnd != 0) { - Optional snapRecoveryFlag = self->txnStateStore->readValue(writeRecoveryKey).get(); - TraceEvent("MasterRecoverySnapshotCheck") - .detail("SnapRecoveryFlag", snapRecoveryFlag.present() ? snapRecoveryFlag.get().toString() : "N/A") - .detail("LastEpochEnd", self->lastEpochEnd); - if (snapRecoveryFlag.present()) { - TEST(true); // Recovering from snapshot, writing to snapShotEndVersionKey - BinaryWriter bw(Unversioned()); - tr.set(recoveryCommitRequest.arena, snapshotEndVersionKey, (bw << self->lastEpochEnd).toValue()); - // Pause the backups that got restored in this snapshot to avoid data corruption - // Requires further operational work to abort the backup - TraceEvent("MasterRecoveryPauseBackupAgents").log(); - Key backupPauseKey = FileBackupAgent::getPauseKey(); - tr.set(recoveryCommitRequest.arena, backupPauseKey, StringRef()); - // Clear the key so multiple recoveries will not overwrite the first version recorded - tr.clear(recoveryCommitRequest.arena, singleKeyRange(writeRecoveryKey)); - } - if (self->forceRecovery) { - BinaryWriter bw(Unversioned()); - tr.set(recoveryCommitRequest.arena, killStorageKey, (bw << self->safeLocality).toValue()); - } - - // This transaction sets \xff/lastEpochEnd, which the shard servers can use to roll back speculatively - // processed semi-committed transactions from the previous epoch. - // It also guarantees the shard servers and tlog servers eventually get versions in the new epoch, which - // clients might rely on. - // This transaction is by itself in a batch (has its own version number), which simplifies storage servers - // slightly (they assume there are no modifications to serverKeys in the same batch) The proxy also expects the - // lastEpochEndKey mutation to be first in the transaction - BinaryWriter bw(Unversioned()); - tr.set(recoveryCommitRequest.arena, lastEpochEndKey, (bw << self->lastEpochEnd).toValue()); - - if (self->forceRecovery) { - tr.set(recoveryCommitRequest.arena, rebootWhenDurableKey, StringRef()); - tr.set(recoveryCommitRequest.arena, - moveKeysLockOwnerKey, - BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned())); - } - } else { - // Recruit and seed initial shard servers - // This transaction must be the very first one in the database (version 1) - seedShardServers(recoveryCommitRequest.arena, tr, seedServers); - } - // initialConfChanges have not been conflict checked against any earlier writes in the recovery transaction, so do - // this as early as possible in the recovery transaction but see above comments as to why it can't be absolutely - // first. Theoretically emergency transactions should conflict check against the lastEpochEndKey. - for (auto& itr : initialConfChanges) { - tr.mutations.append_deep(recoveryCommitRequest.arena, itr.mutations.begin(), itr.mutations.size()); - tr.write_conflict_ranges.append_deep( - recoveryCommitRequest.arena, itr.write_conflict_ranges.begin(), itr.write_conflict_ranges.size()); - } - - tr.set( - recoveryCommitRequest.arena, primaryLocalityKey, BinaryWriter::toValue(self->primaryLocality, Unversioned())); - tr.set(recoveryCommitRequest.arena, backupVersionKey, backupVersionValue); - tr.set(recoveryCommitRequest.arena, coordinatorsKey, self->coordinators.ccr->getConnectionString().toString()); - tr.set(recoveryCommitRequest.arena, logsKey, self->logSystem->getLogsValue()); - tr.set(recoveryCommitRequest.arena, - primaryDatacenterKey, - self->myInterface.locality.dcId().present() ? self->myInterface.locality.dcId().get() : StringRef()); - - tr.clear(recoveryCommitRequest.arena, tLogDatacentersKeys); - for (auto& dc : self->primaryDcId) { - tr.set(recoveryCommitRequest.arena, tLogDatacentersKeyFor(dc), StringRef()); - } - if (self->configuration.usableRegions > 1) { - for (auto& dc : self->remoteDcIds) { - tr.set(recoveryCommitRequest.arena, tLogDatacentersKeyFor(dc), StringRef()); - } - } - - // Write cluster ID into txnStateStore if it is missing. - if (!clusterIdExists) { - tr.set(recoveryCommitRequest.arena, clusterIdKey, BinaryWriter::toValue(self->clusterId, Unversioned())); - } - - applyMetadataMutations(SpanID(), - self->dbgid, - recoveryCommitRequest.arena, - tr.mutations.slice(mmApplied, tr.mutations.size()), - self->txnStateStore); - mmApplied = tr.mutations.size(); - - tr.read_snapshot = self->recoveryTransactionVersion; // lastEpochEnd would make more sense, but isn't in the initial - // window of the resolver(s) - - TraceEvent("MasterRecoveryCommit", self->dbgid).log(); - state Future> recoveryCommit = self->commitProxies[0].commit.tryGetReply(recoveryCommitRequest); - self->addActor.send(self->logSystem->onError()); - self->addActor.send(waitResolverFailure(self->resolvers)); - self->addActor.send(waitCommitProxyFailure(self->commitProxies)); - self->addActor.send(waitGrvProxyFailure(self->grvProxies)); - self->addActor.send(provideVersions(self)); - self->addActor.send(serveLiveCommittedVersion(self)); - self->addActor.send(reportErrors(updateRegistration(self, self->logSystem), "UpdateRegistration", self->dbgid)); - self->registrationTrigger.trigger(); - - wait(discardCommit(self->txnStateStore, self->txnStateLogAdapter)); - - // Wait for the recovery transaction to complete. - // SOMEDAY: For faster recovery, do this and setDBState asynchronously and don't wait for them - // unless we want to change TLogs - wait((success(recoveryCommit) && sendInitialCommitToResolvers(self))); - if (recoveryCommit.isReady() && recoveryCommit.get().isError()) { - TEST(true); // Master recovery failed because of the initial commit failed - throw master_recovery_failed(); - } - - ASSERT(self->recoveryTransactionVersion != 0); - - self->recoveryState = RecoveryState::WRITING_CSTATE; - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::writing_coordinated_state) - .detail("Status", RecoveryStatus::names[RecoveryStatus::writing_coordinated_state]) - .detail("TLogList", self->logSystem->describe()) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - // Multiple masters prevent conflicts between themselves via CoordinatedState (self->cstate) - // 1. If SetMaster succeeds, then by CS's contract, these "new" Tlogs are the immediate - // successors of the "old" ones we are replacing - // 2. logSystem->recoverAndEndEpoch ensured that a co-quorum of the "old" tLogs were stopped at - // versions <= self->lastEpochEnd, so no versions > self->lastEpochEnd could be (fully) committed to them. - // 3. No other master will attempt to commit anything to our "new" Tlogs - // because it didn't recruit them - // 4. Therefore, no full commit can come between self->lastEpochEnd and the first commit - // we made to the new Tlogs (self->recoveryTransactionVersion), and only our own semi-commits can come between - // our first commit and the next new TLogs - - self->addActor.send(trackTlogRecovery(self, oldLogSystems, minRecoveryDuration)); - debug_advanceMaxCommittedVersion(UID(), self->recoveryTransactionVersion); - wait(self->recoveryReadyForCommits.getFuture()); - debug_advanceMinCommittedVersion(UID(), self->recoveryTransactionVersion); - - if (debugResult) { - TraceEvent(self->forceRecovery ? SevWarn : SevError, "DBRecoveryDurabilityError").log(); - } - - TraceEvent("MasterCommittedTLogs", self->dbgid) - .detail("TLogs", self->logSystem->describe()) - .detail("RecoveryCount", self->cstate.myDBState.recoveryCount) - .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); - - TraceEvent(recoveryInterval.end(), self->dbgid) - .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion); - - self->recoveryState = RecoveryState::ACCEPTING_COMMITS; - double recoveryDuration = now() - recoverStartTime; - - TraceEvent((recoveryDuration > 4 && !g_network->isSimulated()) ? SevWarnAlways : SevInfo, - "MasterRecoveryDuration", - self->dbgid) - .detail("RecoveryDuration", recoveryDuration) - .trackLatest(self->masterRecoveryDurationEventHolder->trackingKey); - - TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::accepting_commits) - .detail("Status", RecoveryStatus::names[RecoveryStatus::accepting_commits]) - .detail("StoreType", self->configuration.storageServerStoreType) - .detail("RecoveryDuration", recoveryDuration) - .trackLatest(self->masterRecoveryStateEventHolder->trackingKey); - - TraceEvent("MasterRecoveryAvailable", self->dbgid) - .detail("AvailableAtVersion", self->version) - .trackLatest(self->masterRecoveryAvailableEventHolder->trackingKey); - - if (self->resolvers.size() > 1) - self->addActor.send(resolutionBalancing(self)); - - self->addActor.send(changeCoordinators(self)); - Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True); - self->addActor.send(configurationMonitor(self, cx)); - if (self->configuration.backupWorkerEnabled) { - self->addActor.send(recruitBackupWorkers(self, cx)); - } else { - self->logSystem->setOldestBackupEpoch(self->cstate.myDBState.recoveryCount); - } - - wait(Future(Never())); - throw internal_error(); -} - ACTOR Future masterServer(MasterInterface mi, Reference const> db, Reference> const> ccInterface, @@ -2049,16 +309,18 @@ ACTOR Future masterServer(MasterInterface mi, state PromiseStream> addActor; state Reference self(new MasterData( db, mi, coordinators, db->get().clusterInterface, LiteralStringRef(""), addActor, forceRecovery)); - state Future collection = actorCollection(self->addActor.getFuture()); - self->addActor.send(traceRole(Role::MASTER, mi.id())); + state Future collection = actorCollection(addActor.getFuture()); + + addActor.send(traceRole(Role::MASTER, mi.id())); + addActor.send(provideVersions(self)); + addActor.send(serveLiveCommittedVersion(self)); + addActor.send(updateRecoveryData(self)); TEST(!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id())); // Master born doomed TraceEvent("MasterLifetime", self->dbgid).detail("LifetimeToken", lifetime.toString()); try { - state Future core = masterCore(self); loop choose { - when(wait(core)) { break; } when(wait(onDBChange)) { onDBChange = db->onChange(); if (!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id())) { @@ -2072,13 +334,6 @@ ACTOR Future masterServer(MasterInterface mi, throw worker_removed(); } } - when(BackupWorkerDoneRequest req = waitNext(mi.notifyBackupWorkerDone.getFuture())) { - if (self->logSystem.isValid() && self->logSystem->removeBackupWorker(req)) { - self->registrationTrigger.trigger(); - } - ++self->backupWorkerDoneRequests; - req.reply.send(Void()); - } when(wait(collection)) { ASSERT(false); throw internal_error(); @@ -2089,16 +344,15 @@ ACTOR Future masterServer(MasterInterface mi, if (e.code() != error_code_actor_cancelled) { wait(delay(0.0)); } - - while (!self->addActor.isEmpty()) { - self->addActor.getFuture().pop(); + while (!addActor.isEmpty()) { + addActor.getFuture().pop(); } - TEST(err.code() == error_code_master_tlog_failed); // Master: terminated due to tLog failure + TEST(err.code() == error_code_tlog_failed); // Master: terminated due to tLog failure TEST(err.code() == error_code_commit_proxy_failed); // Master: terminated due to commit proxy failure TEST(err.code() == error_code_grv_proxy_failed); // Master: terminated due to GRV proxy failure - TEST(err.code() == error_code_master_resolver_failed); // Master: terminated due to resolver failure - TEST(err.code() == error_code_master_backup_worker_failed); // Master: terminated due to backup worker failure + TEST(err.code() == error_code_resolver_failed); // Master: terminated due to resolver failure + TEST(err.code() == error_code_backup_worker_failed); // Master: terminated due to backup worker failure if (normalMasterErrors().count(err.code())) { TraceEvent("MasterTerminated", mi.id()).error(err); @@ -2106,5 +360,4 @@ ACTOR Future masterServer(MasterInterface mi, } throw err; } - return Void(); } diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 8ddad4707a..f709aa6e92 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -700,8 +700,8 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimaryDc") { // Manually set up a master address. NetworkAddress testAddress(IPAddress(0x13131313), 1); - testDbInfo.master.changeCoordinators = - RequestStream(Endpoint({ testAddress }, UID(1, 2))); + testDbInfo.master.getCommitVersion = + RequestStream(Endpoint({ testAddress }, UID(1, 2))); // First, create an empty TLogInterface, and check that it shouldn't be considered as in primary DC. testDbInfo.logSystemConfig.tLogs.push_back(TLogSet()); @@ -1772,12 +1772,10 @@ ACTOR Future workerServer(Reference connRecord, startRole(Role::MASTER, recruited.id(), interf.id()); DUMPTOKEN(recruited.waitFailure); - DUMPTOKEN(recruited.tlogRejoin); - DUMPTOKEN(recruited.changeCoordinators); DUMPTOKEN(recruited.getCommitVersion); DUMPTOKEN(recruited.getLiveCommittedVersion); DUMPTOKEN(recruited.reportLiveCommittedVersion); - DUMPTOKEN(recruited.notifyBackupWorkerDone); + DUMPTOKEN(recruited.updateRecoveryData); // printf("Recruited as masterServer\n"); Future masterProcess = masterServer( diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 5406911205..c25a386fdb 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -94,21 +94,22 @@ ERROR( never_reply, 1104, "Never reply to the request" ) ERROR( recruitment_failed, 1200, "Recruitment of a server failed" ) // Be careful, catching this will delete the data of a storage server or tlog permanently ERROR( move_to_removed_server, 1201, "Attempt to move keys to a storage server that was removed" ) ERROR( worker_removed, 1202, "Normal worker shut down" ) // Be careful, catching this will delete the data of a storage server or tlog permanently -ERROR( master_recovery_failed, 1203, "Master recovery failed") +ERROR( cluster_recovery_failed, 1203, "Cluster recovery failed") ERROR( master_max_versions_in_flight, 1204, "Master hit maximum number of versions in flight" ) -ERROR( master_tlog_failed, 1205, "Master terminating because a TLog failed" ) // similar to tlog_stopped, but the tlog has actually died +ERROR( tlog_failed, 1205, "Cluster recovery terminating because a TLog failed" ) // similar to tlog_stopped, but the tlog has actually died ERROR( worker_recovery_failed, 1206, "Recovery of a worker process failed" ) ERROR( please_reboot, 1207, "Reboot of server process requested" ) ERROR( please_reboot_delete, 1208, "Reboot of server process requested, with deletion of state" ) ERROR( commit_proxy_failed, 1209, "Master terminating because a CommitProxy failed" ) -ERROR( master_resolver_failed, 1210, "Master terminating because a Resolver failed" ) +ERROR( resolver_failed, 1210, "Cluster recovery terminating because a Resolver failed" ) ERROR( server_overloaded, 1211, "Server is under too much load and cannot respond" ) -ERROR( master_backup_worker_failed, 1212, "Master terminating because a backup worker failed") +ERROR( backup_worker_failed, 1212, "Cluster recovery terminating because a backup worker failed") ERROR( tag_throttled, 1213, "Transaction tag is being throttled" ) -ERROR( grv_proxy_failed, 1214, "Master terminating because a GRVProxy failed" ) +ERROR( grv_proxy_failed, 1214, "Cluster recovery terminating because a GRVProxy failed" ) ERROR( dd_tracker_cancelled, 1215, "The data distribution tracker has been cancelled" ) ERROR( failed_to_progress, 1216, "Process has failed to make sufficient progress" ) ERROR( invalid_cluster_id, 1217, "Attempted to join cluster with a different cluster ID" ) +ERROR( restart_cluster_controller, 1218, "Restart cluster controller process" ) // 15xx Platform errors ERROR( platform_error, 1500, "Platform error" ) diff --git a/flow/network.h b/flow/network.h index b753a23ec6..5d5b316d0d 100644 --- a/flow/network.h +++ b/flow/network.h @@ -84,6 +84,7 @@ enum class TaskPriority { GetConsistentReadVersion = 8500, GetLiveCommittedVersionReply = 8490, GetLiveCommittedVersion = 8480, + UpdateRecoveryTransactionVersion = 8470, DefaultPromiseEndpoint = 8000, DefaultOnMainThread = 7500, DefaultDelay = 7010,