diff --git a/contrib/alloc_instrumentation_traces.py b/contrib/alloc_instrumentation_traces.py new file mode 100755 index 0000000000..42268dc700 --- /dev/null +++ b/contrib/alloc_instrumentation_traces.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# alloc_instrumentation_traces.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" +Example trace: +{ "Severity": "10", "Time": "194.878474", "DateTime": "2022-02-01T16:28:27Z", "Type": "MemSample", "Machine": "2.1.1.0:2", "ID": "0000000000000000", "Count": "943", "TotalSize": "540000000", "SampleCount": "54", "Hash": "980074757", "Bt": "addr2line -e fdbserver.debug -p -C -f -i 0x1919b72 0x3751d43 0x37518cc 0x19930f8 0x199dac3 0x1999e7c 0x21a1061 0x31e8fc5 0x31e784a 0x10ab3a8 0x36bf4c6 0x36bf304 0x36beea4 0x36bf352 0x36bfa1c 0x10ab3a8 0x37b22fe 0x37a16ee 0x368c754 0x19202d5 0x7fb3fe2d6555 0x1077029", "ThreadID": "10074331651862410074", "LogGroup": "default" } +""" + + +# This program analyzes MemSample trace events produced by setting ALLOC_INSTRUMENTATION in FastAlloc.h +# It outputs the top memory users by total size as well as number of allocations. + +# Example usage: cat trace.* | ./alloc_instrumentation_traces.py + +import sys +import json + +byCnt = [] +bySize = [] +totalSize = 0 + +lastTimestamp = "" + +for line in sys.stdin: + ev = json.loads(line.rstrip()) + type = ev["Type"] + + if (type != 'MemSample'): + continue + bt = ev["Bt"] + + if (bt == "na"): + continue + + timestamp = ev["Time"] + cnt = int(ev["Count"]) + scnt = int(ev["SampleCount"]) + size = int(ev["TotalSize"]) + h = ev["Hash"] + + if (timestamp != lastTimestamp): + byCnt = [] + bySize = [] + totalSize = 0 + lastTimestamp = timestamp + + + # print(str(cnt) + " " + str(scnt) + " " + str(size) + " " + h) + + byCnt.append( (cnt, scnt, size, h, bt) ) + bySize.append( (size, cnt, size, h, bt) ) + totalSize += size + +byCnt.sort(reverse=True) +bySize.sort(reverse=True) + +btByHash = {} + +byte_suffix = ["Bytes", "KB", "MB", "GB", "TB"] +def byte_str(bytes): + suffix_idx = 0 + while (bytes >= 1024 * 10): + suffix_idx += 1 + bytes //= 1024 + return str(bytes) + ' ' + byte_suffix[suffix_idx] + +print("By Size") +print("-------\r\n") +for x in bySize[:10]: + # print(str(x[0]) + ": " + x[3]) + print(str(x[1]) + " / " + byte_str(x[0]) + " (" + byte_str(x[0] // x[1]) + " per alloc):\r\n" + x[4] + "\r\n") + btByHash[x[3]] = x[4] + +print() +print("By Count") +print("--------\r\n") +for x in byCnt[:5]: + # print(str(x[0]) + ": " + x[3]) + print(str(x[0]) + " / " + byte_str(x[2]) + " (" + byte_str(x[2] // x[0]) + " per alloc):\r\n" + x[4] + "\r\n") + btByHash[x[3]] = x[4] + diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp index acb02843b2..6662d7a7ac 100644 --- a/fdbcli/ConfigureCommand.actor.cpp +++ b/fdbcli/ConfigureCommand.actor.cpp @@ -265,7 +265,7 @@ CommandFactory configureFactory( "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" "count=|perpetual_storage_wiggle=|perpetual_storage_wiggle_locality=" "<:|0>|storage_migration_type={disabled|gradual|aggressive}" - "|tenant_mode={disabled|optional_experimental|required_experimental}", + "|tenant_mode={disabled|optional_experimental|required_experimental}|blob_granules_enabled={0|1}", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing " "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index d875be6f9e..c956af3dd4 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -790,6 +790,7 @@ void configureGenerator(const char* text, const char* line, std::vectornumRangesAssigned; }); specialCounter(cc, "MutationBytesBuffered", [this]() { return this->mutationBytesBuffered; }); specialCounter(cc, "ActiveReadRequests", [this]() { return this->activeReadRequests; }); diff --git a/fdbclient/BlobWorkerInterface.h b/fdbclient/BlobWorkerInterface.h index 48fd92ddb3..f69b73e1bc 100644 --- a/fdbclient/BlobWorkerInterface.h +++ b/fdbclient/BlobWorkerInterface.h @@ -34,6 +34,7 @@ struct BlobWorkerInterface { RequestStream blobGranuleFileRequest; RequestStream assignBlobRangeRequest; RequestStream revokeBlobRangeRequest; + RequestStream granuleAssignmentsRequest; RequestStream granuleStatusStreamRequest; RequestStream haltBlobWorker; @@ -58,6 +59,7 @@ struct BlobWorkerInterface { blobGranuleFileRequest, assignBlobRangeRequest, revokeBlobRangeRequest, + granuleAssignmentsRequest, granuleStatusStreamRequest, haltBlobWorker, locality, @@ -94,19 +96,6 @@ struct BlobGranuleFileRequest { } }; -struct AssignBlobRangeReply { - constexpr static FileIdentifier file_identifier = 6431923; - bool epochOk; // false if the worker has seen a new manager - - AssignBlobRangeReply() {} - explicit AssignBlobRangeReply(bool epochOk) : epochOk(epochOk) {} - - template - void serialize(Ar& ar) { - serializer(ar, epochOk); - } -}; - struct RevokeBlobRangeRequest { constexpr static FileIdentifier file_identifier = 4844288; Arena arena; @@ -114,7 +103,7 @@ struct RevokeBlobRangeRequest { int64_t managerEpoch; int64_t managerSeqno; bool dispose; - ReplyPromise reply; + ReplyPromise reply; RevokeBlobRangeRequest() {} @@ -124,6 +113,12 @@ struct RevokeBlobRangeRequest { } }; +/* + * Continue: Blob worker should continue handling a granule that was evaluated for a split + * Normal: Blob worker should open the granule and start processing it + */ +enum AssignRequestType { Normal = 0, Continue = 1 }; + struct AssignBlobRangeRequest { constexpr static FileIdentifier file_identifier = 905381; Arena arena; @@ -133,16 +128,15 @@ struct AssignBlobRangeRequest { // If continueAssignment is true, this is just to instruct the worker that it *still* owns the range, so it should // re-snapshot it and continue. - // For an initial assignment, reassignent, split, or merge, continueAssignment==false. - bool continueAssignment; + AssignRequestType type; - ReplyPromise reply; + ReplyPromise reply; AssignBlobRangeRequest() {} template void serialize(Ar& ar) { - serializer(ar, keyRange, managerEpoch, managerSeqno, continueAssignment, reply, arena); + serializer(ar, keyRange, managerEpoch, managerSeqno, type, reply, arena); } }; @@ -153,22 +147,22 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply { KeyRange granuleRange; bool doSplit; + bool writeHotSplit; int64_t epoch; int64_t seqno; UID granuleID; Version startVersion; - Version latestVersion; GranuleStatusReply() {} explicit GranuleStatusReply(KeyRange range, bool doSplit, + bool writeHotSplit, int64_t epoch, int64_t seqno, UID granuleID, - Version startVersion, - Version latestVersion) - : granuleRange(range), doSplit(doSplit), epoch(epoch), seqno(seqno), granuleID(granuleID), - startVersion(startVersion), latestVersion(latestVersion) {} + Version startVersion) + : granuleRange(range), doSplit(doSplit), writeHotSplit(writeHotSplit), epoch(epoch), seqno(seqno), + granuleID(granuleID), startVersion(startVersion) {} int expectedSize() const { return sizeof(GranuleStatusReply) + granuleRange.expectedSize(); } @@ -179,11 +173,11 @@ struct GranuleStatusReply : public ReplyPromiseStreamReply { ReplyPromiseStreamReply::sequence, granuleRange, doSplit, + writeHotSplit, epoch, seqno, granuleID, - startVersion, - latestVersion); + startVersion); } }; @@ -220,4 +214,42 @@ struct HaltBlobWorkerRequest { } }; +struct GranuleAssignmentRef { + KeyRangeRef range; + int64_t epochAssigned; + int64_t seqnoAssigned; + + GranuleAssignmentRef() {} + + explicit GranuleAssignmentRef(KeyRangeRef range, int64_t epochAssigned, int64_t seqnoAssigned) + : range(range), epochAssigned(epochAssigned), seqnoAssigned(seqnoAssigned) {} + + template + void serialize(Ar& ar) { + serializer(ar, range, epochAssigned, seqnoAssigned); + } +}; + +struct GetGranuleAssignmentsReply { + constexpr static FileIdentifier file_identifier = 9191718; + Arena arena; + VectorRef assignments; + + template + void serialize(Ar& ar) { + serializer(ar, assignments, arena); + } +}; + +struct GetGranuleAssignmentsRequest { + constexpr static FileIdentifier file_identifier = 4121494; + int64_t managerEpoch; + ReplyPromise reply; + + template + void serialize(Ar& ar) { + serializer(ar, managerEpoch, reply); + } +}; + #endif diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index a092a57285..5f098ffa82 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -80,6 +80,7 @@ void ClientKnobs::initialize(Randomize randomize) { init( CHANGE_FEED_LOCATION_LIMIT, 10000 ); init( CHANGE_FEED_CACHE_SIZE, 100000 ); if( randomize && BUGGIFY ) CHANGE_FEED_CACHE_SIZE = 1; init( CHANGE_FEED_POP_TIMEOUT, 5.0 ); + init( CHANGE_FEED_STREAM_MIN_BYTES, 1e4 ); if( randomize && BUGGIFY ) CHANGE_FEED_STREAM_MIN_BYTES = 1; init( MAX_BATCH_SIZE, 1000 ); if( randomize && BUGGIFY ) MAX_BATCH_SIZE = 1; init( GRV_BATCH_TIMEOUT, 0.005 ); if( randomize && BUGGIFY ) GRV_BATCH_TIMEOUT = 0.1; @@ -275,8 +276,9 @@ void ClientKnobs::initialize(Randomize randomize) { init( BUSYNESS_SPIKE_START_THRESHOLD, 0.100 ); init( BUSYNESS_SPIKE_SATURATED_THRESHOLD, 0.500 ); - // blob granules - init( ENABLE_BLOB_GRANULES, false ); + // multi-version client control + init( MVC_CLIENTLIB_CHUNK_SIZE, 8*1024 ); + init( MVC_CLIENTLIB_CHUNKS_PER_TRANSACTION, 32 ); // clang-format on } diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h index 82b73464ad..8d6afa6d7b 100644 --- a/fdbclient/ClientKnobs.h +++ b/fdbclient/ClientKnobs.h @@ -79,6 +79,7 @@ public: int64_t CHANGE_FEED_LOCATION_LIMIT; int64_t CHANGE_FEED_CACHE_SIZE; double CHANGE_FEED_POP_TIMEOUT; + int64_t CHANGE_FEED_STREAM_MIN_BYTES; int MAX_BATCH_SIZE; double GRV_BATCH_TIMEOUT; @@ -267,8 +268,9 @@ public: double BUSYNESS_SPIKE_START_THRESHOLD; double BUSYNESS_SPIKE_SATURATED_THRESHOLD; - // blob granules - bool ENABLE_BLOB_GRANULES; + // multi-version client control + int MVC_CLIENTLIB_CHUNK_SIZE; + int MVC_CLIENTLIB_CHUNKS_PER_TRANSACTION; ClientKnobs(Randomize randomize); void initialize(Randomize randomize); diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index 2508991919..e0205b8628 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -50,6 +50,7 @@ void DatabaseConfiguration::resetInternal() { perpetualStorageWiggleSpeed = 0; perpetualStorageWiggleLocality = "0"; storageMigrationType = StorageMigrationType::DEFAULT; + blobGranulesEnabled = false; tenantMode = TenantMode::DISABLED; } @@ -404,6 +405,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed; result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality; result["storage_migration_type"] = storageMigrationType.toString(); + result["blob_granules_enabled"] = (int32_t)blobGranulesEnabled; result["tenant_mode"] = tenantMode.toString(); return result; } @@ -633,6 +635,9 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { tenantMode = (TenantMode::Mode)type; } else if (ck == LiteralStringRef("proxies")) { overwriteProxiesCount(); + } else if (ck == LiteralStringRef("blob_granules_enabled")) { + parse((&type), value); + blobGranulesEnabled = (type != 0); } else { return false; } diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 9d7012fc18..192d399bc2 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -250,6 +250,8 @@ struct DatabaseConfiguration { // Storage Migration Type StorageMigrationType storageMigrationType; + // Blob Granules + bool blobGranulesEnabled; TenantMode tenantMode; // Excluded servers (no state should be here) diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 2911137058..80a238327d 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -181,6 +181,7 @@ struct ChangeFeedStorageData : ReferenceCounted { NotifiedVersion version; NotifiedVersion desired; Promise destroyed; + UID interfToken; ~ChangeFeedStorageData() { destroyed.send(Void()); } }; @@ -196,6 +197,10 @@ struct ChangeFeedData : ReferenceCounted { std::vector> storageData; AsyncVar notAtLatest; Promise refresh; + Version maxSeenVersion; + Version endVersion = invalidVersion; + Version popVersion = + invalidVersion; // like TLog pop version, set by SS and client can check it to see if they missed data ChangeFeedData() : notAtLatest(1) {} }; @@ -292,6 +297,10 @@ public: StorageMetrics const& permittedError, int shardLimit, int expectedShardCount); + Future splitStorageMetricsStream(PromiseStream const& resultsStream, + KeyRange const& keys, + StorageMetrics const& limit, + StorageMetrics const& estimated); Future>> splitStorageMetrics(KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated); @@ -355,7 +364,9 @@ public: Key rangeID, Version begin = 0, Version end = std::numeric_limits::max(), - KeyRange range = allKeys); + KeyRange range = allKeys, + int replyBufferSize = -1, + bool canReadPopped = true); Future> getOverlappingChangeFeeds(KeyRangeRef ranges, Version minVersion); Future popChangeFeedMutations(Key rangeID, Version version); diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index 1b37a9ec4e..02f8882072 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -175,6 +175,17 @@ std::map configForToken(std::string const& mode) { } out[p + key] = format("%d", type); } + + if (key == "blob_granules_enabled") { + int enabled = std::stoi(value); + if (enabled != 0 && enabled != 1) { + printf("Error: Only 0 or 1 are valid values for blob_granules_enabled. " + "1 enables blob granules and 0 disables them.\n"); + return out; + } + out[p + key] = value; + } + if (key == "tenant_mode") { TenantMode tenantMode; if (value == "disabled") { diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 694b2dc569..a46c16641d 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -79,6 +79,7 @@ #include "flow/TLSConfig.actor.h" #include "flow/Tracing.h" #include "flow/UnitTest.h" +#include "flow/network.h" #include "flow/serialize.h" #ifdef ADDRESS_SANITIZER @@ -303,7 +304,6 @@ std::string printable(const VectorRef& val) { std::string printable(const StringRef& val) { return val.printable(); } - std::string printable(const std::string& str) { return StringRef(str).printable(); } @@ -457,7 +457,7 @@ ACTOR Future tssLogger(DatabaseContext* cx) { // Log each TSS pair separately for (const auto& it : cx->tssMetrics) { - if (it.second->mismatches.getIntervalDelta()) { + if (it.second->detailedMismatches.size()) { cx->tssMismatchStream.send( std::pair>(it.first, it.second->detailedMismatches)); } @@ -499,39 +499,41 @@ ACTOR Future databaseLogger(DatabaseContext* cx) { loop { wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace)); - TraceEvent ev("TransactionMetrics", cx->dbId); + if (!g_network->isSimulated()) { + TraceEvent ev("TransactionMetrics", cx->dbId); - ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged) - .detail("Cluster", - cx->getConnectionRecord() - ? cx->getConnectionRecord()->getConnectionString().clusterKeyName().toString() - : "") - .detail("Internal", cx->internal); + ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged) + .detail("Cluster", + cx->getConnectionRecord() + ? cx->getConnectionRecord()->getConnectionString().clusterKeyName().toString() + : "") + .detail("Internal", cx->internal); - cx->cc.logToTraceEvent(ev); + cx->cc.logToTraceEvent(ev); - ev.detail("LocationCacheEntryCount", cx->locationCache.size()); - ev.detail("MeanLatency", cx->latencies.mean()) - .detail("MedianLatency", cx->latencies.median()) - .detail("Latency90", cx->latencies.percentile(0.90)) - .detail("Latency98", cx->latencies.percentile(0.98)) - .detail("MaxLatency", cx->latencies.max()) - .detail("MeanRowReadLatency", cx->readLatencies.mean()) - .detail("MedianRowReadLatency", cx->readLatencies.median()) - .detail("MaxRowReadLatency", cx->readLatencies.max()) - .detail("MeanGRVLatency", cx->GRVLatencies.mean()) - .detail("MedianGRVLatency", cx->GRVLatencies.median()) - .detail("MaxGRVLatency", cx->GRVLatencies.max()) - .detail("MeanCommitLatency", cx->commitLatencies.mean()) - .detail("MedianCommitLatency", cx->commitLatencies.median()) - .detail("MaxCommitLatency", cx->commitLatencies.max()) - .detail("MeanMutationsPerCommit", cx->mutationsPerCommit.mean()) - .detail("MedianMutationsPerCommit", cx->mutationsPerCommit.median()) - .detail("MaxMutationsPerCommit", cx->mutationsPerCommit.max()) - .detail("MeanBytesPerCommit", cx->bytesPerCommit.mean()) - .detail("MedianBytesPerCommit", cx->bytesPerCommit.median()) - .detail("MaxBytesPerCommit", cx->bytesPerCommit.max()) - .detail("NumLocalityCacheEntries", cx->locationCache.size()); + ev.detail("LocationCacheEntryCount", cx->locationCache.size()); + ev.detail("MeanLatency", cx->latencies.mean()) + .detail("MedianLatency", cx->latencies.median()) + .detail("Latency90", cx->latencies.percentile(0.90)) + .detail("Latency98", cx->latencies.percentile(0.98)) + .detail("MaxLatency", cx->latencies.max()) + .detail("MeanRowReadLatency", cx->readLatencies.mean()) + .detail("MedianRowReadLatency", cx->readLatencies.median()) + .detail("MaxRowReadLatency", cx->readLatencies.max()) + .detail("MeanGRVLatency", cx->GRVLatencies.mean()) + .detail("MedianGRVLatency", cx->GRVLatencies.median()) + .detail("MaxGRVLatency", cx->GRVLatencies.max()) + .detail("MeanCommitLatency", cx->commitLatencies.mean()) + .detail("MedianCommitLatency", cx->commitLatencies.median()) + .detail("MaxCommitLatency", cx->commitLatencies.max()) + .detail("MeanMutationsPerCommit", cx->mutationsPerCommit.mean()) + .detail("MedianMutationsPerCommit", cx->mutationsPerCommit.median()) + .detail("MaxMutationsPerCommit", cx->mutationsPerCommit.max()) + .detail("MeanBytesPerCommit", cx->bytesPerCommit.mean()) + .detail("MedianBytesPerCommit", cx->bytesPerCommit.median()) + .detail("MaxBytesPerCommit", cx->bytesPerCommit.max()) + .detail("NumLocalityCacheEntries", cx->locationCache.size()); + } cx->latencies.clear(); cx->readLatencies.clear(); @@ -996,6 +998,8 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { loop { // state std::pair> data = waitNext(cx->tssMismatchStream.getFuture()); + // return to calling actor, don't do this as part of metrics loop + wait(delay(0)); // find ss pair id so we can remove it from the mapping state UID tssPairID; bool found = false; @@ -7288,9 +7292,7 @@ ACTOR Future>> getBlobGranuleRangesActor(Trans state KeyRange currentRange = keyRange; state Standalone> results; if (BG_REQUEST_DEBUG) { - printf("Getting Blob Granules for [%s - %s)\n", - keyRange.begin.printable().c_str(), - keyRange.end.printable().c_str()); + fmt::print("Getting Blob Granules for [{0} - {1})\n", keyRange.begin.printable(), keyRange.end.printable()); } self->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); loop { @@ -7303,6 +7305,7 @@ ACTOR Future>> getBlobGranuleRangesActor(Trans KeyRangeRef(blobGranuleMapping[i].key, blobGranuleMapping[i + 1].key)); } } + results.arena().dependsOn(blobGranuleMapping.arena()); if (blobGranuleMapping.more) { currentRange = KeyRangeRef(blobGranuleMapping.back().key, currentRange.end); } else { @@ -7334,17 +7337,17 @@ ACTOR Future>> readBlobGranulesActor( state KeyRange keyRange = range; state UID workerId; state int i; + state Version rv; state Standalone> results; if (read.present()) { - *readVersionOut = read.get(); + rv = read.get(); } else { Version _end = wait(self->getReadVersion()); - *readVersionOut = _end; + rv = _end; } self->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - // Right now just read whole blob range assignments from DB // FIXME: eventually we probably want to cache this and invalidate similarly to storage servers. // Cache misses could still read from the DB, or we could add it to the Transaction State Store and @@ -7365,11 +7368,11 @@ ACTOR Future>> readBlobGranulesActor( if (BG_REQUEST_DEBUG) { printf("no blob worker assignments yet\n"); } - throw transaction_too_old(); + throw blob_granule_transaction_too_old(); } if (BG_REQUEST_DEBUG) { - fmt::print("Doing blob granule request @ {}\n", *readVersionOut); + fmt::print("Doing blob granule request @ {}\n", rv); fmt::print("blob worker assignments:\n"); } @@ -7378,31 +7381,42 @@ ACTOR Future>> readBlobGranulesActor( granuleEndKey = blobGranuleMapping[i + 1].key; if (!blobGranuleMapping[i].value.size()) { if (BG_REQUEST_DEBUG) { - printf("Key range [%s - %s) missing worker assignment!\n", - granuleStartKey.printable().c_str(), - granuleEndKey.printable().c_str()); + fmt::print("Key range [{0} - {1}) missing worker assignment!\n", + granuleStartKey.printable(), + granuleEndKey.printable()); // TODO probably new exception type instead } - throw transaction_too_old(); + throw blob_granule_transaction_too_old(); } workerId = decodeBlobGranuleMappingValue(blobGranuleMapping[i].value); + if (workerId == UID()) { + if (BG_REQUEST_DEBUG) { + fmt::print("Key range [{0} - {1}) has no assigned worker yet!\n", + granuleStartKey.printable(), + granuleEndKey.printable()); + } + throw blob_granule_transaction_too_old(); + } if (BG_REQUEST_DEBUG) { - printf(" [%s - %s): %s\n", - granuleStartKey.printable().c_str(), - granuleEndKey.printable().c_str(), - workerId.toString().c_str()); + fmt::print( + " [{0} - {1}): {2}\n", granuleStartKey.printable(), granuleEndKey.printable(), workerId.toString()); } if (!self->trState->cx->blobWorker_interf.count(workerId)) { Optional workerInterface = wait(self->get(blobWorkerListKeyFor(workerId))); + // from the time the mapping was read from the db, the associated blob worker + // could have died and so its interface wouldn't be present as part of the blobWorkerList + // we persist in the db. So throw wrong_shard_server to get the new mapping if (!workerInterface.present()) { - throw wrong_shard_server(); + // need to re-read mapping, throw transaction_too_old so client retries. TODO better error? + // throw wrong_shard_server(); + throw transaction_too_old(); } // FIXME: maybe just want to insert here if there are racing queries for the same worker or something? self->trState->cx->blobWorker_interf[workerId] = decodeBlobWorkerListValue(workerInterface.get()); if (BG_REQUEST_DEBUG) { - printf(" decoded worker interface for %s\n", workerId.toString().c_str()); + fmt::print(" decoded worker interface for {0}\n", workerId.toString()); } } } @@ -7427,7 +7441,7 @@ ACTOR Future>> readBlobGranulesActor( state BlobGranuleFileRequest req; req.keyRange = KeyRangeRef(StringRef(req.arena, granuleStartKey), StringRef(req.arena, granuleEndKey)); req.beginVersion = begin; - req.readVersion = *readVersionOut; + req.readVersion = rv; std::vector>> v; v.push_back( @@ -7435,44 +7449,74 @@ ACTOR Future>> readBlobGranulesActor( state Reference>> location = makeReference(v); // use load balance with one option for now for retry and error handling - BlobGranuleFileReply rep = wait(loadBalance(location, - &BlobWorkerInterface::blobGranuleFileRequest, - req, - TaskPriority::DefaultPromiseEndpoint, - AtMostOnce::False, - nullptr)); + try { + choose { + when(BlobGranuleFileReply rep = wait(loadBalance(location, + &BlobWorkerInterface::blobGranuleFileRequest, + req, + TaskPriority::DefaultPromiseEndpoint, + AtMostOnce::False, + nullptr))) { + if (BG_REQUEST_DEBUG) { + fmt::print("Blob granule request for [{0} - {1}) @ {2} - {3} got reply from {4}:\n", + granuleStartKey.printable(), + granuleEndKey.printable(), + begin, + rv, + workerId.toString()); + } + results.arena().dependsOn(rep.arena); + for (auto& chunk : rep.chunks) { + if (BG_REQUEST_DEBUG) { + fmt::print( + "[{0} - {1})\n", chunk.keyRange.begin.printable(), chunk.keyRange.end.printable()); - if (BG_REQUEST_DEBUG) { - fmt::print("Blob granule request for [{0} - {1}) @ {2} - {3} got reply from {4}:\n", - granuleStartKey.printable(), - granuleEndKey.printable(), - begin, - *readVersionOut, - workerId.toString()); - } - results.arena().dependsOn(rep.arena); - for (auto& chunk : rep.chunks) { - if (BG_REQUEST_DEBUG) { - fmt::print("[{0} - {1})\n", chunk.keyRange.begin.printable(), chunk.keyRange.end.printable()); + fmt::print(" SnapshotFile: {0}\n \n DeltaFiles:\n", + chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() + : ""); + for (auto& df : chunk.deltaFiles) { + fmt::print(" {0}\n", df.toString()); + } + fmt::print(" Deltas: ({0})", chunk.newDeltas.size()); + if (chunk.newDeltas.size() > 0) { + fmt::print(" with version [{0} - {1}]", + chunk.newDeltas[0].version, + chunk.newDeltas[chunk.newDeltas.size() - 1].version); + } + fmt::print(" IncludedVersion: {0}\n\n\n", chunk.includedVersion); + } - fmt::print(" SnapshotFile: {0}\n \n DeltaFiles:\n", - chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : ""); - for (auto& df : chunk.deltaFiles) { - fmt::print(" {0}\n", df.toString()); + results.push_back(results.arena(), chunk); + keyRange = KeyRangeRef(std::min(chunk.keyRange.end, keyRange.end), keyRange.end); + } } - fmt::print(" Deltas: ({0})", chunk.newDeltas.size()); - if (chunk.newDeltas.size() > 0) { - fmt::print(" with version [{0} - {1}]", - chunk.newDeltas[0].version, - chunk.newDeltas[chunk.newDeltas.size() - 1].version); + // if we detect that this blob worker fails, cancel the request, as otherwise load balance will + // retry indefinitely with one option + when(wait(IFailureMonitor::failureMonitor().onStateEqual( + location->get(0, &BlobWorkerInterface::blobGranuleFileRequest).getEndpoint(), + FailureStatus(true)))) { + if (BG_REQUEST_DEBUG) { + fmt::print("readBlobGranules got BW {0} failed\n", workerId.toString()); + } + + throw connection_failed(); } - fmt::print(" IncludedVersion: {0}\n\n\n", chunk.includedVersion); } - - results.push_back(results.arena(), chunk); - keyRange = KeyRangeRef(std::min(chunk.keyRange.end, keyRange.end), keyRange.end); + } catch (Error& e) { + if (BG_REQUEST_DEBUG) { + fmt::print("BGReq got error {}\n", e.name()); + } + // worker is up but didn't actually have granule, or connection failed + if (e.code() == error_code_wrong_shard_server || e.code() == error_code_connection_failed) { + // need to re-read mapping, throw transaction_too_old so client retries. TODO better error? + throw transaction_too_old(); + } + throw e; } } + if (readVersionOut != nullptr) { + *readVersionOut = rv; + } return results; } @@ -7528,6 +7572,109 @@ ACTOR Future>> readStorageWiggleV return res; } +ACTOR Future splitStorageMetricsStream(PromiseStream resultStream, + Database cx, + KeyRange keys, + StorageMetrics limit, + StorageMetrics estimated) { + state Span span("NAPI:SplitStorageMetricsStream"_loc); + state Key beginKey = keys.begin; + state Key globalLastKey = beginKey; + resultStream.send(beginKey); + // track used across loops + state StorageMetrics globalUsed; + loop { + state std::vector locations = + wait(getKeyRangeLocations(cx, + Optional(), + KeyRangeRef(beginKey, keys.end), + CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT, + Reverse::False, + &StorageServerInterface::splitMetrics, + span.context, + Optional(), + UseProvisionalProxies::False, + latestVersion)); + try { + //TraceEvent("SplitStorageMetrics").detail("Locations", locations.size()); + + state StorageMetrics localUsed = globalUsed; + state Key localLastKey = globalLastKey; + state Standalone> results; + state int i = 0; + for (; i < locations.size(); i++) { + SplitMetricsRequest req(locations[i].range, + limit, + localUsed, + estimated, + i == locations.size() - 1 && keys.end <= locations.back().range.end); + SplitMetricsReply res = wait(loadBalance(locations[i].locations->locations(), + &StorageServerInterface::splitMetrics, + req, + TaskPriority::DataDistribution)); + if (res.splits.size() && + res.splits[0] <= localLastKey) { // split points are out of order, possibly because + // of moving data, throw error to retry + ASSERT_WE_THINK(false); // FIXME: This seems impossible and doesn't seem to be covered by testing + throw all_alternatives_failed(); + } + + if (res.splits.size()) { + results.append(results.arena(), res.splits.begin(), res.splits.size()); + results.arena().dependsOn(res.splits.arena()); + localLastKey = res.splits.back(); + } + localUsed = res.used; + + //TraceEvent("SplitStorageMetricsResult").detail("Used", used.bytes).detail("Location", i).detail("Size", res.splits.size()); + } + + globalUsed = localUsed; + + // only truncate split at end + if (keys.end <= locations.back().range.end && + globalUsed.allLessOrEqual(limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT) && + results.size() > 1) { + results.resize(results.arena(), results.size() - 1); + localLastKey = results.back(); + } + globalLastKey = localLastKey; + + for (auto& splitKey : results) { + resultStream.send(splitKey); + } + + if (keys.end <= locations.back().range.end) { + resultStream.send(keys.end); + resultStream.sendError(end_of_stream()); + break; + } else { + beginKey = locations.back().range.end; + } + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw e; + } + if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { + TraceEvent(SevError, "SplitStorageMetricsStreamError").error(e); + resultStream.sendError(e); + throw; + } + cx->invalidateCache(Key(), keys); + wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution)); + } + } + return Void(); +} + +Future DatabaseContext::splitStorageMetricsStream(const PromiseStream& resultStream, + KeyRange const& keys, + StorageMetrics const& limit, + StorageMetrics const& estimated) { + return ::splitStorageMetricsStream( + resultStream, Database(Reference::addRef(this)), keys, limit, estimated); +} + ACTOR Future>> splitStorageMetrics(Database cx, KeyRange keys, StorageMetrics limit, @@ -7566,8 +7713,8 @@ ACTOR Future>> splitStorageMetrics(Database cx, req, TaskPriority::DataDistribution)); if (res.splits.size() && - res.splits[0] <= results.back()) { // split points are out of order, possibly because of moving - // data, throw error to retry + res.splits[0] <= results.back()) { // split points are out of order, possibly because of + // moving data, throw error to retry ASSERT_WE_THINK( false); // FIXME: This seems impossible and doesn't seem to be covered by testing throw all_alternatives_failed(); @@ -7581,11 +7728,14 @@ ACTOR Future>> splitStorageMetrics(Database cx, //TraceEvent("SplitStorageMetricsResult").detail("Used", used.bytes).detail("Location", i).detail("Size", res.splits.size()); } - if (used.allLessOrEqual(limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT)) { + if (used.allLessOrEqual(limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT) && + results.size() > 1) { results.resize(results.arena(), results.size() - 1); } - results.push_back_deep(results.arena(), keys.end); + if (keys.end <= locations.back().range.end) { + results.push_back_deep(results.arena(), keys.end); + } return results; } catch (Error& e) { if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { @@ -7986,10 +8136,22 @@ ACTOR Future storageFeedVersionUpdater(StorageServerInterface interf, Chan return Void(); } if (self->version.get() < self->desired.get()) { - ChangeFeedVersionUpdateReply rep = wait(brokenPromiseToNever( - interf.changeFeedVersionUpdate.getReply(ChangeFeedVersionUpdateRequest(self->desired.get())))); - if (rep.version > self->version.get()) { - self->version.set(rep.version); + try { + ChangeFeedVersionUpdateReply rep = wait(brokenPromiseToNever( + interf.changeFeedVersionUpdate.getReply(ChangeFeedVersionUpdateRequest(self->desired.get())))); + + if (rep.version > self->version.get()) { + self->version.set(rep.version); + } + } catch (Error& e) { + if (e.code() == error_code_server_overloaded) { + if (FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY > CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME) { + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY - + CLIENT_KNOBS->CHANGE_FEED_EMPTY_BATCH_TIME)); + } + } else { + throw e; + } } } } else { @@ -7999,93 +8161,135 @@ ACTOR Future storageFeedVersionUpdater(StorageServerInterface interf, Chan } Reference DatabaseContext::getStorageData(StorageServerInterface interf) { - auto it = changeFeedUpdaters.find(interf.id()); + // use token from interface since that changes on SS restart + UID token = interf.waitFailure.getEndpoint().token; + auto it = changeFeedUpdaters.find(token); if (it == changeFeedUpdaters.end()) { Reference newStorageUpdater = makeReference(); newStorageUpdater->id = interf.id(); + newStorageUpdater->interfToken = token; newStorageUpdater->updater = storageFeedVersionUpdater(interf, newStorageUpdater.getPtr()); - changeFeedUpdaters[interf.id()] = newStorageUpdater; + changeFeedUpdaters[token] = newStorageUpdater; return newStorageUpdater; } return it->second; } Version ChangeFeedData::getVersion() { - if (notAtLatest.get() == 0 && mutations.isEmpty()) { - Version v = storageData[0]->version.get(); - for (int i = 1; i < storageData.size(); i++) { - if (storageData[i]->version.get() < v) { - v = storageData[i]->version.get(); - } - } - return std::max(v, lastReturnedVersion.get()); - } return lastReturnedVersion.get(); } -ACTOR Future changeFeedWhenAtLatest(ChangeFeedData* self, Version version) { - state Future lastReturned = self->lastReturnedVersion.whenAtLeast(version); - loop { - if (self->notAtLatest.get() == 0) { - std::vector> allAtLeast; - for (auto& it : self->storageData) { - if (it->version.get() < version) { - if (version > it->desired.get()) { - it->desired.set(version); - } - allAtLeast.push_back(it->version.whenAtLeast(version)); - } - } - choose { - when(wait(lastReturned)) { return Void(); } - when(wait(waitForAll(allAtLeast))) { - std::vector> onEmpty; - if (!self->mutations.isEmpty()) { - onEmpty.push_back(self->mutations.onEmpty()); - } - for (auto& it : self->streams) { - if (!it.isEmpty()) { - onEmpty.push_back(it.onEmpty()); - } - } - if (!onEmpty.size()) { - return Void(); - } - choose { - when(wait(waitForAll(onEmpty))) { - wait(delay(0)); - return Void(); - } - when(wait(lastReturned)) { return Void(); } - when(wait(self->refresh.getFuture())) {} - when(wait(self->notAtLatest.onChange())) {} - } - } - when(wait(self->refresh.getFuture())) {} - when(wait(self->notAtLatest.onChange())) {} - } - } else { - choose { - when(wait(lastReturned)) { return Void(); } - when(wait(self->notAtLatest.onChange())) {} - when(wait(self->refresh.getFuture())) {} +// This function is essentially bubbling the information about what has been processed from the server through the +// change feed client. First it makes sure the server has returned all mutations up through the target version, the +// native api has consumed and processed, them, and then the fdb client has consumed all of the mutations. +ACTOR Future changeFeedWaitLatest(Reference self, Version version) { + // wait on SS to have sent up through version + int desired = 0; + int waiting = 0; + std::vector> allAtLeast; + for (auto& it : self->storageData) { + if (it->version.get() < version) { + waiting++; + if (version > it->desired.get()) { + it->desired.set(version); + desired++; } + allAtLeast.push_back(it->version.whenAtLeast(version)); } } + + wait(waitForAll(allAtLeast)); + + // then, wait on ss streams to have processed up through version + std::vector> onEmpty; + for (auto& it : self->streams) { + if (!it.isEmpty()) { + onEmpty.push_back(it.onEmpty()); + } + } + + if (onEmpty.size()) { + wait(waitForAll(onEmpty)); + } + + if (self->mutations.isEmpty()) { + wait(delay(0)); + } + + // wait for merge cursor to fully process everything it read from its individual promise streams, either until it is + // done processing or we have up through the desired version + while (self->lastReturnedVersion.get() < self->maxSeenVersion && self->lastReturnedVersion.get() < version) { + Version target = std::min(self->maxSeenVersion, version); + wait(self->lastReturnedVersion.whenAtLeast(target)); + } + + // then, wait for client to have consumed up through version + if (self->maxSeenVersion >= version) { + // merge cursor may have something buffered but has not yet sent it to self->mutations, just wait for + // lastReturnedVersion + wait(self->lastReturnedVersion.whenAtLeast(version)); + } else { + // all mutations <= version are in self->mutations, wait for empty + while (!self->mutations.isEmpty()) { + wait(self->mutations.onEmpty()); + wait(delay(0)); + } + } + + return Void(); +} + +ACTOR Future changeFeedWhenAtLatest(Reference self, Version version) { + if (version >= self->endVersion) { + return Never(); + } + if (version <= self->getVersion()) { + return Void(); + } + state Future lastReturned = self->lastReturnedVersion.whenAtLeast(version); + loop { + // only allowed to use empty versions if you're caught up + Future waitEmptyVersion = (self->notAtLatest.get() == 0) ? changeFeedWaitLatest(self, version) : Never(); + choose { + when(wait(waitEmptyVersion)) { break; } + when(wait(lastReturned)) { break; } + when(wait(self->refresh.getFuture())) {} + when(wait(self->notAtLatest.onChange())) {} + } + } + + if (self->lastReturnedVersion.get() < version) { + self->lastReturnedVersion.set(version); + } + ASSERT(self->getVersion() >= version); + return Void(); } Future ChangeFeedData::whenAtLeast(Version version) { - return changeFeedWhenAtLatest(this, version); + return changeFeedWhenAtLatest(Reference::addRef(this), version); } -ACTOR Future singleChangeFeedStream(StorageServerInterface interf, - PromiseStream> results, - ReplyPromiseStream replyStream, - Version end, - Reference feedData, - Reference storageData) { +#define DEBUG_CF_CLIENT_TRACE false + +ACTOR Future partialChangeFeedStream(StorageServerInterface interf, + PromiseStream> results, + ReplyPromiseStream replyStream, + Version begin, + Version end, + Reference feedData, + Reference storageData, + UID debugUID) { + + // calling lastReturnedVersion's callbacks could cause us to be cancelled + state Promise refresh = feedData->refresh; state bool atLatestVersion = false; - state Version nextVersion = 0; + state Version nextVersion = begin; + // We don't need to force every other partial stream to do an empty if we get an empty, but if we get actual + // mutations back after sending an empty, we may need the other partial streams to get an empty, to advance the + // merge cursor, so we can send the mutations we just got. + // if lastEmpty != invalidVersion, we need to update the desired versions of the other streams BEFORE waiting + // onReady once getting a reply + state Version lastEmpty = invalidVersion; try { loop { if (nextVersion >= end) { @@ -8094,30 +8298,81 @@ ACTOR Future singleChangeFeedStream(StorageServerInterface interf, } choose { when(state ChangeFeedStreamReply rep = waitNext(replyStream.getFuture())) { + // handle first empty mutation on stream establishment explicitly + if (nextVersion == begin && rep.mutations.size() == 1 && rep.mutations[0].mutations.size() == 0 && + rep.mutations[0].version == begin - 1) { + continue; + } + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorReply", debugUID) + .detail("SSID", storageData->id) + .detail("AtLatest", atLatestVersion) + .detail("FirstVersion", rep.mutations.front().version) + .detail("LastVersion", rep.mutations.back().version) + .detail("Count", rep.mutations.size()) + .detail("MinStreamVersion", rep.minStreamVersion) + .detail("PopVersion", rep.popVersion) + .detail("RepAtLatest", rep.atLatestVersion); + } + + if (rep.mutations.back().version > feedData->maxSeenVersion) { + feedData->maxSeenVersion = rep.mutations.back().version; + } + if (rep.popVersion > feedData->popVersion) { + feedData->popVersion = rep.popVersion; + } + + if (lastEmpty != invalidVersion && !results.isEmpty()) { + for (auto& it : feedData->storageData) { + if (refresh.canBeSet() && lastEmpty > it->desired.get()) { + it->desired.set(lastEmpty); + } + } + lastEmpty = invalidVersion; + } + state int resultLoc = 0; while (resultLoc < rep.mutations.size()) { wait(results.onEmpty()); if (rep.mutations[resultLoc].version >= nextVersion) { results.send(rep.mutations[resultLoc]); + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSend", debugUID) + .detail("Version", rep.mutations[resultLoc].version) + .detail("Size", rep.mutations[resultLoc].mutations.size()); + } + + // check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we + // just skip to the next wait and get actor_cancelled + // FIXME: this is somewhat expensive to do every mutation. + for (auto& it : feedData->storageData) { + if (refresh.canBeSet() && rep.mutations[resultLoc].version > it->desired.get()) { + it->desired.set(rep.mutations[resultLoc].version); + } + } } else { ASSERT(rep.mutations[resultLoc].mutations.empty()); } resultLoc++; } - nextVersion = rep.mutations.back().version + 1; - if (!atLatestVersion && rep.atLatestVersion) { + // if we got the empty version that went backwards, don't decrease nextVersion + if (rep.mutations.back().version + 1 > nextVersion) { + nextVersion = rep.mutations.back().version + 1; + } + + if (refresh.canBeSet() && !atLatestVersion && rep.atLatestVersion) { atLatestVersion = true; feedData->notAtLatest.set(feedData->notAtLatest.get() - 1); } - if (rep.minStreamVersion > storageData->version.get()) { + if (refresh.canBeSet() && rep.minStreamVersion > storageData->version.get()) { storageData->version.set(rep.minStreamVersion); } - - for (auto& it : feedData->storageData) { - if (rep.mutations.back().version > it->desired.get()) { - it->desired.set(rep.mutations.back().version); - } + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorReplyDone", debugUID) + .detail("AtLatestNow", atLatestVersion); } } when(wait(atLatestVersion && replyStream.isEmpty() && results.isEmpty() @@ -8127,12 +8382,20 @@ ACTOR Future singleChangeFeedStream(StorageServerInterface interf, empty.version = storageData->version.get(); results.send(empty); nextVersion = storageData->version.get() + 1; + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSendEmpty", debugUID) + .detail("Version", empty.version); + } + lastEmpty = empty.version; } when(wait(atLatestVersion && replyStream.isEmpty() && !results.isEmpty() ? results.onEmpty() : Future(Never()))) {} } } } catch (Error& e) { + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorError", debugUID).errorUnsuppressed(e); + } if (e.code() == error_code_actor_cancelled) { throw; } @@ -8141,16 +8404,137 @@ ACTOR Future singleChangeFeedStream(StorageServerInterface interf, } } +ACTOR Future mergeChangeFeedStreamInternal(Reference results, + std::vector> interfs, + std::vector streams, + Version* begin, + Version end, + UID mergeCursorUID) { + state Promise refresh = results->refresh; + // with empty version handling in the partial cursor, all streams will always have a next element with version >= + // the minimum version of any stream's next element + state std::priority_queue> mutations; + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorStart", mergeCursorUID) + .detail("StreamCount", interfs.size()) + .detail("Begin", *begin) + .detail("End", end); + } + + // previous version of change feed may have put a mutation in the promise stream and then immediately died. Wait for + // that mutation first, so the promise stream always starts empty + wait(results->mutations.onEmpty()); + wait(delay(0)); + ASSERT(results->mutations.isEmpty()); + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorGotEmpty", mergeCursorUID); + } + + // update lastReturned once the previous mutation has been consumed + if (*begin - 1 > results->lastReturnedVersion.get()) { + results->lastReturnedVersion.set(*begin - 1); + } + + state int interfNum = 0; + + state std::vector streamsUsed; + // initially, pull from all streams + for (auto& stream : streams) { + streamsUsed.push_back(stream); + } + + state Version nextVersion; + loop { + // bring all of the streams up to date to ensure we have the latest element from each stream in mutations + interfNum = 0; + while (interfNum < streamsUsed.size()) { + try { + Standalone res = waitNext(streamsUsed[interfNum].results.getFuture()); + streamsUsed[interfNum].next = res; + mutations.push(streamsUsed[interfNum]); + } catch (Error& e) { + if (e.code() != error_code_end_of_stream) { + throw e; + } + } + interfNum++; + } + + if (mutations.empty()) { + throw end_of_stream(); + } + + streamsUsed.clear(); + + // Without this delay, weird issues with the last stream getting on another stream's callstack can happen + wait(delay(0)); + + // pop first item off queue - this will be mutation with the lowest version + Standalone> nextOut; + nextVersion = mutations.top().next.version; + + streamsUsed.push_back(mutations.top()); + nextOut.push_back_deep(nextOut.arena(), mutations.top().next); + mutations.pop(); + + // for each other stream that has mutations with the same version, add it to nextOut + while (!mutations.empty() && mutations.top().next.version == nextVersion) { + if (mutations.top().next.mutations.size() && + mutations.top().next.mutations.front().param1 != lastEpochEndPrivateKey) { + nextOut.back().mutations.append_deep( + nextOut.arena(), mutations.top().next.mutations.begin(), mutations.top().next.mutations.size()); + } + streamsUsed.push_back(mutations.top()); + mutations.pop(); + } + + ASSERT(nextOut.size() == 1); + ASSERT(nextVersion >= *begin); + + *begin = nextVersion + 1; + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorSending", mergeCursorUID) + .detail("Count", streamsUsed.size()) + .detail("Version", nextVersion); + } + + // send mutations at nextVersion to the client + if (nextOut.back().mutations.empty()) { + ASSERT(results->mutations.isEmpty()); + } else { + ASSERT(nextOut.back().version > results->lastReturnedVersion.get()); + + results->mutations.send(nextOut); + wait(results->mutations.onEmpty()); + wait(delay(0)); + } + + if (nextVersion > results->lastReturnedVersion.get()) { + results->lastReturnedVersion.set(nextVersion); + } + } +} + ACTOR Future mergeChangeFeedStream(Reference db, std::vector> interfs, Reference results, Key rangeID, Version* begin, - Version end) { - state std::priority_queue> mutations; + Version end, + int replyBufferSize, + bool canReadPopped) { state std::vector> fetchers(interfs.size()); + state std::vector> onErrors(interfs.size()); state std::vector streams(interfs.size()); + TEST(interfs.size() > 10); // Large change feed merge cursor + TEST(interfs.size() > 100); // Very large change feed merge cursor + + state UID mergeCursorUID = UID(); + state std::vector debugUIDs; results->streams.clear(); for (auto& it : interfs) { ChangeFeedStreamRequest req; @@ -8158,14 +8542,26 @@ ACTOR Future mergeChangeFeedStream(Reference db, req.begin = *begin; req.end = end; req.range = it.second; + req.canReadPopped = canReadPopped; + // divide total buffer size among sub-streams, but keep individual streams large enough to be efficient + req.replyBufferSize = replyBufferSize / interfs.size(); + if (replyBufferSize != -1 && req.replyBufferSize < CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES) { + req.replyBufferSize = CLIENT_KNOBS->CHANGE_FEED_STREAM_MIN_BYTES; + } + req.debugUID = deterministicRandom()->randomUniqueID(); + debugUIDs.push_back(req.debugUID); + mergeCursorUID = + UID(mergeCursorUID.first() ^ req.debugUID.first(), mergeCursorUID.second() ^ req.debugUID.second()); + results->streams.push_back(it.first.changeFeedStream.getReplyStream(req)); } for (auto& it : results->storageData) { if (it->debugGetReferenceCount() == 2) { - db->changeFeedUpdaters.erase(it->id); + db->changeFeedUpdaters.erase(it->interfToken); } } + results->maxSeenVersion = invalidVersion; results->storageData.clear(); Promise refresh = results->refresh; results->refresh = Promise(); @@ -8176,61 +8572,31 @@ ACTOR Future mergeChangeFeedStream(Reference db, refresh.send(Void()); for (int i = 0; i < interfs.size(); i++) { - fetchers[i] = singleChangeFeedStream( - interfs[i].first, streams[i].results, results->streams[i], end, results, results->storageData[i]); - } - state int interfNum = 0; - while (interfNum < interfs.size()) { - try { - Standalone res = waitNext(streams[interfNum].results.getFuture()); - streams[interfNum].next = res; - mutations.push(streams[interfNum]); - } catch (Error& e) { - if (e.code() != error_code_end_of_stream) { - throw e; - } + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientMergeCursorInit", debugUIDs[i]) + .detail("CursorDebugUID", mergeCursorUID) + .detail("Idx", i) + .detail("FeedID", rangeID) + .detail("MergeRange", KeyRangeRef(interfs.front().second.begin, interfs.back().second.end)) + .detail("PartialRange", interfs[i].second) + .detail("Begin", *begin) + .detail("End", end) + .detail("CanReadPopped", canReadPopped); } - interfNum++; + onErrors[i] = results->streams[i].onError(); + fetchers[i] = partialChangeFeedStream(interfs[i].first, + streams[i].results, + results->streams[i], + *begin, + end, + results, + results->storageData[i], + debugUIDs[i]); } - state Version checkVersion = invalidVersion; - state Standalone> nextOut; - while (mutations.size()) { - state MutationAndVersionStream nextStream = mutations.top(); - mutations.pop(); - ASSERT(nextStream.next.version >= checkVersion); - if (nextStream.next.version != checkVersion) { - if (nextOut.size()) { - *begin = checkVersion + 1; - results->mutations.send(nextOut); - results->lastReturnedVersion.set(nextOut.back().version); - nextOut = Standalone>(); - } - checkVersion = nextStream.next.version; - } - if (nextOut.size() && nextStream.next.version == nextOut.back().version) { - if (nextStream.next.mutations.size() && - nextStream.next.mutations.front().param1 != lastEpochEndPrivateKey) { - nextOut.back().mutations.append_deep( - nextOut.arena(), nextStream.next.mutations.begin(), nextStream.next.mutations.size()); - } - } else { - nextOut.push_back_deep(nextOut.arena(), nextStream.next); - } - try { - Standalone res = waitNext(nextStream.results.getFuture()); - nextStream.next = res; - mutations.push(nextStream); - } catch (Error& e) { - if (e.code() != error_code_end_of_stream) { - throw e; - } - } - } - if (nextOut.size()) { - results->mutations.send(nextOut); - results->lastReturnedVersion.set(nextOut.back().version); - } - throw end_of_stream(); + + wait(waitForAny(onErrors) || mergeChangeFeedStreamInternal(results, interfs, streams, begin, end, mergeCursorUID)); + + return Void(); } ACTOR Future getChangeFeedRange(Reference db, Database cx, Key rangeID, Version begin = 0) { @@ -8267,18 +8633,142 @@ ACTOR Future getChangeFeedRange(Reference db, Databas } } +ACTOR Future singleChangeFeedStreamInternal(KeyRange range, + Reference results, + Key rangeID, + Version* begin, + Version end) { + + state Promise refresh = results->refresh; + ASSERT(results->streams.size() == 1); + ASSERT(results->storageData.size() == 1); + state bool atLatest = false; + + // wait for any previous mutations in stream to be consumed + wait(results->mutations.onEmpty()); + wait(delay(0)); + ASSERT(results->mutations.isEmpty()); + // update lastReturned once the previous mutation has been consumed + if (*begin - 1 > results->lastReturnedVersion.get()) { + results->lastReturnedVersion.set(*begin - 1); + } + + loop { + + state ChangeFeedStreamReply feedReply = waitNext(results->streams[0].getFuture()); + *begin = feedReply.mutations.back().version + 1; + + if (feedReply.popVersion > results->popVersion) { + results->popVersion = feedReply.popVersion; + } + + // don't send completely empty set of mutations to promise stream + bool anyMutations = false; + for (auto& it : feedReply.mutations) { + if (!it.mutations.empty()) { + anyMutations = true; + break; + } + } + if (anyMutations) { + // empty versions can come out of order, as we sometimes send explicit empty versions when restarting a + // stream. Anything with mutations should be strictly greater than lastReturnedVersion + ASSERT(feedReply.mutations.front().version > results->lastReturnedVersion.get()); + + results->mutations.send( + Standalone>(feedReply.mutations, feedReply.arena)); + + // Because onEmpty returns here before the consuming process, we must do a delay(0) + wait(results->mutations.onEmpty()); + wait(delay(0)); + } + + // check refresh.canBeSet so that, if we are killed after calling one of these callbacks, we just + // skip to the next wait and get actor_cancelled + if (feedReply.mutations.back().version > results->lastReturnedVersion.get()) { + results->lastReturnedVersion.set(feedReply.mutations.back().version); + } + + if (refresh.canBeSet() && !atLatest && feedReply.atLatestVersion) { + atLatest = true; + results->notAtLatest.set(0); + } + if (refresh.canBeSet() && feedReply.minStreamVersion > results->storageData[0]->version.get()) { + results->storageData[0]->version.set(feedReply.minStreamVersion); + } + } +} + +ACTOR Future singleChangeFeedStream(Reference db, + StorageServerInterface interf, + KeyRange range, + Reference results, + Key rangeID, + Version* begin, + Version end, + int replyBufferSize, + bool canReadPopped) { + state Database cx(db); + state ChangeFeedStreamRequest req; + req.rangeID = rangeID; + req.begin = *begin; + req.end = end; + req.range = range; + req.canReadPopped = canReadPopped; + req.replyBufferSize = replyBufferSize; + req.debugUID = deterministicRandom()->randomUniqueID(); + + if (DEBUG_CF_CLIENT_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedClientSingleCursor", req.debugUID) + .detail("FeedID", rangeID) + .detail("Range", range) + .detail("Begin", *begin) + .detail("End", end) + .detail("CanReadPopped", canReadPopped); + } + + results->streams.clear(); + + for (auto& it : results->storageData) { + if (it->debugGetReferenceCount() == 2) { + db->changeFeedUpdaters.erase(it->interfToken); + } + } + results->streams.push_back(interf.changeFeedStream.getReplyStream(req)); + + results->maxSeenVersion = invalidVersion; + results->storageData.clear(); + results->storageData.push_back(db->getStorageData(interf)); + Promise refresh = results->refresh; + results->refresh = Promise(); + results->notAtLatest.set(1); + refresh.send(Void()); + + wait(results->streams[0].onError() || singleChangeFeedStreamInternal(range, results, rangeID, begin, end)); + + return Void(); +} + ACTOR Future getChangeFeedStreamActor(Reference db, Reference results, Key rangeID, Version begin, Version end, - KeyRange range) { + KeyRange range, + int replyBufferSize, + bool canReadPopped) { state Database cx(db); state Span span("NAPI:GetChangeFeedStream"_loc); + results->endVersion = end; + + state double sleepWithBackoff = CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY; + state Version lastBeginVersion = invalidVersion; + loop { state KeyRange keys; try { + lastBeginVersion = begin; KeyRange fullRange = wait(getChangeFeedRange(db, cx, rangeID, begin)); keys = fullRange & range; state std::vector locations = @@ -8346,59 +8836,34 @@ ACTOR Future getChangeFeedStreamActor(Reference db, interfs.emplace_back(locations[i].locations->getInterface(chosenLocations[i]), locations[i].range & range); } - wait(mergeChangeFeedStream(db, interfs, results, rangeID, &begin, end) || cx->connectionFileChanged()); + TEST(true); // Change feed merge cursor + // TODO (jslocum): validate connectionFileChanged behavior + wait( + mergeChangeFeedStream(db, interfs, results, rangeID, &begin, end, replyBufferSize, canReadPopped) || + cx->connectionFileChanged()); } else { - state ChangeFeedStreamRequest req; - req.rangeID = rangeID; - req.begin = begin; - req.end = end; - req.range = range; + TEST(true); // Change feed single cursor StorageServerInterface interf = locations[0].locations->getInterface(chosenLocations[0]); - state ReplyPromiseStream replyStream = - interf.changeFeedStream.getReplyStream(req); - for (auto& it : results->storageData) { - if (it->debugGetReferenceCount() == 2) { - db->changeFeedUpdaters.erase(it->id); - } - } - results->streams.clear(); - results->storageData.clear(); - results->storageData.push_back(db->getStorageData(interf)); - Promise refresh = results->refresh; - results->refresh = Promise(); - results->notAtLatest.set(1); - refresh.send(Void()); - state bool atLatest = false; - loop { - wait(results->mutations.onEmpty()); - choose { - when(wait(cx->connectionFileChanged())) { break; } - when(ChangeFeedStreamReply rep = waitNext(replyStream.getFuture())) { - begin = rep.mutations.back().version + 1; - results->mutations.send( - Standalone>(rep.mutations, rep.arena)); - results->lastReturnedVersion.set(rep.mutations.back().version); - if (!atLatest && rep.atLatestVersion) { - atLatest = true; - results->notAtLatest.set(0); - } - if (rep.minStreamVersion > results->storageData[0]->version.get()) { - results->storageData[0]->version.set(rep.minStreamVersion); - } - } - } - } + wait(singleChangeFeedStream( + db, interf, range, results, rangeID, &begin, end, replyBufferSize, canReadPopped) || + cx->connectionFileChanged()); } } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) { + if (e.code() == error_code_actor_cancelled || e.code() == error_code_change_feed_popped) { for (auto& it : results->storageData) { if (it->debugGetReferenceCount() == 2) { - db->changeFeedUpdaters.erase(it->id); + db->changeFeedUpdaters.erase(it->interfToken); } } results->streams.clear(); results->storageData.clear(); - results->refresh.sendError(change_feed_cancelled()); + if (e.code() == error_code_change_feed_popped) { + TEST(true); // getChangeFeedStreamActor got popped + results->mutations.sendError(e); + results->refresh.sendError(e); + } else { + results->refresh.sendError(change_feed_cancelled()); + } throw; } if (results->notAtLatest.get() == 0) { @@ -8410,13 +8875,20 @@ ACTOR Future getChangeFeedStreamActor(Reference db, e.code() == error_code_broken_promise) { db->changeFeedCache.erase(rangeID); cx->invalidateCache(Key(), keys); - wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY)); + if (begin == lastBeginVersion) { + // We didn't read anything since the last failure before failing again. + // Do exponential backoff, up to 1 second + sleepWithBackoff = std::min(1.0, sleepWithBackoff * 1.5); + } else { + sleepWithBackoff = CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY; + } + wait(delay(sleepWithBackoff)); } else { results->mutations.sendError(e); results->refresh.sendError(change_feed_cancelled()); for (auto& it : results->storageData) { if (it->debugGetReferenceCount() == 2) { - db->changeFeedUpdaters.erase(it->id); + db->changeFeedUpdaters.erase(it->interfToken); } } results->streams.clear(); @@ -8431,8 +8903,11 @@ Future DatabaseContext::getChangeFeedStream(Reference resu Key rangeID, Version begin, Version end, - KeyRange range) { - return getChangeFeedStreamActor(Reference::addRef(this), results, rangeID, begin, end, range); + KeyRange range, + int replyBufferSize, + bool canReadPopped) { + return getChangeFeedStreamActor( + Reference::addRef(this), results, rangeID, begin, end, range, replyBufferSize, canReadPopped); } ACTOR Future> singleLocationOverlappingChangeFeeds( @@ -8599,7 +9074,8 @@ ACTOR Future popChangeFeedMutationsActor(Reference db, Ke } } catch (Error& e) { if (e.code() != error_code_unknown_change_feed && e.code() != error_code_wrong_shard_server && - e.code() != error_code_all_alternatives_failed) { + e.code() != error_code_all_alternatives_failed && e.code() != error_code_broken_promise && + e.code() != error_code_server_overloaded) { throw; } db->changeFeedCache.erase(rangeID); diff --git a/fdbclient/Notified.h b/fdbclient/Notified.h index fe9e40e59a..b0686f50d1 100644 --- a/fdbclient/Notified.h +++ b/fdbclient/Notified.h @@ -80,6 +80,8 @@ struct Notified { val = std::move(r.val); } + int numWaiting() { return waiting.size(); } + private: using Item = std::pair>; struct ItemCompare { diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 79f7974f71..ab2b18ec57 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -810,6 +810,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "aggressive", "gradual" ]}, + "blob_granules_enabled":0, "tenant_mode": { "$enum":[ "disabled", diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index f50aca30a9..0951498d52 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -292,7 +292,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( SQLITE_CHUNK_SIZE_PAGES_SIM, 1024 ); // 4MB init( SQLITE_READER_THREADS, 64 ); // number of read threads init( SQLITE_WRITE_WINDOW_SECONDS, -1 ); - init( SQLITE_CURSOR_MAX_LIFETIME_BYTES, 1e6 ); if( randomize && BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0; + init( SQLITE_CURSOR_MAX_LIFETIME_BYTES, 1e6 ); if (buggifySmallShards || simulationMediumShards) SQLITE_CURSOR_MAX_LIFETIME_BYTES = MIN_SHARD_BYTES; if( randomize && BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0; init( SQLITE_WRITE_WINDOW_LIMIT, -1 ); if( randomize && BUGGIFY ) { // Choose an window between .01 and 1.01 seconds. @@ -538,6 +538,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( POLICY_GENERATIONS, 100 ); if( randomize && BUGGIFY ) POLICY_GENERATIONS = 10; init( DBINFO_SEND_AMOUNT, 5 ); init( DBINFO_BATCH_DELAY, 0.1 ); + init( SINGLETON_RECRUIT_BME_DELAY, 10.0 ); //Move Keys init( SHARD_READY_DELAY, 0.25 ); @@ -650,6 +651,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( FETCH_KEYS_PARALLELISM_BYTES, 4e6 ); if( randomize && BUGGIFY ) FETCH_KEYS_PARALLELISM_BYTES = 3e6; init( FETCH_KEYS_PARALLELISM, 2 ); init( FETCH_KEYS_LOWER_PRIORITY, 0 ); + init( FETCH_CHANGEFEED_PARALLELISM, 2 ); init( BUGGIFY_BLOCK_BYTES, 10000 ); init( STORAGE_COMMIT_BYTES, 10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000; init( STORAGE_FETCH_BYTES, 2500000 ); if( randomize && BUGGIFY ) STORAGE_FETCH_BYTES = 500000; @@ -680,6 +682,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( FETCH_KEYS_TOO_LONG_TIME_CRITERIA, 300.0 ); init( MAX_STORAGE_COMMIT_TIME, 120.0 ); //The max fsync stall time on the storage server and tlog before marking a disk as failed init( RANGESTREAM_LIMIT_BYTES, 2e6 ); if( randomize && BUGGIFY ) RANGESTREAM_LIMIT_BYTES = 1; + init( CHANGEFEEDSTREAM_LIMIT_BYTES, 1e6 ); if( randomize && BUGGIFY ) CHANGEFEEDSTREAM_LIMIT_BYTES = 1; + init( BLOBWORKERSTATUSSTREAM_LIMIT_BYTES, 1e4 ); if( randomize && BUGGIFY ) BLOBWORKERSTATUSSTREAM_LIMIT_BYTES = 1; init( ENABLE_CLEAR_RANGE_EAGER_READS, true ); init( CHECKPOINT_TRANSFER_BLOCK_BYTES, 40e6 ); init( QUICK_GET_VALUE_FALLBACK, true ); @@ -826,12 +830,23 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ENABLE_ENCRYPTION, false ); // Blob granlues - init( BG_URL, "" ); // TODO: store in system key space, eventually - init( BG_SNAPSHOT_FILE_TARGET_BYTES, 10000000 ); if( randomize && BUGGIFY ) { deterministicRandom()->random01() < 0.1 ? BG_SNAPSHOT_FILE_TARGET_BYTES /= 100 : BG_SNAPSHOT_FILE_TARGET_BYTES /= 10; } + init( BG_URL, isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually + init( BG_SNAPSHOT_FILE_TARGET_BYTES, 10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (simulationMediumShards || (randomize && BUGGIFY) ) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000; init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 ); init( BG_DELTA_FILE_TARGET_BYTES, BG_DELTA_BYTES_BEFORE_COMPACT/10 ); + init( BG_MAX_SPLIT_FANOUT, 10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15); + init( BG_HOT_SNAPSHOT_VERSIONS, 5000000 ); + init( BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM, 8 ); if( randomize && BUGGIFY ) BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM = 1; init( BLOB_WORKER_TIMEOUT, 10.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0; + init( BLOB_WORKER_REQUEST_TIMEOUT, 5.0 ); if( randomize && BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0; + init( BLOB_WORKERLIST_FETCH_INTERVAL, 1.0 ); + init( BLOB_WORKER_BATCH_GRV_INTERVAL, 0.1 ); + + + init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN, 0.1 ); + init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX, 5.0 ); + init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT, 1.5 ); // clang-format on diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index f636f91cd4..d120ed3986 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -470,6 +470,7 @@ public: double RECRUITMENT_TIMEOUT; int DBINFO_SEND_AMOUNT; double DBINFO_BATCH_DELAY; + double SINGLETON_RECRUIT_BME_DELAY; // Move Keys double SHARD_READY_DELAY; @@ -586,6 +587,7 @@ public: int FETCH_KEYS_PARALLELISM_BYTES; int FETCH_KEYS_PARALLELISM; int FETCH_KEYS_LOWER_PRIORITY; + int FETCH_CHANGEFEED_PARALLELISM; int BUGGIFY_BLOCK_BYTES; double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD; double STORAGE_DURABILITY_LAG_MIN_RATE; @@ -616,6 +618,8 @@ public: double FETCH_KEYS_TOO_LONG_TIME_CRITERIA; double MAX_STORAGE_COMMIT_TIME; int64_t RANGESTREAM_LIMIT_BYTES; + int64_t CHANGEFEEDSTREAM_LIMIT_BYTES; + int64_t BLOBWORKERSTATUSSTREAM_LIMIT_BYTES; bool ENABLE_CLEAR_RANGE_EAGER_READS; bool QUICK_GET_VALUE_FALLBACK; bool QUICK_GET_KEY_VALUES_FALLBACK; @@ -781,8 +785,18 @@ public: int BG_SNAPSHOT_FILE_TARGET_BYTES; int BG_DELTA_FILE_TARGET_BYTES; int BG_DELTA_BYTES_BEFORE_COMPACT; + int BG_MAX_SPLIT_FANOUT; + int BG_HOT_SNAPSHOT_VERSIONS; + int BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM; double BLOB_WORKER_TIMEOUT; // Blob Manager's reaction time to a blob worker failure + double BLOB_WORKER_REQUEST_TIMEOUT; // Blob Worker's server-side request timeout + double BLOB_WORKERLIST_FETCH_INTERVAL; + double BLOB_WORKER_BATCH_GRV_INTERVAL; + + double BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN; + double BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX; + double BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT; ServerKnobs(Randomize, ClientKnobs*, IsSimulated); void initialize(Randomize, ClientKnobs*, IsSimulated); diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 4417d36800..382b38cf18 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -773,6 +773,7 @@ struct ChangeFeedStreamReply : public ReplyPromiseStreamReply { VectorRef mutations; bool atLatestVersion = false; Version minStreamVersion = invalidVersion; + Version popVersion = invalidVersion; ChangeFeedStreamReply() {} @@ -786,6 +787,7 @@ struct ChangeFeedStreamReply : public ReplyPromiseStreamReply { mutations, atLatestVersion, minStreamVersion, + popVersion, arena); } }; @@ -798,12 +800,18 @@ struct ChangeFeedStreamRequest { Version begin = 0; Version end = 0; KeyRange range; + int replyBufferSize = -1; + bool canReadPopped = true; + UID debugUID; // This is only used for debugging and tracing, but being able to link a client + server side stream + // is so useful for testing, and this is such small overhead compared to streaming large amounts of + // change feed data, it is left in the interface + ReplyPromiseStream reply; ChangeFeedStreamRequest() {} template void serialize(Ar& ar) { - serializer(ar, rangeID, begin, end, range, reply, spanContext, arena); + serializer(ar, rangeID, begin, end, range, reply, spanContext, replyBufferSize, canReadPopped, debugUID, arena); } }; @@ -881,19 +889,21 @@ struct FetchCheckpointRequest { struct OverlappingChangeFeedEntry { Key rangeId; KeyRange range; - bool stopped = false; + Version emptyVersion; + Version stopVersion; bool operator==(const OverlappingChangeFeedEntry& r) const { - return rangeId == r.rangeId && range == r.range && stopped == r.stopped; + return rangeId == r.rangeId && range == r.range && emptyVersion == r.emptyVersion && + stopVersion == r.stopVersion; } OverlappingChangeFeedEntry() {} - OverlappingChangeFeedEntry(Key const& rangeId, KeyRange const& range, bool stopped) - : rangeId(rangeId), range(range), stopped(stopped) {} + OverlappingChangeFeedEntry(Key const& rangeId, KeyRange const& range, Version emptyVersion, Version stopVersion) + : rangeId(rangeId), range(range), emptyVersion(emptyVersion), stopVersion(stopVersion) {} template void serialize(Ar& ar) { - serializer(ar, rangeId, range, stopped); + serializer(ar, rangeId, range, emptyVersion, stopVersion); } }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index c044559e75..5c24441f32 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1153,30 +1153,33 @@ const KeyRangeRef blobGranuleMappingKeys(LiteralStringRef("\xff\x02/bgm/"), Lite const KeyRangeRef blobGranuleLockKeys(LiteralStringRef("\xff\x02/bgl/"), LiteralStringRef("\xff\x02/bgl0")); const KeyRangeRef blobGranuleSplitKeys(LiteralStringRef("\xff\x02/bgs/"), LiteralStringRef("\xff\x02/bgs0")); const KeyRangeRef blobGranuleHistoryKeys(LiteralStringRef("\xff\x02/bgh/"), LiteralStringRef("\xff\x02/bgh0")); +const KeyRangeRef blobGranulePruneKeys(LiteralStringRef("\xff\x02/bgp/"), LiteralStringRef("\xff\x02/bgp0")); +const KeyRangeRef blobGranuleVersionKeys(LiteralStringRef("\xff\x02/bgv/"), LiteralStringRef("\xff\x02/bgv0")); +const KeyRef blobGranulePruneChangeKey = LiteralStringRef("\xff\x02/bgpChange"); const uint8_t BG_FILE_TYPE_DELTA = 'D'; const uint8_t BG_FILE_TYPE_SNAPSHOT = 'S'; -const Key blobGranuleFileKeyFor(UID granuleID, uint8_t fileType, Version fileVersion) { +const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t fileType) { ASSERT(fileType == 'D' || fileType == 'S'); BinaryWriter wr(AssumeVersion(ProtocolVersion::withBlobGranule())); wr.serializeBytes(blobGranuleFileKeys.begin); wr << granuleID; - wr << fileType; wr << bigEndian64(fileVersion); + wr << fileType; return wr.toValue(); } -std::tuple decodeBlobGranuleFileKey(KeyRef const& key) { +std::tuple decodeBlobGranuleFileKey(KeyRef const& key) { UID granuleID; - uint8_t fileType; Version fileVersion; + uint8_t fileType; BinaryReader reader(key.removePrefix(blobGranuleFileKeys.begin), AssumeVersion(ProtocolVersion::withBlobGranule())); reader >> granuleID; - reader >> fileType; reader >> fileVersion; + reader >> fileType; ASSERT(fileType == 'D' || fileType == 'S'); - return std::tuple(granuleID, fileType, bigEndian64(fileVersion)); + return std::tuple(granuleID, bigEndian64(fileVersion), fileType); } const KeyRange blobGranuleFileKeyRangeFor(UID granuleID) { @@ -1206,6 +1209,25 @@ std::tuple, int64_t, int64_t> decodeBlobGranuleFileValue(V return std::tuple(filename, offset, length); } +const Value blobGranulePruneValueFor(Version version, KeyRange range, bool force) { + BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule())); + wr << version; + wr << range; + wr << force; + return wr.toValue(); +} + +std::tuple decodeBlobGranulePruneValue(ValueRef const& value) { + Version version; + KeyRange range; + bool force; + BinaryReader reader(value, IncludeVersion()); + reader >> version; + reader >> range; + reader >> force; + return std::tuple(version, range, force); +} + const Value blobGranuleMappingValueFor(UID const& workerID) { BinaryWriter wr(IncludeVersion(ProtocolVersion::withBlobGranule())); wr << workerID; @@ -1284,7 +1306,8 @@ std::pair decodeBlobGranuleSplitValue(const Valu BinaryReader reader(value, IncludeVersion()); reader >> st; reader >> v; - return std::pair(st, v); + + return std::pair(st, bigEndian64(v)); } const Key blobGranuleHistoryKeyFor(KeyRangeRef const& range, Version version) { diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index bc926c8227..171130559e 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -563,13 +563,21 @@ extern const KeyRangeRef blobGranuleSplitKeys; // \xff\x02/bgh/(beginKey,endKey,startVersion) = { granuleUID, [parentGranuleHistoryKeys] } extern const KeyRangeRef blobGranuleHistoryKeys; -const Key blobGranuleFileKeyFor(UID granuleID, uint8_t fileType, Version fileVersion); -std::tuple decodeBlobGranuleFileKey(ValueRef const& value); +// \xff\x02/bgp/(start,end) = (version, force) +extern const KeyRangeRef blobGranulePruneKeys; +extern const KeyRangeRef blobGranuleVersionKeys; +extern const KeyRef blobGranulePruneChangeKey; + +const Key blobGranuleFileKeyFor(UID granuleID, Version fileVersion, uint8_t fileType); +std::tuple decodeBlobGranuleFileKey(KeyRef const& key); const KeyRange blobGranuleFileKeyRangeFor(UID granuleID); const Value blobGranuleFileValueFor(StringRef const& filename, int64_t offset, int64_t length); std::tuple, int64_t, int64_t> decodeBlobGranuleFileValue(ValueRef const& value); +const Value blobGranulePruneValueFor(Version version, KeyRange range, bool force); +std::tuple decodeBlobGranulePruneValue(ValueRef const& value); + const Value blobGranuleMappingValueFor(UID const& workerID); UID decodeBlobGranuleMappingValue(ValueRef const& value); @@ -587,7 +595,7 @@ const Value blobGranuleSplitValueFor(BlobGranuleSplitState st); std::pair decodeBlobGranuleSplitValue(ValueRef const& value); const Key blobGranuleHistoryKeyFor(KeyRangeRef const& range, Version version); -std::pair decodeBlobGranuleHistoryKey(KeyRef const& value); +std::pair decodeBlobGranuleHistoryKey(KeyRef const& key); const KeyRange blobGranuleHistoryKeyRangeFor(KeyRangeRef const& range); const Value blobGranuleHistoryValueFor(Standalone const& historyValue); diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp index fe32bd1adb..bac1a9b145 100644 --- a/fdbrpc/FlowTransport.actor.cpp +++ b/fdbrpc/FlowTransport.actor.cpp @@ -45,6 +45,7 @@ #include "flow/actorcompiler.h" // This must be the last #include. static NetworkAddressList g_currentDeliveryPeerAddress = NetworkAddressList(); +static Future g_currentDeliveryPeerDisconnect; constexpr int PACKET_LEN_WIDTH = sizeof(uint32_t); const uint64_t TOKEN_STREAM_FLAG = 1; @@ -545,28 +546,20 @@ ACTOR Future connectionWriter(Reference self, Reference } } -ACTOR Future delayedHealthUpdate(NetworkAddress address) { +ACTOR Future delayedHealthUpdate(NetworkAddress address, bool* tooManyConnectionsClosed) { state double start = now(); - state bool delayed = false; loop { if (FLOW_KNOBS->HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS && FlowTransport::transport().healthMonitor()->tooManyConnectionsClosed(address) && address.isPublic()) { - if (!delayed) { - TraceEvent("TooManyConnectionsClosedMarkFailed") - .detail("Dest", address) - .detail("StartTime", start) - .detail("ClosedCount", FlowTransport::transport().healthMonitor()->closedConnectionsCount(address)); - IFailureMonitor::failureMonitor().setStatus(address, FailureStatus(true)); - } - delayed = true; wait(delayJittered(FLOW_KNOBS->MAX_RECONNECTION_TIME * 2.0)); } else { - if (delayed) { + if (*tooManyConnectionsClosed) { TraceEvent("TooManyConnectionsClosedMarkAvailable") .detail("Dest", address) .detail("StartTime", start) .detail("TimeElapsed", now() - start) .detail("ClosedCount", FlowTransport::transport().healthMonitor()->closedConnectionsCount(address)); + *tooManyConnectionsClosed = false; } IFailureMonitor::failureMonitor().setStatus(address, FailureStatus(false)); break; @@ -586,6 +579,7 @@ ACTOR Future connectionKeeper(Reference self, state Future delayedHealthUpdateF; state Optional firstConnFailedTime = Optional(); state int retryConnect = false; + state bool tooManyConnectionsClosed = false; loop { try { @@ -635,7 +629,8 @@ ACTOR Future connectionKeeper(Reference self, IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(false)); } if (self->unsent.empty()) { - delayedHealthUpdateF = delayedHealthUpdate(self->destination); + delayedHealthUpdateF = + delayedHealthUpdate(self->destination, &tooManyConnectionsClosed); choose { when(wait(delayedHealthUpdateF)) { conn->close(); @@ -675,7 +670,7 @@ ACTOR Future connectionKeeper(Reference self, try { self->transport->countConnEstablished++; if (!delayedHealthUpdateF.isValid()) - delayedHealthUpdateF = delayedHealthUpdate(self->destination); + delayedHealthUpdateF = delayedHealthUpdate(self->destination, &tooManyConnectionsClosed); wait(connectionWriter(self, conn) || reader || connectionMonitor(self) || self->resetConnection.onTrigger()); TraceEvent("ConnectionReset", conn ? conn->getDebugID() : UID()) @@ -761,6 +756,17 @@ ACTOR Future connectionKeeper(Reference self, if (conn) { if (self->destination.isPublic() && e.code() == error_code_connection_failed) { FlowTransport::transport().healthMonitor()->reportPeerClosed(self->destination); + if (FLOW_KNOBS->HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS && + FlowTransport::transport().healthMonitor()->tooManyConnectionsClosed(self->destination) && + self->destination.isPublic()) { + TraceEvent("TooManyConnectionsClosedMarkFailed") + .detail("Dest", self->destination) + .detail( + "ClosedCount", + FlowTransport::transport().healthMonitor()->closedConnectionsCount(self->destination)); + tooManyConnectionsClosed = true; + IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(true)); + } } conn->close(); @@ -776,6 +782,9 @@ ACTOR Future connectionKeeper(Reference self, // Clients might send more packets in response, which needs to go out on the next connection IFailureMonitor::failureMonitor().notifyDisconnect(self->destination); + Promise disconnect = self->disconnect; + self->disconnect = Promise(); + disconnect.send(Void()); if (e.code() == error_code_actor_cancelled) throw; @@ -918,7 +927,8 @@ ACTOR static void deliver(TransportData* self, Endpoint destination, TaskPriority priority, ArenaReader reader, - bool inReadSocket) { + bool inReadSocket, + Future disconnect) { // We want to run the task at the right priority. If the priority is higher than the current priority (which is // ReadSocket) we can just upgrade. Otherwise we'll context switch so that we don't block other tasks that might run // with a higher priority. ReplyPromiseStream needs to guarantee that messages are received in the order they were @@ -937,13 +947,16 @@ ACTOR static void deliver(TransportData* self, } try { g_currentDeliveryPeerAddress = destination.addresses; + g_currentDeliveryPeerDisconnect = disconnect; StringRef data = reader.arenaReadAll(); ASSERT(data.size() > 8); ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll(), AssumeVersion(reader.protocolVersion())); receiver->receive(objReader); g_currentDeliveryPeerAddress = { NetworkAddress() }; + g_currentDeliveryPeerDisconnect = Future(); } catch (Error& e) { g_currentDeliveryPeerAddress = { NetworkAddress() }; + g_currentDeliveryPeerDisconnect = Future(); TraceEvent(SevError, "ReceiverError") .error(e) .detail("Token", destination.token.toString()) @@ -977,7 +990,8 @@ static void scanPackets(TransportData* transport, const uint8_t* e, Arena& arena, NetworkAddress const& peerAddress, - ProtocolVersion peerProtocolVersion) { + ProtocolVersion peerProtocolVersion, + Future disconnect) { // Find each complete packet in the given byte range and queue a ready task to deliver it. // Remove the complete packets from the range by increasing unprocessed_begin. // There won't be more than 64K of data plus one packet, so this shouldn't take a long time. @@ -1090,7 +1104,7 @@ static void scanPackets(TransportData* transport, // we have many messages to UnknownEndpoint we want to optimize earlier. As deliver is an actor it // will allocate some state on the heap and this prevents it from doing that. if (priority != TaskPriority::UnknownEndpoint || (token.first() & TOKEN_STREAM_FLAG) != 0) { - deliver(transport, Endpoint({ peerAddress }, token), priority, std::move(reader), true); + deliver(transport, Endpoint({ peerAddress }, token), priority, std::move(reader), true, disconnect); } unprocessed_begin = p = p + packetLen; @@ -1285,8 +1299,13 @@ ACTOR static Future connectionReader(TransportData* transport, if (!expectConnectPacket) { if (compatible || peerProtocolVersion.hasStableInterfaces()) { - scanPackets( - transport, unprocessed_begin, unprocessed_end, arena, peerAddress, peerProtocolVersion); + scanPackets(transport, + unprocessed_begin, + unprocessed_end, + arena, + peerAddress, + peerProtocolVersion, + peer->disconnect.getFuture()); } else { unprocessed_begin = unprocessed_end; peer->resetPing.trigger(); @@ -1483,6 +1502,10 @@ Endpoint FlowTransport::loadedEndpoint(const UID& token) { return Endpoint(g_currentDeliveryPeerAddress, token); } +Future FlowTransport::loadedDisconnect() { + return g_currentDeliveryPeerDisconnect; +} + void FlowTransport::addPeerReference(const Endpoint& endpoint, bool isStream) { if (!isStream || !endpoint.getPrimaryAddress().isValid() || !endpoint.getPrimaryAddress().isPublic()) return; @@ -1556,8 +1579,12 @@ static void sendLocal(TransportData* self, ISerializeSource const& what, const E ASSERT(copy.size() > 0); TaskPriority priority = self->endpoints.getPriority(destination.token); if (priority != TaskPriority::UnknownEndpoint || (destination.token.first() & TOKEN_STREAM_FLAG) != 0) { - deliver( - self, destination, priority, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false); + deliver(self, + destination, + priority, + ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), + false, + Never()); } } diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h index 3a84b4162a..d70b98c401 100644 --- a/fdbrpc/FlowTransport.h +++ b/fdbrpc/FlowTransport.h @@ -169,6 +169,7 @@ struct Peer : public ReferenceCounted { int connectIncomingCount; int connectFailedCount; ContinuousSample connectLatencies; + Promise disconnect; explicit Peer(TransportData* transport, NetworkAddress const& destination); @@ -269,6 +270,7 @@ public: static NetworkAddressList getGlobalLocalAddresses() { return transport().getLocalAddresses(); } Endpoint loadedEndpoint(const UID& token); + Future loadedDisconnect(); HealthMonitor* healthMonitor(); diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h index e97eb82acf..0c289b5c90 100644 --- a/fdbrpc/fdbrpc.h +++ b/fdbrpc/fdbrpc.h @@ -326,14 +326,14 @@ struct NetNotifiedQueueWithAcknowledgements final : NotifiedQueue, AcknowledgementReceiver acknowledgements; Endpoint requestStreamEndpoint; bool sentError = false; + Promise onConnect; - NetNotifiedQueueWithAcknowledgements(int futures, int promises) : NotifiedQueue(futures, promises) {} + NetNotifiedQueueWithAcknowledgements(int futures, int promises) + : NotifiedQueue(futures, promises), onConnect(nullptr) {} NetNotifiedQueueWithAcknowledgements(int futures, int promises, const Endpoint& remoteEndpoint) - : NotifiedQueue(futures, promises), FlowReceiver(remoteEndpoint, true) { + : NotifiedQueue(futures, promises), FlowReceiver(remoteEndpoint, true), onConnect(nullptr) { // A ReplyPromiseStream will be terminated on the server side if the network connection with the client breaks - acknowledgements.failures = tagError( - makeDependent(IFailureMonitor::failureMonitor()).onDisconnect(remoteEndpoint.getPrimaryAddress()), - operation_obsolete()); + acknowledgements.failures = tagError(FlowTransport::transport().loadedDisconnect(), operation_obsolete()); } void destroy() override { delete this; } @@ -350,11 +350,17 @@ struct NetNotifiedQueueWithAcknowledgements final : NotifiedQueue, // GetKeyValuesStream requests on the same endpoint will fail IFailureMonitor::failureMonitor().endpointNotFound(requestStreamEndpoint); } + if (onConnect.isValid() && onConnect.canBeSet()) { + onConnect.send(Void()); + } this->sendError(message.getError()); } else { if (message.get().asUnderlyingType().acknowledgeToken.present()) { acknowledgements = AcknowledgementReceiver( FlowTransport::transport().loadedEndpoint(message.get().asUnderlyingType().acknowledgeToken.get())); + if (onConnect.isValid() && onConnect.canBeSet()) { + onConnect.send(Void()); + } } if (acknowledgements.sequence != message.get().asUnderlyingType().sequence) { TraceEvent(SevError, "StreamSequenceMismatch") @@ -487,6 +493,18 @@ public: void setRequestStreamEndpoint(const Endpoint& endpoint) { queue->requestStreamEndpoint = endpoint; } + bool connected() { return queue->acknowledgements.getRawEndpoint().isValid() || queue->error.isValid(); } + + Future onConnected() { + if (connected()) { + return Void(); + } + if (!queue->onConnect.isValid()) { + queue->onConnect = Promise(); + } + return queue->onConnect.getFuture(); + } + ~ReplyPromiseStream() { if (queue) queue->delPromiseRef(); @@ -513,6 +531,19 @@ public: return queue->onEmpty.getFuture(); } + bool isError() const { return !queue->isError(); } + + // throws, used to short circuit waiting on the queue if there has been an unexpected error + Future onError() { + if (queue->hasError() && queue->error.code() != error_code_end_of_stream) { + throw queue->error; + } + if (!queue->onError.isValid()) { + queue->onError = Promise(); + } + return queue->onError.getFuture(); + } + uint32_t size() const { return queue->size(); } // Must be called on the server before sending results on the stream to ratelimit the amount of data outstanding to @@ -729,10 +760,13 @@ public: Future disc = makeDependent(IFailureMonitor::failureMonitor()).onDisconnectOrFailure(getEndpoint()); auto& p = getReplyPromiseStream(value); - Reference peer = - FlowTransport::transport().sendUnreliable(SerializeSource(value), getEndpoint(), true); - // FIXME: defer sending the message until we know the connection is established - endStreamOnDisconnect(disc, p, getEndpoint(), peer); + if (disc.isReady()) { + p.sendError(request_maybe_delivered()); + } else { + Reference peer = + FlowTransport::transport().sendUnreliable(SerializeSource(value), getEndpoint(), true); + endStreamOnDisconnect(disc, p, getEndpoint(), peer); + } return p; } else { send(value); diff --git a/fdbrpc/genericactors.actor.h b/fdbrpc/genericactors.actor.h index 9bf08ceac9..e2fd1885fd 100644 --- a/fdbrpc/genericactors.actor.h +++ b/fdbrpc/genericactors.actor.h @@ -210,9 +210,21 @@ void endStreamOnDisconnect(Future signal, Reference peer = Reference()) { state PeerHolder holder = PeerHolder(peer); stream.setRequestStreamEndpoint(endpoint); - choose { - when(wait(signal)) { stream.sendError(connection_failed()); } - when(wait(stream.getErrorFutureAndDelPromiseRef())) {} + try { + choose { + when(wait(signal)) { stream.sendError(connection_failed()); } + when(wait(peer.isValid() ? peer->disconnect.getFuture() : Never())) { + stream.sendError(connection_failed()); + } + when(wait(stream.getErrorFutureAndDelPromiseRef())) {} + } + } catch (Error& e) { + if (e.code() == error_code_broken_promise) { + // getErrorFutureAndDelPromiseRef returned, wait on stream connect or error + if (!stream.connected()) { + wait(signal || stream.onConnected()); + } + } } } diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp new file mode 100644 index 0000000000..c47ed13199 --- /dev/null +++ b/fdbserver/BlobGranuleServerCommon.actor.cpp @@ -0,0 +1,104 @@ +/* + * BlobGranuleServerCommon.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/SystemData.h" +#include "fdbclient/BlobGranuleCommon.h" +#include "fdbserver/BlobGranuleServerCommon.actor.h" +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/ReadYourWrites.h" +#include "flow/Arena.h" +#include "flow/actorcompiler.h" // has to be last include + +// Gets the latest granule history node for range that was persisted +ACTOR Future> getLatestGranuleHistory(Transaction* tr, KeyRange range) { + state KeyRange historyRange = blobGranuleHistoryKeyRangeFor(range); + state RangeResult result = wait(tr->getRange(historyRange, 1, Snapshot::False, Reverse::True)); + + ASSERT(result.size() <= 1); + + Optional history; + if (!result.empty()) { + std::pair decodedKey = decodeBlobGranuleHistoryKey(result[0].key); + ASSERT(range == decodedKey.first); + history = GranuleHistory(range, decodedKey.second, decodeBlobGranuleHistoryValue(result[0].value)); + } + return history; +} + +// Gets the files based on the file key range [startKey, endKey) +// and populates the files object accordingly +ACTOR Future readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, GranuleFiles* files, UID granuleID) { + + loop { + int lim = BUGGIFY ? 2 : 1000; + RangeResult res = wait(tr->getRange(KeyRangeRef(*startKey, endKey), lim)); + for (auto& it : res) { + UID gid; + uint8_t fileType; + Version version; + + Standalone filename; + int64_t offset; + int64_t length; + + std::tie(gid, version, fileType) = decodeBlobGranuleFileKey(it.key); + ASSERT(gid == granuleID); + + std::tie(filename, offset, length) = decodeBlobGranuleFileValue(it.value); + + BlobFileIndex idx(version, filename.toString(), offset, length); + if (fileType == 'S') { + ASSERT(files->snapshotFiles.empty() || files->snapshotFiles.back().version < idx.version); + files->snapshotFiles.push_back(idx); + } else { + ASSERT(fileType == 'D'); + ASSERT(files->deltaFiles.empty() || files->deltaFiles.back().version < idx.version); + files->deltaFiles.push_back(idx); + } + } + if (res.more) { + *startKey = keyAfter(res.back().key); + } else { + break; + } + } + return Void(); +} + +// Wrapper around readGranuleFiles +// Gets all files belonging to the granule with id granule ID +ACTOR Future loadHistoryFiles(Database cx, UID granuleID) { + state KeyRange range = blobGranuleFileKeyRangeFor(granuleID); + state Key startKey = range.begin; + state GranuleFiles files; + state Transaction tr(cx); + + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID)); + return files; + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} diff --git a/fdbserver/BlobGranuleServerCommon.actor.h b/fdbserver/BlobGranuleServerCommon.actor.h new file mode 100644 index 0000000000..d48418c951 --- /dev/null +++ b/fdbserver/BlobGranuleServerCommon.actor.h @@ -0,0 +1,70 @@ +/* + * BlobGranuleServerCommon.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BLOBGRANULESERVERCOMMON_ACTOR_G_H) +#define FDBSERVER_BLOBGRANULESERVERCOMMON_ACTOR_G_H +#include "fdbserver/BlobGranuleServerCommon.actor.g.h" +#elif !defined(FDBSERVER_BLOBGRANULESERVERCOMMON_ACTOR_H) +#define FDBSERVER_BLOBGRANULESERVERCOMMON_ACTOR_H + +#pragma once + +#include "flow/flow.h" +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/BlobGranuleCommon.h" +#include "flow/actorcompiler.h" // has to be last include + +struct GranuleHistory { + KeyRange range; + Version version; + Standalone value; + + GranuleHistory() {} + + GranuleHistory(KeyRange range, Version version, Standalone value) + : range(range), version(version), value(value) {} +}; + +// Stores info about a file in blob storage +struct BlobFileIndex { + Version version; + std::string filename; + int64_t offset; + int64_t length; + + BlobFileIndex() {} + + BlobFileIndex(Version version, std::string filename, int64_t offset, int64_t length) + : version(version), filename(filename), offset(offset), length(length) {} +}; + +// Stores the files that comprise a blob granule +struct GranuleFiles { + std::deque snapshotFiles; + std::deque deltaFiles; +}; + +class Transaction; +ACTOR Future> getLatestGranuleHistory(Transaction* tr, KeyRange range); +ACTOR Future readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, GranuleFiles* files, UID granuleID); + +ACTOR Future loadHistoryFiles(Database cx, UID granuleID); +#endif diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index 82e71eb914..2b6a4da2bf 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -18,10 +18,14 @@ * limitations under the License. */ +#include +#include +#include #include #include #include "contrib/fmt-8.1.1/include/fmt/format.h" +#include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BlobGranuleCommon.h" #include "fdbclient/BlobWorkerInterface.h" #include "fdbclient/KeyRangeMap.h" @@ -30,17 +34,21 @@ #include "fdbclient/SystemData.h" #include "fdbserver/BlobManagerInterface.h" #include "fdbserver/Knobs.h" +#include "fdbserver/BlobGranuleServerCommon.actor.h" +#include "fdbserver/QuietDatabase.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" +#include "flow/Error.h" #include "flow/IRandom.h" #include "flow/UnitTest.h" #include "flow/actorcompiler.h" // has to be last include +/* + * The Blob Manager is responsible for managing range granules, and recruiting and monitoring Blob Workers. + */ + #define BM_DEBUG false -// FIXME: change all BlobManagerData* to Reference to avoid segfaults if core loop gets error - -// TODO add comments + documentation void handleClientBlobRange(KeyRangeMap* knownBlobRanges, Arena& ar, VectorRef* rangesToAdd, @@ -49,10 +57,8 @@ void handleClientBlobRange(KeyRangeMap* knownBlobRanges, KeyRef rangeEnd, bool rangeActive) { if (BM_DEBUG) { - printf("db range [%s - %s): %s\n", - rangeStart.printable().c_str(), - rangeEnd.printable().c_str(), - rangeActive ? "T" : "F"); + fmt::print( + "db range [{0} - {1}): {2}\n", rangeStart.printable(), rangeEnd.printable(), rangeActive ? "T" : "F"); } KeyRange keyRange(KeyRangeRef(rangeStart, rangeEnd)); auto allRanges = knownBlobRanges->intersectingRanges(keyRange); @@ -63,16 +69,16 @@ void handleClientBlobRange(KeyRangeMap* knownBlobRanges, KeyRangeRef overlap(overlapStart, overlapEnd); if (rangeActive) { if (BM_DEBUG) { - printf("BM Adding client range [%s - %s)\n", - overlapStart.printable().c_str(), - overlapEnd.printable().c_str()); + fmt::print("BM Adding client range [{0} - {1})\n", + overlapStart.printable().c_str(), + overlapEnd.printable().c_str()); } rangesToAdd->push_back_deep(ar, overlap); } else { if (BM_DEBUG) { - printf("BM Removing client range [%s - %s)\n", - overlapStart.printable().c_str(), - overlapEnd.printable().c_str()); + fmt::print("BM Removing client range [{0} - {1})\n", + overlapStart.printable().c_str(), + overlapEnd.printable().c_str()); } rangesToRemove->push_back_deep(ar, overlap); } @@ -87,9 +93,9 @@ void updateClientBlobRanges(KeyRangeMap* knownBlobRanges, VectorRef* rangesToAdd, VectorRef* rangesToRemove) { if (BM_DEBUG) { - printf("Updating %d client blob ranges", dbBlobRanges.size() / 2); + fmt::print("Updating {0} client blob ranges", dbBlobRanges.size() / 2); for (int i = 0; i < dbBlobRanges.size() - 1; i += 2) { - printf(" [%s - %s)", dbBlobRanges[i].key.printable().c_str(), dbBlobRanges[i + 1].key.printable().c_str()); + fmt::print(" [{0} - {1})", dbBlobRanges[i].key.printable(), dbBlobRanges[i + 1].key.printable()); } printf("\n"); } @@ -100,7 +106,7 @@ void updateClientBlobRanges(KeyRangeMap* knownBlobRanges, // worker. for any range that isn't set in results that is set in ranges, revoke the range from the // worker. and, update ranges to match results as you go - // FIXME: could change this to O(N) instead of O(NLogN) by doing a sorted merge instead of requesting the + // SOMEDAY: could change this to O(N) instead of O(NLogN) by doing a sorted merge instead of requesting the // intersection for each insert, but this operation is pretty infrequent so it's probably not necessary if (dbBlobRanges.size() == 0) { // special case. Nothing in the DB, reset knownBlobRanges and revoke all existing ranges from workers @@ -114,25 +120,24 @@ void updateClientBlobRanges(KeyRangeMap* knownBlobRanges, for (int i = 0; i < dbBlobRanges.size() - 1; i++) { if (dbBlobRanges[i].key >= normalKeys.end) { if (BM_DEBUG) { - printf("Found invalid blob range start %s\n", dbBlobRanges[i].key.printable().c_str()); + fmt::print("Found invalid blob range start {0}\n", dbBlobRanges[i].key.printable()); } break; } bool active = dbBlobRanges[i].value == LiteralStringRef("1"); if (active) { - ASSERT(dbBlobRanges[i + 1].value == StringRef()); if (BM_DEBUG) { - printf("BM sees client range [%s - %s)\n", - dbBlobRanges[i].key.printable().c_str(), - dbBlobRanges[i + 1].key.printable().c_str()); + fmt::print("BM sees client range [{0} - {1})\n", + dbBlobRanges[i].key.printable(), + dbBlobRanges[i + 1].key.printable()); } } KeyRef endKey = dbBlobRanges[i + 1].key; if (endKey > normalKeys.end) { if (BM_DEBUG) { - printf("Removing system keyspace from blob range [%s - %s)\n", - dbBlobRanges[i].key.printable().c_str(), - endKey.printable().c_str()); + fmt::print("Removing system keyspace from blob range [{0} - {1})\n", + dbBlobRanges[i].key.printable(), + endKey.printable()); } endKey = normalKeys.end; } @@ -160,17 +165,16 @@ void getRanges(std::vector>& results, KeyRangeMap revoke; }; -// TODO: track worker's reads/writes eventually +// SOMEDAY: track worker's reads/writes eventually struct BlobWorkerStats { int numGranulesAssigned; BlobWorkerStats(int numGranulesAssigned = 0) : numGranulesAssigned(numGranulesAssigned) {} }; -struct BlobManagerData { +struct SplitEvaluation { + int64_t epoch; + int64_t seqno; + Future inProgress; + + SplitEvaluation() : epoch(0), seqno(0) {} + SplitEvaluation(int64_t epoch, int64_t seqno, Future inProgress) + : epoch(epoch), seqno(seqno), inProgress(inProgress) {} +}; + +struct BlobManagerData : NonCopyable, ReferenceCounted { UID id; Database db; + Optional dcId; PromiseStream> addActor; + Promise doLockCheck; + + Reference bstore; std::unordered_map workersById; std::unordered_map workerStats; // mapping between workerID -> workerStats + std::unordered_set workerAddresses; + std::unordered_set deadWorkers; KeyRangeMap workerAssignments; + KeyRangeActorMap assignsInProgress; + KeyRangeMap splitEvaluations; KeyRangeMap knownBlobRanges; + AsyncTrigger startRecruiting; Debouncer restartRecruiting; std::set recruitingLocalities; // the addrs of the workers being recruited on + AsyncVar recruitingStream; + Promise foundBlobWorkers; + Promise doneRecovering; int64_t epoch = -1; int64_t seqNo = 1; @@ -219,59 +245,106 @@ struct BlobManagerData { // assigned sequence numbers PromiseStream rangesToAssign; - BlobManagerData(UID id, Database db) - : id(id), db(db), knownBlobRanges(false, normalKeys.end), - restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY) {} - ~BlobManagerData() { printf("Destroying blob manager data for %s\n", id.toString().c_str()); } + BlobManagerData(UID id, Database db, Optional dcId) + : id(id), db(db), dcId(dcId), knownBlobRanges(false, normalKeys.end), + restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), recruitingStream(0) {} }; -ACTOR Future>> splitRange(Reference tr, KeyRange range) { - // TODO is it better to just pass empty metrics to estimated? - // redo split if previous txn failed to calculate it - loop { - try { - if (BM_DEBUG) { - printf( - "Splitting new range [%s - %s)\n", range.begin.printable().c_str(), range.end.printable().c_str()); - } - StorageMetrics estimated = - wait(tr->getTransaction().getDatabase()->getStorageMetrics(range, CLIENT_KNOBS->TOO_MANY)); - - if (BM_DEBUG) { - fmt::print("Estimated bytes for [{0} - {1}): {2}\n", - range.begin.printable(), - range.end.printable(), - estimated.bytes); - } - - if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES) { - // printf(" Splitting range\n"); - // only split on bytes - StorageMetrics splitMetrics; - splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES; - splitMetrics.bytesPerKSecond = splitMetrics.infinity; - splitMetrics.iosPerKSecond = splitMetrics.infinity; - splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth - - Standalone> keys = - wait(tr->getTransaction().getDatabase()->splitStorageMetrics(range, splitMetrics, estimated)); - return keys; - } else { - // printf(" Not splitting range\n"); - Standalone> keys; - keys.push_back_deep(keys.arena(), range.begin); - keys.push_back_deep(keys.arena(), range.end); - return keys; - } - } catch (Error& e) { - wait(tr->onError(e)); +ACTOR Future>> splitRange(Reference bmData, + KeyRange range, + bool writeHot) { + try { + if (BM_DEBUG) { + fmt::print("Splitting new range [{0} - {1}): {2}\n", + range.begin.printable(), + range.end.printable(), + writeHot ? "hot" : "normal"); } + state StorageMetrics estimated = wait(bmData->db->getStorageMetrics(range, CLIENT_KNOBS->TOO_MANY)); + + if (BM_DEBUG) { + fmt::print("Estimated bytes for [{0} - {1}): {2}\n", + range.begin.printable(), + range.end.printable(), + estimated.bytes); + } + + TEST(writeHot); // Change feed write hot split + if (estimated.bytes > SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES || writeHot) { + // only split on bytes and write rate + state StorageMetrics splitMetrics; + splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES; + splitMetrics.bytesPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC; + if (writeHot) { + splitMetrics.bytesPerKSecond = std::min(splitMetrics.bytesPerKSecond, estimated.bytesPerKSecond / 2); + splitMetrics.bytesPerKSecond = + std::max(splitMetrics.bytesPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC); + } + splitMetrics.iosPerKSecond = splitMetrics.infinity; + splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; + + state PromiseStream resultStream; + state Standalone> keys; + state Future streamFuture = + bmData->db->splitStorageMetricsStream(resultStream, range, splitMetrics, estimated); + loop { + try { + Key k = waitNext(resultStream.getFuture()); + keys.push_back_deep(keys.arena(), k); + } catch (Error& e) { + if (e.code() != error_code_end_of_stream) { + throw; + } + break; + } + } + + ASSERT(keys.size() >= 2); + ASSERT(keys.front() == range.begin); + ASSERT(keys.back() == range.end); + return keys; + } else { + if (BM_DEBUG) { + printf("Not splitting range\n"); + } + Standalone> keys; + keys.push_back_deep(keys.arena(), range.begin); + keys.push_back_deep(keys.arena(), range.end); + return keys; + } + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw e; + } + // SplitStorageMetrics explicitly has a SevError if it gets an error, so no errors should propagate here + TraceEvent(SevError, "BlobManagerUnexpectedErrorSplitRange", bmData->id) + .error(e) + .detail("Epoch", bmData->epoch); + ASSERT_WE_THINK(false); + + // if not simulation, kill the BM + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.sendError(e); + } + throw e; } } // Picks a worker with the fewest number of already assigned ranges. // If there is a tie, picks one such worker at random. -static UID pickWorkerForAssign(BlobManagerData* bmData) { +ACTOR Future pickWorkerForAssign(Reference bmData) { + // wait until there are BWs to pick from + while (bmData->workerStats.size() == 0) { + TEST(true); // BM wants to assign range, but no workers available + if (BM_DEBUG) { + fmt::print("BM {0} waiting for blob workers before assigning granules\n", bmData->epoch); + } + bmData->restartRecruiting.trigger(); + wait(bmData->recruitingStream.onChange() || bmData->foundBlobWorkers.getFuture()); + // FIXME: may want to have some buffer here so zero-worker recruiting case doesn't assign every single pending + // range to the first worker recruited + } + int minGranulesAssigned = INT_MAX; std::vector eligibleWorkers; @@ -292,28 +365,53 @@ static UID pickWorkerForAssign(BlobManagerData* bmData) { ASSERT(eligibleWorkers.size() > 0); int idx = deterministicRandom()->randomInt(0, eligibleWorkers.size()); if (BM_DEBUG) { - printf("picked worker %s, which has a minimal number (%d) of granules assigned\n", - eligibleWorkers[idx].toString().c_str(), - minGranulesAssigned); + fmt::print("picked worker {0}, which has a minimal number ({1}) of granules assigned\n", + eligibleWorkers[idx].toString(), + minGranulesAssigned); } return eligibleWorkers[idx]; } -ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment assignment, UID workerID, int64_t seqNo) { +ACTOR Future doRangeAssignment(Reference bmData, + RangeAssignment assignment, + Optional workerID, + int64_t seqNo) { + // WorkerId is set, except in case of assigning to any worker. Then we pick the worker to assign to in here + + // inject delay into range assignments + if (BUGGIFY_WITH_PROB(0.05)) { + wait(delay(deterministicRandom()->random01())); + } + + if (!workerID.present()) { + ASSERT(assignment.isAssign && assignment.assign.get().type != AssignRequestType::Continue); + UID _workerId = wait(pickWorkerForAssign(bmData)); + if (BM_DEBUG) { + fmt::print("Chose BW {0} for seqno {1} in BM {2}\n", _workerId.toString(), seqNo, bmData->epoch); + } + workerID = _workerId; + // We don't have to check for races with an overlapping assignment because it would insert over us in the actor + // map, cancelling this actor before it got here + bmData->workerAssignments.insert(assignment.keyRange, workerID.get()); + + if (bmData->workerStats.count(workerID.get())) { + bmData->workerStats[workerID.get()].numGranulesAssigned += 1; + } + } if (BM_DEBUG) { - fmt::print("BM {0} {1} range [{2} - {3}) @ ({4}, {5})\n", - bmData->id.toString(), + fmt::print("BM {0} {1} range [{2} - {3}) @ ({4}, {5}) to {6}\n", + bmData->epoch, assignment.isAssign ? "assigning" : "revoking", assignment.keyRange.begin.printable(), assignment.keyRange.end.printable(), bmData->epoch, - seqNo); + seqNo, + workerID.get().toString()); } try { - state AssignBlobRangeReply rep; if (assignment.isAssign) { ASSERT(assignment.assign.present()); ASSERT(!assignment.revoke.present()); @@ -323,14 +421,13 @@ ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment as StringRef(req.arena, assignment.keyRange.end)); req.managerEpoch = bmData->epoch; req.managerSeqno = seqNo; - req.continueAssignment = assignment.assign.get().continueAssignment; + req.type = assignment.assign.get().type; // if that worker isn't alive anymore, add the range back into the stream - if (bmData->workersById.count(workerID) == 0) { + if (bmData->workersById.count(workerID.get()) == 0) { throw no_more_servers(); } - AssignBlobRangeReply _rep = wait(bmData->workersById[workerID].assignBlobRangeRequest.getReply(req)); - rep = _rep; + wait(bmData->workersById[workerID.get()].assignBlobRangeRequest.getReply(req)); } else { ASSERT(!assignment.assign.present()); ASSERT(assignment.revoke.present()); @@ -343,31 +440,64 @@ ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment as req.dispose = assignment.revoke.get().dispose; // if that worker isn't alive anymore, this is a noop - if (bmData->workersById.count(workerID)) { - AssignBlobRangeReply _rep = wait(bmData->workersById[workerID].revokeBlobRangeRequest.getReply(req)); - rep = _rep; + if (bmData->workersById.count(workerID.get())) { + wait(bmData->workersById[workerID.get()].revokeBlobRangeRequest.getReply(req)); } else { return Void(); } } - if (!rep.epochOk) { - if (BM_DEBUG) { - printf("BM heard from BW that there is a new manager with higher epoch\n"); - } + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw; + } + if (e.code() == error_code_blob_manager_replaced) { if (bmData->iAmReplaced.canBeSet()) { bmData->iAmReplaced.send(Void()); } + return Void(); } - } catch (Error& e) { - // TODO confirm: using reliable delivery this should only trigger if the worker is marked as failed, right? - // So assignment needs to be retried elsewhere, and a revoke is trivially complete + if (e.code() == error_code_granule_assignment_conflict) { + // Another blob worker already owns the range, don't retry. + // And, if it was us that send the request to another worker for this range, this actor should have been + // cancelled. So if it wasn't, it's likely that the conflict is from a new blob manager. Trigger the lock + // check to make sure, and die if so. + if (BM_DEBUG) { + fmt::print("BM {0} got conflict assigning [{1} - {2}) to worker {3}, ignoring\n", + bmData->epoch, + assignment.keyRange.begin.printable(), + assignment.keyRange.end.printable(), + workerID.get().toString()); + } + if (bmData->doLockCheck.canBeSet()) { + bmData->doLockCheck.send(Void()); + } + return Void(); + } + + if (e.code() != error_code_broken_promise && e.code() != error_code_no_more_servers) { + TraceEvent(SevWarn, "BlobManagerUnexpectedErrorDoRangeAssignment", bmData->id) + .error(e) + .detail("Epoch", bmData->epoch); + ASSERT_WE_THINK(false); + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.sendError(e); + } + throw; + } + + TEST(true); // BM retrying range assign + + // We use reliable delivery (getReply), so the broken_promise means the worker is dead, and we may need to retry + // somewhere else if (assignment.isAssign) { if (BM_DEBUG) { - printf("BM got error assigning range [%s - %s) to worker %s, requeueing\n", - assignment.keyRange.begin.printable().c_str(), - assignment.keyRange.end.printable().c_str(), - workerID.toString().c_str()); + fmt::print("BM got error {0} assigning range [{1} - {2}) to worker {3}, requeueing\n", + e.name(), + assignment.keyRange.begin.printable(), + assignment.keyRange.end.printable(), + workerID.get().toString()); } + // re-send revoke to queue to handle range being un-assigned from that worker before the new one RangeAssignment revokeOld; revokeOld.isAssign = false; @@ -377,14 +507,18 @@ ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment as bmData->rangesToAssign.send(revokeOld); // send assignment back to queue as is, clearing designated worker if present + // if we failed to send continue to the worker we thought owned the shard, it should be retried + // as a normal assign + ASSERT(assignment.assign.present()); + assignment.assign.get().type = AssignRequestType::Normal; assignment.worker.reset(); bmData->rangesToAssign.send(assignment); // FIXME: improvement would be to add history of failed workers to assignment so it can try other ones first } else { if (BM_DEBUG) { - printf("BM got error revoking range [%s - %s) from worker", - assignment.keyRange.begin.printable().c_str(), - assignment.keyRange.end.printable().c_str()); + fmt::print("BM got error revoking range [{0} - {1}) from worker", + assignment.keyRange.begin.printable(), + assignment.keyRange.end.printable()); } if (assignment.revoke.get().dispose) { @@ -405,69 +539,109 @@ ACTOR Future doRangeAssignment(BlobManagerData* bmData, RangeAssignment as return Void(); } -ACTOR Future rangeAssigner(BlobManagerData* bmData) { +ACTOR Future rangeAssigner(Reference bmData) { loop { - // inject delay into range assignments - if (BUGGIFY_WITH_PROB(0.05)) { - wait(delay(deterministicRandom()->random01())); - } - RangeAssignment assignment = waitNext(bmData->rangesToAssign.getFuture()); - int64_t seqNo = bmData->seqNo; + + state RangeAssignment assignment = waitNext(bmData->rangesToAssign.getFuture()); + state int64_t seqNo = bmData->seqNo; bmData->seqNo++; // modify the in-memory assignment data structures, and send request off to worker - UID workerId; + state UID workerId; if (assignment.isAssign) { + bool skip = false; // Ensure range isn't currently assigned anywhere, and there is only 1 intersecting range auto currentAssignments = bmData->workerAssignments.intersectingRanges(assignment.keyRange); int count = 0; - for (auto& it : currentAssignments) { - if (assignment.assign.get().continueAssignment) { + for (auto i = currentAssignments.begin(); i != currentAssignments.end(); ++i) { + if (assignment.assign.get().type == AssignRequestType::Continue) { ASSERT(assignment.worker.present()); - ASSERT(it.value() == assignment.worker.get()); - } else { - ASSERT(it.value() == UID()); + if (i.range() != assignment.keyRange || i.cvalue() != assignment.worker.get()) { + TEST(true); // BM assignment out of date + if (BM_DEBUG) { + fmt::print("Out of date re-assign for ({0}, {1}). Assignment must have changed while " + "checking split.\n Reassign: [{2} - {3}): {4}\n Existing: [{5} - {6}): {7}\n", + bmData->epoch, + seqNo, + assignment.keyRange.begin.printable(), + assignment.keyRange.end.printable(), + assignment.worker.get().toString().substr(0, 5), + i.begin().printable(), + i.end().printable(), + i.cvalue().toString().substr(0, 5)); + } + skip = true; + } } count++; } ASSERT(count == 1); - - workerId = assignment.worker.present() ? assignment.worker.get() : pickWorkerForAssign(bmData); - bmData->workerAssignments.insert(assignment.keyRange, workerId); - - ASSERT(bmData->workerStats.count(workerId)); - if (!assignment.assign.get().continueAssignment) { - bmData->workerStats[workerId].numGranulesAssigned += 1; + if (skip) { + continue; } - // FIXME: if range is assign, have some sort of semaphore for outstanding assignments so we don't assign - // a ton ranges at once and blow up FDB with reading initial snapshots. - bmData->addActor.send(doRangeAssignment(bmData, assignment, workerId, seqNo)); - } else { - // Revoking a range could be a large range that contains multiple ranges. - auto currentAssignments = bmData->workerAssignments.intersectingRanges(assignment.keyRange); - for (auto& it : currentAssignments) { - // ensure range doesn't truncate existing ranges - ASSERT(it.begin() >= assignment.keyRange.begin); - ASSERT(it.end() <= assignment.keyRange.end); - - // It is fine for multiple disjoint sub-ranges to have the same sequence number since they were part of - // the same logical change - - if (bmData->workerStats.count(it.value())) { - bmData->workerStats[it.value()].numGranulesAssigned -= 1; + if (assignment.worker.present() && assignment.worker.get().isValid()) { + if (BM_DEBUG) { + fmt::print("BW {0} already chosen for seqno {1} in BM {2}\n", + assignment.worker.get().toString(), + seqNo, + bmData->id.toString()); } + workerId = assignment.worker.get(); - // revoke the range for the worker that owns it, not the worker specified in the revoke - bmData->addActor.send(doRangeAssignment(bmData, assignment, it.value(), seqNo)); + bmData->workerAssignments.insert(assignment.keyRange, workerId); + bmData->assignsInProgress.insert(assignment.keyRange, + doRangeAssignment(bmData, assignment, workerId, seqNo)); + // If we know about the worker and this is not a continue, then this is a new range for the worker + if (bmData->workerStats.count(workerId) && + assignment.assign.get().type != AssignRequestType::Continue) { + bmData->workerStats[workerId].numGranulesAssigned += 1; + } + } else { + // Ensure the key boundaries are updated before we pick a worker + bmData->workerAssignments.insert(assignment.keyRange, UID()); + bmData->assignsInProgress.insert(assignment.keyRange, + doRangeAssignment(bmData, assignment, Optional(), seqNo)); } - bmData->workerAssignments.insert(assignment.keyRange, UID()); + } else { + if (assignment.worker.present()) { + // revoke this specific range from this specific worker. Either part of recovery or failing a worker + if (bmData->workerStats.count(assignment.worker.get())) { + bmData->workerStats[assignment.worker.get()].numGranulesAssigned -= 1; + } + // if this revoke matches the worker assignment state, mark the range as unassigned + auto existingRange = bmData->workerAssignments.rangeContaining(assignment.keyRange.begin); + if (existingRange.range() == assignment.keyRange && existingRange.cvalue() == assignment.worker.get()) { + bmData->workerAssignments.insert(assignment.keyRange, UID()); + } + bmData->addActor.send(doRangeAssignment(bmData, assignment, assignment.worker.get(), seqNo)); + } else { + auto currentAssignments = bmData->workerAssignments.intersectingRanges(assignment.keyRange); + for (auto& it : currentAssignments) { + // ensure range doesn't truncate existing ranges + ASSERT(it.begin() >= assignment.keyRange.begin); + ASSERT(it.end() <= assignment.keyRange.end); + + // It is fine for multiple disjoint sub-ranges to have the same sequence number since they were part + // of the same logical change + + if (bmData->workerStats.count(it.value())) { + bmData->workerStats[it.value()].numGranulesAssigned -= 1; + } + + // revoke the range for the worker that owns it, not the worker specified in the revoke + bmData->addActor.send(doRangeAssignment(bmData, assignment, it.value(), seqNo)); + } + bmData->workerAssignments.insert(assignment.keyRange, UID()); + } + + bmData->assignsInProgress.cancel(assignment.keyRange); } } } -ACTOR Future checkManagerLock(Reference tr, BlobManagerData* bmData) { +ACTOR Future checkManagerLock(Reference tr, Reference bmData) { Optional currentLockValue = wait(tr->get(blobManagerEpochKey)); ASSERT(currentLockValue.present()); int64_t currentEpoch = decodeBlobManagerEpochValue(currentLockValue.get()); @@ -482,16 +656,52 @@ ACTOR Future checkManagerLock(Reference tr, Blo bmData->iAmReplaced.send(Void()); } - throw granule_assignment_conflict(); + throw blob_manager_replaced(); } tr->addReadConflictRange(singleKeyRange(blobManagerEpochKey)); + tr->addWriteConflictRange(singleKeyRange(blobManagerEpochKey)); return Void(); } -// FIXME: this does all logic in one transaction. Adding a giant range to an existing database to blobify would -// require doing a ton of storage metrics calls, which we should split up across multiple transactions likely. -ACTOR Future monitorClientRanges(BlobManagerData* bmData) { +ACTOR Future writeInitialGranuleMapping(Reference bmData, + Standalone> boundaries) { + state Reference tr = makeReference(bmData->db); + // don't do too many in one transaction + state int i = 0; + state int transactionChunkSize = BUGGIFY ? deterministicRandom()->randomInt(2, 5) : 1000; + while (i < boundaries.size() - 1) { + TEST(i > 0); // multiple transactions for large granule split + tr->reset(); + state int j = 0; + loop { + try { + tr->setOption(FDBTransactionOptions::Option::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::Option::ACCESS_SYSTEM_KEYS); + wait(checkManagerLock(tr, bmData)); + while (i + j < boundaries.size() - 1 && j < transactionChunkSize) { + // set to empty UID - no worker assigned yet + wait(krmSetRange(tr, + blobGranuleMappingKeys.begin, + KeyRangeRef(boundaries[i + j], boundaries[i + j + 1]), + blobGranuleMappingValueFor(UID()))); + j++; + } + wait(tr->commit()); + break; + } catch (Error& e) { + wait(tr->onError(e)); + j = 0; + } + } + i += j; + } + return Void(); +} + +ACTOR Future monitorClientRanges(Reference bmData) { + state Optional lastChangeKeyValue; + state bool needToCoalesce = bmData->epoch > 1; loop { state Reference tr = makeReference(bmData->db); @@ -503,10 +713,25 @@ ACTOR Future monitorClientRanges(BlobManagerData* bmData) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - // TODO probably knobs here? This should always be pretty small though - RangeResult results = wait(krmGetRanges( - tr, blobRangeKeys.begin, KeyRange(normalKeys), 10000, GetRangeLimits::BYTE_LIMIT_UNLIMITED)); - ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY); + // read change key at this point along with ranges + state Optional ckvBegin = wait(tr->get(blobRangeChangeKey)); + + state RangeResult results = wait(krmGetRanges(tr, + blobRangeKeys.begin, + KeyRange(normalKeys), + CLIENT_KNOBS->TOO_MANY, + GetRangeLimits::BYTE_LIMIT_UNLIMITED)); + ASSERT_WE_THINK(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY); + if (results.more || results.size() >= CLIENT_KNOBS->TOO_MANY) { + TraceEvent(SevError, "BlobManagerTooManyClientRanges", bmData->id) + .detail("Epoch", bmData->epoch) + .detail("ClientRanges", results.size() - 1); + wait(delay(600)); + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.sendError(internal_error()); + } + throw internal_error(); + } state Arena ar; ar.dependsOn(results.arena()); @@ -514,11 +739,23 @@ ACTOR Future monitorClientRanges(BlobManagerData* bmData) { VectorRef rangesToRemove; updateClientBlobRanges(&bmData->knownBlobRanges, results, ar, &rangesToAdd, &rangesToRemove); + if (needToCoalesce) { + // recovery has granules instead of known ranges in here. We need to do so to identify any parts of + // known client ranges the last manager didn't finish blob-ifying. + // To coalesce the map, we simply override known ranges with the current DB ranges after computing + // rangesToAdd + rangesToRemove + needToCoalesce = false; + + for (int i = 0; i < results.size() - 1; i++) { + bool active = results[i].value == LiteralStringRef("1"); + bmData->knownBlobRanges.insert(KeyRangeRef(results[i].key, results[i + 1].key), active); + } + } + for (KeyRangeRef range : rangesToRemove) { if (BM_DEBUG) { - printf("BM Got range to revoke [%s - %s)\n", - range.begin.printable().c_str(), - range.end.printable().c_str()); + fmt::print( + "BM Got range to revoke [{0} - {1})\n", range.begin.printable(), range.end.printable()); } RangeAssignment ra; @@ -531,43 +768,64 @@ ACTOR Future monitorClientRanges(BlobManagerData* bmData) { state std::vector>>> splitFutures; // Divide new ranges up into equal chunks by using SS byte sample for (KeyRangeRef range : rangesToAdd) { - // assert that this range contains no currently assigned ranges in this - splitFutures.push_back(splitRange(tr, range)); + splitFutures.push_back(splitRange(bmData, range, false)); } for (auto f : splitFutures) { - Standalone> splits = wait(f); + state Standalone> splits = wait(f); if (BM_DEBUG) { - printf("Split client range [%s - %s) into %d ranges:\n", - splits[0].printable().c_str(), - splits[splits.size() - 1].printable().c_str(), - splits.size() - 1); + fmt::print("Split client range [{0} - {1}) into {2} ranges:\n", + splits[0].printable(), + splits[splits.size() - 1].printable(), + splits.size() - 1); } + // Write to DB BEFORE sending assign requests, so that if manager dies before/during, new manager + // picks up the same ranges + wait(writeInitialGranuleMapping(bmData, splits)); + for (int i = 0; i < splits.size() - 1; i++) { KeyRange range = KeyRange(KeyRangeRef(splits[i], splits[i + 1])); + // only add the client range if this is the first BM or it's not already assigned if (BM_DEBUG) { - printf(" [%s - %s)\n", range.begin.printable().c_str(), range.end.printable().c_str()); + fmt::print( + " [{0} - {1})\n", range.begin.printable().c_str(), range.end.printable().c_str()); } RangeAssignment ra; ra.isAssign = true; ra.keyRange = range; - ra.assign = RangeAssignmentData(false); // continue=false + ra.assign = RangeAssignmentData(); // type=normal bmData->rangesToAssign.send(ra); } } - state Future watchFuture = tr->watch(blobRangeChangeKey); - wait(tr->commit()); - if (BM_DEBUG) { - printf("Blob manager done processing client ranges, awaiting update\n"); + lastChangeKeyValue = + ckvBegin; // the version of the ranges we processed is the one read alongside the ranges + + // do a new transaction, check for change in change key, watch if none + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + state Future watchFuture; + + Optional ckvEnd = wait(tr->get(blobRangeChangeKey)); + + if (ckvEnd == lastChangeKeyValue) { + watchFuture = tr->watch(blobRangeChangeKey); // watch for change in key + wait(tr->commit()); + if (BM_DEBUG) { + printf("Blob manager done processing client ranges, awaiting update\n"); + } + } else { + watchFuture = Future(Void()); // restart immediately } + wait(watchFuture); break; } catch (Error& e) { if (BM_DEBUG) { - printf("Blob manager got error looking for range updates %s\n", e.name()); + fmt::print("Blob manager got error looking for range updates {}\n", e.name()); } wait(tr->onError(e)); } @@ -575,60 +833,179 @@ ACTOR Future monitorClientRanges(BlobManagerData* bmData) { } } -ACTOR Future maybeSplitRange(BlobManagerData* bmData, +// split recursively in the middle to guarantee roughly equal splits across different parts of key space +static void downsampleSplit(const Standalone>& splits, + Standalone>& out, + int startIdx, + int endIdx, + int remaining) { + ASSERT(endIdx - startIdx >= remaining); + ASSERT(remaining >= 0); + if (remaining == 0) { + return; + } + if (endIdx - startIdx == remaining) { + out.append(out.arena(), splits.begin() + startIdx, remaining); + } else { + int mid = (startIdx + endIdx) / 2; + int startCount = (remaining - 1) / 2; + int endCount = remaining - startCount - 1; + // ensure no infinite recursion + ASSERT(mid != endIdx); + ASSERT(mid + 1 != startIdx); + downsampleSplit(splits, out, startIdx, mid, startCount); + out.push_back(out.arena(), splits[mid]); + downsampleSplit(splits, out, mid + 1, endIdx, endCount); + } +} + +ACTOR Future maybeSplitRange(Reference bmData, UID currentWorkerId, KeyRange granuleRange, UID granuleID, Version granuleStartVersion, - Version latestVersion) { + bool writeHot) { state Reference tr = makeReference(bmData->db); state Standalone> newRanges; - state int64_t newLockSeqno = -1; // first get ranges to split - if (newRanges.empty()) { - Standalone> _newRanges = wait(splitRange(tr, granuleRange)); - newRanges = _newRanges; - } + Standalone> _newRanges = wait(splitRange(bmData, granuleRange, writeHot)); + newRanges = _newRanges; + ASSERT(newRanges.size() >= 2); if (newRanges.size() == 2) { // not large enough to split, just reassign back to worker if (BM_DEBUG) { - printf("Not splitting existing range [%s - %s). Continuing assignment to %s\n", - granuleRange.begin.printable().c_str(), - granuleRange.end.printable().c_str(), - currentWorkerId.toString().c_str()); + fmt::print("Not splitting existing range [{0} - {1}). Continuing assignment to {2}\n", + granuleRange.begin.printable(), + granuleRange.end.printable(), + currentWorkerId.toString()); } RangeAssignment raContinue; raContinue.isAssign = true; raContinue.worker = currentWorkerId; raContinue.keyRange = granuleRange; - raContinue.assign = RangeAssignmentData(true); // continue assignment and re-snapshot + raContinue.assign = RangeAssignmentData(AssignRequestType::Continue); // continue assignment and re-snapshot bmData->rangesToAssign.send(raContinue); return Void(); } - // Need to split range. Persist intent to split and split metadata to DB BEFORE sending split requests + // Enforce max split fanout for performance reasons. This mainly happens when a blob worker is behind. + if (newRanges.size() >= + SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2) { // +2 because this is boundaries, so N keys would have N+1 bounaries. + TEST(true); // downsampling granule split because fanout too high + Standalone> coalescedRanges; + coalescedRanges.arena().dependsOn(newRanges.arena()); + coalescedRanges.push_back(coalescedRanges.arena(), newRanges.front()); + + // since we include start + end boundaries here, only need maxSplitFanout-1 split boundaries to produce + // maxSplitFanout granules + downsampleSplit(newRanges, coalescedRanges, 1, newRanges.size() - 1, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT - 1); + + coalescedRanges.push_back(coalescedRanges.arena(), newRanges.back()); + ASSERT(coalescedRanges.size() == SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1); + if (BM_DEBUG) { + fmt::print("Downsampled split from {0} -> {1} granules\n", + newRanges.size() - 1, + SERVER_KNOBS->BG_MAX_SPLIT_FANOUT); + } + + newRanges = coalescedRanges; + ASSERT(newRanges.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1); + } + + ASSERT(granuleRange.begin == newRanges.front()); + ASSERT(granuleRange.end == newRanges.back()); + + // Have to make set of granule ids deterministic across retries to not end up with extra UIDs in the split + // state, which could cause recovery to fail and resources to not be cleaned up. + // This entire transaction must be idempotent across retries for all splitting state + state std::vector newGranuleIDs; + newGranuleIDs.reserve(newRanges.size() - 1); + for (int i = 0; i < newRanges.size() - 1; i++) { + newGranuleIDs.push_back(deterministicRandom()->randomUniqueID()); + } + + if (BM_DEBUG) { + fmt::print("Splitting range [{0} - {1}) into {2} granules:\n", + granuleRange.begin.printable(), + granuleRange.end.printable(), + newRanges.size() - 1); + for (int i = 0; i < newRanges.size(); i++) { + fmt::print(" {}:{}\n", + (i < newGranuleIDs.size() ? newGranuleIDs[i] : UID()).toString().substr(0, 6).c_str(), + newRanges[i].printable()); + } + } + + state Version splitVersion; + + // Need to split range. Persist intent to split and split metadata to DB BEFORE sending split assignments to blob + // workers, so that nothing is lost on blob manager recovery loop { try { tr->reset(); tr->setOption(FDBTransactionOptions::Option::PRIORITY_SYSTEM_IMMEDIATE); tr->setOption(FDBTransactionOptions::Option::ACCESS_SYSTEM_KEYS); - ASSERT(newRanges.size() >= 2); + ASSERT(newRanges.size() > 2); // make sure we're still manager when this transaction gets committed wait(checkManagerLock(tr, bmData)); + // TODO can do this + lock in parallel + // Read splitState to see if anything was committed instead of reading granule mapping because we don't want + // to conflict with mapping changes/reassignments + state RangeResult existingState = + wait(tr->getRange(blobGranuleSplitKeyRangeFor(granuleID), SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 2)); + ASSERT_WE_THINK(!existingState.more && existingState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1); + // maybe someone decreased the knob, we should gracefully handle it not in simulation + if (existingState.more || existingState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT) { + RangeResult tryAgain = wait(tr->getRange(blobGranuleSplitKeyRangeFor(granuleID), 10000)); + ASSERT(!tryAgain.more); + existingState = tryAgain; + } + if (!existingState.empty()) { + // Something was previously committed, we must go with that decision. + // Read its boundaries and override our planned split boundaries + TEST(true); // Overriding split ranges with existing ones from DB + RangeResult existingBoundaries = + wait(tr->getRange(KeyRangeRef(granuleRange.begin.withPrefix(blobGranuleMappingKeys.begin), + keyAfter(granuleRange.end).withPrefix(blobGranuleMappingKeys.begin)), + existingState.size() + 2)); + // +2 because this is boundaries and existingState was granules, and to ensure it doesn't set more + ASSERT(!existingBoundaries.more); + ASSERT(existingBoundaries.size() == existingState.size() + 1); + newRanges.clear(); + newRanges.arena().dependsOn(existingBoundaries.arena()); + for (auto& it : existingBoundaries) { + newRanges.push_back(newRanges.arena(), it.key.removePrefix(blobGranuleMappingKeys.begin)); + } + ASSERT(newRanges.front() == granuleRange.begin); + ASSERT(newRanges.back() == granuleRange.end); + if (BM_DEBUG) { + fmt::print("Replaced old range splits for [{0} - {1}) with {2}:\n", + granuleRange.begin.printable(), + granuleRange.end.printable(), + newRanges.size() - 1); + for (int i = 0; i < newRanges.size(); i++) { + fmt::print(" {}\n", newRanges[i].printable()); + } + } + break; + } + // acquire lock for old granule to make sure nobody else modifies it state Key lockKey = blobGranuleLockKeyFor(granuleRange); Optional lockValue = wait(tr->get(lockKey)); ASSERT(lockValue.present()); std::tuple prevGranuleLock = decodeBlobGranuleLockValue(lockValue.get()); - if (std::get<0>(prevGranuleLock) > bmData->epoch) { + int64_t ownerEpoch = std::get<0>(prevGranuleLock); + + if (ownerEpoch > bmData->epoch) { if (BM_DEBUG) { fmt::print("BM {0} found a higher epoch {1} than {2} for granule lock of [{3} - {4})\n", - bmData->id.toString(), - std::get<0>(prevGranuleLock), + bmData->epoch, + ownerEpoch, bmData->epoch, granuleRange.begin.printable(), granuleRange.end.printable()); @@ -639,41 +1016,68 @@ ACTOR Future maybeSplitRange(BlobManagerData* bmData, } return Void(); } - if (newLockSeqno == -1) { - newLockSeqno = bmData->seqNo; - bmData->seqNo++; - ASSERT(newLockSeqno > std::get<1>(prevGranuleLock)); - } else { - // previous transaction could have succeeded but got commit_unknown_result - ASSERT(newLockSeqno >= std::get<1>(prevGranuleLock)); + + // Set lock to max value for this manager, so other reassignments can't race with this transaction + // and existing owner can't modify it further. + // FIXME: Implementing merging may require us to make lock go backwards if we later merge other granules + // back to this same range, but I think that's fine + tr->set(lockKey, + blobGranuleLockValueFor( + bmData->epoch, std::numeric_limits::max(), std::get<2>(prevGranuleLock))); + + // get last delta file version written, to make that the split version + RangeResult lastDeltaFile = + wait(tr->getRange(blobGranuleFileKeyRangeFor(granuleID), 1, Snapshot::False, Reverse::True)); + ASSERT(lastDeltaFile.size() == 1); + std::tuple k = decodeBlobGranuleFileKey(lastDeltaFile[0].key); + ASSERT(std::get<0>(k) == granuleID); + ASSERT(std::get<2>(k) == 'D'); + splitVersion = std::get<1>(k); + + if (BM_DEBUG) { + fmt::print("BM {0} found version {1} for splitting [{2} - {3})\n", + bmData->epoch, + splitVersion, + granuleRange.begin.printable(), + granuleRange.end.printable()); } - // acquire granule lock so nobody else can make changes to this granule. - tr->set(lockKey, blobGranuleLockValueFor(bmData->epoch, newLockSeqno, std::get<2>(prevGranuleLock))); - - // set up split metadata - for (int i = 0; i < newRanges.size() - 1; i++) { - UID newGranuleID = deterministicRandom()->randomUniqueID(); - - Key splitKey = blobGranuleSplitKeyFor(granuleID, newGranuleID); + // set up splits in granule mapping, but point each part to the old owner (until they get reassigned) + state int i; + for (i = 0; i < newRanges.size() - 1; i++) { + Key splitKey = blobGranuleSplitKeyFor(granuleID, newGranuleIDs[i]); tr->atomicOp(splitKey, - blobGranuleSplitValueFor(BlobGranuleSplitState::Started), + blobGranuleSplitValueFor(BlobGranuleSplitState::Initialized), MutationRef::SetVersionstampedValue); - Key historyKey = blobGranuleHistoryKeyFor(KeyRangeRef(newRanges[i], newRanges[i + 1]), latestVersion); + Key historyKey = blobGranuleHistoryKeyFor(KeyRangeRef(newRanges[i], newRanges[i + 1]), splitVersion); Standalone historyValue; - historyValue.granuleID = newGranuleID; + historyValue.granuleID = newGranuleIDs[i]; historyValue.parentGranules.push_back(historyValue.arena(), std::pair(granuleRange, granuleStartVersion)); tr->set(historyKey, blobGranuleHistoryValueFor(historyValue)); + + // split the assignment but still pointing to the same worker + // FIXME: could pick new random workers here, they'll get overridden shortly unless the BM immediately + // restarts + wait(krmSetRange(tr, + blobGranuleMappingKeys.begin, + KeyRangeRef(newRanges[i], newRanges[i + 1]), + blobGranuleMappingValueFor(currentWorkerId))); } wait(tr->commit()); break; } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw; + } + if (BM_DEBUG) { + fmt::print("BM {0} Persisting granule split got error {1}\n", bmData->epoch, e.name()); + } if (e.code() == error_code_granule_assignment_conflict) { if (bmData->iAmReplaced.canBeSet()) { bmData->iAmReplaced.send(Void()); @@ -685,20 +1089,17 @@ ACTOR Future maybeSplitRange(BlobManagerData* bmData, } if (BM_DEBUG) { - printf("Splitting range [%s - %s) into (%d):\n", - granuleRange.begin.printable().c_str(), - granuleRange.end.printable().c_str(), - newRanges.size() - 1); - for (int i = 0; i < newRanges.size() - 1; i++) { - printf(" [%s - %s)\n", newRanges[i].printable().c_str(), newRanges[i + 1].printable().c_str()); - } + fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} done, sending assignments:\n", + granuleRange.begin.printable(), + granuleRange.end.printable(), + newRanges.size() - 1, + splitVersion); } // transaction committed, send range assignments - // revoke from current worker + // range could have been moved since split eval started, so just revoke from whoever has it RangeAssignment raRevoke; raRevoke.isAssign = false; - raRevoke.worker = currentWorkerId; raRevoke.keyRange = granuleRange; raRevoke.revoke = RangeRevokeData(false); // not a dispose bmData->rangesToAssign.send(raRevoke); @@ -708,60 +1109,147 @@ ACTOR Future maybeSplitRange(BlobManagerData* bmData, RangeAssignment raAssignSplit; raAssignSplit.isAssign = true; raAssignSplit.keyRange = KeyRangeRef(newRanges[i], newRanges[i + 1]); - raAssignSplit.assign = RangeAssignmentData(false); + raAssignSplit.assign = RangeAssignmentData(); // don't care who this range gets assigned to bmData->rangesToAssign.send(raAssignSplit); } + if (BM_DEBUG) { + fmt::print("Splitting range [{0} - {1}) into {2} granules @ {3} got assignments processed\n", + granuleRange.begin.printable(), + granuleRange.end.printable(), + newRanges.size() - 1, + splitVersion); + } + return Void(); } -void killBlobWorker(BlobManagerData* bmData, BlobWorkerInterface bwInterf) { - UID bwId = bwInterf.id(); +ACTOR Future deregisterBlobWorker(Reference bmData, BlobWorkerInterface interf) { + state Reference tr = makeReference(bmData->db); + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + try { + wait(checkManagerLock(tr, bmData)); + Key blobWorkerListKey = blobWorkerListKeyFor(interf.id()); + // FIXME: should be able to remove this conflict range + tr->addReadConflictRange(singleKeyRange(blobWorkerListKey)); + tr->clear(blobWorkerListKey); + + wait(tr->commit()); + + if (BM_DEBUG) { + fmt::print("Deregistered blob worker {0}\n", interf.id().toString()); + } + return Void(); + } catch (Error& e) { + if (BM_DEBUG) { + fmt::print("Deregistering blob worker {0} got error {1}\n", interf.id().toString(), e.name()); + } + wait(tr->onError(e)); + } + } +} + +ACTOR Future haltBlobWorker(Reference bmData, BlobWorkerInterface bwInterf) { + loop { + try { + wait(bwInterf.haltBlobWorker.getReply(HaltBlobWorkerRequest(bmData->epoch, bmData->id))); + break; + } catch (Error& e) { + // throw other errors instead of returning? + if (e.code() == error_code_operation_cancelled) { + throw; + } + if (e.code() != error_code_blob_manager_replaced) { + break; + } + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.send(Void()); + } + } + } + + return Void(); +} + +ACTOR Future killBlobWorker(Reference bmData, BlobWorkerInterface bwInterf, bool registered) { + state UID bwId = bwInterf.id(); // Remove blob worker from stats map so that when we try to find a worker to takeover the range, // the one we just killed isn't considered. // Remove it from workersById also since otherwise that worker addr will remain excluded // when we try to recruit new blob workers. - bmData->workerStats.erase(bwId); - bmData->workersById.erase(bwId); + + if (registered) { + bmData->deadWorkers.insert(bwId); + bmData->workerStats.erase(bwId); + bmData->workersById.erase(bwId); + bmData->workerAddresses.erase(bwInterf.stableAddress()); + } + + // Remove blob worker from persisted list of blob workers + Future deregister = deregisterBlobWorker(bmData, bwInterf); // for every range owned by this blob worker, we want to // - send a revoke request for that range // - add the range back to the stream of ranges to be assigned if (BM_DEBUG) { - printf("Taking back ranges from BW %s\n", bwId.toString().c_str()); + fmt::print("Taking back ranges from BW {0}\n", bwId.toString()); } + // copy ranges into vector before sending, because send then modifies workerAssignments + state std::vector rangesToMove; for (auto& it : bmData->workerAssignments.ranges()) { if (it.cvalue() == bwId) { - // Send revoke request - RangeAssignment raRevoke; - raRevoke.isAssign = false; - raRevoke.keyRange = it.range(); - raRevoke.revoke = RangeRevokeData(false); - bmData->rangesToAssign.send(raRevoke); - - // Add range back into the stream of ranges to be assigned - RangeAssignment raAssign; - raAssign.isAssign = true; - raAssign.worker = Optional(); - raAssign.keyRange = it.range(); - raAssign.assign = RangeAssignmentData(false); // not a continue - bmData->rangesToAssign.send(raAssign); + rangesToMove.push_back(it.range()); } } + for (auto& it : rangesToMove) { + // Send revoke request + RangeAssignment raRevoke; + raRevoke.isAssign = false; + raRevoke.keyRange = it; + raRevoke.revoke = RangeRevokeData(false); + bmData->rangesToAssign.send(raRevoke); + + // Add range back into the stream of ranges to be assigned + RangeAssignment raAssign; + raAssign.isAssign = true; + raAssign.worker = Optional(); + raAssign.keyRange = it; + raAssign.assign = RangeAssignmentData(); // not a continue + bmData->rangesToAssign.send(raAssign); + } // Send halt to blob worker, with no expectation of hearing back if (BM_DEBUG) { - printf("Sending halt to BW %s\n", bwId.toString().c_str()); + fmt::print("Sending halt to BW {}\n", bwId.toString()); } - bmData->addActor.send( - brokenPromiseToNever(bwInterf.haltBlobWorker.getReply(HaltBlobWorkerRequest(bmData->epoch, bmData->id)))); + bmData->addActor.send(haltBlobWorker(bmData, bwInterf)); + + // wait for blob worker to be removed from DB and in-memory mapping to have reassigned all shards from this worker + // before removing it from deadWorkers, to avoid a race with checkBlobWorkerList + wait(deregister); + + // restart recruiting to replace the dead blob worker + bmData->restartRecruiting.trigger(); + + if (registered) { + bmData->deadWorkers.erase(bwInterf.id()); + } + + return Void(); } -ACTOR Future monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerInterface bwInterf) { - state KeyRangeMap> lastSeenSeqno; +ACTOR Future monitorBlobWorkerStatus(Reference bmData, BlobWorkerInterface bwInterf) { // outer loop handles reconstructing stream if it got a retryable error + // do backoff, we can get a lot of retries in a row + + // wait for blob manager to be done recovering, so it has initial granule mapping and worker data + wait(bmData->doneRecovering.getFuture()); + + state double backoff = SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN; loop { try { state ReplyPromiseStream statusStream = @@ -771,27 +1259,30 @@ ACTOR Future monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerIn GranuleStatusReply rep = waitNext(statusStream.getFuture()); if (BM_DEBUG) { - fmt::print("BM {0} got status of [{1} - {2}) @ ({3}, {4}) from BW {5}: {6}\n", + fmt::print("BM {0} got status of [{1} - {2}) @ ({3}, {4}) from BW {5}: {6} {7}\n", bmData->epoch, rep.granuleRange.begin.printable(), rep.granuleRange.end.printable(), rep.epoch, rep.seqno, bwInterf.id().toString(), - rep.doSplit ? "split" : ""); + rep.doSplit ? "split" : "", + rep.writeHotSplit ? "hot" : "normal"); } + // if we get a reply from the stream, reset backoff + backoff = SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN; if (rep.epoch > bmData->epoch) { if (BM_DEBUG) { - printf("BM heard from BW %s that there is a new manager with higher epoch\n", - bwInterf.id().toString().c_str()); + fmt::print("BM heard from BW {0} that there is a new manager with higher epoch\n", + bwInterf.id().toString()); } if (bmData->iAmReplaced.canBeSet()) { bmData->iAmReplaced.send(Void()); } } - // TODO maybe this won't be true eventually, but right now the only time the blob worker reports back is - // to split the range. + // This won't be true eventually, but right now the only time the blob worker reports + // back is to split the range. ASSERT(rep.doSplit); // only evaluate for split if this worker currently owns the granule in this blob manager's mapping @@ -799,59 +1290,118 @@ ACTOR Future monitorBlobWorkerStatus(BlobManagerData* bmData, BlobWorkerIn if (!(currGranuleAssignment.begin() == rep.granuleRange.begin && currGranuleAssignment.end() == rep.granuleRange.end && currGranuleAssignment.cvalue() == bwInterf.id())) { + if (BM_DEBUG) { + fmt::print("Manager {0} ignoring status from BW {1} for granule [{2} - {3}) since BW {4} owns " + "[{5} - {6}).\n", + bmData->epoch, + bwInterf.id().toString().substr(0, 5), + rep.granuleRange.begin.printable(), + rep.granuleRange.end.printable(), + currGranuleAssignment.cvalue().toString().substr(0, 5), + currGranuleAssignment.begin().printable(), + currGranuleAssignment.end().printable()); + } + // FIXME: could send revoke request continue; } - auto lastReqForGranule = lastSeenSeqno.rangeContaining(rep.granuleRange.begin); - if (rep.granuleRange.begin == lastReqForGranule.begin() && - rep.granuleRange.end == lastReqForGranule.end() && rep.epoch == lastReqForGranule.value().first && - rep.seqno == lastReqForGranule.value().second) { + // FIXME: We will need to go over all splits in the range once we're doing merges, instead of first one + auto lastSplitEval = bmData->splitEvaluations.rangeContaining(rep.granuleRange.begin); + if (rep.granuleRange.begin == lastSplitEval.begin() && rep.granuleRange.end == lastSplitEval.end() && + rep.epoch == lastSplitEval.cvalue().epoch && rep.seqno == lastSplitEval.cvalue().seqno) { if (BM_DEBUG) { - fmt::print("Manager {0} received repeat status for the same granule [{1} - {2}), ignoring.", + fmt::print("Manager {0} received repeat status for the same granule [{1} - {2}), ignoring.\n", bmData->epoch, rep.granuleRange.begin.printable(), rep.granuleRange.end.printable()); } } else { - if (BM_DEBUG) { - fmt::print("Manager {0} evaluating [{1} - {2}) for split\n", - bmData->epoch, - rep.granuleRange.begin.printable().c_str(), - rep.granuleRange.end.printable().c_str()); + ASSERT(lastSplitEval.cvalue().epoch < rep.epoch || + (lastSplitEval.cvalue().epoch == rep.epoch && lastSplitEval.cvalue().seqno < rep.seqno)); + if (lastSplitEval.cvalue().inProgress.isValid() && !lastSplitEval.cvalue().inProgress.isReady()) { + TEST(true); // racing BM splits + // For example, one worker asked BM to split, then died, granule was moved, new worker asks to + // split on recovery. We need to ensure that they are semantically the same split. + // We will just rely on the in-progress split to finish + if (BM_DEBUG) { + fmt::print("Manager {0} got split request for [{1} - {2}) @ ({3}, {4}), but already in " + "progress from [{5} - {6}) @ ({7}, {8})\n", + bmData->epoch, + rep.granuleRange.begin.printable().c_str(), + rep.granuleRange.end.printable().c_str(), + rep.epoch, + rep.seqno, + lastSplitEval.begin().printable().c_str(), + lastSplitEval.end().printable().c_str(), + lastSplitEval.cvalue().epoch, + lastSplitEval.cvalue().seqno); + } + // ignore the request, they will retry + } else { + if (BM_DEBUG) { + fmt::print("Manager {0} evaluating [{1} - {2}) @ ({3}, {4}) for split\n", + bmData->epoch, + rep.granuleRange.begin.printable().c_str(), + rep.granuleRange.end.printable().c_str(), + rep.epoch, + rep.seqno); + } + Future doSplitEval = maybeSplitRange(bmData, + bwInterf.id(), + rep.granuleRange, + rep.granuleID, + rep.startVersion, + rep.writeHotSplit); + bmData->splitEvaluations.insert(rep.granuleRange, + SplitEvaluation(rep.epoch, rep.seqno, doSplitEval)); } - lastSeenSeqno.insert(rep.granuleRange, std::pair(rep.epoch, rep.seqno)); - bmData->addActor.send(maybeSplitRange( - bmData, bwInterf.id(), rep.granuleRange, rep.granuleID, rep.startVersion, rep.latestVersion)); } } } catch (Error& e) { if (e.code() == error_code_operation_cancelled) { throw e; } + + // on known network errors or stream close errors, throw + if (e.code() == error_code_broken_promise) { + throw e; + } + + // if manager is replaced, die + if (e.code() == error_code_blob_manager_replaced) { + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.send(Void()); + } + return Void(); + } + // if we got an error constructing or reading from stream that is retryable, wait and retry. + // Sometimes we get connection_failed without the failure monitor tripping. One example is simulation's + // rollRandomClose. In this case, just reconstruct the stream. If it was a transient failure, it works, and + // if it is permanent, the failure monitor will eventually trip. ASSERT(e.code() != error_code_end_of_stream); - if (e.code() == error_code_connection_failed || e.code() == error_code_request_maybe_delivered) { - // FIXME: this could throw connection_failed and we could handle catch this the same as the failure - // detection triggering - wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); + if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_connection_failed) { + TEST(true); // BM retrying BW monitoring + wait(delay(backoff)); + backoff = std::min(backoff * SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT, + SERVER_KNOBS->BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX); continue; } else { - if (BM_DEBUG) { - printf("BM got unexpected error %s monitoring BW %s status\n", - e.name(), - bwInterf.id().toString().c_str()); - } - // TODO change back from SevError? - TraceEvent(SevError, "BWStatusMonitoringFailed", bmData->id) + TraceEvent(SevError, "BlobManagerUnexpectedErrorStatusMonitoring", bmData->id) .error(e) - .detail("BlobWorkerID", bwInterf.id()); + .detail("Epoch", bmData->epoch); + ASSERT_WE_THINK(false); + // if not simulation, kill the BM + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.sendError(e); + } throw e; } } } } -ACTOR Future monitorBlobWorker(BlobManagerData* bmData, BlobWorkerInterface bwInterf) { +ACTOR Future monitorBlobWorker(Reference bmData, BlobWorkerInterface bwInterf) { try { state Future waitFailure = waitFailureClient(bwInterf.waitFailure, SERVER_KNOBS->BLOB_WORKER_TIMEOUT); state Future monitorStatus = monitorBlobWorkerStatus(bmData, bwInterf); @@ -864,40 +1414,444 @@ ACTOR Future monitorBlobWorker(BlobManagerData* bmData, BlobWorkerInterfac TraceEvent("BlobWorkerFailed", bmData->id).detail("BlobWorkerID", bwInterf.id()); } when(wait(monitorStatus)) { - ASSERT(false); - throw internal_error(); + // should only return when manager got replaced + ASSERT(!bmData->iAmReplaced.canBeSet()); } } } catch (Error& e) { + // will blob worker get cleaned up in this case? if (e.code() == error_code_operation_cancelled) { throw e; } - // FIXME: forward errors somewhere from here + if (BM_DEBUG) { - printf("BM got unexpected error %s monitoring BW %s\n", e.name(), bwInterf.id().toString().c_str()); + fmt::print( + "BM {0} got monitoring error {1} from BW {2}\n", bmData->epoch, e.name(), bwInterf.id().toString()); + } + + // Expected errors here are: [broken_promise] + if (e.code() != error_code_broken_promise) { + if (BM_DEBUG) { + fmt::print("BM got unexpected error {0} monitoring BW {1}\n", e.name(), bwInterf.id().toString()); + } + TraceEvent(SevError, "BlobManagerUnexpectedErrorMonitorBW", bmData->id) + .error(e) + .detail("Epoch", bmData->epoch); + ASSERT_WE_THINK(false); + // if not simulation, kill the BM + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.sendError(e); + } + throw e; } - // TODO change back from SevError? - TraceEvent(SevError, "BWMonitoringFailed", bmData->id).error(e).detail("BlobWorkerID", bwInterf.id()); - throw e; } // kill the blob worker - killBlobWorker(bmData, bwInterf); - - // Trigger recruitment for a new blob worker - if (BM_DEBUG) { - printf("Restarting recruitment to replace dead BW %s\n", bwInterf.id().toString().c_str()); - } - bmData->restartRecruiting.trigger(); + wait(killBlobWorker(bmData, bwInterf, true)); if (BM_DEBUG) { - printf("No longer monitoring BW %s\n", bwInterf.id().toString().c_str()); + fmt::print("No longer monitoring BW {0}\n", bwInterf.id().toString()); } return Void(); } -ACTOR Future chaosRangeMover(BlobManagerData* bmData) { +ACTOR Future checkBlobWorkerList(Reference bmData, Promise workerListReady) { + + try { + loop { + // Get list of last known blob workers + // note: the list will include every blob worker that the old manager knew about, + // but it might also contain blob workers that died while the new manager was being recruited + std::vector blobWorkers = wait(getBlobWorkers(bmData->db)); + // add all blob workers to this new blob manager's records and start monitoring it + bool foundAnyNew = false; + for (auto& worker : blobWorkers) { + if (!bmData->deadWorkers.count(worker.id())) { + if (!bmData->workerAddresses.count(worker.stableAddress()) && + worker.locality.dcId() == bmData->dcId) { + bmData->workerAddresses.insert(worker.stableAddress()); + bmData->workersById[worker.id()] = worker; + bmData->workerStats[worker.id()] = BlobWorkerStats(); + bmData->addActor.send(monitorBlobWorker(bmData, worker)); + foundAnyNew = true; + } else if (!bmData->workersById.count(worker.id())) { + bmData->addActor.send(killBlobWorker(bmData, worker, false)); + } + } + } + if (workerListReady.canBeSet()) { + workerListReady.send(Void()); + } + // if any assigns are stuck on workers, and we have workers, wake them + if (foundAnyNew || !bmData->workersById.empty()) { + Promise hold = bmData->foundBlobWorkers; + bmData->foundBlobWorkers = Promise(); + hold.send(Void()); + } + wait(delay(SERVER_KNOBS->BLOB_WORKERLIST_FETCH_INTERVAL)); + } + } catch (Error& e) { + if (BM_DEBUG) { + fmt::print("BM {0} got error {1} reading blob worker list!!\n", bmData->epoch, e.name()); + } + throw e; + } +} +// Shared code for handling KeyRangeMap that is used several places in blob manager recovery +// when there can be conflicting sources of what assignments exist or which workers owns a granule. +// Resolves these conflicts by comparing the epoch + seqno for the range +// Special epoch/seqnos: +// (0,0): range is not mapped +static void addAssignment(KeyRangeMap>& map, + const KeyRangeRef& newRange, + UID newId, + int64_t newEpoch, + int64_t newSeqno, + std::vector>& outOfDate) { + std::vector>> newer; + auto intersecting = map.intersectingRanges(newRange); + bool allExistingNewer = true; + bool anyConflicts = false; + for (auto& old : intersecting) { + UID oldWorker = std::get<0>(old.value()); + int64_t oldEpoch = std::get<1>(old.value()); + int64_t oldSeqno = std::get<2>(old.value()); + if (oldEpoch > newEpoch || (oldEpoch == newEpoch && oldSeqno > newSeqno)) { + newer.push_back(std::pair(old.range(), std::tuple(oldWorker, oldEpoch, oldSeqno))); + if (old.range() != newRange) { + TEST(true); // BM Recovery: BWs disagree on range boundaries + anyConflicts = true; + } + } else { + allExistingNewer = false; + if (newId != UID() && newEpoch != std::numeric_limits::max()) { + // different workers can't have same epoch and seqno for granule assignment + ASSERT(oldEpoch != newEpoch || oldSeqno != newSeqno); + } + if (newEpoch == std::numeric_limits::max() && (oldWorker != newId || old.range() != newRange)) { + TEST(true); // BM Recovery: DB disagrees with workers + // new one is from DB (source of truth on boundaries) and existing mapping disagrees on boundary or + // assignment, do explicit revoke and re-assign to converge + anyConflicts = true; + // if ranges don't match, need to explicitly reassign all parts of old range, as it could be from a + // yet-unassigned split + if (old.range() != newRange) { + std::get<0>(old.value()) = UID(); + } + if (oldWorker != UID() && + (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range())))) { + + outOfDate.push_back(std::pair(oldWorker, old.range())); + } + } else if (oldWorker != UID() && oldWorker != newId && + (oldEpoch < newEpoch || (oldEpoch == newEpoch && oldSeqno < newSeqno))) { + // 2 blob workers reported conflicting mappings, add old one to out of date (if not already added by a + // previous intersecting range in the split case) + // if ranges don't match, need to explicitly reassign all parts of old range, as it could be from a + // partially-assigned split + if (old.range() != newRange) { + std::get<0>(old.value()) = UID(); + } + if (outOfDate.empty() || outOfDate.back() != std::pair(oldWorker, KeyRange(old.range()))) { + TEST(true); // BM Recovery: Two workers claim ownership of same granule + outOfDate.push_back(std::pair(oldWorker, old.range())); + } + } + } + } + + if (!allExistingNewer) { + // if this range supercedes an old range insert it over that + map.insert(newRange, std::tuple(anyConflicts ? UID() : newId, newEpoch, newSeqno)); + + // then, if there were any ranges superceded by this one, insert them over this one + if (newer.size()) { + if (newId != UID()) { + outOfDate.push_back(std::pair(newId, newRange)); + } + for (auto& it : newer) { + map.insert(it.first, it.second); + } + } + } else { + if (newId != UID()) { + outOfDate.push_back(std::pair(newId, newRange)); + } + } +} + +ACTOR Future recoverBlobManager(Reference bmData) { + state Promise workerListReady; + bmData->addActor.send(checkBlobWorkerList(bmData, workerListReady)); + wait(workerListReady.getFuture()); + + state std::vector startingWorkers; + for (auto& it : bmData->workersById) { + startingWorkers.push_back(it.second); + } + + // Once we acknowledge the existing blob workers, we can go ahead and recruit new ones + bmData->startRecruiting.trigger(); + + // skip the rest of the algorithm for the first blob manager + if (bmData->epoch == 1) { + bmData->doneRecovering.send(Void()); + return Void(); + } + + TEST(true); // BM doing recovery + + wait(delay(0)); + + // At this point, bmData->workersById is a list of all alive blob workers, but could also include some dead BWs. + // The algorithm below works as follows: + // + // 1. We get the existing granule mappings. We do this by asking all active blob workers for their current granule + // assignments. This guarantees a consistent snapshot of the state of that worker's assignments: Any request it + // recieved and processed from the old manager before the granule assignment request will be included in the + // assignments, and any request it recieves from the old manager afterwards will be rejected with + // blob_manager_replaced. We will then read any gaps in the mapping from the database. We will reconcile the set + // of ongoing splits to this mapping, and any ranges that are not already assigned to existing blob workers will + // be reassigned. + // + // 2. For every range in our granuleAssignments, we send an assign request to the stream of requests, + // ultimately giving every range back to some worker (trying to mimic the state of the old BM). + // If the worker already had the range, this is a no-op. If the worker didn't have it, it will + // begin persisting it. The worker that had the same range before will now be at a lower seqno. + + state KeyRangeMap> workerAssignments; + workerAssignments.insert(normalKeys, std::tuple(UID(), 0, 0)); + state Reference tr = makeReference(bmData->db); + + // FIXME: use range stream instead + state int rowLimit = BUGGIFY ? deterministicRandom()->randomInt(2, 10) : 10000; + + if (BM_DEBUG) { + fmt::print("BM {0} recovering:\n", bmData->epoch); + } + + // Step 1. Get the latest known mapping of granules to blob workers (i.e. assignments) + // This must happen causally AFTER reading the split boundaries, since the blob workers can clear the split + // boundaries for a granule as part of persisting their assignment. + + // First, ask existing workers for their mapping + if (BM_DEBUG) { + fmt::print("BM {0} requesting assignments from {1} workers:\n", bmData->epoch, startingWorkers.size()); + } + state std::vector>> aliveAssignments; + aliveAssignments.reserve(startingWorkers.size()); + for (auto& it : startingWorkers) { + GetGranuleAssignmentsRequest req; + req.managerEpoch = bmData->epoch; + aliveAssignments.push_back(timeout(brokenPromiseToNever(it.granuleAssignmentsRequest.getReply(req)), + SERVER_KNOBS->BLOB_WORKER_TIMEOUT)); + } + + state std::vector> outOfDateAssignments; + state int successful = 0; + state int assignIdx = 0; + + for (; assignIdx < aliveAssignments.size(); assignIdx++) { + Optional reply = wait(aliveAssignments[assignIdx]); + UID workerId = startingWorkers[assignIdx].id(); + + if (reply.present()) { + if (BM_DEBUG) { + fmt::print(" Worker {}: ({})\n", workerId.toString().substr(0, 5), reply.get().assignments.size()); + } + successful++; + for (auto& assignment : reply.get().assignments) { + if (BM_DEBUG) { + fmt::print(" [{0} - {1}): ({2}, {3})\n", + assignment.range.begin.printable(), + assignment.range.end.printable(), + assignment.epochAssigned, + assignment.seqnoAssigned); + } + bmData->knownBlobRanges.insert(assignment.range, true); + addAssignment(workerAssignments, + assignment.range, + workerId, + assignment.epochAssigned, + assignment.seqnoAssigned, + outOfDateAssignments); + } + if (bmData->workerStats.count(workerId)) { + bmData->workerStats[workerId].numGranulesAssigned = reply.get().assignments.size(); + } + } else { + TEST(true); // BM Recovery: BW didn't respond to assignments request + // SOMEDAY: mark as failed and kill it + if (BM_DEBUG) { + fmt::print(" Worker {}: failed\n", workerId.toString().substr(0, 5)); + } + } + } + + if (BM_DEBUG) { + fmt::print("BM {0} got assignments from {1}/{2} workers:\n", bmData->epoch, successful, startingWorkers.size()); + } + + if (BM_DEBUG) { + fmt::print("BM {0} found old assignments:\n", bmData->epoch); + } + + // DB is the source of truth, so read from here, and resolve any conflicts with current worker mapping + // We don't have a consistent snapshot of the mapping ACROSS blob workers, so we need the DB to reconcile any + // differences (eg blob manager revoked from worker A, assigned to B, the revoke from A was processed but the assign + // to B wasn't, meaning in the snapshot nobody owns the granule). This also handles races with a BM persisting a + // boundary change, then dying before notifying the workers + state Key beginKey = blobGranuleMappingKeys.begin; + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + KeyRange nextRange(KeyRangeRef(beginKey, blobGranuleMappingKeys.end)); + // using the krm functions can produce incorrect behavior here as it does weird stuff with beginKey + state GetRangeLimits limits(rowLimit, GetRangeLimits::BYTE_LIMIT_UNLIMITED); + limits.minRows = 2; + RangeResult results = wait(tr->getRange(nextRange, limits)); + + // Add the mappings to our in memory key range map + for (int rangeIdx = 0; rangeIdx < results.size() - 1; rangeIdx++) { + Key granuleStartKey = results[rangeIdx].key.removePrefix(blobGranuleMappingKeys.begin); + Key granuleEndKey = results[rangeIdx + 1].key.removePrefix(blobGranuleMappingKeys.begin); + if (results[rangeIdx].value.size()) { + // note: if the old owner is dead, we handle this in rangeAssigner + UID existingOwner = decodeBlobGranuleMappingValue(results[rangeIdx].value); + // use (max int64_t, 0) to be higher than anything that existing workers have + addAssignment(workerAssignments, + KeyRangeRef(granuleStartKey, granuleEndKey), + existingOwner, + std::numeric_limits::max(), + 0, + outOfDateAssignments); + + bmData->knownBlobRanges.insert(KeyRangeRef(granuleStartKey, granuleEndKey), true); + if (BM_DEBUG) { + fmt::print(" [{0} - {1})={2}\n", + granuleStartKey.printable(), + granuleEndKey.printable(), + existingOwner.toString().substr(0, 5)); + } + } else { + if (BM_DEBUG) { + fmt::print(" [{0} - {1})\n", granuleStartKey.printable(), granuleEndKey.printable()); + } + } + } + + if (!results.more || results.size() <= 1) { + break; + } + + // re-read last key to get range that starts there + beginKey = results.back().key; + } catch (Error& e) { + if (BM_DEBUG) { + fmt::print("BM {0} got error reading granule mapping during recovery: {1}\n", bmData->epoch, e.name()); + } + wait(tr->onError(e)); + } + } + + // Step 2. Send assign requests for all the granules and transfer assignments + // from local workerAssignments to bmData + // before we take ownership of all of the ranges, check the manager lock again + tr->reset(); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + wait(checkManagerLock(tr, bmData)); + wait(tr->commit()); + break; + } catch (Error& e) { + if (BM_DEBUG) { + fmt::print("BM {0} got error checking lock after recovery: {1}\n", bmData->epoch, e.name()); + } + wait(tr->onError(e)); + } + } + + // Get set of workers again. Some could have died after reporting assignments + std::unordered_set endingWorkers; + for (auto& it : bmData->workersById) { + endingWorkers.insert(it.first); + } + + // revoke assignments that are old and incorrect + TEST(!outOfDateAssignments.empty()); // BM resolved conflicting assignments on recovery + for (auto& it : outOfDateAssignments) { + if (BM_DEBUG) { + fmt::print("BM {0} revoking out of date assignment [{1} - {2}): {3}:\n", + bmData->epoch, + it.second.begin.printable().c_str(), + it.second.end.printable().c_str(), + it.first.toString().c_str()); + } + RangeAssignment raRevoke; + raRevoke.isAssign = false; + raRevoke.worker = it.first; + raRevoke.keyRange = it.second; + raRevoke.revoke = RangeRevokeData(false); + bmData->rangesToAssign.send(raRevoke); + } + + if (BM_DEBUG) { + fmt::print("BM {0} final ranges:\n", bmData->epoch); + } + + int explicitAssignments = 0; + for (auto& range : workerAssignments.intersectingRanges(normalKeys)) { + int64_t epoch = std::get<1>(range.value()); + int64_t seqno = std::get<2>(range.value()); + if (epoch == 0 && seqno == 0) { + continue; + } + + UID workerId = std::get<0>(range.value()); + bmData->workerAssignments.insert(range.range(), workerId); + + if (BM_DEBUG) { + fmt::print(" [{0} - {1}): {2}\n", + range.begin().printable(), + range.end().printable(), + workerId == UID() || epoch == 0 ? " (?)" : workerId.toString().substr(0, 5).c_str()); + } + + // if worker id is already set to a known worker that replied with it in the mapping, range is already assigned + // there. If not, need to explicitly assign it to someone + if (workerId == UID() || epoch == 0 || !endingWorkers.count(workerId)) { + RangeAssignment raAssign; + raAssign.isAssign = true; + raAssign.worker = workerId; + raAssign.keyRange = range.range(); + raAssign.assign = RangeAssignmentData(AssignRequestType::Normal); + bmData->rangesToAssign.send(raAssign); + explicitAssignments++; + } + } + + TraceEvent("BlobManagerRecovered", bmData->id) + .detail("Epoch", bmData->epoch) + .detail("Granules", bmData->workerAssignments.size()) + .detail("Assigned", explicitAssignments) + .detail("Revoked", outOfDateAssignments.size()); + + ASSERT(bmData->doneRecovering.canBeSet()); + bmData->doneRecovering.send(Void()); + + return Void(); +} + +ACTOR Future chaosRangeMover(Reference bmData) { + // Only move each granule once during the test, otherwise it can cause availability issues + // KeyRange isn't hashable and this is only for simulation, so just use toString of range + state std::unordered_set alreadyMoved; ASSERT(g_network->isSimulated()); + TEST(true); // BM chaos range mover enabled loop { wait(delay(30.0)); @@ -913,27 +1867,29 @@ ACTOR Future chaosRangeMover(BlobManagerData* bmData) { while (tries > 0) { tries--; auto randomRange = bmData->workerAssignments.randomRange(); - if (randomRange.value() != UID()) { + if (randomRange.value() != UID() && !alreadyMoved.count(randomRange.range().toString())) { if (BM_DEBUG) { - printf("Range mover moving range [%s - %s): %s\n", - randomRange.begin().printable().c_str(), - randomRange.end().printable().c_str(), - randomRange.value().toString().c_str()); + fmt::print("Range mover moving range [{0} - {1}): {2}\n", + randomRange.begin().printable().c_str(), + randomRange.end().printable().c_str(), + randomRange.value().toString().c_str()); } + alreadyMoved.insert(randomRange.range().toString()); - // FIXME: with low probability, could immediately revoke it from the new assignment and move it back - // right after to test that race + // FIXME: with low probability, could immediately revoke it from the new assignment and move + // it back right after to test that race + state KeyRange range = randomRange.range(); RangeAssignment revokeOld; revokeOld.isAssign = false; - revokeOld.keyRange = randomRange.range(); + revokeOld.keyRange = range; revokeOld.revoke = RangeRevokeData(false); bmData->rangesToAssign.send(revokeOld); RangeAssignment assignNew; assignNew.isAssign = true; - assignNew.keyRange = randomRange.range(); - assignNew.assign = RangeAssignmentData(false); // not a continue + assignNew.keyRange = range; + assignNew.assign = RangeAssignmentData(); // not a continue bmData->rangesToAssign.send(assignNew); break; } @@ -942,13 +1898,13 @@ ACTOR Future chaosRangeMover(BlobManagerData* bmData) { printf("Range mover couldn't find random range to move, skipping\n"); } } else if (BM_DEBUG) { - printf("Range mover found %d workers, skipping\n", bmData->workerAssignments.size()); + fmt::print("Range mover found {0} workers, skipping\n", bmData->workerAssignments.size()); } } } // Returns the number of blob workers on addr -int numExistingBWOnAddr(BlobManagerData* self, const AddressExclusion& addr) { +int numExistingBWOnAddr(Reference self, const AddressExclusion& addr) { int numExistingBW = 0; for (auto& server : self->workersById) { const NetworkAddress& netAddr = server.second.stableAddress(); @@ -962,9 +1918,11 @@ int numExistingBWOnAddr(BlobManagerData* self, const AddressExclusion& addr) { } // Tries to recruit a blob worker on the candidateWorker process -ACTOR Future initializeBlobWorker(BlobManagerData* self, RecruitBlobWorkerReply candidateWorker) { +ACTOR Future initializeBlobWorker(Reference self, RecruitBlobWorkerReply candidateWorker) { const NetworkAddress& netAddr = candidateWorker.worker.stableAddress(); AddressExclusion workerAddr(netAddr.ip, netAddr.port); + self->recruitingStream.set(self->recruitingStream.get() + 1); + // Ask the candidateWorker to initialize a BW only if the worker does not have a pending request if (numExistingBWOnAddr(self, workerAddr) == 0 && self->recruitingLocalities.count(candidateWorker.worker.stableAddress()) == 0) { @@ -996,6 +1954,7 @@ ACTOR Future initializeBlobWorker(BlobManagerData* self, RecruitBlobWorker // if it failed in an expected way, add some delay before we try to recruit again // on this worker if (newBlobWorker.isError()) { + TEST(true); // BM got error recruiting BW TraceEvent(SevWarn, "BMRecruitmentError").error(newBlobWorker.getError()); if (!newBlobWorker.isError(error_code_recruitment_failed) && !newBlobWorker.isError(error_code_request_maybe_delivered)) { @@ -1009,9 +1968,16 @@ ACTOR Future initializeBlobWorker(BlobManagerData* self, RecruitBlobWorker if (newBlobWorker.present()) { BlobWorkerInterface bwi = newBlobWorker.get().interf; - self->workersById[bwi.id()] = bwi; - self->workerStats[bwi.id()] = BlobWorkerStats(); - self->addActor.send(monitorBlobWorker(self, bwi)); + if (!self->deadWorkers.count(bwi.id())) { + if (!self->workerAddresses.count(bwi.stableAddress()) && bwi.locality.dcId() == self->dcId) { + self->workerAddresses.insert(bwi.stableAddress()); + self->workersById[bwi.id()] = bwi; + self->workerStats[bwi.id()] = BlobWorkerStats(); + self->addActor.send(monitorBlobWorker(self, bwi)); + } else if (!self->workersById.count(bwi.id())) { + self->addActor.send(killBlobWorker(self, bwi, false)); + } + } TraceEvent("BMRecruiting") .detail("State", "Finished request") @@ -1028,17 +1994,23 @@ ACTOR Future initializeBlobWorker(BlobManagerData* self, RecruitBlobWorker } // try to recruit more blob workers + self->recruitingStream.set(self->recruitingStream.get() - 1); self->restartRecruiting.trigger(); return Void(); } // Recruits blob workers in a loop ACTOR Future blobWorkerRecruiter( - BlobManagerData* self, + Reference self, Reference>> recruitBlobWorker) { state Future fCandidateWorker; state RecruitBlobWorkerRequest lastRequest; + // wait until existing blob workers have been acknowledged so we don't break recruitment invariants + loop choose { + when(wait(self->startRecruiting.onTrigger())) { break; } + } + loop { try { state RecruitBlobWorkerRequest recruitReq; @@ -1066,7 +2038,8 @@ ACTOR Future blobWorkerRecruiter( } choose { - // when we get back a worker we can use, we will try to initialize a blob worker onto that process + // when we get back a worker we can use, we will try to initialize a blob worker onto that + // process when(RecruitBlobWorkerReply candidateWorker = wait(fCandidateWorker)) { self->addActor.send(initializeBlobWorker(self, candidateWorker)); } @@ -1087,66 +2060,701 @@ ACTOR Future blobWorkerRecruiter( } } -ACTOR Future blobManager(BlobManagerInterface bmInterf, - Reference const> dbInfo, - int64_t epoch) { - state BlobManagerData self(deterministicRandom()->randomUniqueID(), - openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True)); +ACTOR Future haltBlobGranules(Reference bmData) { + std::vector blobWorkers = wait(getBlobWorkers(bmData->db)); + std::vector> deregisterBlobWorkers; + for (auto& worker : blobWorkers) { + bmData->addActor.send(haltBlobWorker(bmData, worker)); + deregisterBlobWorkers.emplace_back(deregisterBlobWorker(bmData, worker)); + } + waitForAll(deregisterBlobWorkers); - state Future collection = actorCollection(self.addActor.getFuture()); + return Void(); +} +ACTOR Future loadHistoryFiles(Reference bmData, UID granuleID) { + state Transaction tr(bmData->db); + state KeyRange range = blobGranuleFileKeyRangeFor(granuleID); + state Key startKey = range.begin; + state GranuleFiles files; + loop { + try { + wait(readGranuleFiles(&tr, &startKey, range.end, &files, granuleID)); + return files; + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + +/* + * Deletes all files pertaining to the granule with id granuleId and + * also removes the history entry for this granule from the system keyspace + * TODO: ensure cannot fully delete granule that is still splitting! + */ +ACTOR Future fullyDeleteGranule(Reference self, UID granuleId, Key historyKey) { if (BM_DEBUG) { - printf("Blob manager starting...\n"); + fmt::print("Fully deleting granule {0}: init\n", granuleId.toString()); } - self.epoch = epoch; + // get files + GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId)); - // make sure the epoch hasn't gotten stale - state Reference tr = makeReference(self.db); + std::vector> deletions; + std::vector filesToDelete; // TODO: remove, just for debugging - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - try { - wait(checkManagerLock(tr, &self)); - } catch (Error& e) { - if (BM_DEBUG) { - printf("Blob manager lock check got unexpected error %s. Dying...\n", e.name()); + for (auto snapshotFile : files.snapshotFiles) { + std::string fname = snapshotFile.filename; + deletions.emplace_back(self->bstore->deleteFile(fname)); + filesToDelete.emplace_back(fname); + } + + for (auto deltaFile : files.deltaFiles) { + std::string fname = deltaFile.filename; + deletions.emplace_back(self->bstore->deleteFile(fname)); + filesToDelete.emplace_back(fname); + } + + if (BM_DEBUG) { + fmt::print("Fully deleting granule {0}: deleting {1} files\n", granuleId.toString(), deletions.size()); + for (auto filename : filesToDelete) { + fmt::print(" - {}\n", filename.c_str()); } + } + + // delete the files before the corresponding metadata. + // this could lead to dangling pointers in fdb, but this granule should + // never be read again anyways, and we can clean up the keys the next time around. + // deleting files before corresponding metadata reduces the # of orphaned files. + wait(waitForAll(deletions)); + + // delete metadata in FDB (history entry and file keys) + if (BM_DEBUG) { + fmt::print("Fully deleting granule {0}: deleting history and file keys\n", granuleId.toString()); + } + + state Transaction tr(self->db); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + loop { + try { + KeyRange fileRangeKey = blobGranuleFileKeyRangeFor(granuleId); + tr.clear(historyKey); + tr.clear(fileRangeKey); + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + if (BM_DEBUG) { + fmt::print("Fully deleting granule {0}: success\n", granuleId.toString()); + } + + return Void(); +} + +/* + * For the granule with id granuleId, finds the first snapshot file at a + * version <= pruneVersion and deletes all files older than it. + * + * Assumption: this granule's startVersion might change because the first snapshot + * file might be deleted. We will need to ensure we don't rely on the granule's startVersion + * (that's persisted as part of the key), but rather use the granule's first snapshot's version when needed + */ +ACTOR Future partiallyDeleteGranule(Reference self, UID granuleId, Version pruneVersion) { + if (BM_DEBUG) { + fmt::print("Partially deleting granule {0}: init\n", granuleId.toString()); + } + + // get files + GranuleFiles files = wait(loadHistoryFiles(self->db, granuleId)); + + // represents the version of the latest snapshot file in this granule with G.version < pruneVersion + Version latestSnapshotVersion = invalidVersion; + + state std::vector> deletions; // deletion work per file + state std::vector deletedFileKeys; // keys for deleted files + state std::vector filesToDelete; // TODO: remove evenutally, just for debugging + + // TODO: binary search these snapshot files for latestSnapshotVersion + for (int idx = files.snapshotFiles.size() - 1; idx >= 0; --idx) { + // if we already found the latestSnapshotVersion, this snapshot can be deleted + if (latestSnapshotVersion != invalidVersion) { + std::string fname = files.snapshotFiles[idx].filename; + deletions.emplace_back(self->bstore->deleteFile(fname)); + deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, files.snapshotFiles[idx].version, 'S')); + filesToDelete.emplace_back(fname); + } else if (files.snapshotFiles[idx].version <= pruneVersion) { + // otherwise if this is the FIRST snapshot file with version < pruneVersion, + // then we found our latestSnapshotVersion (FIRST since we are traversing in reverse) + latestSnapshotVersion = files.snapshotFiles[idx].version; + } + } + + if (latestSnapshotVersion == invalidVersion) { return Void(); } - if (BM_DEBUG) { - fmt::print("Blob manager acquired lock at epoch {}\n", epoch); + // delete all delta files older than latestSnapshotVersion + for (auto deltaFile : files.deltaFiles) { + // traversing in fwd direction, so stop once we find the first delta file past the latestSnapshotVersion + if (deltaFile.version > latestSnapshotVersion) { + break; + } + + // otherwise deltaFile.version <= latestSnapshotVersion so delete it + // == should also be deleted because the last delta file before a snapshot would have the same version + std::string fname = deltaFile.filename; + deletions.emplace_back(self->bstore->deleteFile(fname)); + deletedFileKeys.emplace_back(blobGranuleFileKeyFor(granuleId, deltaFile.version, 'D')); + filesToDelete.emplace_back(fname); } - // needed to pick up changes to dbinfo in case new CC comes along + if (BM_DEBUG) { + fmt::print("Partially deleting granule {0}: deleting {1} files\n", granuleId.toString(), deletions.size()); + for (auto filename : filesToDelete) { + fmt::print(" - {0}\n", filename); + } + } + + // TODO: the following comment relies on the assumption that BWs will not get requests to + // read data that was already pruned. confirm assumption is fine. otherwise, we'd need + // to communicate with BWs here and have them ack the pruneVersion + + // delete the files before the corresponding metadata. + // this could lead to dangling pointers in fdb, but we should never read data older than + // pruneVersion anyways, and we can clean up the keys the next time around. + // deleting files before corresponding metadata reduces the # of orphaned files. + wait(waitForAll(deletions)); + + // delete metadata in FDB (deleted file keys) + if (BM_DEBUG) { + fmt::print("Partially deleting granule {0}: deleting file keys\n", granuleId.toString()); + } + + state Transaction tr(self->db); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + loop { + try { + for (auto& key : deletedFileKeys) { + tr.clear(key); + } + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + if (BM_DEBUG) { + fmt::print("Partially deleting granule {0}: success\n", granuleId.toString()); + } + return Void(); +} + +/* + * This method is used to prune the range [startKey, endKey) at (and including) pruneVersion. + * To do this, we do a BFS traversal starting at the active granules. Then we classify granules + * in the history as nodes that can be fully deleted (i.e. their files and history can be deleted) + * and nodes that can be partially deleted (i.e. some of their files can be deleted). + * Once all this is done, we finally clear the pruneIntent key, if possible, to indicate we are done + * processing this prune intent. + */ +ACTOR Future pruneRange(Reference self, KeyRangeRef range, Version pruneVersion, bool force) { + if (BM_DEBUG) { + fmt::print("pruneRange starting for range [{0} - {1}) @ pruneVersion={2}, force={3}\n", + range.begin.printable(), + range.end.printable(), + pruneVersion, + force); + } + + // queue of for BFS traversal of history + state std::queue> historyEntryQueue; + + // stacks of and to track which granules to delete + state std::vector> toFullyDelete; + state std::vector toPartiallyDelete; + + // track which granules we have already added to traversal + // note: (startKey, startVersion) uniquely identifies a granule + state std::unordered_set, boost::hash>> + visited; + + // find all active granules (that comprise the range) and add to the queue + state KeyRangeMap::Ranges activeRanges = self->workerAssignments.intersectingRanges(range); + + state Transaction tr(self->db); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + state KeyRangeMap::iterator activeRange; + for (activeRange = activeRanges.begin(); activeRange != activeRanges.end(); ++activeRange) { + if (BM_DEBUG) { + fmt::print("Checking if active range [{0} - {1}), owned by BW {2}, should be pruned\n", + activeRange.begin().printable(), + activeRange.end().printable(), + activeRange.value().toString()); + } + + // assumption: prune boundaries must respect granule boundaries + if (activeRange.begin() < range.begin || activeRange.end() > range.end) { + continue; + } + + // TODO: if this is a force prune, then revoke the assignment from the corresponding BW first + // so that it doesn't try to interact with the granule (i.e. force it to give up gLock). + // we'll need some way to ack that the revoke was successful + + loop { + try { + if (BM_DEBUG) { + fmt::print("Fetching latest history entry for range [{0} - {1})\n", + activeRange.begin().printable(), + activeRange.end().printable()); + } + Optional history = wait(getLatestGranuleHistory(&tr, activeRange.range())); + // TODO: can we tell from the krm that this range is not valid, so that we don't need to do a + // get + if (history.present()) { + if (BM_DEBUG) { + printf("Adding range to history queue\n"); + } + visited.insert({ activeRange.range().begin.begin(), history.get().version }); + historyEntryQueue.push({ activeRange.range(), history.get().version, MAX_VERSION }); + } + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + + if (BM_DEBUG) { + printf("Beginning BFS traversal of history\n"); + } + while (!historyEntryQueue.empty()) { + // process the node at the front of the queue and remove it + KeyRange currRange; + state Version startVersion; + state Version endVersion; + std::tie(currRange, startVersion, endVersion) = historyEntryQueue.front(); + historyEntryQueue.pop(); + + if (BM_DEBUG) { + fmt::print("Processing history node [{0} - {1}) with versions [{2}, {3})\n", + currRange.begin.printable(), + currRange.end.printable(), + startVersion, + endVersion); + } + + // get the persisted history entry for this granule + state Standalone currHistoryNode; + state Key historyKey = blobGranuleHistoryKeyFor(currRange, startVersion); + state bool foundHistory = false; + loop { + try { + Optional persistedHistory = wait(tr.get(historyKey)); + if (persistedHistory.present()) { + currHistoryNode = decodeBlobGranuleHistoryValue(persistedHistory.get()); + foundHistory = true; + } + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + if (!foundHistory) { + continue; + } + + if (BM_DEBUG) { + fmt::print("Found history entry for this node. It's granuleID is {0}\n", + currHistoryNode.granuleID.toString()); + } + + // There are three cases this granule can fall into: + // - if the granule's end version is at or before the prune version or this is a force delete, + // this granule should be completely deleted + // - else if the startVersion <= pruneVersion, then G.startVersion < pruneVersion < G.endVersion + // and so this granule should be partially deleted + // - otherwise, this granule is active, so don't schedule it for deletion + if (force || endVersion <= pruneVersion) { + if (BM_DEBUG) { + fmt::print("Granule {0} will be FULLY deleted\n", currHistoryNode.granuleID.toString()); + } + toFullyDelete.push_back({ currHistoryNode.granuleID, historyKey }); + } else if (startVersion < pruneVersion) { + if (BM_DEBUG) { + fmt::print("Granule {0} will be partially deleted\n", currHistoryNode.granuleID.toString()); + } + toPartiallyDelete.push_back({ currHistoryNode.granuleID }); + } + + // add all of the node's parents to the queue + for (auto& parent : currHistoryNode.parentGranules) { + // if we already added this node to queue, skip it; otherwise, mark it as visited + if (visited.count({ parent.first.begin.begin(), parent.second })) { + if (BM_DEBUG) { + fmt::print("Already added {0} to queue, so skipping it\n", currHistoryNode.granuleID.toString()); + } + continue; + } + visited.insert({ parent.first.begin.begin(), parent.second }); + + if (BM_DEBUG) { + fmt::print("Adding parent [{0} - {1}) with versions [{2} - {3}) to queue\n", + parent.first.begin.printable(), + parent.first.end.printable(), + parent.second, + startVersion); + } + + // the parent's end version is this node's startVersion, + // since this node must have started where it's parent finished + historyEntryQueue.push({ parent.first, parent.second, startVersion }); + } + } + + // The top of the stacks have the oldest ranges. This implies that for a granule located at + // index i, it's parent must be located at some index j, where j > i. For this reason, + // we delete granules in reverse order; this way, we will never end up with unreachable + // nodes in the persisted history. Moreover, for any node that must be fully deleted, + // any node that must be partially deleted must occur later on in the history. Thus, + // we delete the 'toFullyDelete' granules first. + // + // Unfortunately we can't do parallelize _full_ deletions because they might + // race and we'll end up with unreachable nodes in the case of a crash. + // Since partial deletions only occur for "leafs", they can be done in parallel + // + // Note about file deletions: although we might be retrying a deletion of a granule, + // we won't run into any issues with trying to "re-delete" a blob file since deleting + // a file that doesn't exist is considered successful + + state int i; + if (BM_DEBUG) { + fmt::print("{0} granules to fully delete\n", toFullyDelete.size()); + } + for (i = toFullyDelete.size() - 1; i >= 0; --i) { + UID granuleId; + Key historyKey; + std::tie(granuleId, historyKey) = toFullyDelete[i]; + // FIXME: consider batching into a single txn (need to take care of txn size limit) + if (BM_DEBUG) { + fmt::print("About to fully delete granule {0}\n", granuleId.toString()); + } + wait(fullyDeleteGranule(self, granuleId, historyKey)); + } + + if (BM_DEBUG) { + fmt::print("{0} granules to partially delete\n", toPartiallyDelete.size()); + } + std::vector> partialDeletions; + for (i = toPartiallyDelete.size() - 1; i >= 0; --i) { + UID granuleId = toPartiallyDelete[i]; + if (BM_DEBUG) { + fmt::print("About to partially delete granule {0}\n", granuleId.toString()); + } + partialDeletions.emplace_back(partiallyDeleteGranule(self, granuleId, pruneVersion)); + } + + wait(waitForAll(partialDeletions)); + + // Now that all the necessary granules and their files have been deleted, we can + // clear the pruneIntent key to signify that the work is done. However, there could have been + // another pruneIntent that got written for this table while we were processing this one. + // If that is the case, we should not clear the key. Otherwise, we can just clear the key. + + if (BM_DEBUG) { + fmt::print("Successfully pruned range [{0} - {1}) at pruneVersion={2}\n", + range.begin.printable(), + range.end.printable(), + pruneVersion); + } + return Void(); +} + +/* + * This monitor watches for changes to a key K that gets updated whenever there is a new prune intent. + * On this change, we scan through all blobGranulePruneKeys (which look like =) and prune any intents. + * + * Once the prune has succeeded, we clear the key IF the version is still the same one that was pruned. + * That way, if another prune intent arrived for the same range while we were working on an older one, + * we wouldn't end up clearing the intent. + * + * When watching for changes, we might end up in scenarios where we failed to do the work + * for a prune intent even though the watch was triggered (maybe the BM had a blip). This is problematic + * if the intent is a force and there isn't another prune intent for quite some time. To remedy this, + * if we don't see a watch change in X (configurable) seconds, we will just sweep through the prune intents, + * consolidating any work we might have missed before. + * + * Note: we could potentially use a changefeed here to get the exact pruneIntent that was added + * rather than iterating through all of them, but this might have too much overhead for latency + * improvements we don't really need here (also we need to go over all prune intents anyways in the + * case that the timer is up before any new prune intents arrive). + */ +ACTOR Future monitorPruneKeys(Reference self) { + // setup bstore + if (BM_DEBUG) { + fmt::print("BM constructing backup container from {}\n", SERVER_KNOBS->BG_URL.c_str()); + } + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + if (BM_DEBUG) { + printf("BM constructed backup container\n"); + } + + loop { + state Reference tr = makeReference(self->db); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + // Wait for the watch to change, or some time to expire (whichever comes first) + // before checking through the prune intents. We write a UID into the change key value + // so that we can still recognize when the watch key has been changed while we weren't + // monitoring it + + state Key lastPruneKey = blobGranulePruneKeys.begin; + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + state std::vector> prunes; + state CoalescedKeyRangeMap> pruneMap; + pruneMap.insert(allKeys, std::make_pair(0, false)); + try { + // TODO: replace 10000 with a knob + state RangeResult pruneIntents = wait(tr->getRange(blobGranulePruneKeys, BUGGIFY ? 1 : 10000)); + if (pruneIntents.size()) { + int rangeIdx = 0; + for (; rangeIdx < pruneIntents.size(); ++rangeIdx) { + Version pruneVersion; + KeyRange range; + bool force; + std::tie(pruneVersion, range, force) = + decodeBlobGranulePruneValue(pruneIntents[rangeIdx].value); + auto ranges = pruneMap.intersectingRanges(range); + bool foundConflict = false; + for (auto it : ranges) { + if ((it.value().second && !force && it.value().first < pruneVersion) || + (!it.value().second && force && pruneVersion < it.value().first)) { + foundConflict = true; + break; + } + } + if (foundConflict) { + break; + } + pruneMap.insert(range, std::make_pair(pruneVersion, force)); + + fmt::print("about to prune range [{0} - {1}) @ {2}, force={3}\n", + range.begin.printable(), + range.end.printable(), + pruneVersion, + force ? "T" : "F"); + } + lastPruneKey = pruneIntents[rangeIdx - 1].key; + + for (auto it : pruneMap.ranges()) { + if (it.value().first > 0) { + prunes.emplace_back(pruneRange(self, it.range(), it.value().first, it.value().second)); + } + } + + // wait for this set of prunes to complete before starting the next ones since if we + // prune a range R at version V and while we are doing that, the time expires, we will + // end up trying to prune the same range again since the work isn't finished and the + // prunes will race + // + // TODO: this isn't that efficient though. Instead we could keep metadata as part of the + // BM's memory that tracks which prunes are active. Once done, we can mark that work as + // done. If the BM fails then all prunes will fail and so the next BM will have a clear + // set of metadata (i.e. no work in progress) so we will end up doing the work in the + // new BM + + wait(waitForAll(prunes)); + break; + } else { + state Future watchPruneIntentsChange = tr->watch(blobGranulePruneChangeKey); + wait(tr->commit()); + wait(watchPruneIntentsChange); + tr->reset(); + } + } catch (Error& e) { + wait(tr->onError(e)); + } + } + + tr->reset(); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->clear(KeyRangeRef(blobGranulePruneKeys.begin, keyAfter(lastPruneKey))); + wait(tr->commit()); + break; + } catch (Error& e) { + wait(tr->onError(e)); + } + } + + if (BM_DEBUG) { + printf("Done pruning current set of prune intents.\n"); + } + } +} + +ACTOR Future doLockChecks(Reference bmData) { + loop { + Promise check = bmData->doLockCheck; + wait(check.getFuture()); + wait(delay(0.5)); // don't do this too often if a lot of conflict + + TEST(true); // BM doing lock checks after getting conflicts + + state Reference tr = makeReference(bmData->db); + + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + wait(checkManagerLock(tr, bmData)); + wait(tr->commit()); + break; + } catch (Error& e) { + if (e.code() == error_code_granule_assignment_conflict) { + if (BM_DEBUG) { + fmt::print("BM {0} got lock out of date in lock check on conflict! Dying\n", bmData->epoch); + } + if (bmData->iAmReplaced.canBeSet()) { + bmData->iAmReplaced.send(Void()); + } + return Void(); + } + wait(tr->onError(e)); + if (BM_DEBUG) { + fmt::print("BM {0} still ok after checking lock on conflict\n", bmData->epoch); + } + } + } + bmData->doLockCheck = Promise(); + } +} + +static void blobManagerExclusionSafetyCheck(Reference self, + BlobManagerExclusionSafetyCheckRequest req) { + TraceEvent("BMExclusionSafetyCheckBegin", self->id).log(); + BlobManagerExclusionSafetyCheckReply reply(true); + // make sure at least one blob worker remains after exclusions + if (self->workersById.empty()) { + TraceEvent("BMExclusionSafetyCheckNoWorkers", self->id).log(); + reply.safe = false; + } else { + std::set remainingWorkers; + for (auto& worker : self->workersById) { + remainingWorkers.insert(worker.first); + } + for (const AddressExclusion& excl : req.exclusions) { + for (auto& worker : self->workersById) { + if (excl.excludes(worker.second.address())) { + remainingWorkers.erase(worker.first); + } + } + } + + TraceEvent("BMExclusionSafetyChecked", self->id).detail("RemainingWorkers", remainingWorkers.size()).log(); + reply.safe = !remainingWorkers.empty(); + } + + TraceEvent("BMExclusionSafetyCheckEnd", self->id).log(); + req.reply.send(reply); +} + +// Simulation validation that multiple blob managers aren't started with the same epoch +static std::map managerEpochsSeen; + +ACTOR Future blobManager(BlobManagerInterface bmInterf, + Reference const> dbInfo, + int64_t epoch) { + if (g_network->isSimulated()) { + bool managerEpochAlreadySeen = managerEpochsSeen.count(epoch); + if (managerEpochAlreadySeen) { + TraceEvent(SevError, "DuplicateBlobManagersAtEpoch") + .detail("Epoch", epoch) + .detail("BMID1", bmInterf.id()) + .detail("BMID2", managerEpochsSeen.at(epoch)); + } + ASSERT(!managerEpochAlreadySeen); + managerEpochsSeen[epoch] = bmInterf.id(); + } + state Reference self = + makeReference(deterministicRandom()->randomUniqueID(), + openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True), + bmInterf.locality.dcId()); + + state Future collection = actorCollection(self->addActor.getFuture()); + + if (BM_DEBUG) { + fmt::print("Blob manager {0} starting...\n", epoch); + } + TraceEvent("BlobManagerInit", bmInterf.id()).detail("Epoch", epoch).log(); + + self->epoch = epoch; + + // start rangeAssigner first since other actors can send messages to it + self->addActor.send(rangeAssigner(self)); + // although we start the recruiter, we wait until existing workers are ack'd auto recruitBlobWorker = IAsyncListener>::create( dbInfo, [](auto const& info) { return info.clusterInterface.recruitBlobWorker; }); - self.addActor.send(blobWorkerRecruiter(&self, recruitBlobWorker)); - self.addActor.send(monitorClientRanges(&self)); - self.addActor.send(rangeAssigner(&self)); + self->addActor.send(blobWorkerRecruiter(self, recruitBlobWorker)); + + // we need to recover the old blob manager's state (e.g. granule assignments) before + // before the new blob manager does anything + wait(recoverBlobManager(self)); + + self->addActor.send(doLockChecks(self)); + self->addActor.send(monitorClientRanges(self)); + self->addActor.send(monitorPruneKeys(self)); if (BUGGIFY) { - self.addActor.send(chaosRangeMover(&self)); + self->addActor.send(chaosRangeMover(self)); } - // TODO probably other things here eventually try { loop choose { - when(wait(self.iAmReplaced.getFuture())) { + when(wait(self->iAmReplaced.getFuture())) { if (BM_DEBUG) { - printf("Blob Manager exiting because it is replaced\n"); + fmt::print("BM {} exiting because it is replaced\n", self->epoch); } + TraceEvent("BlobManagerReplaced", bmInterf.id()).detail("Epoch", epoch); break; } when(HaltBlobManagerRequest req = waitNext(bmInterf.haltBlobManager.getFuture())) { req.reply.send(Void()); - TraceEvent("BlobManagerHalted", bmInterf.id()).detail("ReqID", req.requesterID); + TraceEvent("BlobManagerHalted", bmInterf.id()).detail("Epoch", epoch).detail("ReqID", req.requesterID); break; } + when(state HaltBlobGranulesRequest req = waitNext(bmInterf.haltBlobGranules.getFuture())) { + wait(haltBlobGranules(self)); + req.reply.send(Void()); + TraceEvent("BlobGranulesHalted", bmInterf.id()).detail("Epoch", epoch).detail("ReqID", req.requesterID); + break; + } + when(BlobManagerExclusionSafetyCheckRequest exclCheckReq = + waitNext(bmInterf.blobManagerExclCheckReq.getFuture())) { + blobManagerExclusionSafetyCheck(self, exclCheckReq); + } when(wait(collection)) { - TraceEvent("BlobManagerActorCollectionError"); + TraceEvent(SevError, "BlobManagerActorCollectionError"); ASSERT(false); throw internal_error(); } @@ -1168,8 +2776,8 @@ ACTOR Future blobManager(BlobManagerInterface bmInterf, // DB has [B - D). It should show up coalesced in knownBlobRanges, and [C - D) should be removed. // DB has [A - D). It should show up coalesced in knownBlobRanges, and [A - B) should be removed. // DB has [A - B) and [C - D). They should show up in knownBlobRanges, and [B - C) should be in removed. -// DB has [B - C). It should show up in knownBlobRanges, [B - C) should be in added, and [A - B) and [C - D) should -// be in removed. +// DB has [B - C). It should show up in knownBlobRanges, [B - C) should be in added, and [A - B) and [C - D) +// should be in removed. TEST_CASE(":/blobmanager/updateranges") { KeyRangeMap knownBlobRanges(false, normalKeys.end); Arena ar; diff --git a/fdbserver/BlobManagerInterface.h b/fdbserver/BlobManagerInterface.h index fc030ec8ca..7fb73a220d 100644 --- a/fdbserver/BlobManagerInterface.h +++ b/fdbserver/BlobManagerInterface.h @@ -30,6 +30,8 @@ struct BlobManagerInterface { constexpr static FileIdentifier file_identifier = 369169; RequestStream> waitFailure; RequestStream haltBlobManager; + RequestStream haltBlobGranules; + RequestStream blobManagerExclCheckReq; struct LocalityData locality; UID myId; @@ -44,7 +46,7 @@ struct BlobManagerInterface { template void serialize(Archive& ar) { - serializer(ar, waitFailure, haltBlobManager, locality, myId); + serializer(ar, waitFailure, haltBlobManager, haltBlobGranules, blobManagerExclCheckReq, locality, myId); } }; @@ -62,4 +64,46 @@ struct HaltBlobManagerRequest { } }; +struct HaltBlobGranulesRequest { + constexpr static FileIdentifier file_identifier = 904267; + UID requesterID; + ReplyPromise reply; + + HaltBlobGranulesRequest() {} + explicit HaltBlobGranulesRequest(UID uid) : requesterID(uid) {} + + template + void serialize(Ar& ar) { + serializer(ar, requesterID, reply); + } +}; + +struct BlobManagerExclusionSafetyCheckReply { + constexpr static FileIdentifier file_identifier = 8068627; + bool safe; + + BlobManagerExclusionSafetyCheckReply() : safe(false) {} + explicit BlobManagerExclusionSafetyCheckReply(bool safe) : safe(safe) {} + + template + void serialize(Ar& ar) { + serializer(ar, safe); + } +}; + +struct BlobManagerExclusionSafetyCheckRequest { + constexpr static FileIdentifier file_identifier = 1996387; + std::vector exclusions; + ReplyPromise reply; + + BlobManagerExclusionSafetyCheckRequest() {} + explicit BlobManagerExclusionSafetyCheckRequest(std::vector exclusions) + : exclusions(exclusions) {} + + template + void serialize(Ar& ar) { + serializer(ar, exclusions, reply); + } +}; + #endif diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index e1cc499db4..e939d71ec6 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -35,45 +35,25 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/Notified.h" #include "fdbserver/Knobs.h" +#include "fdbserver/BlobGranuleServerCommon.actor.h" #include "fdbserver/MutationTracking.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/ServerDBInfo.h" #include "flow/Arena.h" #include "flow/Error.h" #include "flow/IRandom.h" +#include "flow/Trace.h" #include "flow/actorcompiler.h" // has to be last include +#include "flow/network.h" #define BW_DEBUG false #define BW_REQUEST_DEBUG false -// TODO add comments + documentation -struct BlobFileIndex { - Version version; - std::string filename; - int64_t offset; - int64_t length; - - BlobFileIndex() {} - - BlobFileIndex(Version version, std::string filename, int64_t offset, int64_t length) - : version(version), filename(filename), offset(offset), length(length) {} -}; - -struct GranuleFiles { - std::deque snapshotFiles; - std::deque deltaFiles; -}; - -struct GranuleHistory { - KeyRange range; - Version version; - Standalone value; - - GranuleHistory() {} - - GranuleHistory(KeyRange range, Version version, Standalone value) - : range(range), version(version), value(value) {} -}; +/* + * The Blob Worker is a stateless role assigned a set of granules by the Blob Manager. + * It is responsible for managing the change feeds for those granules, and for consuming the mutations from those change + * feeds and writing them out as files to blob storage. + */ struct GranuleStartState { UID granuleID; @@ -86,25 +66,25 @@ struct GranuleStartState { Optional history; }; +// FIXME: add global byte limit for pending and buffered deltas struct GranuleMetadata : NonCopyable, ReferenceCounted { KeyRange keyRange; GranuleFiles files; - GranuleDeltas currentDeltas; // only contain deltas in pendingDeltaVersion + 1, bufferedDeltaVersion - // TODO get rid of this and do Reference>? - Arena deltaArena; + Standalone + currentDeltas; // only contain deltas in pendingDeltaVersion + 1 through bufferedDeltaVersion uint64_t bytesInNewDeltaFiles = 0; uint64_t bufferedDeltaBytes = 0; // for client to know when it is safe to read a certain version and from where (check waitForVersion) - NotifiedVersion bufferedDeltaVersion; // largest delta version in currentDeltas (including empty versions) + Version bufferedDeltaVersion; // largest delta version in currentDeltas (including empty versions) Version pendingDeltaVersion = 0; // largest version in progress writing to s3/fdb NotifiedVersion durableDeltaVersion; // largest version persisted in s3/fdb NotifiedVersion durableSnapshotVersion; // same as delta vars, except for snapshots Version pendingSnapshotVersion = 0; - - AsyncVar rollbackCount; + Version initialSnapshotVersion = invalidVersion; + Version knownCommittedVersion; int64_t originalEpoch; int64_t originalSeqno; @@ -117,15 +97,17 @@ struct GranuleMetadata : NonCopyable, ReferenceCounted { Promise resumeSnapshot; + AsyncVar> activeCFData; + AssignBlobRangeRequest originalReq; void resume() { - ASSERT(resumeSnapshot.canBeSet()); - resumeSnapshot.send(Void()); + if (resumeSnapshot.canBeSet()) { + resumeSnapshot.send(Void()); + } } }; -// TODO: rename this struct struct GranuleRangeMetadata { int64_t lastEpoch; int64_t lastSeqno; @@ -135,12 +117,22 @@ struct GranuleRangeMetadata { Future fileUpdaterFuture; Future historyLoaderFuture; + void cancel() { + if (activeMetadata->cancelled.canBeSet()) { + activeMetadata->cancelled.send(Void()); + } + activeMetadata.clear(); + assignFuture.cancel(); + historyLoaderFuture.cancel(); + fileUpdaterFuture.cancel(); + } + GranuleRangeMetadata() : lastEpoch(0), lastSeqno(0) {} GranuleRangeMetadata(int64_t epoch, int64_t seqno, Reference activeMetadata) : lastEpoch(epoch), lastSeqno(seqno), activeMetadata(activeMetadata) {} }; -// represents a previous version of a granule, and optionally the files that compose it +// represents a previous version of a granule, and optionally the files that compose it. struct GranuleHistoryEntry : NonCopyable, ReferenceCounted { KeyRange range; UID granuleID; @@ -148,6 +140,7 @@ struct GranuleHistoryEntry : NonCopyable, ReferenceCounted Version endVersion; // version of the last delta file // load files lazily, and allows for clearing old cold-queried files to save memory + // FIXME: add memory limit and evictor for old cached files Future files; // FIXME: do skip pointers with single back-pointer and neighbor pointers @@ -171,6 +164,7 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { int64_t currentManagerEpoch = -1; AsyncVar> currentManagerStatusStream; + bool statusStreamInitialized = false; // FIXME: refactor out the parts of this that are just for interacting with blob stores from the backup business // logic @@ -182,19 +176,24 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { // FIXME: expire from map after a delay when granule is revoked and the history is no longer needed KeyRangeMap> granuleHistory; - AsyncVar pendingDeltaFileCommitChecks; - AsyncVar knownCommittedVersion; - uint64_t knownCommittedCheckCount = 0; - PromiseStream granuleUpdateErrors; - BlobWorkerData(UID id, Database db) : id(id), db(db), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL) {} - ~BlobWorkerData() { printf("Destroying blob worker data for %s\n", id.toString().c_str()); } + Promise doGRVCheck; + NotifiedVersion grvVersion; + Promise fatalError; + + FlowLock initialSnapshotLock; + + int changeFeedStreamReplyBufferSize = SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / 2; + + BlobWorkerData(UID id, Database db) + : id(id), db(db), stats(id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL), + initialSnapshotLock(SERVER_KNOBS->BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM) {} bool managerEpochOk(int64_t epoch) { if (epoch < currentManagerEpoch) { if (BW_DEBUG) { - fmt::print("BW {0} got request from old epoch {1}, notifying manager it is out of date\n", + fmt::print("BW {0} got request from old epoch {1}, notifying them they are out of date\n", id.toString(), epoch); } @@ -212,6 +211,18 @@ struct BlobWorkerData : NonCopyable, ReferenceCounted { } }; +// serialize change feed key as UID bytes, to use 16 bytes on disk +static Key granuleIDToCFKey(UID granuleID) { + BinaryWriter wr(Unversioned()); + wr << granuleID; + return wr.toValue(); +} + +// parse change feed key back to UID, to be human-readable +static UID cfKeyToGranuleID(Key cfKey) { + return BinaryReader::fromStringRef(cfKey, Unversioned()); +} + // returns true if we can acquire it static void acquireGranuleLock(int64_t epoch, int64_t seqno, int64_t prevOwnerEpoch, int64_t prevOwnerSeqno) { // returns true if our lock (E, S) >= (Eprev, Sprev) @@ -229,11 +240,6 @@ static void acquireGranuleLock(int64_t epoch, int64_t seqno, int64_t prevOwnerEp static void checkGranuleLock(int64_t epoch, int64_t seqno, int64_t ownerEpoch, int64_t ownerSeqno) { // sanity check - lock value should never go backwards because of acquireGranuleLock - /* - printf( - "Checking granule lock: \n mine: (%lld, %lld)\n owner: (%lld, %lld)\n", epoch, seqno, ownerEpoch, - ownerSeqno); - */ ASSERT(epoch <= ownerEpoch); ASSERT(epoch < ownerEpoch || (epoch == ownerEpoch && seqno <= ownerSeqno)); @@ -262,56 +268,12 @@ ACTOR Future readAndCheckGranuleLock(Reference checkGranuleLock(epoch, seqno, std::get<0>(currentOwner), std::get<1>(currentOwner)); // if we still own the lock, add a conflict range in case anybody else takes it over while we add this file + // FIXME: we don't need these conflict ranges tr->addReadConflictRange(singleKeyRange(lockKey)); return Void(); } -// used for "business logic" of both versions of loading granule files -ACTOR Future readGranuleFiles(Transaction* tr, Key* startKey, Key endKey, GranuleFiles* files, UID granuleID) { - - loop { - int lim = BUGGIFY ? 2 : 1000; - RangeResult res = wait(tr->getRange(KeyRangeRef(*startKey, endKey), lim)); - for (auto& it : res) { - UID gid; - uint8_t fileType; - Version version; - - Standalone filename; - int64_t offset; - int64_t length; - - std::tie(gid, fileType, version) = decodeBlobGranuleFileKey(it.key); - ASSERT(gid == granuleID); - - std::tie(filename, offset, length) = decodeBlobGranuleFileValue(it.value); - - BlobFileIndex idx(version, filename.toString(), offset, length); - if (fileType == 'S') { - ASSERT(files->snapshotFiles.empty() || files->snapshotFiles.back().version < idx.version); - files->snapshotFiles.push_back(idx); - } else { - ASSERT(fileType == 'D'); - ASSERT(files->deltaFiles.empty() || files->deltaFiles.back().version < idx.version); - files->deltaFiles.push_back(idx); - } - } - if (res.more) { - *startKey = keyAfter(res.back().key); - } else { - break; - } - } - if (BW_DEBUG) { - fmt::print("Loaded {0} snapshot and {1} delta files for {2}\n", - files->snapshotFiles.size(), - files->deltaFiles.size(), - granuleID.toString()); - } - return Void(); -} - // Read snapshot and delta files for granule history, for completed granule // Retries on error local to this function ACTOR Future loadHistoryFiles(Reference bwData, UID granuleID) { @@ -369,15 +331,21 @@ ACTOR Future updateGranuleSplitState(Transaction* tr, BlobGranuleSplitState newState) { state KeyRange currentRange = blobGranuleSplitKeyRangeFor(parentGranuleID); - RangeResult totalState = wait(tr->getRange(currentRange, 100)); - // TODO is this explicit conflit range necessary with the above read? + state RangeResult totalState = wait(tr->getRange(currentRange, SERVER_KNOBS->BG_MAX_SPLIT_FANOUT + 1)); + // FIXME: remove above conflict range? tr->addWriteConflictRange(currentRange); - ASSERT(!totalState.more); + ASSERT_WE_THINK(!totalState.more && totalState.size() <= SERVER_KNOBS->BG_MAX_SPLIT_FANOUT); + // maybe someone decreased the knob, we should gracefully handle it not in simulation + if (totalState.more || totalState.size() > SERVER_KNOBS->BG_MAX_SPLIT_FANOUT) { + RangeResult tryAgain = wait(tr->getRange(currentRange, 10000)); + ASSERT(!tryAgain.more); + totalState = tryAgain; + } if (totalState.empty()) { ASSERT(newState == BlobGranuleSplitState::Done); if (BW_DEBUG) { - printf("Found empty split state for parent granule %s\n", parentGranuleID.toString().c_str()); + fmt::print("Found empty split state for parent granule {0}\n", parentGranuleID.toString()); } // must have retried and successfully nuked everything return Void(); @@ -398,7 +366,7 @@ ACTOR Future updateGranuleSplitState(Transaction* tr, BlobGranuleSplitState st = decodeBlobGranuleSplitValue(it.value).first; ASSERT(st != BlobGranuleSplitState::Unknown); - if (st == BlobGranuleSplitState::Started) { + if (st == BlobGranuleSplitState::Initialized) { totalStarted++; } else if (st == BlobGranuleSplitState::Done) { totalDone++; @@ -413,11 +381,11 @@ ACTOR Future updateGranuleSplitState(Transaction* tr, if (currentState < newState) { if (BW_DEBUG) { - printf("Updating granule %s split state from %s %d -> %d\n", - currentGranuleID.toString().c_str(), - parentGranuleID.toString().c_str(), - currentState, - newState); + fmt::print("Updating granule {0} split state from {1} {2} -> {3}\n", + currentGranuleID.toString(), + parentGranuleID.toString(), + currentState, + newState); } Key myStateKey = blobGranuleSplitKeyFor(parentGranuleID, currentGranuleID); @@ -426,50 +394,63 @@ ACTOR Future updateGranuleSplitState(Transaction* tr, // we are the last one to change from Assigned -> Done, so everything can be cleaned up for the old // change feed and splitting state if (BW_DEBUG) { - printf("%s destroying old granule %s\n", - currentGranuleID.toString().c_str(), - parentGranuleID.toString().c_str()); + fmt::print("{0} destroying old granule {1}\n", currentGranuleID.toString(), parentGranuleID.toString()); } - wait(updateChangeFeed(tr, KeyRef(parentGranuleID.toString()), ChangeFeedStatus::CHANGE_FEED_DESTROY)); + // FIXME: appears change feed destroy isn't working! ADD BACK + // wait(updateChangeFeed(tr, granuleIDToCFKey(parentGranuleID), ChangeFeedStatus::CHANGE_FEED_DESTROY)); + Key oldGranuleLockKey = blobGranuleLockKeyFor(parentGranuleRange); - tr->clear(singleKeyRange(oldGranuleLockKey)); + // FIXME: deleting granule lock can cause races where another granule with the same range starts way later + // and thinks it can own the granule! Need to change file cleanup to destroy these, if there is no more + // granule in the history with that exact key range! + // Alternative fix could be to, on granule open, query for all overlapping granule locks and ensure none of + // them have higher (epoch, seqno), but that is much more expensive + + // tr->clear(singleKeyRange(oldGranuleLockKey)); tr->clear(currentRange); + TEST(true); // Granule split cleanup on last delta file persisted } else { - if (newState == BlobGranuleSplitState::Assigned && currentState == BlobGranuleSplitState::Started && - totalStarted == 1) { - if (BW_DEBUG) { - printf("%s WOULD BE stopping change feed for old granule %s\n", - currentGranuleID.toString().c_str(), - parentGranuleID.toString().c_str()); - } - // FIXME: enable - // wait(updateChangeFeed(tr, KeyRef(parentGranuleID.toString()), - // ChangeFeedStatus::CHANGE_FEED_DESTROY)); - } tr->atomicOp(myStateKey, blobGranuleSplitValueFor(newState), MutationRef::SetVersionstampedValue); + if (newState == BlobGranuleSplitState::Assigned && currentState == BlobGranuleSplitState::Initialized && + totalStarted == 1) { + // We are the last one to change from Start -> Assigned, so we can stop the parent change feed. + if (BW_DEBUG) { + fmt::print("{0} stopping change feed for old granule {1}\n", + currentGranuleID.toString().c_str(), + parentGranuleID.toString().c_str()); + } + + wait(updateChangeFeed( + tr, KeyRef(granuleIDToCFKey(parentGranuleID)), ChangeFeedStatus::CHANGE_FEED_STOP)); + } + TEST(true); // Granule split stopping change feed } } else if (BW_DEBUG) { - printf("Ignoring granule %s split state from %s %d -> %d\n", - currentGranuleID.toString().c_str(), - parentGranuleID.toString().c_str(), - currentState, - newState); + TEST(true); // Out of order granule split state updates ignored + fmt::print("Ignoring granule {0} split state from {1} {2} -> {3}\n", + currentGranuleID.toString(), + parentGranuleID.toString(), + currentState, + newState); } return Void(); } -// returns the split state for a given granule on granule reassignment. Assumes granule is in fact splitting, by the -// presence of the previous granule's lock key +// Returns the split state for a given granule on granule reassignment, or unknown if it doesn't exist (meaning the +// granule splitting finished) ACTOR Future> getGranuleSplitState(Transaction* tr, UID parentGranuleID, UID currentGranuleID) { Key myStateKey = blobGranuleSplitKeyFor(parentGranuleID, currentGranuleID); Optional st = wait(tr->get(myStateKey)); - ASSERT(st.present()); - return decodeBlobGranuleSplitValue(st.get()); + if (st.present()) { + return decodeBlobGranuleSplitValue(st.get()); + } else { + return std::pair(BlobGranuleSplitState::Unknown, invalidVersion); + } } // writeDelta file writes speculatively in the common case to optimize throughput. It creates the s3 object even though @@ -481,48 +462,41 @@ ACTOR Future writeDeltaFile(Reference bwData, UID granuleID, int64_t epoch, int64_t seqno, - Arena deltaArena, - GranuleDeltas deltasToWrite, + Standalone deltasToWrite, Version currentDeltaVersion, Future previousDeltaFileFuture, + Future waitCommitted, Optional> oldGranuleComplete) { wait(delay(0, TaskPriority::BlobWorkerUpdateStorage)); - // potentially kick off delta file commit check, if our version isn't already known to be committed - state uint64_t checkCount = -1; - if (bwData->knownCommittedVersion.get() < currentDeltaVersion) { - bwData->pendingDeltaFileCommitChecks.set(bwData->pendingDeltaFileCommitChecks.get() + 1); - checkCount = bwData->knownCommittedCheckCount; - } - // TODO some sort of directory structure would be useful? - state std::string fname = deterministicRandom()->randomUniqueID().toString() + "_T" + - std::to_string((uint64_t)(1000.0 * now())) + "_V" + std::to_string(currentDeltaVersion) + - ".delta"; + // Prefix filename with random chars both to avoid hotspotting on granuleID, and to have unique file names if + // multiple blob workers try to create the exact same file at the same millisecond (which observably happens) + state std::string fname = deterministicRandom()->randomUniqueID().shortString() + "_" + granuleID.toString() + + "_T" + std::to_string((uint64_t)(1000.0 * now())) + "_V" + + std::to_string(currentDeltaVersion) + ".delta"; state Value serialized = ObjectWriter::toValue(deltasToWrite, Unversioned()); + state size_t serializedSize = serialized.size(); - // FIXME: technically we can free up deltaArena here to reduce memory + // Free up deltasToWrite here to reduce memory + deltasToWrite = Standalone(); state Reference objectFile = wait(bwData->bstore->writeFile(fname)); ++bwData->stats.s3PutReqs; ++bwData->stats.deltaFilesWritten; - bwData->stats.deltaBytesWritten += serialized.size(); + bwData->stats.deltaBytesWritten += serializedSize; - wait(objectFile->append(serialized.begin(), serialized.size())); + wait(objectFile->append(serialized.begin(), serializedSize)); wait(objectFile->finish()); + // free serialized since it is persisted in blob + serialized = Value(); + state int numIterations = 0; try { // before updating FDB, wait for the delta file version to be committed and previous delta files to finish - while (bwData->knownCommittedVersion.get() < currentDeltaVersion) { - if (bwData->knownCommittedCheckCount != checkCount) { - checkCount = bwData->knownCommittedCheckCount; - // a check happened between the start and now, and the version is still lower. Kick off another one. - bwData->pendingDeltaFileCommitChecks.set(bwData->pendingDeltaFileCommitChecks.get() + 1); - } - wait(bwData->knownCommittedVersion.onChange()); - } + wait(waitCommitted); BlobFileIndex prev = wait(previousDeltaFileFuture); wait(delay(0, TaskPriority::BlobWorkerUpdateFDB)); @@ -532,9 +506,10 @@ ACTOR Future writeDeltaFile(Reference bwData, tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno)); + numIterations++; - Key dfKey = blobGranuleFileKeyFor(granuleID, 'D', currentDeltaVersion); - Value dfValue = blobGranuleFileValueFor(fname, 0, serialized.size()); + Key dfKey = blobGranuleFileKeyFor(granuleID, currentDeltaVersion, 'D'); + Value dfValue = blobGranuleFileValueFor(fname, 0, serializedSize); tr->set(dfKey, dfValue); if (oldGranuleComplete.present()) { @@ -553,7 +528,7 @@ ACTOR Future writeDeltaFile(Reference bwData, keyRange.begin.printable(), keyRange.end.printable(), fname, - serialized.size(), + serializedSize, currentDeltaVersion, tr->getCommittedVersion()); } @@ -561,31 +536,26 @@ ACTOR Future writeDeltaFile(Reference bwData, if (BUGGIFY_WITH_PROB(0.01)) { wait(delay(deterministicRandom()->random01())); } - return BlobFileIndex(currentDeltaVersion, fname, 0, serialized.size()); + return BlobFileIndex(currentDeltaVersion, fname, 0, serializedSize); } catch (Error& e) { - numIterations++; wait(tr->onError(e)); } } } catch (Error& e) { - if (e.code() == error_code_operation_cancelled) { + // If this actor was cancelled, doesn't own the granule anymore, or got some other error before trying to + // commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB + // with file and cannot safely delete it. + if (numIterations > 0) { + TEST(true); // Granule potentially leaving orphaned delta file throw e; } - - // if commit failed the first time due to granule assignment conflict (which is non-retryable), - // then the file key was persisted and we should delete it. Otherwise, the commit failed - // for some other reason and the key wasn't persisted, so we should just propogate the error - if (numIterations != 1 || e.code() != error_code_granule_assignment_conflict) { - throw e; - } - if (BW_DEBUG) { - printf("deleting s3 delta file %s after error %s\n", fname.c_str(), e.name()); + fmt::print("deleting delta file {0} after error {1}\n", fname, e.name()); } - state Error eState = e; + TEST(true); // Granule cleaning up delta file after error ++bwData->stats.s3DeleteReqs; - wait(bwData->bstore->deleteFile(fname)); - throw eState; + bwData->addActor.send(bwData->bstore->deleteFile(fname)); + throw e; } } @@ -597,19 +567,20 @@ ACTOR Future writeSnapshot(Reference bwData, Version version, PromiseStream rows, bool createGranuleHistory) { - // TODO some sort of directory structure would be useful maybe? - state std::string fname = deterministicRandom()->randomUniqueID().toString() + "_T" + - std::to_string((uint64_t)(1000.0 * now())) + "_V" + std::to_string(version) + ".snapshot"; - state Arena arena; - state GranuleSnapshot snapshot; + // Prefix filename with random chars both to avoid hotspotting on granuleID, and to have unique file names if + // multiple blob workers try to create the exact same file at the same millisecond (which observably happens) + state std::string fname = deterministicRandom()->randomUniqueID().shortString() + "_" + granuleID.toString() + + "_T" + std::to_string((uint64_t)(1000.0 * now())) + "_V" + std::to_string(version) + + ".snapshot"; + state Standalone snapshot; wait(delay(0, TaskPriority::BlobWorkerUpdateStorage)); loop { try { RangeResult res = waitNext(rows.getFuture()); - arena.dependsOn(res.arena()); - snapshot.append(arena, res.begin(), res.size()); + snapshot.arena().dependsOn(res.arena()); + snapshot.append(snapshot.arena(), res.begin(), res.size()); wait(yield(TaskPriority::BlobWorkerUpdateStorage)); } catch (Error& e) { if (e.code() == error_code_end_of_stream) { @@ -622,40 +593,46 @@ ACTOR Future writeSnapshot(Reference bwData, wait(delay(0, TaskPriority::BlobWorkerUpdateStorage)); if (BW_DEBUG) { - printf("Granule [%s - %s) read %d snapshot rows\n", - keyRange.begin.printable().c_str(), - keyRange.end.printable().c_str(), - snapshot.size()); + fmt::print("Granule [{0} - {1}) read {2} snapshot rows\n", + keyRange.begin.printable(), + keyRange.end.printable(), + snapshot.size()); } - // TODO REMOVE sanity checks! - if (snapshot.size() > 0) { - ASSERT(keyRange.begin <= snapshot[0].key); - ASSERT(keyRange.end > snapshot[snapshot.size() - 1].key); - } - for (int i = 0; i < snapshot.size() - 1; i++) { - if (snapshot[i].key >= snapshot[i + 1].key) { - printf("SORT ORDER VIOLATION IN SNAPSHOT FILE: %s, %s\n", - snapshot[i].key.printable().c_str(), - snapshot[i + 1].key.printable().c_str()); + if (g_network->isSimulated()) { + if (snapshot.size() > 0) { + ASSERT(keyRange.begin <= snapshot[0].key); + ASSERT(keyRange.end > snapshot[snapshot.size() - 1].key); + } + for (int i = 0; i < snapshot.size() - 1; i++) { + if (snapshot[i].key >= snapshot[i + 1].key) { + fmt::print("SORT ORDER VIOLATION IN SNAPSHOT FILE: {0}, {1}\n", + snapshot[i].key.printable(), + snapshot[i + 1].key.printable()); + } + ASSERT(snapshot[i].key < snapshot[i + 1].key); } - ASSERT(snapshot[i].key < snapshot[i + 1].key); } - // TODO is this easy to read as a flatbuffer from reader? Need to be sure about this data format state Value serialized = ObjectWriter::toValue(snapshot, Unversioned()); + state size_t serializedSize = serialized.size(); - // write to s3 using multi part upload + // free snapshot to reduce memory + snapshot = Standalone(); + + // write to blob using multi part upload state Reference objectFile = wait(bwData->bstore->writeFile(fname)); ++bwData->stats.s3PutReqs; ++bwData->stats.snapshotFilesWritten; - bwData->stats.snapshotBytesWritten += serialized.size(); + bwData->stats.snapshotBytesWritten += serializedSize; - // TODO: inject write error - wait(objectFile->append(serialized.begin(), serialized.size())); + wait(objectFile->append(serialized.begin(), serializedSize)); wait(objectFile->finish()); + // free serialized since it is persisted in blob + serialized = Value(); + wait(delay(0, TaskPriority::BlobWorkerUpdateFDB)); // object uploaded successfully, save it to system key space @@ -667,8 +644,9 @@ ACTOR Future writeSnapshot(Reference bwData, tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { wait(readAndCheckGranuleLock(tr, keyRange, epoch, seqno)); - Key snapshotFileKey = blobGranuleFileKeyFor(granuleID, 'S', version); - Key snapshotFileValue = blobGranuleFileValueFor(fname, 0, serialized.size()); + numIterations++; + Key snapshotFileKey = blobGranuleFileKeyFor(granuleID, version, 'S'); + Key snapshotFileValue = blobGranuleFileValueFor(fname, 0, serializedSize); tr->set(snapshotFileKey, snapshotFileValue); // create granule history at version if this is a new granule with the initial dump from FDB if (createGranuleHistory) { @@ -680,61 +658,65 @@ ACTOR Future writeSnapshot(Reference bwData, wait(tr->commit()); break; } catch (Error& e) { - numIterations++; wait(tr->onError(e)); } } } catch (Error& e) { - if (e.code() == error_code_operation_cancelled) { + // If this actor was cancelled, doesn't own the granule anymore, or got some other error before trying to + // commit a transaction, we can and want to safely delete the file we wrote. Otherwise, we may have updated FDB + // with file and cannot safely delete it. + if (numIterations > 0) { + TEST(true); // Granule potentially leaving orphaned snapshot file throw e; } - - // if commit failed the first time due to granule assignment conflict (which is non-retryable), - // then the file key was persisted and we should delete it. Otherwise, the commit failed - // for some other reason and the key wasn't persisted, so we should just propogate the error - if (numIterations != 1 || e.code() != error_code_granule_assignment_conflict) { - throw e; - } - if (BW_DEBUG) { - printf("deleting s3 snapshot file %s after error %s\n", fname.c_str(), e.name()); + fmt::print("deleting snapshot file {0} after error {1}\n", fname, e.name()); } - state Error eState = e; + TEST(true); // Granule deleting snapshot file after error ++bwData->stats.s3DeleteReqs; - wait(bwData->bstore->deleteFile(fname)); - throw eState; + bwData->addActor.send(bwData->bstore->deleteFile(fname)); + throw e; } if (BW_DEBUG) { - printf("Granule [%s - %s) committed new snapshot file %s with %d bytes\n\n", - keyRange.begin.printable().c_str(), - keyRange.end.printable().c_str(), - fname.c_str(), - serialized.size()); + fmt::print("Granule [{0} - {1}) committed new snapshot file {2} with {3} bytes\n\n", + keyRange.begin.printable(), + keyRange.end.printable(), + fname, + serializedSize); } if (BUGGIFY_WITH_PROB(0.1)) { wait(delay(deterministicRandom()->random01())); } - return BlobFileIndex(version, fname, 0, serialized.size()); + return BlobFileIndex(version, fname, 0, serializedSize); } ACTOR Future dumpInitialSnapshotFromFDB(Reference bwData, Reference metadata, - UID granuleID) { + UID granuleID, + Key cfKey) { if (BW_DEBUG) { - printf("Dumping snapshot from FDB for [%s - %s)\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str()); + fmt::print("Dumping snapshot from FDB for [{0} - {1})\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable()); } + wait(bwData->initialSnapshotLock.take()); + state FlowLock::Releaser holdingDVL(bwData->initialSnapshotLock); + state Reference tr = makeReference(bwData->db); + state int64_t bytesRead = 0; + state int retries = 0; + state Version lastReadVersion = invalidVersion; + state Version readVersion = invalidVersion; loop { - state Key beginKey = metadata->keyRange.begin; tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { - state Version readVersion = wait(tr->getReadVersion()); + Version rv = wait(tr->getReadVersion()); + readVersion = rv; + ASSERT(lastReadVersion <= readVersion); state PromiseStream rowsStream; state Future snapshotWriter = writeSnapshot(bwData, metadata->keyRange, @@ -744,33 +726,45 @@ ACTOR Future dumpInitialSnapshotFromFDB(Reference readVersion, rowsStream, true); - - loop { - // TODO: use streaming range read - // TODO: inject read error - // TODO knob for limit? - int lim = BUGGIFY ? 2 : 1000; - RangeResult res = wait(tr->getRange(KeyRangeRef(beginKey, metadata->keyRange.end), lim)); - bwData->stats.bytesReadFromFDBForInitialSnapshot += res.size(); - rowsStream.send(res); - if (res.more) { - beginKey = keyAfter(res.back().key); - } else { - rowsStream.sendError(end_of_stream()); - break; - } - } - BlobFileIndex f = wait(snapshotWriter); + Future streamFuture = + tr->getTransaction().getRangeStream(rowsStream, metadata->keyRange, GetRangeLimits(), Snapshot::True); + wait(streamFuture && success(snapshotWriter)); + TraceEvent("BlobGranuleSnapshotFile", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("Version", readVersion); DEBUG_KEY_RANGE("BlobWorkerFDBSnapshot", readVersion, metadata->keyRange, bwData->id); - return f; + + // initial snapshot is committed in fdb, we can pop the change feed up to this version + bwData->addActor.send(bwData->db->popChangeFeedMutations(cfKey, readVersion)); + return snapshotWriter.get(); } catch (Error& e) { - if (BW_DEBUG) { - printf("Dumping snapshot from FDB for [%s - %s) got error %s\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str(), - e.name()); + if (e.code() == error_code_operation_cancelled) { + throw e; } + if (BW_DEBUG) { + fmt::print("Dumping snapshot {0} from FDB for [{1} - {2}) got error {3} after {4} bytes\n", + retries + 1, + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + e.name(), + bytesRead); + } + state Error err = e; wait(tr->onError(e)); + retries++; + TEST(true); // Granule initial snapshot failed + TraceEvent(SevWarn, "BlobGranuleInitialSnapshotRetry", bwData->id) + .error(err) + .detail("Granule", metadata->keyRange) + .detail("Count", retries); + bytesRead = 0; + lastReadVersion = readVersion; + // Pop change feed up to readVersion, because that data will be before the next snapshot + // Do this to prevent a large amount of CF data from accumulating if we have consecutive failures to + // snapshot + // Also somewhat servers as a rate limiting function and checking that the database is available for this + // key range + wait(bwData->db->popChangeFeedMutations(cfKey, readVersion)); } } } @@ -780,17 +774,17 @@ ACTOR Future dumpInitialSnapshotFromFDB(Reference ACTOR Future compactFromBlob(Reference bwData, Reference metadata, UID granuleID, - GranuleFiles files) { + GranuleFiles files, + Version version) { wait(delay(0, TaskPriority::BlobWorkerUpdateStorage)); if (BW_DEBUG) { - printf("Compacting snapshot from blob for [%s - %s)\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str()); + fmt::print("Compacting snapshot from blob for [{0} - {1})\n", + metadata->keyRange.begin.printable().c_str(), + metadata->keyRange.end.printable().c_str()); } ASSERT(!files.snapshotFiles.empty()); ASSERT(!files.deltaFiles.empty()); - state Version version = files.deltaFiles.back().version; state Arena filenameArena; state BlobGranuleChunkRef chunk; @@ -798,6 +792,9 @@ ACTOR Future compactFromBlob(Reference bwData, state int64_t compactBytesRead = 0; state Version snapshotVersion = files.snapshotFiles.back().version; BlobFileIndex snapshotF = files.snapshotFiles.back(); + + ASSERT(snapshotVersion < version); + chunk.snapshotFile = BlobFilePointerRef(filenameArena, snapshotF.filename, snapshotF.offset, snapshotF.length); compactBytesRead += snapshotF.length; int deltaIdx = files.deltaFiles.size() - 1; @@ -805,12 +802,15 @@ ACTOR Future compactFromBlob(Reference bwData, deltaIdx--; } deltaIdx++; - while (deltaIdx < files.deltaFiles.size()) { + Version lastDeltaVersion = invalidVersion; + while (deltaIdx < files.deltaFiles.size() && files.deltaFiles[deltaIdx].version <= version) { BlobFileIndex deltaF = files.deltaFiles[deltaIdx]; chunk.deltaFiles.emplace_back_deep(filenameArena, deltaF.filename, deltaF.offset, deltaF.length); compactBytesRead += deltaF.length; + lastDeltaVersion = files.deltaFiles[deltaIdx].version; deltaIdx++; } + ASSERT(lastDeltaVersion == version); chunk.includedVersion = version; if (BW_DEBUG) { @@ -818,12 +818,6 @@ ACTOR Future compactFromBlob(Reference bwData, metadata->keyRange.begin.printable(), metadata->keyRange.end.printable(), version); - - /*printf(" SnapshotFile:\n %s\n", chunk.snapshotFile.get().toString().c_str()); - printf(" DeltaFiles:\n"); - for (auto& df : chunk.deltaFiles) { - printf(" %s\n", df.toString().c_str()); - }*/ } loop { @@ -840,8 +834,6 @@ ACTOR Future compactFromBlob(Reference bwData, RangeResult newGranule = wait(readBlobGranule(chunk, metadata->keyRange, version, bwData->bstore, &bwData->stats)); - // TODO: inject read error - bwData->stats.bytesReadFromS3ForCompaction += compactBytesRead; rowsStream.send(std::move(newGranule)); rowsStream.sendError(end_of_stream()); @@ -851,85 +843,168 @@ ACTOR Future compactFromBlob(Reference bwData, return f; } catch (Error& e) { if (BW_DEBUG) { - printf("Compacting snapshot from blob for [%s - %s) got error %s\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str(), - e.name()); + fmt::print("Compacting snapshot from blob for [{0} - {1}) got error {2}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + e.name()); } throw e; } } } -// When reading from a prior change feed, the prior change feed may contain mutations that don't belong in the new -// granule. And, we only want to read the prior change feed up to the start of the new change feed. -static bool filterOldMutations(const KeyRange& range, - const Standalone>* oldMutations, - Standalone>* mutations, - Version maxVersion) { - Standalone> filteredMutations; - mutations->arena().dependsOn(range.arena()); - mutations->arena().dependsOn(oldMutations->arena()); - for (auto& delta : *oldMutations) { - if (delta.version >= maxVersion) { - return true; - } - MutationsAndVersionRef filteredDelta; - filteredDelta.version = delta.version; - for (auto& m : delta.mutations) { - ASSERT(m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange); - if (m.type == MutationRef::SetValue) { - if (m.param1 >= range.begin && m.param1 < range.end) { - filteredDelta.mutations.push_back(mutations->arena(), m); +ACTOR Future checkSplitAndReSnapshot(Reference bwData, + Reference metadata, + UID granuleID, + int64_t bytesInNewDeltaFiles, + Future lastDeltaBeforeSnapshot, + int64_t versionsSinceLastSnapshot) { + + BlobFileIndex lastDeltaIdx = wait(lastDeltaBeforeSnapshot); + state Version reSnapshotVersion = lastDeltaIdx.version; + while (!bwData->statusStreamInitialized) { + wait(bwData->currentManagerStatusStream.onChange()); + } + + wait(delay(0, TaskPriority::BlobWorkerUpdateFDB)); + + if (BW_DEBUG) { + fmt::print("Granule [{0} - {1}) checking with BM for re-snapshot after {2} bytes\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + metadata->bytesInNewDeltaFiles); + } + + TraceEvent("BlobGranuleSnapshotCheck", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("Version", reSnapshotVersion); + + // Save these from the start so repeated requests are idempotent + // Need to retry in case response is dropped or manager changes. Eventually, a manager will + // either reassign the range with continue=true, or will revoke the range. But, we will keep the + // range open at this version for reads until that assignment change happens + metadata->resumeSnapshot.reset(); + state int64_t statusEpoch = metadata->continueEpoch; + state int64_t statusSeqno = metadata->continueSeqno; + + // If two snapshots happen without a split within a low time interval, this granule is "write-hot" + // FIXME: If a rollback happens, this could incorrectly identify a hot granule as not hot. This should be rare + // though and is just less efficient. + state bool writeHot = versionsSinceLastSnapshot <= SERVER_KNOBS->BG_HOT_SNAPSHOT_VERSIONS; + // FIXME: could probably refactor all of this logic into one large choose/when state machine that's less complex + loop { + loop { + try { + // wait for manager stream to become ready, and send a message + loop { + choose { + when(wait(bwData->currentManagerStatusStream.get().onReady())) { break; } + when(wait(bwData->currentManagerStatusStream.onChange())) {} + when(wait(metadata->resumeSnapshot.getFuture())) { break; } + } } - } else { - if (m.param2 >= range.begin && m.param1 < range.end) { - // clamp clear range down to sub-range - MutationRef m2 = m; - if (range.begin > m.param1) { - m2.param1 = range.begin; - } - if (range.end < m.param2) { - m2.param2 = range.end; - } - filteredDelta.mutations.push_back(mutations->arena(), m2); + if (metadata->resumeSnapshot.isSet()) { + break; + } + + bwData->currentManagerStatusStream.get().send(GranuleStatusReply(metadata->keyRange, + true, + writeHot, + statusEpoch, + statusSeqno, + granuleID, + metadata->initialSnapshotVersion)); + break; + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw e; + } + TEST(true); // Blob worker re-sending split evaluation to manager after not error/not hearing back + // if we got broken promise while waiting, the old stream was killed, so we don't need to wait on + // change, just retry + if (e.code() == error_code_broken_promise) { + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); + } else { + wait(bwData->currentManagerStatusStream.onChange()); } } } - mutations->push_back(mutations->arena(), filteredDelta); + + // wait for manager reply (which will either cancel this future or call resumeSnapshot), or re-send on manager + // change/no response + choose { + when(wait(bwData->currentManagerStatusStream.onChange())) {} + when(wait(metadata->resumeSnapshot.getFuture())) { break; } + when(wait(delay(1.0))) {} + } + + if (BW_DEBUG) { + fmt::print("Granule [{0} - {1})\n, hasn't heard back from BM in BW {2}, re-sending status\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + bwData->id.toString()); + } } - return false; + + if (BW_DEBUG) { + fmt::print("Granule [{0} - {1}) re-snapshotting after {2} bytes\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + bytesInNewDeltaFiles); + } + TraceEvent("BlobGranuleSnapshotFile", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("Version", metadata->durableDeltaVersion.get()); + + // wait for file updater to make sure that last delta file is in the metadata before + while (metadata->files.deltaFiles.empty() || metadata->files.deltaFiles.back().version < reSnapshotVersion) { + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); + } + BlobFileIndex reSnapshotIdx = + wait(compactFromBlob(bwData, metadata, granuleID, metadata->files, reSnapshotVersion)); + return reSnapshotIdx; } -ACTOR Future handleCompletedDeltaFile(Reference bwData, - Reference metadata, - BlobFileIndex completedDeltaFile, - Key cfKey, - Version cfStartVersion, - std::deque> rollbacksInProgress) { +static void handleCompletedDeltaFile(Reference bwData, + Reference metadata, + BlobFileIndex completedDeltaFile, + Key cfKey, + Version cfStartVersion, + std::deque>* rollbacksCompleted) { metadata->files.deltaFiles.push_back(completedDeltaFile); ASSERT(metadata->durableDeltaVersion.get() < completedDeltaFile.version); metadata->durableDeltaVersion.set(completedDeltaFile.version); if (completedDeltaFile.version > cfStartVersion) { if (BW_DEBUG) { - fmt::print("Popping change feed {0} at {1}\n", cfKey.printable(), completedDeltaFile.version); + fmt::print("Popping change feed {0} at {1}\n", + cfKeyToGranuleID(cfKey).toString().c_str(), + completedDeltaFile.version); } // FIXME: for a write-hot shard, we could potentially batch these and only pop the largest one after several // have completed - // FIXME: also have these be async, have each pop change feed wait on the prior one, wait on them before - // re-snapshotting + // FIXME: we actually want to pop at this version + 1 because pop is exclusive? + // FIXME: since this is async, and worker could die, new blob worker that opens granule should probably kick off + // an async pop at its previousDurableVersion after opening the granule to guarantee it is eventually popped? Future popFuture = bwData->db->popChangeFeedMutations(cfKey, completedDeltaFile.version); - wait(popFuture); + // Do pop asynchronously + bwData->addActor.send(popFuture); } - while (!rollbacksInProgress.empty() && completedDeltaFile.version >= rollbacksInProgress.front().first) { - rollbacksInProgress.pop_front(); + while (!rollbacksCompleted->empty() && completedDeltaFile.version >= rollbacksCompleted->front().second) { + if (BW_DEBUG) { + fmt::print("Granule [{0} - {1}) on BW {2} completed rollback {3} -> {4} with delta file {5}\n", + metadata->keyRange.begin.printable().c_str(), + metadata->keyRange.end.printable().c_str(), + bwData->id.toString().substr(0, 5).c_str(), + rollbacksCompleted->front().second, + rollbacksCompleted->front().first, + completedDeltaFile.version); + } + rollbacksCompleted->pop_front(); } - return Void(); } // if we get an i/o error updating files, or a rollback, reassign the granule to ourselves and start fresh -// FIXME: is this the correct set of errors? static bool granuleCanRetry(const Error& e) { switch (e.code()) { case error_code_please_reboot: @@ -942,65 +1017,106 @@ static bool granuleCanRetry(const Error& e) { }; } -struct InFlightDeltaFile { +struct InFlightFile { Future future; Version version; uint64_t bytes; + bool snapshot; - InFlightDeltaFile(Future future, Version version, uint64_t bytes) - : future(future), version(version), bytes(bytes) {} + InFlightFile(Future future, Version version, uint64_t bytes, bool snapshot) + : future(future), version(version), bytes(bytes), snapshot(snapshot) {} }; static Version doGranuleRollback(Reference metadata, Version mutationVersion, Version rollbackVersion, - std::deque& inFlightDeltaFiles, + std::deque& inFlightFiles, std::deque>& rollbacksInProgress, std::deque>& rollbacksCompleted) { Version cfRollbackVersion; if (metadata->pendingDeltaVersion > rollbackVersion) { - // if we already started writing mutations to a delta file with version > rollbackVersion, + // if we already started writing mutations to a delta or snapshot file with version > rollbackVersion, // we need to rescind those delta file writes - ASSERT(!inFlightDeltaFiles.empty()); + ASSERT(!inFlightFiles.empty()); cfRollbackVersion = metadata->durableDeltaVersion.get(); + metadata->pendingSnapshotVersion = metadata->durableSnapshotVersion.get(); int toPop = 0; - for (auto& df : inFlightDeltaFiles) { - if (df.version > rollbackVersion) { - df.future.cancel(); - metadata->bytesInNewDeltaFiles -= df.bytes; - toPop++; - if (BW_DEBUG) { - fmt::print("[{0} - {1}) rollback cancelling delta file @ {2}\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - df.version); + bool pendingSnapshot = false; + for (auto& f : inFlightFiles) { + if (f.snapshot) { + if (f.version > rollbackVersion) { + TEST(true); // Granule rollback cancelling snapshot file + if (BW_DEBUG) { + fmt::print("[{0} - {1}) rollback cancelling snapshot file @ {2}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + f.version); + } + f.future.cancel(); + toPop++; + } else { + metadata->pendingSnapshotVersion = f.version; + metadata->bytesInNewDeltaFiles = 0; + pendingSnapshot = true; } } else { - ASSERT(df.version > cfRollbackVersion); - cfRollbackVersion = df.version; + if (f.version > rollbackVersion) { + f.future.cancel(); + if (!pendingSnapshot) { + metadata->bytesInNewDeltaFiles -= f.bytes; + } + toPop++; + TEST(true); // Granule rollback cancelling delta file + if (BW_DEBUG) { + fmt::print("[{0} - {1}) rollback cancelling delta file @ {2}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + f.version); + } + } else { + ASSERT(f.version > cfRollbackVersion); + cfRollbackVersion = f.version; + if (pendingSnapshot) { + metadata->bytesInNewDeltaFiles += f.bytes; + } + } } } ASSERT(toPop > 0); while (toPop > 0) { - inFlightDeltaFiles.pop_back(); + inFlightFiles.pop_back(); toPop--; } metadata->pendingDeltaVersion = cfRollbackVersion; if (BW_DEBUG) { - printf("[%s - %s) rollback discarding all %d in-memory mutations\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str(), - metadata->currentDeltas.size()); + fmt::print("[{0} - {1}) rollback discarding all {2} in-memory mutations\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + metadata->currentDeltas.size()); } // discard all in-memory mutations - metadata->deltaArena = Arena(); - metadata->currentDeltas = GranuleDeltas(); + metadata->currentDeltas = Standalone(); metadata->bufferedDeltaBytes = 0; - metadata->bufferedDeltaVersion.set(cfRollbackVersion); + metadata->bufferedDeltaVersion = cfRollbackVersion; + + // Track that this rollback happened, since we have to re-read mutations up to the rollback + // Add this rollback to in progress, and put all completed ones back in progress + rollbacksInProgress.push_back(std::pair(rollbackVersion, mutationVersion)); + while (!rollbacksCompleted.empty()) { + if (rollbacksCompleted.back().first >= cfRollbackVersion) { + rollbacksInProgress.push_front(rollbacksCompleted.back()); + rollbacksCompleted.pop_back(); + } else { + // some rollbacks in completed could still have a delta file in flight after this rollback, they should + // remain in completed + break; + } + } } else { // No pending delta files to discard, just in-memory mutations + TEST(true); // Granule rollback discarding in memory mutations // FIXME: could binary search? int mIdx = metadata->currentDeltas.size() - 1; @@ -1023,12 +1139,14 @@ static Version doGranuleRollback(Reference metadata, metadata->bufferedDeltaBytes); } - metadata->currentDeltas.resize(metadata->deltaArena, mIdx); + metadata->currentDeltas.resize(metadata->currentDeltas.arena(), mIdx); // delete all deltas in rollback range, but we can optimize here to just skip the uncommitted mutations - // directly and immediately pop the rollback out of inProgress - metadata->bufferedDeltaVersion.set(rollbackVersion); + // directly and immediately pop the rollback out of inProgress to completed + + metadata->bufferedDeltaVersion = rollbackVersion; cfRollbackVersion = mutationVersion; + rollbacksCompleted.push_back(std::pair(rollbackVersion, mutationVersion)); } if (BW_DEBUG) { @@ -1038,36 +1156,84 @@ static Version doGranuleRollback(Reference metadata, cfRollbackVersion); } - metadata->rollbackCount.set(metadata->rollbackCount.get() + 1); - - // add this rollback to in progress, and put all completed ones back in progress - rollbacksInProgress.push_back(std::pair(rollbackVersion, mutationVersion)); - for (int i = rollbacksCompleted.size() - 1; i >= 0; i--) { - rollbacksInProgress.push_front(rollbacksCompleted[i]); - } - rollbacksCompleted.clear(); - return cfRollbackVersion; } +ACTOR Future waitOnCFVersion(Reference metadata, Version waitVersion) { + loop { + try { + // if not valid, we're about to be cancelled anyway + state Future atLeast = metadata->activeCFData.get().isValid() + ? metadata->activeCFData.get()->whenAtLeast(waitVersion) + : Never(); + choose { + when(wait(atLeast)) { break; } + when(wait(metadata->activeCFData.onChange())) {} + } + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled || e.code() == error_code_change_feed_popped) { + throw e; + } + + // if waiting on a parent granule change feed and we change to the child, the parent will get end_of_stream, + // which could cause this waiting whenAtLeast to get change_feed_cancelled. We should simply retry and wait + // a bit, as blobGranuleUpdateFiles will switch to the new change feed + wait(delay(0.05)); + } + } + + // stop after change feed callback + wait(delay(0, TaskPriority::BlobWorkerReadChangeFeed)); + + return Void(); +} + +ACTOR Future waitCommittedGrv(Reference bwData, + Reference metadata, + Version version) { + if (version > bwData->grvVersion.get()) { + // this order is important, since we need to register a waiter on the notified version before waking the GRV + // actor + Future grvAtLeast = bwData->grvVersion.whenAtLeast(version); + Promise doGrvCheck = bwData->doGRVCheck; + if (doGrvCheck.canBeSet()) { + doGrvCheck.send(Void()); + } + wait(grvAtLeast); + } + + Version grvVersion = bwData->grvVersion.get(); + wait(waitOnCFVersion(metadata, grvVersion)); + return Void(); +} + +ACTOR Future waitVersionCommitted(Reference bwData, + Reference metadata, + Version version) { + // If GRV is way in the future, we know we can't roll back more than 5 seconds (or whatever this knob is set to) + // worth of versions + wait(waitCommittedGrv(bwData, metadata, version) || + waitOnCFVersion(metadata, version + SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)); + if (version > metadata->knownCommittedVersion) { + metadata->knownCommittedVersion = version; + } + return Void(); +} + // updater for a single granule // TODO: this is getting kind of large. Should try to split out this actor if it continues to grow? -// FIXME: handle errors here (forward errors) ACTOR Future blobGranuleUpdateFiles(Reference bwData, Reference metadata, Future assignFuture) { - state Reference oldChangeFeedStream = makeReference(); - state Reference changeFeedStream = makeReference(); - state Future inFlightBlobSnapshot; - state std::deque inFlightDeltaFiles; + state std::deque inFlightFiles; state Future oldChangeFeedFuture; state Future changeFeedFuture; state GranuleStartState startState; state bool readOldChangeFeed; - state bool lastFromOldChangeFeed = false; state Optional> oldChangeFeedDataComplete; state Key cfKey; state Optional oldCFKey; + state int pendingSnapshots = 0; state std::deque> rollbacksInProgress; state std::deque> rollbacksCompleted; @@ -1085,16 +1251,18 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, wait(delay(0, TaskPriority::BlobWorkerUpdateStorage)); - cfKey = StringRef(startState.granuleID.toString()); + cfKey = granuleIDToCFKey(startState.granuleID); if (startState.parentGranule.present()) { - oldCFKey = StringRef(startState.parentGranule.get().second.toString()); + oldCFKey = granuleIDToCFKey(startState.parentGranule.get().second); } if (BW_DEBUG) { - fmt::print("Granule File Updater Starting for [{0} - {1}):\n", + fmt::print("Granule File Updater Starting for [{0} - {1}) @ ({2}, {3}):\n", metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable()); - fmt::print(" CFID: {}\n", startState.granuleID.toString()); + metadata->keyRange.end.printable(), + metadata->originalEpoch, + metadata->originalSeqno); + fmt::print(" CFID: {} ({})\n", startState.granuleID.toString(), cfKey.printable()); fmt::print(" CF Start Version: {}\n", startState.changeFeedStartVersion); fmt::print(" Previous Durable Version: {}\n", startState.previousDurableVersion); fmt::print(" doSnapshot={}\n", startState.doSnapshot ? "T" : "F"); @@ -1107,8 +1275,6 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, state Version startVersion; state BlobFileIndex newSnapshotFile; - inFlightBlobSnapshot = Future(); // not valid! - // if this is a reassign, calculate how close to a snapshot the previous owner was if (startState.existingFiles.present()) { GranuleFiles files = startState.existingFiles.get(); @@ -1120,97 +1286,112 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, } } } + metadata->files = startState.existingFiles.get(); snapshotEligible = true; } if (!startState.doSnapshot) { + TEST(true); // Granule moved without split startVersion = startState.previousDurableVersion; ASSERT(!metadata->files.snapshotFiles.empty()); metadata->pendingSnapshotVersion = metadata->files.snapshotFiles.back().version; metadata->durableSnapshotVersion.set(metadata->pendingSnapshotVersion); + metadata->initialSnapshotVersion = metadata->files.snapshotFiles.front().version; } else { if (startState.blobFilesToSnapshot.present()) { - inFlightBlobSnapshot = - compactFromBlob(bwData, metadata, startState.granuleID, startState.blobFilesToSnapshot.get()); startVersion = startState.previousDurableVersion; + Future inFlightBlobSnapshot = compactFromBlob( + bwData, metadata, startState.granuleID, startState.blobFilesToSnapshot.get(), startVersion); + inFlightFiles.push_back(InFlightFile(inFlightBlobSnapshot, startVersion, 0, true)); + pendingSnapshots++; + metadata->durableSnapshotVersion.set(startState.blobFilesToSnapshot.get().snapshotFiles.back().version); } else { ASSERT(startState.previousDurableVersion == invalidVersion); - BlobFileIndex fromFDB = wait(dumpInitialSnapshotFromFDB(bwData, metadata, startState.granuleID)); + BlobFileIndex fromFDB = wait(dumpInitialSnapshotFromFDB(bwData, metadata, startState.granuleID, cfKey)); newSnapshotFile = fromFDB; ASSERT(startState.changeFeedStartVersion <= fromFDB.version); startVersion = newSnapshotFile.version; metadata->files.snapshotFiles.push_back(newSnapshotFile); metadata->durableSnapshotVersion.set(startVersion); - // construct fake history entry so we can store start version for splitting later - startState.history = - GranuleHistory(metadata->keyRange, startVersion, Standalone()); - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); } + metadata->initialSnapshotVersion = startVersion; metadata->pendingSnapshotVersion = startVersion; } metadata->durableDeltaVersion.set(startVersion); metadata->pendingDeltaVersion = startVersion; - metadata->bufferedDeltaVersion.set(startVersion); + metadata->bufferedDeltaVersion = startVersion; + metadata->knownCommittedVersion = startVersion; + + Reference cfData = makeReference(); + + if (startState.parentGranule.present() && startVersion < startState.changeFeedStartVersion) { + // read from parent change feed up until our new change feed is started + // Required to have canReadPopped = false, otherwise another granule can take over the change feed, and pop + // it. That could cause this worker to think it has the full correct set of data if it then reads the data, + // until it checks the granule lock again. + // passing false for canReadPopped means we will get an exception if we try to read any popped data, killing + // this actor + readOldChangeFeed = true; + + oldChangeFeedFuture = bwData->db->getChangeFeedStream(cfData, + oldCFKey.get(), + startVersion + 1, + startState.changeFeedStartVersion, + metadata->keyRange, + bwData->changeFeedStreamReplyBufferSize, + false); + + } else { + readOldChangeFeed = false; + changeFeedFuture = bwData->db->getChangeFeedStream(cfData, + cfKey, + startVersion + 1, + MAX_VERSION, + metadata->keyRange, + bwData->changeFeedStreamReplyBufferSize, + false); + } + + // Start actors BEFORE setting new change feed data to ensure the change feed data is properly initialized by + // the client + metadata->activeCFData.set(cfData); ASSERT(metadata->readable.canBeSet()); metadata->readable.send(Void()); - if (startState.parentGranule.present()) { - // FIXME: once we have empty versions, only include up to startState.changeFeedStartVersion in the read - // stream. Then we can just stop the old stream when we get end_of_stream from this and not handle the - // mutation version truncation stuff - - // FIXME: filtering on key range != change feed range doesn't work - readOldChangeFeed = true; - oldChangeFeedFuture = - bwData->db->getChangeFeedStream(oldChangeFeedStream, - oldCFKey.get(), - startVersion + 1, - MAX_VERSION, - startState.parentGranule.get().first /*metadata->keyRange*/); - } else { - readOldChangeFeed = false; - changeFeedFuture = bwData->db->getChangeFeedStream( - changeFeedStream, cfKey, startVersion + 1, MAX_VERSION, metadata->keyRange); - } - - state Version lastVersion = startVersion + 1; loop { // check outstanding snapshot/delta files for completion - if (inFlightBlobSnapshot.isValid() && inFlightBlobSnapshot.isReady()) { - BlobFileIndex completedSnapshot = wait(inFlightBlobSnapshot); - metadata->files.snapshotFiles.push_back(completedSnapshot); - metadata->durableSnapshotVersion.set(completedSnapshot.version); - inFlightBlobSnapshot = Future(); // not valid! - if (BW_DEBUG) { - printf("Async Blob Snapshot completed for [%s - %s)\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str()); - } + while (inFlightFiles.size() > 0) { + if (inFlightFiles.front().future.isReady()) { + BlobFileIndex completedFile = wait(inFlightFiles.front().future); + if (inFlightFiles.front().snapshot) { + if (metadata->files.deltaFiles.empty()) { + ASSERT(completedFile.version == metadata->initialSnapshotVersion); + } else { + ASSERT(completedFile.version == metadata->files.deltaFiles.back().version); + } - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); - } - if (!inFlightBlobSnapshot.isValid()) { - while (inFlightDeltaFiles.size() > 0) { - if (inFlightDeltaFiles.front().future.isReady()) { - BlobFileIndex completedDeltaFile = wait(inFlightDeltaFiles.front().future); - wait(handleCompletedDeltaFile(bwData, - metadata, - completedDeltaFile, - cfKey, - startState.changeFeedStartVersion, - rollbacksCompleted)); - - inFlightDeltaFiles.pop_front(); - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); + metadata->files.snapshotFiles.push_back(completedFile); + metadata->durableSnapshotVersion.set(completedFile.version); + pendingSnapshots--; } else { - break; + handleCompletedDeltaFile(bwData, + metadata, + completedFile, + cfKey, + startState.changeFeedStartVersion, + &rollbacksCompleted); } + + inFlightFiles.pop_front(); + wait(yield(TaskPriority::BlobWorkerUpdateStorage)); + } else { + break; } } @@ -1218,101 +1399,292 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, if (BUGGIFY_WITH_PROB(0.001)) { wait(delay(deterministicRandom()->random01(), TaskPriority::BlobWorkerReadChangeFeed)); } else { + // FIXME: if we're already BlobWorkerReadChangeFeed, don't do a delay? wait(delay(0, TaskPriority::BlobWorkerReadChangeFeed)); } state Standalone> mutations; - if (readOldChangeFeed) { - Standalone> oldMutations = - waitNext(oldChangeFeedStream->mutations.getFuture()); - // TODO filter old mutations won't be necessary, SS does it already - if (filterOldMutations( - metadata->keyRange, &oldMutations, &mutations, startState.changeFeedStartVersion)) { - // if old change feed has caught up with where new one would start, finish last one and start new - // one + try { + // Even if there are no new mutations, there still might be readers waiting on durableDeltaVersion + // to advance. We need to check whether any outstanding files have finished so we don't wait on + // mutations forever + choose { + when(Standalone> _mutations = + waitNext(metadata->activeCFData.get()->mutations.getFuture())) { + mutations = _mutations; + ASSERT(!mutations.empty()); + if (readOldChangeFeed) { + ASSERT(mutations.back().version < startState.changeFeedStartVersion); + } else { + ASSERT(mutations.front().version >= startState.changeFeedStartVersion); + } - Key cfKey = StringRef(startState.granuleID.toString()); - changeFeedFuture = bwData->db->getChangeFeedStream( - changeFeedStream, cfKey, startState.changeFeedStartVersion, MAX_VERSION, metadata->keyRange); - oldChangeFeedFuture.cancel(); - lastFromOldChangeFeed = true; + if (mutations.front().version <= metadata->bufferedDeltaVersion) { + fmt::print("ERROR: Mutations went backwards for granule [{0} - {1}). " + "bufferedDeltaVersion={2}, mutationVersion={3} !!!\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + metadata->bufferedDeltaVersion, + mutations.front().version); + } + ASSERT(mutations.front().version > metadata->bufferedDeltaVersion); - // now that old change feed is cancelled, clear out any mutations still in buffer by replacing - // promise stream - oldChangeFeedStream = makeReference(); + // If this assert trips we should have gotten change_feed_popped from SS and didn't + ASSERT(mutations.front().version >= metadata->activeCFData.get()->popVersion); + } + when(wait(inFlightFiles.empty() ? Never() : success(inFlightFiles.front().future))) {} } - } else { - Standalone> newMutations = - waitNext(changeFeedStream->mutations.getFuture()); - mutations = newMutations; + } catch (Error& e) { + // only error we should expect here is when we finish consuming old change feed + if (e.code() != error_code_end_of_stream) { + throw; + } + ASSERT(readOldChangeFeed); + + readOldChangeFeed = false; + // set this so next delta file write updates granule split metadata to done + ASSERT(startState.parentGranule.present()); + oldChangeFeedDataComplete = startState.parentGranule.get(); + if (BW_DEBUG) { + fmt::print("Granule [{0} - {1}) switching to new change feed {2} @ {3}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + startState.granuleID.toString(), + metadata->bufferedDeltaVersion); + } + + Reference cfData = makeReference(); + + changeFeedFuture = bwData->db->getChangeFeedStream(cfData, + cfKey, + startState.changeFeedStartVersion, + MAX_VERSION, + metadata->keyRange, + bwData->changeFeedStreamReplyBufferSize, + false); + + // Start actors BEFORE setting new change feed data to ensure the change feed data is properly + // initialized by the client + metadata->activeCFData.set(cfData); } // process mutations - for (MutationsAndVersionRef d : mutations) { - state MutationsAndVersionRef deltas = d; - ASSERT(deltas.version >= lastVersion); - ASSERT(lastVersion > metadata->bufferedDeltaVersion.get()); + if (!mutations.empty()) { + bool processedAnyMutations = false; + Version lastDeltaVersion = invalidVersion; + for (MutationsAndVersionRef deltas : mutations) { - // if lastVersion is complete, update buffered version and potentially write a delta file with - // everything up to lastVersion - if (deltas.version > lastVersion) { - metadata->bufferedDeltaVersion.set(lastVersion); + // Buffer mutations at this version. There should not be multiple MutationsAndVersionRef with the + // same version + ASSERT(deltas.version > metadata->bufferedDeltaVersion); + ASSERT(deltas.version > lastDeltaVersion); + // FIXME: this assert isn't true - why + // ASSERT(!deltas.mutations.empty()); + if (!deltas.mutations.empty()) { + if (deltas.mutations.size() == 1 && deltas.mutations.back().param1 == lastEpochEndPrivateKey) { + // Note rollbackVerision is durable, [rollbackVersion+1 - deltas.version] needs to be tossed + // For correctness right now, there can be no waits and yields either in rollback handling + // or in handleBlobGranuleFileRequest once waitForVersion has succeeded, otherwise this will + // race and clobber results + Version rollbackVersion; + BinaryReader br(deltas.mutations[0].param2, Unversioned()); + br >> rollbackVersion; + + ASSERT(rollbackVersion >= metadata->durableDeltaVersion.get()); + + if (!rollbacksInProgress.empty()) { + ASSERT(rollbacksInProgress.front().first == rollbackVersion); + ASSERT(rollbacksInProgress.front().second == deltas.version); + if (BW_DEBUG) { + fmt::print("Passed rollback {0} -> {1}\n", deltas.version, rollbackVersion); + } + rollbacksCompleted.push_back(rollbacksInProgress.front()); + rollbacksInProgress.pop_front(); + } else { + // FIXME: add counter for granule rollbacks and rollbacks skipped? + // explicitly check last delta in currentDeltas because lastVersion and + // bufferedDeltaVersion include empties + if (metadata->pendingDeltaVersion <= rollbackVersion && + (metadata->currentDeltas.empty() || + metadata->currentDeltas.back().version <= rollbackVersion)) { + TEST(true); // Granule ignoring rollback + + if (BW_DEBUG) { + fmt::print( + "Granule [{0} - {1}) on BW {2} skipping rollback {3} -> {4} completely\n", + metadata->keyRange.begin.printable().c_str(), + metadata->keyRange.end.printable().c_str(), + bwData->id.toString().substr(0, 5).c_str(), + deltas.version, + rollbackVersion); + } + // Still have to add to rollbacksCompleted. If we later roll the granule back past + // this because of cancelling a delta file, we need to count this as in progress so + // we can match the rollback mutation to a rollbackInProgress when we restart the + // stream. + rollbacksCompleted.push_back(std::pair(rollbackVersion, deltas.version)); + } else { + TEST(true); // Granule processing rollback + if (BW_DEBUG) { + fmt::print("[{0} - {1}) on BW {2} ROLLBACK @ {3} -> {4}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + bwData->id.toString().substr(0, 5).c_str(), + deltas.version, + rollbackVersion); + TraceEvent(SevWarn, "GranuleRollback", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("Version", deltas.version) + .detail("RollbackVersion", rollbackVersion); + } + + Version cfRollbackVersion = doGranuleRollback(metadata, + deltas.version, + rollbackVersion, + inFlightFiles, + rollbacksInProgress, + rollbacksCompleted); + + Reference cfData = makeReference(); + + if (!readOldChangeFeed && cfRollbackVersion < startState.changeFeedStartVersion) { + // It isn't possible to roll back across the parent/child feed boundary, but as + // part of rolling back we may need to cancel in-flight delta files, and those + // delta files may include stuff from before the parent/child boundary. So we + // have to go back to reading the old change feed + ASSERT(cfRollbackVersion >= startState.previousDurableVersion); + ASSERT(cfRollbackVersion >= metadata->durableDeltaVersion.get()); + TEST(true); // rollback crossed change feed boundaries + readOldChangeFeed = true; + oldChangeFeedDataComplete.reset(); + } + + if (readOldChangeFeed) { + ASSERT(cfRollbackVersion < startState.changeFeedStartVersion); + oldChangeFeedFuture = + bwData->db->getChangeFeedStream(cfData, + oldCFKey.get(), + cfRollbackVersion + 1, + startState.changeFeedStartVersion, + metadata->keyRange, + bwData->changeFeedStreamReplyBufferSize, + false); + + } else { + if (cfRollbackVersion < startState.changeFeedStartVersion) { + fmt::print("Rollback past CF start??. rollback={0}, start={1}\n", + cfRollbackVersion, + startState.changeFeedStartVersion); + } + ASSERT(cfRollbackVersion >= startState.changeFeedStartVersion); + + changeFeedFuture = + bwData->db->getChangeFeedStream(cfData, + cfKey, + cfRollbackVersion + 1, + MAX_VERSION, + metadata->keyRange, + bwData->changeFeedStreamReplyBufferSize, + false); + } + + // Start actors BEFORE setting new change feed data to ensure the change feed data + // is properly initialized by the client + metadata->activeCFData.set(cfData); + + justDidRollback = true; + break; + } + } + } else if (!rollbacksInProgress.empty() && rollbacksInProgress.front().first < deltas.version && + rollbacksInProgress.front().second > deltas.version) { + TEST(true); // Granule skipping mutations b/c prior rollback + if (BW_DEBUG) { + fmt::print("Skipping mutations @ {} b/c prior rollback\n", deltas.version); + } + } else { + for (auto& delta : deltas.mutations) { + metadata->bufferedDeltaBytes += delta.totalSize(); + bwData->stats.changeFeedInputBytes += delta.totalSize(); + bwData->stats.mutationBytesBuffered += delta.totalSize(); + + DEBUG_MUTATION("BlobWorkerBuffer", deltas.version, delta, bwData->id) + .detail("Granule", metadata->keyRange) + .detail("ChangeFeedID", + cfKeyToGranuleID(readOldChangeFeed ? oldCFKey.get() : cfKey)) + .detail("OldChangeFeed", readOldChangeFeed ? "T" : "F"); + } + metadata->currentDeltas.push_back_deep(metadata->currentDeltas.arena(), deltas); + + processedAnyMutations = true; + ASSERT(deltas.version != invalidVersion); + ASSERT(deltas.version > lastDeltaVersion); + lastDeltaVersion = deltas.version; + } + } + if (justDidRollback) { + break; + } } - // Write a new delta file IF we have enough bytes, and we have all of the previous version's stuff - // there to ensure no versions span multiple delta files. Check this by ensuring the version of this - // new delta is larger than the previous largest seen version - if (metadata->bufferedDeltaBytes >= SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES && - deltas.version > lastVersion) { + if (!justDidRollback && processedAnyMutations) { + // update buffered version + ASSERT(lastDeltaVersion != invalidVersion); + ASSERT(lastDeltaVersion > metadata->bufferedDeltaVersion); + + // Update buffered delta version so new waitForVersion checks can bypass waiting entirely + metadata->bufferedDeltaVersion = lastDeltaVersion; + } + justDidRollback = false; + + // Write a new delta file IF we have enough bytes + if (metadata->bufferedDeltaBytes >= SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES) { if (BW_DEBUG) { - fmt::print("Granule [{0} - {1}) flushing delta file after {2} bytes @ {3} {4}{5}\n", + fmt::print("Granule [{0} - {1}) flushing delta file after {2} bytes @ {3} {4}\n", metadata->keyRange.begin.printable(), metadata->keyRange.end.printable(), metadata->bufferedDeltaBytes, - lastVersion, - deltas.version, + lastDeltaVersion, oldChangeFeedDataComplete.present() ? ". Finalizing " : ""); } TraceEvent("BlobGranuleDeltaFile", bwData->id) .detail("Granule", metadata->keyRange) - .detail("Version", lastVersion); + .detail("Version", lastDeltaVersion); // sanity check for version order - ASSERT(lastVersion >= metadata->currentDeltas.back().version); + ASSERT(lastDeltaVersion >= metadata->currentDeltas.back().version); ASSERT(metadata->pendingDeltaVersion < metadata->currentDeltas.front().version); // launch pipelined, but wait for previous operation to complete before persisting to FDB - Future previousDeltaFileFuture; - if (inFlightBlobSnapshot.isValid() && inFlightDeltaFiles.empty()) { - previousDeltaFileFuture = inFlightBlobSnapshot; - } else if (!inFlightDeltaFiles.empty()) { - previousDeltaFileFuture = inFlightDeltaFiles.back().future; + Future previousFuture; + if (!inFlightFiles.empty()) { + previousFuture = inFlightFiles.back().future; } else { - previousDeltaFileFuture = Future(BlobFileIndex()); + previousFuture = Future(BlobFileIndex()); } - Future dfFuture = writeDeltaFile(bwData, - metadata->keyRange, - startState.granuleID, - metadata->originalEpoch, - metadata->originalSeqno, - metadata->deltaArena, - metadata->currentDeltas, - lastVersion, - previousDeltaFileFuture, - oldChangeFeedDataComplete); - inFlightDeltaFiles.push_back( - InFlightDeltaFile(dfFuture, lastVersion, metadata->bufferedDeltaBytes)); + Future dfFuture = + writeDeltaFile(bwData, + metadata->keyRange, + startState.granuleID, + metadata->originalEpoch, + metadata->originalSeqno, + metadata->currentDeltas, + lastDeltaVersion, + previousFuture, + waitVersionCommitted(bwData, metadata, lastDeltaVersion), + oldChangeFeedDataComplete); + inFlightFiles.push_back( + InFlightFile(dfFuture, lastDeltaVersion, metadata->bufferedDeltaBytes, false)); oldChangeFeedDataComplete.reset(); // add new pending delta file - ASSERT(metadata->pendingDeltaVersion < lastVersion); - metadata->pendingDeltaVersion = lastVersion; + ASSERT(metadata->pendingDeltaVersion < lastDeltaVersion); + metadata->pendingDeltaVersion = lastDeltaVersion; metadata->bytesInNewDeltaFiles += metadata->bufferedDeltaBytes; bwData->stats.mutationBytesBuffered -= metadata->bufferedDeltaBytes; // reset current deltas - metadata->deltaArena = Arena(); - metadata->currentDeltas = GranuleDeltas(); + metadata->currentDeltas = Standalone(); metadata->bufferedDeltaBytes = 0; // if we just wrote a delta file, check if we need to compact here. @@ -1323,254 +1695,125 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, // FIXME: if we're still reading from old change feed, we should probably compact if we're making a // bunch of extra delta files at some point, even if we don't consider it for a split yet + + // If we have enough delta files, try to re-snapshot if (snapshotEligible && metadata->bytesInNewDeltaFiles >= SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT && - !readOldChangeFeed) { - if (BW_DEBUG && (inFlightBlobSnapshot.isValid() || !inFlightDeltaFiles.empty())) { - fmt::print( - "Granule [{0} - {1}) ready to re-snapshot, waiting for outstanding {2} snapshot and {3} " - "deltas to " - "finish\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - inFlightBlobSnapshot.isValid() ? 1 : 0, - inFlightDeltaFiles.size()); - } - // wait for all in flight snapshot/delta files - if (inFlightBlobSnapshot.isValid()) { - BlobFileIndex completedSnapshot = wait(inFlightBlobSnapshot); - metadata->files.snapshotFiles.push_back(completedSnapshot); - metadata->durableSnapshotVersion.set(completedSnapshot.version); - inFlightBlobSnapshot = Future(); // not valid! - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); - } - for (auto& it : inFlightDeltaFiles) { - BlobFileIndex completedDeltaFile = wait(it.future); - wait(handleCompletedDeltaFile(bwData, - metadata, - completedDeltaFile, - cfKey, - startState.changeFeedStartVersion, - rollbacksCompleted)); - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); - } - inFlightDeltaFiles.clear(); - - if (BW_DEBUG) { - fmt::print("Granule [{0} - {1}) checking with BM for re-snapshot after {2} bytes\n", + metadata->pendingDeltaVersion >= startState.changeFeedStartVersion) { + if (BW_DEBUG && !inFlightFiles.empty()) { + fmt::print("Granule [{0} - {1}) ready to re-snapshot at {2} after {3} > {4} bytes, waiting for " + "outstanding {5} files to finish\n", metadata->keyRange.begin.printable(), metadata->keyRange.end.printable(), - metadata->bytesInNewDeltaFiles); + metadata->pendingDeltaVersion, + metadata->bytesInNewDeltaFiles, + SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT, + inFlightFiles.size()); } - TraceEvent("BlobGranuleSnapshotCheck", bwData->id) - .detail("Granule", metadata->keyRange) - .detail("Version", metadata->durableDeltaVersion.get()); + // Speculatively assume we will get the range back. This is both a performance optimization, and + // necessary to keep consuming versions from the change feed so that we can realize + // our last delta file is committed and write it - // Save these from the start so repeated requests are idempotent - // Need to retry in case response is dropped or manager changes. Eventually, a manager will - // either reassign the range with continue=true, or will revoke the range. But, we will keep the - // range open at this version for reads until that assignment change happens - metadata->resumeSnapshot.reset(); - state int64_t statusEpoch = metadata->continueEpoch; - state int64_t statusSeqno = metadata->continueSeqno; - loop { - loop { - try { - wait(bwData->currentManagerStatusStream.get().onReady()); - bwData->currentManagerStatusStream.get().send( - GranuleStatusReply(metadata->keyRange, - true, - statusEpoch, - statusSeqno, - startState.granuleID, - startState.history.get().version, - metadata->durableDeltaVersion.get())); - break; - } catch (Error& e) { - wait(bwData->currentManagerStatusStream.onChange()); - } - } - - choose { - when(wait(metadata->resumeSnapshot.getFuture())) { break; } - when(wait(delay(1.0))) {} - when(wait(bwData->currentManagerStatusStream.onChange())) {} - } - - if (BW_DEBUG) { - fmt::print( - "Granule [{0} - {1})\n, hasn't heard back from BM in BW {2}, re-sending status\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - bwData->id.toString()); - } + Future previousFuture; + if (!inFlightFiles.empty()) { + previousFuture = inFlightFiles.back().future; + ASSERT(!inFlightFiles.back().snapshot); + } else { + previousFuture = Future(metadata->files.deltaFiles.back()); } + int64_t versionsSinceLastSnapshot = + metadata->pendingDeltaVersion - metadata->pendingSnapshotVersion; + Future inFlightBlobSnapshot = checkSplitAndReSnapshot(bwData, + metadata, + startState.granuleID, + metadata->bytesInNewDeltaFiles, + previousFuture, + versionsSinceLastSnapshot); + inFlightFiles.push_back(InFlightFile(inFlightBlobSnapshot, metadata->pendingDeltaVersion, 0, true)); + pendingSnapshots++; - if (BW_DEBUG) { - fmt::print("Granule [{0} - {1}) re-snapshotting after {2} bytes\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - metadata->bytesInNewDeltaFiles); - } - TraceEvent("BlobGranuleSnapshotFile", bwData->id) - .detail("Granule", metadata->keyRange) - .detail("Version", metadata->durableDeltaVersion.get()); - // TODO: this could read from FDB instead if it knew there was a large range clear at the end or - // it knew the granule was small, or something - - // Have to copy files object so that adding to it as we start writing new delta files in - // parallel doesn't conflict. We could also pass the snapshot version and ignore any snapshot - // files >= version and any delta files > version, but that's more complicated - inFlightBlobSnapshot = compactFromBlob(bwData, metadata, startState.granuleID, metadata->files); - metadata->pendingSnapshotVersion = metadata->durableDeltaVersion.get(); + metadata->pendingSnapshotVersion = metadata->pendingDeltaVersion; // reset metadata metadata->bytesInNewDeltaFiles = 0; + + // If we have more than one snapshot file and that file is unblocked (committedVersion >= + // snapshotVersion), wait for it to finish + + if (pendingSnapshots > 1) { + state int waitIdx = 0; + int idx = 0; + Version safeVersion = + std::max(metadata->knownCommittedVersion, + metadata->bufferedDeltaVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS); + for (auto& f : inFlightFiles) { + if (f.snapshot && f.version < metadata->pendingSnapshotVersion && + f.version <= safeVersion) { + if (BW_DEBUG) { + fmt::print("[{0} - {1}) Waiting on previous snapshot file @ {2} <= {3}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + f.version, + safeVersion); + } + waitIdx = idx + 1; + } + idx++; + } + while (waitIdx > 0) { + TEST(true); // Granule blocking on previous snapshot + // TODO don't duplicate code + BlobFileIndex completedFile = wait(inFlightFiles.front().future); + if (inFlightFiles.front().snapshot) { + if (metadata->files.deltaFiles.empty()) { + ASSERT(completedFile.version == metadata->initialSnapshotVersion); + } else { + ASSERT(completedFile.version == metadata->files.deltaFiles.back().version); + } + metadata->files.snapshotFiles.push_back(completedFile); + metadata->durableSnapshotVersion.set(completedFile.version); + pendingSnapshots--; + } else { + handleCompletedDeltaFile(bwData, + metadata, + completedFile, + cfKey, + startState.changeFeedStartVersion, + &rollbacksCompleted); + } + + inFlightFiles.pop_front(); + waitIdx--; + wait(yield(TaskPriority::BlobWorkerUpdateStorage)); + } + } + } else if (snapshotEligible && metadata->bytesInNewDeltaFiles >= SERVER_KNOBS->BG_DELTA_BYTES_BEFORE_COMPACT) { // if we're in the old change feed case and can't snapshot but we have enough data to, don't - // queue too many delta files in parallel - while (inFlightDeltaFiles.size() > 10) { + // queue too many files in parallel, and slow down change feed consuming to let file writing + // catch up + + TEST(true); // Granule processing long tail of old change feed + if (inFlightFiles.size() > 10 && inFlightFiles.front().version <= metadata->knownCommittedVersion) { if (BW_DEBUG) { - printf("[%s - %s) Waiting on delta file b/c old change feed\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str()); + fmt::print("[{0} - {1}) Waiting on delta file b/c old change feed\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable()); } - BlobFileIndex completedDeltaFile = wait(inFlightDeltaFiles.front().future); - if (BW_DEBUG) { - printf(" [%s - %s) Got completed delta file\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str()); + choose { + when(BlobFileIndex completedDeltaFile = wait(inFlightFiles.front().future)) {} + when(wait(delay(0.1))) {} } - wait(handleCompletedDeltaFile(bwData, - metadata, - completedDeltaFile, - cfKey, - startState.changeFeedStartVersion, - rollbacksCompleted)); - wait(yield(TaskPriority::BlobWorkerUpdateStorage)); - inFlightDeltaFiles.pop_front(); } } snapshotEligible = false; - - wait(yield(TaskPriority::BlobWorkerReadChangeFeed)); - - // finally, after we optionally write delta and snapshot files, add new mutations to buffer - if (!deltas.mutations.empty()) { - if (deltas.mutations.size() == 1 && deltas.mutations.back().param1 == lastEpochEndPrivateKey) { - // Note rollbackVerision is durable, [rollbackVersion+1 - deltas.version] needs to be tossed - // For correctness right now, there can be no waits and yields either in rollback handling - // or in handleBlobGranuleFileRequest once waitForVersion has succeeded, otherwise this will - // race and clobber results - Version rollbackVersion; - BinaryReader br(deltas.mutations[0].param2, Unversioned()); - br >> rollbackVersion; - - // FIXME: THIS IS FALSE!! delta can commit by getting committed version out of band, without - // seeing rollback mutation. - ASSERT(rollbackVersion >= metadata->durableDeltaVersion.get()); - - if (!rollbacksInProgress.empty()) { - ASSERT(rollbacksInProgress.front().first == rollbackVersion); - ASSERT(rollbacksInProgress.front().second == deltas.version); - fmt::print("Passed rollback {0} -> {1}\n", deltas.version, rollbackVersion); - rollbacksCompleted.push_back(rollbacksInProgress.front()); - rollbacksInProgress.pop_front(); - } else { - // FIXME: add counter for granule rollbacks and rollbacks skipped? - // explicitly check last delta in currentDeltas because lastVersion and bufferedDeltaVersion - // include empties - if (metadata->pendingDeltaVersion <= rollbackVersion && - (metadata->currentDeltas.empty() || - metadata->currentDeltas.back().version <= rollbackVersion)) { - - if (BW_DEBUG) { - fmt::print("BW skipping rollback {0} -> {1} completely\n", - deltas.version, - rollbackVersion); - } - } else { - if (BW_DEBUG) { - fmt::print("BW [{0} - {1}) ROLLBACK @ {2} -> {3}\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - deltas.version, - rollbackVersion); - TraceEvent(SevWarn, "GranuleRollback", bwData->id) - .detail("Granule", metadata->keyRange) - .detail("Version", deltas.version) - .detail("RollbackVersion", rollbackVersion); - } - Version cfRollbackVersion = doGranuleRollback(metadata, - deltas.version, - rollbackVersion, - inFlightDeltaFiles, - rollbacksInProgress, - rollbacksCompleted); - - // reset change feeds to cfRollbackVersion - if (readOldChangeFeed) { - oldChangeFeedStream = makeReference(); - oldChangeFeedFuture = bwData->db->getChangeFeedStream( - oldChangeFeedStream, - oldCFKey.get(), - cfRollbackVersion + 1, - MAX_VERSION, - startState.parentGranule.get().first /*metadata->keyRange*/); - } else { - changeFeedStream = makeReference(); - changeFeedFuture = bwData->db->getChangeFeedStream(changeFeedStream, - cfKey, - cfRollbackVersion + 1, - MAX_VERSION, - metadata->keyRange); - } - justDidRollback = true; - break; - } - } - } else if (!rollbacksInProgress.empty() && rollbacksInProgress.front().first < deltas.version && - rollbacksInProgress.front().second > deltas.version) { - if (BW_DEBUG) { - fmt::print("Skipping mutations @ {} b/c prior rollback\n", deltas.version); - } - } else { - for (auto& delta : deltas.mutations) { - metadata->bufferedDeltaBytes += delta.totalSize(); - bwData->stats.changeFeedInputBytes += delta.totalSize(); - bwData->stats.mutationBytesBuffered += delta.totalSize(); - - DEBUG_MUTATION("BlobWorkerBuffer", deltas.version, delta, bwData->id) - .detail("Granule", metadata->keyRange) - .detail("ChangeFeedID", readOldChangeFeed ? oldCFKey.get() : cfKey) - .detail("OldChangeFeed", readOldChangeFeed ? "T" : "F"); - } - metadata->currentDeltas.push_back_deep(metadata->deltaArena, deltas); - } - } - if (justDidRollback) { - break; - } - lastVersion = deltas.version; } - if (lastFromOldChangeFeed && !justDidRollback) { - readOldChangeFeed = false; - lastFromOldChangeFeed = false; - // set this so next delta file write updates granule split metadata to done - ASSERT(startState.parentGranule.present()); - oldChangeFeedDataComplete = startState.parentGranule.get(); - if (BW_DEBUG) { - fmt::print("Granule [{0} - {1}) switching to new change feed {2} @ {3}\n", - metadata->keyRange.begin.printable(), - metadata->keyRange.end.printable(), - startState.granuleID.toString(), - metadata->bufferedDeltaVersion.get()); - } - } - justDidRollback = false; } } catch (Error& e) { + // Free last change feed data + metadata->activeCFData.set(Reference()); + if (e.code() == error_code_operation_cancelled) { throw; } @@ -1580,27 +1823,57 @@ ACTOR Future blobGranuleUpdateFiles(Reference bwData, } if (e.code() == error_code_granule_assignment_conflict) { - TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id).detail("Granule", metadata->keyRange); - } else { - ++bwData->stats.granuleUpdateErrors; - if (BW_DEBUG) { - printf("Granule file updater for [%s - %s) got error %s, exiting\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str(), - e.name()); - } - TraceEvent(SevWarn, "GranuleFileUpdaterError", bwData->id).error(e).detail("Granule", metadata->keyRange); + TraceEvent(SevInfo, "GranuleAssignmentConflict", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("GranuleID", startState.granuleID); + return Void(); + } + if (e.code() == error_code_change_feed_popped) { + TraceEvent(SevInfo, "GranuleGotChangeFeedPopped", bwData->id) + .detail("Granule", metadata->keyRange) + .detail("GranuleID", startState.granuleID); + return Void(); + } + ++bwData->stats.granuleUpdateErrors; + if (BW_DEBUG) { + fmt::print("Granule file updater for [{0} - {1}) got error {2}, exiting\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + e.name()); + } - if (granuleCanRetry(e)) { - // explicitly cancel all outstanding write futures BEFORE updating promise stream, to ensure they - // can't update files after the re-assigned granule acquires the lock - inFlightBlobSnapshot.cancel(); - for (auto& f : inFlightDeltaFiles) { - f.future.cancel(); - } - - bwData->granuleUpdateErrors.send(metadata->originalReq); + if (granuleCanRetry(e)) { + TEST(true); // Granule close and re-open on error + TraceEvent("GranuleFileUpdaterRetriableError", bwData->id) + .error(e) + .detail("Granule", metadata->keyRange) + .detail("GranuleID", startState.granuleID); + // explicitly cancel all outstanding write futures BEFORE updating promise stream, to ensure they + // can't update files after the re-assigned granule acquires the lock + // do it backwards though because future depends on previous one, so it could cause a cascade + for (int i = inFlightFiles.size() - 1; i >= 0; i--) { + inFlightFiles[i].future.cancel(); } + + // if we retry and re-open, we need to use a normal request (no continue) and update the + // seqno + metadata->originalReq.managerEpoch = metadata->continueEpoch; + metadata->originalReq.managerSeqno = metadata->continueSeqno; + metadata->originalReq.type = AssignRequestType::Normal; + + bwData->granuleUpdateErrors.send(metadata->originalReq); + throw e; + } + + TraceEvent(SevError, "GranuleFileUpdaterUnexpectedError", bwData->id) + .error(e) + .detail("Granule", metadata->keyRange) + .detail("GranuleID", startState.granuleID); + ASSERT_WE_THINK(false); + + // if not simulation, kill the BW + if (bwData->fatalError.canBeSet()) { + bwData->fatalError.sendError(e); } throw e; } @@ -1628,25 +1901,34 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, stopVersion = prev.value().isValid() ? prev.value()->startVersion : invalidVersion; state std::vector> historyEntryStack; + state bool foundHistory = true; - // while the start version of the current granule's parent is larger than the last known start version, walk - // backwards + // while the start version of the current granule's parent not past the last known start version, + // walk backwards while (curHistory.value.parentGranules.size() > 0 && - curHistory.value.parentGranules[0].second > stopVersion) { + curHistory.value.parentGranules[0].second >= stopVersion) { state GranuleHistory next; + loop { try { Optional v = wait(tr.get(blobGranuleHistoryKeyFor( curHistory.value.parentGranules[0].first, curHistory.value.parentGranules[0].second))); - ASSERT(v.present()); - next = GranuleHistory(curHistory.value.parentGranules[0].first, - curHistory.value.parentGranules[0].second, - decodeBlobGranuleHistoryValue(v.get())); + if (!v.present()) { + foundHistory = false; + } else { + next = GranuleHistory(curHistory.value.parentGranules[0].first, + curHistory.value.parentGranules[0].second, + decodeBlobGranuleHistoryValue(v.get())); + } + break; } catch (Error& e) { wait(tr.onError(e)); } } + if (!foundHistory) { + break; + } ASSERT(next.version != invalidVersion); // granule next.granuleID goes from the version range [next.version, curHistory.version] @@ -1655,6 +1937,19 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, curHistory = next; } + if (!historyEntryStack.empty()) { + Version oldestStartVersion = historyEntryStack.back()->startVersion; + if (!foundHistory && stopVersion != invalidVersion) { + stopVersion = oldestStartVersion; + } + ASSERT(stopVersion == oldestStartVersion || stopVersion == invalidVersion); + } else { + if (!foundHistory && stopVersion != invalidVersion) { + stopVersion = invalidVersion; + } + ASSERT(stopVersion == invalidVersion); + } + // go back up stack and apply history entries from oldest to newest, skipping ranges that were already // applied by other racing loads. // yielding in this loop would mean we'd need to re-check for load races @@ -1665,6 +1960,7 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, int i = historyEntryStack.size() - 1; while (i >= 0 && historyEntryStack[i]->startVersion <= stopVersion) { + TEST(true); // Granule skipping history entries loaded by parallel reader i--; } int skipped = historyEntryStack.size() - 1 - i; @@ -1672,11 +1968,13 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, while (i >= 0) { auto prevRanges = bwData->granuleHistory.rangeContaining(historyEntryStack[i]->range.begin); - // sanity check - ASSERT(!prevRanges.value().isValid() || - prevRanges.value()->endVersion == historyEntryStack[i]->startVersion); + if (prevRanges.value().isValid() && + prevRanges.value()->endVersion != historyEntryStack[i]->startVersion) { + historyEntryStack[i]->parentGranule = Reference(); + } else { + historyEntryStack[i]->parentGranule = prevRanges.value(); + } - historyEntryStack[i]->parentGranule = prevRanges.value(); bwData->granuleHistory.insert(historyEntryStack[i]->range, historyEntryStack[i]); i--; } @@ -1693,14 +1991,20 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, metadata->historyLoaded.send(Void()); return Void(); } catch (Error& e) { - if (e.code() == error_code_operation_cancelled || e.code() == error_code_granule_assignment_conflict) { + if (e.code() == error_code_operation_cancelled) { throw e; } - if (BW_DEBUG) { - printf("Loading blob granule history got unexpected error %s\n", e.name()); + if (e.code() == error_code_granule_assignment_conflict) { + return Void(); + } + // SplitStorageMetrics explicitly has a SevError if it gets an error, so no errors should propagate here + TraceEvent(SevError, "BlobWorkerUnexpectedErrorLoadGranuleHistory", bwData->id).error(e); + ASSERT_WE_THINK(false); + + // if not simulation, kill the BW + if (bwData->fatalError.canBeSet()) { + bwData->fatalError.sendError(e); } - // TODO this should never happen? - ASSERT(false); throw e; } } @@ -1710,6 +2014,7 @@ ACTOR Future blobGranuleLoadHistory(Reference bwData, namespace { bool canReplyWith(Error e) { switch (e.code()) { + case error_code_blob_granule_transaction_too_old: case error_code_transaction_too_old: case error_code_future_version: // not thrown yet case error_code_wrong_shard_server: @@ -1726,58 +2031,75 @@ ACTOR Future waitForVersion(Reference metadata, Version v // if we don't have to wait for change feed version to catch up or wait for any pending file writes to complete, // nothing to do - /*printf(" [%s - %s) waiting for %lld\n readable:%s\n bufferedDelta=%lld\n pendingDelta=%lld\n " - "durableDelta=%lld\n pendingSnapshot=%lld\n durableSnapshot=%lld\n", - metadata->keyRange.begin.printable().c_str(), - metadata->keyRange.end.printable().c_str(), - v, - metadata->readable.isSet() ? "T" : "F", - metadata->bufferedDeltaVersion.get(), - metadata->pendingDeltaVersion, - metadata->durableDeltaVersion.get(), - metadata->pendingSnapshotVersion, - metadata->durableSnapshotVersion.get());*/ + if (BW_REQUEST_DEBUG) { + fmt::print("WFV {0}) CF={1}, pendingD={2}, durableD={3}, pendingS={4}, durableS={5}\n", + v, + metadata->activeCFData.get()->getVersion(), + metadata->pendingDeltaVersion, + metadata->durableDeltaVersion.get(), + metadata->pendingSnapshotVersion, + metadata->durableSnapshotVersion.get()); + } - if (v <= metadata->bufferedDeltaVersion.get() && + ASSERT(metadata->activeCFData.get().isValid()); + + if (v <= metadata->activeCFData.get()->getVersion() && (v <= metadata->durableDeltaVersion.get() || metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion) && (v <= metadata->durableSnapshotVersion.get() || metadata->durableSnapshotVersion.get() == metadata->pendingSnapshotVersion)) { + TEST(true); // Granule read not waiting return Void(); } // wait for change feed version to catch up to ensure we have all data - if (metadata->bufferedDeltaVersion.get() < v) { - wait(metadata->bufferedDeltaVersion.whenAtLeast(v)); + if (metadata->activeCFData.get()->getVersion() < v) { + wait(metadata->activeCFData.get()->whenAtLeast(v)); + ASSERT(metadata->activeCFData.get()->getVersion() >= v); } - // wait for any pending delta and snapshot files as of the momemt the change feed version caught up. + // wait for any pending delta and snapshot files as of the moment the change feed version caught up. state Version pendingDeltaV = metadata->pendingDeltaVersion; state Version pendingSnapshotV = metadata->pendingSnapshotVersion; - ASSERT(pendingDeltaV <= metadata->bufferedDeltaVersion.get()); - if (pendingDeltaV > metadata->durableDeltaVersion.get()) { + // If there are mutations that are no longer buffered but have not been + // persisted to a delta file that are necessary for the query, wait for them + if (pendingDeltaV > metadata->durableDeltaVersion.get() && v > metadata->durableDeltaVersion.get()) { + TEST(true); // Granule read waiting for pending delta wait(metadata->durableDeltaVersion.whenAtLeast(pendingDeltaV)); + ASSERT(metadata->durableDeltaVersion.get() >= pendingDeltaV); } - // This isn't strictly needed, but if we're in the process of re-snapshotting, we'd likely rather return that - // snapshot file than the previous snapshot file and all its delta files. - if (pendingSnapshotV > metadata->durableSnapshotVersion.get()) { + // This isn't strictly needed, but if we're in the process of re-snapshotting, we'd likely rather + // return that snapshot file than the previous snapshot file and all its delta files. + if (pendingSnapshotV > metadata->durableSnapshotVersion.get() && v > metadata->durableSnapshotVersion.get()) { + TEST(true); // Granule read waiting for pending snapshot wait(metadata->durableSnapshotVersion.whenAtLeast(pendingSnapshotV)); + ASSERT(metadata->durableSnapshotVersion.get() >= pendingSnapshotV); } - // There is a race here - we wait for pending delta files before this to finish, but while we do, we kick off - // another delta file and roll the mutations. In that case, we must return the new delta file instead of in - // memory mutations, so we wait for that delta file to complete + // There is a race here - we wait for pending delta files before this to finish, but while we do, we + // kick off another delta file and roll the mutations. In that case, we must return the new delta + // file instead of in memory mutations, so we wait for that delta file to complete - if (metadata->pendingDeltaVersion != pendingDeltaV) { - wait(metadata->durableDeltaVersion.whenAtLeast(pendingDeltaV + 1)); + if (metadata->pendingDeltaVersion >= v) { + TEST(true); // Granule mutations flushed while waiting for files to complete + wait(metadata->durableDeltaVersion.whenAtLeast(v)); + ASSERT(metadata->durableDeltaVersion.get() >= v); } return Void(); } -ACTOR Future handleBlobGranuleFileRequest(Reference bwData, BlobGranuleFileRequest req) { +ACTOR Future doBlobGranuleFileRequest(Reference bwData, BlobGranuleFileRequest req) { + if (BW_REQUEST_DEBUG) { + fmt::print("BW {0} processing blobGranuleFileRequest for range [{1} - {2}) @ {3}\n", + bwData->id.toString(), + req.keyRange.begin.printable(), + req.keyRange.end.printable(), + req.readVersion); + } + try { // TODO REMOVE in api V2 ASSERT(req.beginVersion == 0); @@ -1785,19 +2107,20 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData state std::vector> granules; auto checkRanges = bwData->granuleMetadata.intersectingRanges(req.keyRange); - // check for gaps as errors and copy references to granule metadata before yielding or doing any work + // check for gaps as errors and copy references to granule metadata before yielding or doing any + // work KeyRef lastRangeEnd = req.keyRange.begin; for (auto& r : checkRanges) { bool isValid = r.value().activeMetadata.isValid(); if (lastRangeEnd < r.begin() || !isValid) { if (BW_REQUEST_DEBUG) { - printf("No %s blob data for [%s - %s) in request range [%s - %s), skipping request\n", - isValid ? "" : "valid", - lastRangeEnd.printable().c_str(), - r.begin().printable().c_str(), - req.keyRange.begin.printable().c_str(), - req.keyRange.end.printable().c_str()); + fmt::print("No {0} blob data for [{1} - {2}) in request range [{3} - {4}), skipping request\n", + isValid ? "" : "valid", + lastRangeEnd.printable(), + r.begin().printable(), + req.keyRange.begin.printable(), + req.keyRange.end.printable()); } throw wrong_shard_server(); @@ -1807,11 +2130,11 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData } if (lastRangeEnd < req.keyRange.end) { if (BW_REQUEST_DEBUG) { - printf("No blob data for [%s - %s) in request range [%s - %s), skipping request\n", - lastRangeEnd.printable().c_str(), - req.keyRange.end.printable().c_str(), - req.keyRange.begin.printable().c_str(), - req.keyRange.end.printable().c_str()); + fmt::print("No blob data for [{0} - {1}) in request range [{2} - {3}), skipping request\n", + lastRangeEnd.printable(), + req.keyRange.end.printable(), + req.keyRange.begin.printable(), + req.keyRange.end.printable()); } throw wrong_shard_server(); @@ -1822,42 +2145,55 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData for (auto m : granules) { if (readThrough >= m->keyRange.end) { // previous read did time travel that already included this granule - // FIXME: this will get more complicated with merges where this could potentially include partial - // boundaries. For now with only splits we can skip the whole range + // FIXME: this will get more complicated with merges where this could potentially + // include partial boundaries. For now with only splits we can skip the whole range continue; } state Reference metadata = m; - // don't do 'if (canBeSet)' - if (metadata->readable.canBeSet()) { - wait(metadata->readable.getFuture()); + choose { + when(wait(metadata->readable.getFuture())) {} + when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); } } - if (metadata->cancelled.isSet()) { + + // in case both readable and cancelled are ready, check cancelled + if (!metadata->cancelled.canBeSet()) { throw wrong_shard_server(); } state KeyRange chunkRange; state GranuleFiles chunkFiles; - if ((!metadata->files.snapshotFiles.empty() && - metadata->files.snapshotFiles.front().version > req.readVersion) || - (metadata->files.snapshotFiles.empty() && metadata->pendingSnapshotVersion > req.readVersion)) { + if (metadata->initialSnapshotVersion > req.readVersion) { + TEST(true); // Granule Time Travel Read // this is a time travel query, find previous granule if (metadata->historyLoaded.canBeSet()) { - wait(metadata->historyLoaded.getFuture()); + choose { + when(wait(metadata->historyLoaded.getFuture())) {} + when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); } + } } - // FIXME: doesn't work once we add granule merging, could be multiple ranges and/or multiple parents - Reference cur = bwData->granuleHistory.rangeContaining(req.keyRange.begin).value(); + // FIXME: doesn't work once we add granule merging, could be multiple ranges and/or + // multiple parents + Key historySearchKey = std::max(req.keyRange.begin, metadata->keyRange.begin); + Reference cur = bwData->granuleHistory.rangeContaining(historySearchKey).value(); + // FIXME: use skip pointers here + Version expectedEndVersion = metadata->initialSnapshotVersion; + if (cur.isValid()) { + ASSERT(cur->endVersion == expectedEndVersion); + } while (cur.isValid() && req.readVersion < cur->startVersion) { + // assert version of history is contiguous + ASSERT(cur->endVersion == expectedEndVersion); + expectedEndVersion = cur->startVersion; cur = cur->parentGranule; } if (!cur.isValid()) { // this request predates blob data - // FIXME: probably want a dedicated exception like blob_range_too_old or something instead - throw transaction_too_old(); + throw blob_granule_transaction_too_old(); } if (BW_REQUEST_DEBUG) { @@ -1872,37 +2208,69 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData cur->endVersion); } + ASSERT(cur->endVersion > req.readVersion); + ASSERT(cur->startVersion <= req.readVersion); + // lazily load files for old granule if not present chunkRange = cur->range; - if (cur->files.isError() || !cur->files.isValid()) { - cur->files = loadHistoryFiles(bwData, cur->granuleID); + if (!cur->files.isValid() || cur->files.isError()) { + cur->files = loadHistoryFiles(bwData->db, cur->granuleID); } - GranuleFiles _f = wait(cur->files); - chunkFiles = _f; + choose { + when(GranuleFiles _f = wait(cur->files)) { chunkFiles = _f; } + when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); } + } + if (chunkFiles.snapshotFiles.empty()) { + // a snapshot file must have been pruned + throw blob_granule_transaction_too_old(); + } + + ASSERT(!chunkFiles.deltaFiles.empty()); + ASSERT(chunkFiles.deltaFiles.back().version > req.readVersion); + if (chunkFiles.snapshotFiles.front().version > req.readVersion) { + // a snapshot file must have been pruned + throw blob_granule_transaction_too_old(); + } } else { + TEST(true); // Granule Active Read // this is an active granule query loop { + if (!metadata->activeCFData.get().isValid() || !metadata->cancelled.canBeSet()) { + throw wrong_shard_server(); + } Future waitForVersionFuture = waitForVersion(metadata, req.readVersion); if (waitForVersionFuture.isReady()) { - // didn't yield, so no need to check rollback stuff + // didn't wait, so no need to check rollback stuff break; } - // rollback resets all of the version information, so we have to redo wait for version on rollback - state int rollbackCount = metadata->rollbackCount.get(); - choose { - when(wait(waitForVersionFuture)) {} - when(wait(metadata->rollbackCount.onChange())) {} - when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); } + // rollback resets all of the version information, so we have to redo wait for + // version on rollback + try { + choose { + when(wait(waitForVersionFuture)) { break; } + when(wait(metadata->activeCFData.onChange())) {} + when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); } + } + } catch (Error& e) { + // We can get change feed cancelled from whenAtLeast. This means the change feed may retry, or + // may be cancelled. Wait a bit and try again to see + if (e.code() == error_code_change_feed_popped) { + TEST(true); // Change feed popped while read waiting + throw wrong_shard_server(); + } + if (e.code() != error_code_change_feed_cancelled) { + throw e; + } + TEST(true); // Change feed switched while read waiting + // wait 1ms and try again + wait(delay(0.001)); } - - if (rollbackCount == metadata->rollbackCount.get()) { - break; - } else if (BW_REQUEST_DEBUG) { - fmt::print("[{0} - {1}) @ {2} hit rollback, restarting waitForVersion\n", - req.keyRange.begin.printable(), - req.keyRange.end.printable(), + if ((BW_REQUEST_DEBUG) && metadata->activeCFData.get().isValid()) { + fmt::print("{0} - {1}) @ {2} hit CF change, restarting waitForVersion\n", + req.keyRange.begin.printable().c_str(), + req.keyRange.end.printable().c_str(), req.readVersion); } } @@ -1910,7 +2278,17 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData chunkRange = metadata->keyRange; } + if (!metadata->cancelled.canBeSet()) { + fmt::print("ERROR: Request [{0} - {1}) @ {2} cancelled for granule [{3} - {4}) after waitForVersion!\n", + req.keyRange.begin.printable(), + req.keyRange.end.printable(), + req.readVersion, + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable()); + } + // granule is up to date, do read + ASSERT(metadata->cancelled.canBeSet()); BlobGranuleChunkRef chunk; // TODO change in V2 @@ -1924,13 +2302,23 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData while (i >= 0 && chunkFiles.snapshotFiles[i].version > req.readVersion) { i--; } - // because of granule history, we should always be able to find the desired snapshot version, and have - // thrown transaction_too_old earlier if not possible. + // because of granule history, we should always be able to find the desired snapshot + // version, and have thrown blob_granule_transaction_too_old earlier if not possible. + if (i < 0) { + fmt::print("req @ {0} >= initial snapshot {1} but can't find snapshot in ({2}) files:\n", + req.readVersion, + metadata->initialSnapshotVersion, + chunkFiles.snapshotFiles.size()); + for (auto& f : chunkFiles.snapshotFiles) { + fmt::print(" {0}", f.version); + } + } ASSERT(i >= 0); BlobFileIndex snapshotF = chunkFiles.snapshotFiles[i]; chunk.snapshotFile = BlobFilePointerRef(rep.arena, snapshotF.filename, snapshotF.offset, snapshotF.length); Version snapshotVersion = chunkFiles.snapshotFiles[i].version; + chunk.snapshotVersion = snapshotVersion; // handle delta files // cast this to an int so i going to -1 still compares properly @@ -1941,8 +2329,8 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData i--; } if (i < lastDeltaFileIdx) { - // we skipped one file at the end with a larger read version, this will actually contain our query - // version, so add it back. + // we skipped one file at the end with a larger read version, this will actually contain + // our query version, so add it back. i++; } // only include delta files after the snapshot file @@ -1951,21 +2339,28 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData j--; } j++; - [[maybe_unused]] Version latestDeltaVersion = invalidVersion; while (j <= i) { BlobFileIndex deltaF = chunkFiles.deltaFiles[j]; chunk.deltaFiles.emplace_back_deep(rep.arena, deltaF.filename, deltaF.offset, deltaF.length); bwData->stats.readReqDeltaBytesReturned += deltaF.length; - latestDeltaVersion = deltaF.version; j++; } // new deltas (if version is larger than version of last delta file) - // FIXME: do trivial key bounds here if key range is not fully contained in request key range + // FIXME: do trivial key bounds here if key range is not fully contained in request key + // range if (req.readVersion > metadata->durableDeltaVersion.get()) { + if (metadata->durableDeltaVersion.get() != metadata->pendingDeltaVersion) { + fmt::print("real-time read [{0} - {1}) @ {2} doesn't have mutations!! durable={3}, pending={4}\n", + metadata->keyRange.begin.printable(), + metadata->keyRange.end.printable(), + req.readVersion, + metadata->durableDeltaVersion.get(), + metadata->pendingDeltaVersion); + } ASSERT(metadata->durableDeltaVersion.get() == metadata->pendingDeltaVersion); - rep.arena.dependsOn(metadata->deltaArena); + rep.arena.dependsOn(metadata->currentDeltas.arena()); for (auto& delta : metadata->currentDeltas) { if (delta.version > req.readVersion) { break; @@ -1981,9 +2376,16 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData wait(yield(TaskPriority::DefaultEndpoint)); } + ASSERT(!req.reply.isSet()); req.reply.send(rep); --bwData->stats.activeReadRequests; } catch (Error& e) { + // fmt::print("Error in BGFRequest {0}\n", e.name()); + if (e.code() == error_code_operation_cancelled) { + req.reply.sendError(wrong_shard_server()); + throw; + } + if (e.code() == error_code_wrong_shard_server) { ++bwData->stats.wrongShardServer; } @@ -1997,30 +2399,42 @@ ACTOR Future handleBlobGranuleFileRequest(Reference bwData return Void(); } -ACTOR Future> getLatestGranuleHistory(Transaction* tr, KeyRange range) { - KeyRange historyRange = blobGranuleHistoryKeyRangeFor(range); - RangeResult result = wait(tr->getRange(historyRange, 1, Snapshot::False, Reverse::True)); - ASSERT(result.size() <= 1); +ACTOR Future handleBlobGranuleFileRequest(Reference bwData, BlobGranuleFileRequest req) { + choose { + when(wait(doBlobGranuleFileRequest(bwData, req))) {} + when(wait(delay(SERVER_KNOBS->BLOB_WORKER_REQUEST_TIMEOUT))) { + if (!req.reply.isSet()) { + TEST(true); // Blob Worker request timeout hit + if (BW_DEBUG) { + fmt::print("BW {0} request [{1} - {2}) @ {3} timed out, sending WSS\n", + bwData->id.toString().substr(0, 5), + req.keyRange.begin.printable(), + req.keyRange.end.printable(), + req.readVersion); + } + --bwData->stats.activeReadRequests; + ++bwData->stats.granuleRequestTimeouts; - Optional history; - if (!result.empty()) { - std::pair decodedKey = decodeBlobGranuleHistoryKey(result[0].key); - ASSERT(range == decodedKey.first); - history = GranuleHistory(range, decodedKey.second, decodeBlobGranuleHistoryValue(result[0].value)); + // return wrong_shard_server because it's possible that someone else actually owns the granule now + req.reply.sendError(wrong_shard_server()); + } + } } - return history; + return Void(); } +// FIXME: move this up by other granule state stuff like BGUF ACTOR Future openGranule(Reference bwData, AssignBlobRangeRequest req) { - ASSERT(!req.continueAssignment); + ASSERT(req.type != AssignRequestType::Continue); state Transaction tr(bwData->db); state Key lockKey = blobGranuleLockKeyFor(req.keyRange); + state UID newGranuleID = deterministicRandom()->randomUniqueID(); if (BW_DEBUG) { - printf("%s [%s - %s) opening\n", - bwData->id.toString().c_str(), - req.keyRange.begin.printable().c_str(), - req.keyRange.end.printable().c_str()); + fmt::print("{0} [{1} - {2}) opening\n", + bwData->id.toString(), + req.keyRange.begin.printable(), + req.keyRange.end.printable()); } loop { @@ -2032,12 +2446,13 @@ ACTOR Future openGranule(Reference bwData, As info.changeFeedStartVersion = invalidVersion; state Future> fLockValue = tr.get(lockKey); - state Future> fHistory = getLatestGranuleHistory(&tr, req.keyRange); + Future> fHistory = getLatestGranuleHistory(&tr, req.keyRange); Optional history = wait(fHistory); info.history = history; Optional prevLockValue = wait(fLockValue); state bool hasPrevOwner = prevLockValue.present(); if (hasPrevOwner) { + TEST(true); // Granule open found previous owner std::tuple prevOwner = decodeBlobGranuleLockValue(prevLockValue.get()); acquireGranuleLock(req.managerEpoch, req.managerSeqno, std::get<0>(prevOwner), std::get<1>(prevOwner)); info.granuleID = std::get<2>(prevOwner); @@ -2052,8 +2467,8 @@ ACTOR Future openGranule(Reference bwData, As info.doSnapshot = false; if (!info.history.present()) { - // the only time history can be not present if a lock already exists is if it's a new granule and it - // died before it could persist the initial snapshot from FDB + // the only time history can be not present if a lock already exists is if it's a + // new granule and it died before it could persist the initial snapshot from FDB ASSERT(info.existingFiles.get().snapshotFiles.empty()); } @@ -2067,25 +2482,22 @@ ACTOR Future openGranule(Reference bwData, As info.previousDurableVersion = info.existingFiles.get().deltaFiles.back().version; } - // for the non-splitting cases, this doesn't need to be 100% accurate, it just needs to be - // smaller than the next delta file write. + // for the non-splitting cases, this doesn't need to be 100% accurate, it just needs to + // be smaller than the next delta file write. info.changeFeedStartVersion = info.previousDurableVersion; } else { // else we are first, no need to check for owner conflict - // FIXME: use actual 16 bytes of UID instead of converting it to 32 character string and then that - // to bytes - if (info.history.present()) { - // if this granule is derived from a split or merge, this history entry is already present (written - // by the blob manager) + // if this granule is derived from a split or merge, this history entry is already + // present (written by the blob manager) info.granuleID = info.history.get().value.granuleID; } else { // FIXME: could avoid max uid for granule ids here - // if this granule is not derived from a split or merge, create the granule id here - info.granuleID = deterministicRandom()->randomUniqueID(); + // if this granule is not derived from a split or merge, use new granule id + info.granuleID = newGranuleID; } wait(updateChangeFeed( - &tr, StringRef(info.granuleID.toString()), ChangeFeedStatus::CHANGE_FEED_CREATE, req.keyRange)); + &tr, granuleIDToCFKey(info.granuleID), ChangeFeedStatus::CHANGE_FEED_CREATE, req.keyRange)); info.doSnapshot = true; info.previousDurableVersion = invalidVersion; } @@ -2093,9 +2505,11 @@ ACTOR Future openGranule(Reference bwData, As tr.set(lockKey, blobGranuleLockValueFor(req.managerEpoch, req.managerSeqno, info.granuleID)); wait(krmSetRange(&tr, blobGranuleMappingKeys.begin, req.keyRange, blobGranuleMappingValueFor(bwData->id))); - // If anything in previousGranules, need to do the handoff logic and set ret.previousChangeFeedId, and - // the previous durable version will come from the previous granules + // If anything in previousGranules, need to do the handoff logic and set + // ret.previousChangeFeedId, and the previous durable version will come from the previous + // granules if (info.history.present() && info.history.get().value.parentGranules.size() > 0) { + TEST(true); // Granule open found parent // TODO change this for merge ASSERT(info.history.get().value.parentGranules.size() == 1); state KeyRange parentGranuleRange = info.history.get().value.parentGranules[0].first; @@ -2105,12 +2519,14 @@ ACTOR Future openGranule(Reference bwData, As std::tuple parentGranuleLock = decodeBlobGranuleLockValue(parentGranuleLockValue.get()); UID parentGranuleID = std::get<2>(parentGranuleLock); - printf(" parent granule id %s\n", parentGranuleID.toString().c_str()); + if (BW_DEBUG) { + fmt::print(" parent granule id {}\n", parentGranuleID.toString()); + } info.parentGranule = std::pair(parentGranuleRange, parentGranuleID); state std::pair granuleSplitState = - std::pair(BlobGranuleSplitState::Started, invalidVersion); + std::pair(BlobGranuleSplitState::Initialized, invalidVersion); if (hasPrevOwner) { std::pair _gss = wait(getGranuleSplitState(&tr, parentGranuleID, info.granuleID)); @@ -2118,20 +2534,22 @@ ACTOR Future openGranule(Reference bwData, As } if (granuleSplitState.first == BlobGranuleSplitState::Assigned) { + TEST(true); // Granule open found granule in assign state // was already assigned, use change feed start version - ASSERT(granuleSplitState.second != invalidVersion); + ASSERT(granuleSplitState.second > 0); info.changeFeedStartVersion = granuleSplitState.second; - } else if (granuleSplitState.first == BlobGranuleSplitState::Started) { + } else if (granuleSplitState.first == BlobGranuleSplitState::Initialized) { + TEST(true); // Granule open found granule in initialized state wait(updateGranuleSplitState(&tr, info.parentGranule.get().first, info.parentGranule.get().second, info.granuleID, BlobGranuleSplitState::Assigned)); - // change feed was created as part of this transaction, changeFeedStartVersion will be - // set later + // change feed was created as part of this transaction, changeFeedStartVersion + // will be set later } else { + TEST(true); // Granule open found granule in done state // this sub-granule is done splitting, no need for split logic. - ASSERT(granuleSplitState.first == BlobGranuleSplitState::Done); info.parentGranule.reset(); } } @@ -2149,7 +2567,6 @@ ACTOR Future openGranule(Reference bwData, As : info.blobFilesToSnapshot.get().deltaFiles.back().version; } } - wait(tr.commit()); if (info.changeFeedStartVersion == invalidVersion) { @@ -2206,27 +2623,35 @@ static bool newerRangeAssignment(GranuleRangeMetadata oldMetadata, int64_t epoch // TODO unit test this assignment, particularly out-of-order insertions! // The contract from the blob manager is: -// If a key range [A, B) was assigned to the worker at seqno S1, no part of the keyspace that intersects [A, B] may -// be re-assigned to the worker until the range has been revoked from this worker. This revoking can either happen -// by the blob manager willingly relinquishing the range, or by the blob manager reassigning it somewhere else. This -// means that if the worker gets an assignment for any range that intersects [A, B) at S3, there must have been a -// revoke message for [A, B) with seqno S3 where S1 < S2 < S3, that was delivered out of order. This means that if -// there are any intersecting but not fully overlapping ranges with a new range assignment, they had already been -// revoked. So the worker will mark them as revoked, but leave the sequence number as S1, so that when the actual -// revoke message comes in, it is a no-op, but updates the sequence number. Similarly, if a worker gets an assign -// message for any range that already has a higher sequence number, that range was either revoked, or revoked and -// then re-assigned. Either way, this assignment is no longer valid. +// If a key range [A, B) was assigned to the worker at seqno S1, no part of the keyspace that intersects +// [A, B] may be re-assigned to the worker until the range has been revoked from this worker. This +// revoking can either happen by the blob manager willingly relinquishing the range, or by the blob +// manager reassigning it somewhere else. This means that if the worker gets an assignment for any range +// that intersects [A, B) at S3, there must have been a revoke message for [A, B) with seqno S3 where S1 +// < S2 < S3, that was delivered out of order. This means that if there are any intersecting but not +// fully overlapping ranges with a new range assignment, they had already been revoked. So the worker +// will mark them as revoked, but leave the sequence number as S1, so that when the actual revoke +// message comes in, it is a no-op, but updates the sequence number. Similarly, if a worker gets an +// assign message for any range that already has a higher sequence number, that range was either +// revoked, or revoked and then re-assigned. Either way, this assignment is no longer valid. + +// Returns future to wait on to ensure prior work of other granules is done before responding to the +// manager with a successful assignment And if the change produced a new granule that needs to start +// doing work, returns the new granule so that the caller can start() it with the appropriate starting +// state. + +// Not an actor because we need to guarantee it changes the synchronously as part of the request +static bool changeBlobRange(Reference bwData, + KeyRange keyRange, + int64_t epoch, + int64_t seqno, + bool active, + bool disposeOnCleanup, + bool selfReassign, + std::vector>& toWaitOut, + Optional assignType = Optional()) { + ASSERT(active == assignType.present()); -// Returns future to wait on to ensure prior work of other granules is done before responding to the manager with a -// successful assignment And if the change produced a new granule that needs to start doing work, returns the new -// granule so that the caller can start() it with the appropriate starting state. -ACTOR Future changeBlobRange(Reference bwData, - KeyRange keyRange, - int64_t epoch, - int64_t seqno, - bool active, - bool disposeOnCleanup, - bool selfReassign) { if (BW_DEBUG) { fmt::print("{0} range for [{1} - {2}): {3} @ ({4}, {5})\n", selfReassign ? "Re-assigning" : "Changing", @@ -2238,38 +2663,38 @@ ACTOR Future changeBlobRange(Reference bwData, } // For each range that intersects this update: - // If the identical range already exists at the same assignment sequence number and it is not a self-reassign, - // this is a noop. Otherwise, this will consist of a series of ranges that are either older, or newer. For each - // older range, cancel it if it is active. Insert the current range. Re-insert all newer ranges over the current - // range. + // If the identical range already exists at the same assignment sequence number and it is not a + // self-reassign, this is a noop. Otherwise, this will consist of a series of ranges that are either + // older, or newer. For each older range, cancel it if it is active. Insert the current range. + // Re-insert all newer ranges over the current range. - state std::vector> futures; - - state std::vector> newerRanges; + std::vector> newerRanges; auto ranges = bwData->granuleMetadata.intersectingRanges(keyRange); bool alreadyAssigned = false; for (auto& r : ranges) { - if (!active) { - if (r.value().activeMetadata.isValid() && r.value().activeMetadata->cancelled.canBeSet()) { - if (BW_DEBUG) { - printf("Cancelling activeMetadata\n"); - } - r.value().activeMetadata->cancelled.send(Void()); - } - } bool thisAssignmentNewer = newerRangeAssignment(r.value(), epoch, seqno); + if (BW_DEBUG) { + fmt::print("thisAssignmentNewer={}\n", thisAssignmentNewer ? "true" : "false"); + } + + if (BW_DEBUG) { + fmt::print("last: ({0}, {1}). now: ({2}, {3})\n", r.value().lastEpoch, r.value().lastSeqno, epoch, seqno); + } + if (r.value().lastEpoch == epoch && r.value().lastSeqno == seqno) { - ASSERT(r.begin() == keyRange.begin); - ASSERT(r.end() == keyRange.end); + // the range in our map can be different if later the range was split, but then an old request gets retried. + // Assume that it's the same as initially if (selfReassign) { thisAssignmentNewer = true; } else { - printf("same assignment\n"); + if (BW_DEBUG) { + printf("same assignment\n"); + } // applied the same assignment twice, make idempotent if (r.value().activeMetadata.isValid()) { - futures.push_back(success(r.value().assignFuture)); + toWaitOut.push_back(success(r.value().assignFuture)); } alreadyAssigned = true; break; @@ -2285,15 +2710,17 @@ ACTOR Future changeBlobRange(Reference bwData, r.value().lastEpoch, r.value().lastSeqno); } - r.value().activeMetadata.clear(); + if (!active) { + bwData->stats.numRangesAssigned--; + } + r.value().cancel(); } else if (!thisAssignmentNewer) { - // this assignment is outdated, re-insert it over the current range + // re-insert the known newer range over this existing range newerRanges.push_back(std::pair(r.range(), r.value())); } } if (alreadyAssigned) { - wait(waitForAll(futures)); // already applied, nothing to do return false; } @@ -2324,9 +2751,7 @@ ACTOR Future changeBlobRange(Reference bwData, bwData->granuleMetadata.insert(it.first, it.second); } - printf("returning from changeblobrange"); - wait(waitForAll(futures)); - return true; + return newerRanges.size() == 0; } static bool resumeBlobRange(Reference bwData, KeyRange keyRange, int64_t epoch, int64_t seqno) { @@ -2367,83 +2792,98 @@ static bool resumeBlobRange(Reference bwData, KeyRange keyRange, return true; } -ACTOR Future registerBlobWorker(Reference bwData, BlobWorkerInterface interf) { - state Reference tr = makeReference(bwData->db); - loop { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - try { - Key blobWorkerListKey = blobWorkerListKeyFor(interf.id()); - tr->addReadConflictRange(singleKeyRange(blobWorkerListKey)); - tr->set(blobWorkerListKey, blobWorkerListValue(interf)); - - wait(tr->commit()); - - if (BW_DEBUG) { - printf("Registered blob worker %s\n", interf.id().toString().c_str()); - } - return Void(); - } catch (Error& e) { - if (BW_DEBUG) { - printf("Registering blob worker %s got error %s\n", interf.id().toString().c_str(), e.name()); - } - wait(tr->onError(e)); - } - } -} - +// the contract of handleRangeAssign and handleRangeRevoke is that they change the mapping before doing any waiting. +// This ensures GetGranuleAssignment returns an up-to-date set of ranges ACTOR Future handleRangeAssign(Reference bwData, AssignBlobRangeRequest req, bool isSelfReassign) { try { - if (req.continueAssignment) { + if (req.type == AssignRequestType::Continue) { resumeBlobRange(bwData, req.keyRange, req.managerEpoch, req.managerSeqno); } else { - bool shouldStart = wait( - changeBlobRange(bwData, req.keyRange, req.managerEpoch, req.managerSeqno, true, false, isSelfReassign)); + std::vector> toWait; + state bool shouldStart = changeBlobRange(bwData, + req.keyRange, + req.managerEpoch, + req.managerSeqno, + true, + false, + isSelfReassign, + toWait, + req.type); + wait(waitForAll(toWait)); if (shouldStart) { + bwData->stats.numRangesAssigned++; auto m = bwData->granuleMetadata.rangeContaining(req.keyRange.begin); ASSERT(m.begin() == req.keyRange.begin && m.end() == req.keyRange.end); - wait(start(bwData, &m.value(), req)); + if (m.value().activeMetadata.isValid()) { + wait(start(bwData, &m.value(), req)); + } } } if (!isSelfReassign) { ASSERT(!req.reply.isSet()); - req.reply.send(AssignBlobRangeReply(true)); + req.reply.send(Void()); } return Void(); } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw e; + } if (BW_DEBUG) { - printf("AssignRange [%s - %s) got error %s\n", - req.keyRange.begin.printable().c_str(), - req.keyRange.end.printable().c_str(), - e.name()); + fmt::print("AssignRange [{0} - {1}) ({2}, {3}) in BW {4} got error {5}\n", + req.keyRange.begin.printable().c_str(), + req.keyRange.end.printable().c_str(), + req.managerEpoch, + req.managerSeqno, + bwData->id.toString().c_str(), + e.name()); } if (!isSelfReassign) { + if (e.code() == error_code_granule_assignment_conflict) { + req.reply.sendError(e); + bwData->stats.numRangesAssigned--; + return Void(); + } + if (canReplyWith(e)) { req.reply.sendError(e); } } - throw; + TraceEvent(SevError, "BlobWorkerUnexpectedErrorRangeAssign", bwData->id) + .error(e) + .detail("Range", req.keyRange) + .detail("ManagerEpoch", req.managerEpoch) + .detail("SeqNo", req.managerSeqno); + ASSERT_WE_THINK(false); + + // if not simulation, kill the BW + if (bwData->fatalError.canBeSet()) { + bwData->fatalError.sendError(e); + } + throw e; } } ACTOR Future handleRangeRevoke(Reference bwData, RevokeBlobRangeRequest req) { try { - bool _shouldStart = - wait(changeBlobRange(bwData, req.keyRange, req.managerEpoch, req.managerSeqno, false, req.dispose, false)); - req.reply.send(AssignBlobRangeReply(true)); + std::vector> toWait; + changeBlobRange(bwData, req.keyRange, req.managerEpoch, req.managerSeqno, false, req.dispose, false, toWait); + wait(waitForAll(toWait)); + req.reply.send(Void()); return Void(); } catch (Error& e) { // FIXME: retry on error if dispose fails? if (BW_DEBUG) { - printf("RevokeRange [%s - %s) got error %s\n", - req.keyRange.begin.printable().c_str(), - req.keyRange.end.printable().c_str(), - e.name()); + fmt::print("RevokeRange [{0} - {1}) ({2}, {3}) got error {4}\n", + req.keyRange.begin.printable(), + req.keyRange.end.printable(), + req.managerEpoch, + req.managerSeqno, + e.name()); } if (canReplyWith(e)) { req.reply.sendError(e); @@ -2452,33 +2892,87 @@ ACTOR Future handleRangeRevoke(Reference bwData, RevokeBlo } } +ACTOR Future registerBlobWorker(Reference bwData, BlobWorkerInterface interf) { + state Reference tr = makeReference(bwData->db); + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + try { + Key blobWorkerListKey = blobWorkerListKeyFor(interf.id()); + // FIXME: should be able to remove this conflict range + tr->addReadConflictRange(singleKeyRange(blobWorkerListKey)); + tr->set(blobWorkerListKey, blobWorkerListValue(interf)); + + // Get manager lock from DB + Optional currentLockValue = wait(tr->get(blobManagerEpochKey)); + ASSERT(currentLockValue.present()); + int64_t currentEpoch = decodeBlobManagerEpochValue(currentLockValue.get()); + bwData->managerEpochOk(currentEpoch); + + wait(tr->commit()); + + if (BW_DEBUG) { + fmt::print("Registered blob worker {}\n", interf.id().toString()); + } + return Void(); + } catch (Error& e) { + if (BW_DEBUG) { + fmt::print("Registering blob worker {0} got error {1}\n", interf.id().toString(), e.name()); + } + wait(tr->onError(e)); + } + } +} + +ACTOR Future monitorRemoval(Reference bwData) { + state Key blobWorkerListKey = blobWorkerListKeyFor(bwData->id); + loop { + loop { + state ReadYourWritesTransaction tr(bwData->db); + try { + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + Optional val = wait(tr.get(blobWorkerListKey)); + if (!val.present()) { + TEST(true); // Blob worker found out BM killed it from reading DB + return Void(); + } + + state Future watchFuture = tr.watch(blobWorkerListKey); + + wait(tr.commit()); + wait(watchFuture); + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } +} + // Because change feeds send uncommitted data and explicit rollback messages, we speculatively buffer/write // uncommitted data. This means we must ensure the data is actually committed before "committing" those writes in // the blob granule. The simplest way to do this is to have the blob worker do a periodic GRV, which is guaranteed -// to be an earlier committed version. -ACTOR Future runCommitVersionChecks(Reference bwData) { +// to be an earlier committed version. Then, once the change feed has consumed up through the GRV's data, we can +// guarantee nothing will roll back the in-memory mutations +ACTOR Future runGRVChecks(Reference bwData) { state Transaction tr(bwData->db); loop { // only do grvs to get committed version if we need it to persist delta files - while (bwData->pendingDeltaFileCommitChecks.get() == 0) { - wait(bwData->pendingDeltaFileCommitChecks.onChange()); + while (bwData->grvVersion.numWaiting() == 0) { + wait(bwData->doGRVCheck.getFuture()); + bwData->doGRVCheck = Promise(); } // batch potentially multiple delta files into one GRV, and also rate limit GRVs for this worker - wait(delay(0.1)); // TODO KNOB? - - state int checksToResolve = bwData->pendingDeltaFileCommitChecks.get(); + wait(delay(SERVER_KNOBS->BLOB_WORKER_BATCH_GRV_INTERVAL)); tr.reset(); try { Version readVersion = wait(tr.getReadVersion()); + ASSERT(readVersion >= bwData->grvVersion.get()); + bwData->grvVersion.set(readVersion); - ASSERT(readVersion >= bwData->knownCommittedVersion.get()); - if (readVersion > bwData->knownCommittedVersion.get()) { - ++bwData->knownCommittedCheckCount; - bwData->knownCommittedVersion.set(readVersion); - bwData->pendingDeltaFileCommitChecks.set(bwData->pendingDeltaFileCommitChecks.get() - checksToResolve); - } ++bwData->stats.commitVersionChecks; } catch (Error& e) { wait(tr.onError(e)); @@ -2486,6 +2980,29 @@ ACTOR Future runCommitVersionChecks(Reference bwData) { } } +static void handleGetGranuleAssignmentsRequest(Reference self, + const GetGranuleAssignmentsRequest& req) { + GetGranuleAssignmentsReply reply; + auto allRanges = self->granuleMetadata.intersectingRanges(normalKeys); + for (auto& it : allRanges) { + if (it.value().activeMetadata.isValid()) { + // range is active, copy into reply's arena + StringRef start = StringRef(reply.arena, it.begin()); + StringRef end = StringRef(reply.arena, it.end()); + + reply.assignments.push_back( + reply.arena, GranuleAssignmentRef(KeyRangeRef(start, end), it.value().lastEpoch, it.value().lastSeqno)); + } + } + if (BW_DEBUG) { + fmt::print("Worker {0} sending {1} granule assignments back to BM {2}\n", + self->id.toString(), + reply.assignments.size(), + req.managerEpoch); + } + req.reply.send(reply); +} + ACTOR Future blobWorker(BlobWorkerInterface bwInterf, ReplyPromise recruitReply, Reference const> dbInfo) { @@ -2501,26 +3018,19 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, } try { - if (g_network->isSimulated()) { - if (BW_DEBUG) { - printf("BW constructing simulated backup container\n"); - } - self->bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/"); - } else { - if (BW_DEBUG) { - printf("BW constructing backup container from %s\n", SERVER_KNOBS->BG_URL.c_str()); - } - self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); - if (BW_DEBUG) { - printf("BW constructed backup container\n"); - } + if (BW_DEBUG) { + fmt::print("BW constructing backup container from {0}\n", SERVER_KNOBS->BG_URL); + } + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + if (BW_DEBUG) { + printf("BW constructed backup container\n"); } // register the blob worker to the system keyspace wait(registerBlobWorker(self, bwInterf)); } catch (Error& e) { if (BW_DEBUG) { - printf("BW got backup container init error %s\n", e.name()); + fmt::print("BW got backup container init error {0}\n", e.name()); } // if any errors came up while initializing the blob worker, let the blob manager know // that recruitment failed @@ -2538,7 +3048,10 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, recruitReply.send(rep); self->addActor.send(waitFailureServer(bwInterf.waitFailure.getFuture())); - self->addActor.send(runCommitVersionChecks(self)); + self->addActor.send(runGRVChecks(self)); + state Future selfRemoved = monitorRemoval(self); + + TraceEvent("BlobWorkerInit", self->id).log(); try { loop choose { @@ -2550,36 +3063,52 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, when(state GranuleStatusStreamRequest req = waitNext(bwInterf.granuleStatusStreamRequest.getFuture())) { if (self->managerEpochOk(req.managerEpoch)) { if (BW_DEBUG) { - printf("Worker %s got new granule status endpoint\n", self->id.toString().c_str()); + fmt::print("Worker {0} got new granule status endpoint {1} from BM {2}\n", + self->id.toString(), + req.reply.getEndpoint().token.toString().c_str(), + req.managerEpoch); + } + + // send an error to the old stream before closing it, so it doesn't get broken_promise and mark this + // endpoint as failed + self->currentManagerStatusStream.get().sendError(connection_failed()); + + // hold a copy of the previous stream if it exists, so any waiting send calls don't get + // proken_promise before onChange + ReplyPromiseStream copy; + if (self->statusStreamInitialized) { + copy = self->currentManagerStatusStream.get(); } // TODO: pick a reasonable byte limit instead of just piggy-backing - req.reply.setByteLimit(SERVER_KNOBS->RANGESTREAM_LIMIT_BYTES); + req.reply.setByteLimit(SERVER_KNOBS->BLOBWORKERSTATUSSTREAM_LIMIT_BYTES); + self->statusStreamInitialized = true; + self->currentManagerStatusStream.set(req.reply); + } else { + req.reply.sendError(blob_manager_replaced()); } } when(AssignBlobRangeRequest _req = waitNext(bwInterf.assignBlobRangeRequest.getFuture())) { ++self->stats.rangeAssignmentRequests; - --self->stats.numRangesAssigned; state AssignBlobRangeRequest assignReq = _req; if (BW_DEBUG) { - fmt::print("Worker {0} assigned range [{1} - {2}) @ ({3}, {4}):\n continue={5}\n", + fmt::print("Worker {0} assigned range [{1} - {2}) @ ({3}, {4}):\n type={5}\n", self->id.toString(), assignReq.keyRange.begin.printable(), assignReq.keyRange.end.printable(), assignReq.managerEpoch, assignReq.managerSeqno, - assignReq.continueAssignment ? "T" : "F"); + assignReq.type); } if (self->managerEpochOk(assignReq.managerEpoch)) { self->addActor.send(handleRangeAssign(self, assignReq, false)); } else { - assignReq.reply.send(AssignBlobRangeReply(false)); + assignReq.reply.sendError(blob_manager_replaced()); } } when(RevokeBlobRangeRequest _req = waitNext(bwInterf.revokeBlobRangeRequest.getFuture())) { state RevokeBlobRangeRequest revokeReq = _req; - --self->stats.numRangesAssigned; if (BW_DEBUG) { fmt::print("Worker {0} revoked range [{1} - {2}) @ ({3}, {4}):\n dispose={5}\n", self->id.toString(), @@ -2593,25 +3122,54 @@ ACTOR Future blobWorker(BlobWorkerInterface bwInterf, if (self->managerEpochOk(revokeReq.managerEpoch)) { self->addActor.send(handleRangeRevoke(self, revokeReq)); } else { - revokeReq.reply.send(AssignBlobRangeReply(false)); + revokeReq.reply.sendError(blob_manager_replaced()); } } when(AssignBlobRangeRequest granuleToReassign = waitNext(self->granuleUpdateErrors.getFuture())) { self->addActor.send(handleRangeAssign(self, granuleToReassign, true)); } - when(HaltBlobWorkerRequest req = waitNext(bwInterf.haltBlobWorker.getFuture())) { - req.reply.send(Void()); + when(GetGranuleAssignmentsRequest req = waitNext(bwInterf.granuleAssignmentsRequest.getFuture())) { if (self->managerEpochOk(req.managerEpoch)) { - TraceEvent("BlobWorkerHalted", bwInterf.id()).detail("ReqID", req.requesterID); - printf("BW %s was halted\n", bwInterf.id().toString().c_str()); + if (BW_DEBUG) { + fmt::print("Worker {0} got granule assignments request from BM {1}\n", + self->id.toString(), + req.managerEpoch); + } + handleGetGranuleAssignmentsRequest(self, req); + } else { + req.reply.sendError(blob_manager_replaced()); + } + } + when(HaltBlobWorkerRequest req = waitNext(bwInterf.haltBlobWorker.getFuture())) { + if (self->managerEpochOk(req.managerEpoch)) { + TraceEvent("BlobWorkerHalted", self->id) + .detail("ReqID", req.requesterID) + .detail("ManagerEpoch", req.managerEpoch); + if (BW_DEBUG) { + fmt::print("BW {0} was halted by manager {1}\n", bwInterf.id().toString(), req.managerEpoch); + } + req.reply.send(Void()); break; + } else { + req.reply.sendError(blob_manager_replaced()); } } when(wait(collection)) { - TraceEvent("BlobWorkerActorCollectionError"); + TraceEvent("BlobWorkerActorCollectionError", self->id); ASSERT(false); throw internal_error(); } + when(wait(selfRemoved)) { + if (BW_DEBUG) { + printf("Blob worker detected removal. Exiting...\n"); + } + TraceEvent("BlobWorkerRemoved", self->id); + break; + } + when(wait(self->fatalError.getFuture())) { + TraceEvent(SevError, "BlobWorkerActorCollectionFatalErrorNotError", self->id); + ASSERT(false); + } } } catch (Error& e) { if (e.code() == error_code_operation_cancelled) { diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 98d9d3eeae..5cd5d148e3 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -5,6 +5,8 @@ set(FDBSERVER_SRCS BackupProgress.actor.cpp BackupProgress.actor.h BackupWorker.actor.cpp + BlobGranuleServerCommon.actor.cpp + BlobGranuleServerCommon.actor.h BlobManager.actor.cpp BlobManagerInterface.h BlobWorker.actor.cpp @@ -178,6 +180,7 @@ set(FDBSERVER_SRCS workloads/BackupToDBAbort.actor.cpp workloads/BackupToDBCorrectness.actor.cpp workloads/BackupToDBUpgrade.actor.cpp + workloads/BlobGranuleCorrectnessWorkload.actor.cpp workloads/BlobGranuleVerifier.actor.cpp workloads/BlobStoreWorkload.h workloads/BulkLoad.actor.cpp diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index ebdf4c8b1e..b51b2182db 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -25,6 +25,7 @@ #include #include +#include "fdbclient/SystemData.h" #include "fdbrpc/FailureMonitor.h" #include "fdbserver/EncryptKeyProxyInterface.h" #include "flow/ActorCollection.h" @@ -105,7 +106,10 @@ struct RatekeeperSingleton : Singleton { brokenPromiseToNever(interface.get().haltRatekeeper.getReply(HaltRatekeeperRequest(cc->id))); } } - void recruit(ClusterControllerData* cc) const { cc->recruitRatekeeper.set(true); } + void recruit(ClusterControllerData* cc) const { + cc->lastRecruitTime = now(); + cc->recruitRatekeeper.set(true); + } }; struct DataDistributorSingleton : Singleton { @@ -127,7 +131,10 @@ struct DataDistributorSingleton : Singleton { brokenPromiseToNever(interface.get().haltDataDistributor.getReply(HaltDataDistributorRequest(cc->id))); } } - void recruit(ClusterControllerData* cc) const { cc->recruitDistributor.set(true); } + void recruit(ClusterControllerData* cc) const { + cc->lastRecruitTime = now(); + cc->recruitDistributor.set(true); + } }; struct BlobManagerSingleton : Singleton { @@ -149,7 +156,17 @@ struct BlobManagerSingleton : Singleton { brokenPromiseToNever(interface.get().haltBlobManager.getReply(HaltBlobManagerRequest(cc->id))); } } - void recruit(ClusterControllerData* cc) const { cc->recruitBlobManager.set(true); } + void recruit(ClusterControllerData* cc) const { + cc->lastRecruitTime = now(); + cc->recruitBlobManager.set(true); + } + + void haltBlobGranules(ClusterControllerData* cc, Optional> pid) const { + if (interface.present()) { + cc->id_worker[pid].haltBlobManager = + brokenPromiseToNever(interface.get().haltBlobGranules.getReply(HaltBlobGranulesRequest(cc->id))); + } + } }; struct EncryptKeyProxySingleton : Singleton { @@ -171,7 +188,10 @@ struct EncryptKeyProxySingleton : Singleton { brokenPromiseToNever(interface.get().haltEncryptKeyProxy.getReply(HaltEncryptKeyProxyRequest(cc->id))); } } - void recruit(ClusterControllerData* cc) const { cc->recruitEncryptKeyProxy.set(true); } + void recruit(ClusterControllerData* cc) const { + cc->lastRecruitTime = now(); + cc->recruitEncryptKeyProxy.set(true); + } }; ACTOR Future handleLeaderReplacement(Reference self, Future leaderFail) { @@ -594,7 +614,7 @@ void checkBetterSingletons(ClusterControllerData* self) { WorkerDetails newDDWorker = findNewProcessForSingleton(self, ProcessClass::DataDistributor, id_used); WorkerDetails newBMWorker; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { newBMWorker = findNewProcessForSingleton(self, ProcessClass::BlobManager, id_used); } @@ -608,7 +628,7 @@ void checkBetterSingletons(ClusterControllerData* self) { auto bestFitnessForDD = findBestFitnessForSingleton(self, newDDWorker, ProcessClass::DataDistributor); ProcessClass::Fitness bestFitnessForBM; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { bestFitnessForBM = findBestFitnessForSingleton(self, newBMWorker, ProcessClass::BlobManager); } @@ -632,7 +652,7 @@ void checkBetterSingletons(ClusterControllerData* self) { self, newDDWorker, ddSingleton, bestFitnessForDD, self->recruitingDistributorID); bool bmHealthy = true; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { bmHealthy = isHealthySingleton( self, newBMWorker, bmSingleton, bestFitnessForBM, self->recruitingBlobManagerID); } @@ -656,7 +676,7 @@ void checkBetterSingletons(ClusterControllerData* self) { Optional> newDDProcessId = newDDWorker.interf.locality.processId(); Optional> currBMProcessId, newBMProcessId; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { currBMProcessId = bmSingleton.interface.get().locality.processId(); newBMProcessId = newBMWorker.interf.locality.processId(); } @@ -669,7 +689,7 @@ void checkBetterSingletons(ClusterControllerData* self) { std::vector>> currPids = { currRKProcessId, currDDProcessId }; std::vector>> newPids = { newRKProcessId, newDDProcessId }; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { currPids.emplace_back(currBMProcessId); newPids.emplace_back(newBMProcessId); } @@ -683,7 +703,7 @@ void checkBetterSingletons(ClusterControllerData* self) { auto newColocMap = getColocCounts(newPids); // if the knob is disabled, the BM coloc counts should have no affect on the coloc counts check below - if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (!self->db.blobGranulesEnabled.get()) { ASSERT(currColocMap[currBMProcessId] == 0); ASSERT(newColocMap[newBMProcessId] == 0); } @@ -704,7 +724,7 @@ void checkBetterSingletons(ClusterControllerData* self) { rkSingleton.recruit(self); } else if (newColocMap[newDDProcessId] < currColocMap[currDDProcessId]) { ddSingleton.recruit(self); - } else if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES && newColocMap[newBMProcessId] < currColocMap[currBMProcessId]) { + } else if (self->db.blobGranulesEnabled.get() && newColocMap[newBMProcessId] < currColocMap[currBMProcessId]) { bmSingleton.recruit(self); } else if (SERVER_KNOBS->ENABLE_ENCRYPTION && newColocMap[newEKPProcessId] < currColocMap[currEKPProcessId]) { ekpSingleton.recruit(self); @@ -715,14 +735,20 @@ void checkBetterSingletons(ClusterControllerData* self) { ACTOR Future doCheckOutstandingRequests(ClusterControllerData* self) { try { wait(delay(SERVER_KNOBS->CHECK_OUTSTANDING_INTERVAL)); - while (!self->goodRecruitmentTime.isReady()) { - wait(self->goodRecruitmentTime); + while (now() - self->lastRecruitTime < SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY || + !self->goodRecruitmentTime.isReady()) { + if (now() - self->lastRecruitTime < SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY) { + wait(delay(SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY + 0.001 - (now() - self->lastRecruitTime))); + } + if (!self->goodRecruitmentTime.isReady()) { + wait(self->goodRecruitmentTime); + } } checkOutstandingRecruitmentRequests(self); checkOutstandingStorageRequests(self); - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (self->db.blobGranulesEnabled.get()) { checkOutstandingBlobWorkerRequests(self); } checkBetterSingletons(self); @@ -1232,7 +1258,7 @@ void registerWorker(RegisterWorkerRequest req, self, w, currSingleton, registeringSingleton, self->recruitingRatekeeperID); } - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES && req.blobManagerInterf.present()) { + if (self->db.blobGranulesEnabled.get() && req.blobManagerInterf.present()) { auto currSingleton = BlobManagerSingleton(self->db.serverInfo->get().blobManager); auto registeringSingleton = BlobManagerSingleton(req.blobManagerInterf); haltRegisteringOrCurrentSingleton( @@ -2096,9 +2122,9 @@ ACTOR Future getNextBMEpoch(ClusterControllerData* self) { tr->set(blobManagerEpochKey, blobManagerEpochValueFor(newEpoch)); wait(tr->commit()); + TraceEvent(SevDebug, "CCNextBlobManagerEpoch", self->id).detail("Epoch", newEpoch); return newEpoch; } catch (Error& e) { - printf("Acquiring blob manager lock got error %s\n", e.name()); wait(tr->onError(e)); } } @@ -2219,6 +2245,11 @@ ACTOR Future startBlobManager(ClusterControllerData* self) { id_used); int64_t nextEpoch = wait(getNextBMEpoch(self)); + if (!self->masterProcessId.present() || + self->masterProcessId != self->db.serverInfo->get().master.locality.processId() || + self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) { + continue; + } InitializeBlobManagerRequest req(deterministicRandom()->randomUniqueID(), nextEpoch); state WorkerDetails worker = bmWorker.worker; if (self->onMasterIsBetter(worker, ProcessClass::BlobManager)) { @@ -2262,6 +2293,30 @@ ACTOR Future startBlobManager(ClusterControllerData* self) { } } +ACTOR Future watchBlobGranulesConfigKey(ClusterControllerData* self) { + state Reference tr = makeReference(self->cx); + state Key blobGranuleConfigKey = configKeysPrefix.withSuffix("blob_granules_enabled"_sr); + + loop { + try { + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + Optional blobConfig = wait(tr->get(blobGranuleConfigKey)); + if (blobConfig.present()) { + self->db.blobGranulesEnabled.set(blobConfig.get() == LiteralStringRef("1")); + } + + state Future watch = tr->watch(blobGranuleConfigKey); + wait(tr->commit()); + wait(watch); + } catch (Error& e) { + wait(tr->onError(e)); + } + } +} + ACTOR Future monitorBlobManager(ClusterControllerData* self) { while (self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) { wait(self->db.serverInfo->onChange()); @@ -2269,17 +2324,34 @@ ACTOR Future monitorBlobManager(ClusterControllerData* self) { loop { if (self->db.serverInfo->get().blobManager.present() && !self->recruitBlobManager.get()) { - choose { - when(wait(waitFailureClient(self->db.serverInfo->get().blobManager.get().waitFailure, - SERVER_KNOBS->BLOB_MANAGER_FAILURE_TIME))) { - TraceEvent("CCBlobManagerDied", self->id) - .detail("BMID", self->db.serverInfo->get().blobManager.get().id()); - self->db.clearInterf(ProcessClass::BlobManagerClass); + state Future wfClient = waitFailureClient(self->db.serverInfo->get().blobManager.get().waitFailure, + SERVER_KNOBS->BLOB_MANAGER_FAILURE_TIME); + loop { + choose { + when(wait(wfClient)) { + TraceEvent("CCBlobManagerDied", self->id) + .detail("BMID", self->db.serverInfo->get().blobManager.get().id()); + self->db.clearInterf(ProcessClass::BlobManagerClass); + break; + } + when(wait(self->recruitBlobManager.onChange())) { break; } + when(wait(self->db.blobGranulesEnabled.onChange())) { + // if there is a blob manager present but blob granules are now disabled, stop the BM + if (!self->db.blobGranulesEnabled.get()) { + const auto& blobManager = self->db.serverInfo->get().blobManager; + BlobManagerSingleton(blobManager) + .haltBlobGranules(self, blobManager.get().locality.processId()); + break; + } + } } - when(wait(self->recruitBlobManager.onChange())) {} } - } else { + } else if (self->db.blobGranulesEnabled.get()) { + // if there is no blob manager present but blob granules are now enabled, recruit a BM wait(startBlobManager(self)); + } else { + // if there is no blob manager present and blob granules are disabled, wait for a config change + wait(self->db.blobGranulesEnabled.onChange()); } } } @@ -2434,10 +2506,8 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, self.addActor.send(handleForcedRecoveries(&self, interf)); self.addActor.send(monitorDataDistributor(&self)); self.addActor.send(monitorRatekeeper(&self)); - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { - self.addActor.send(monitorBlobManager(&self)); - } - // self.addActor.send(monitorTSSMapping(&self)); + self.addActor.send(monitorBlobManager(&self)); + self.addActor.send(watchBlobGranulesConfigKey(&self)); self.addActor.send(dbInfoUpdater(&self)); self.addActor.send(traceCounters("ClusterControllerMetrics", self.id, @@ -3055,4 +3125,4 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer return Void(); } -} // namespace +} // namespace \ No newline at end of file diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h index 2ce2f0a23e..d9e245425b 100644 --- a/fdbserver/ClusterController.actor.h +++ b/fdbserver/ClusterController.actor.h @@ -137,6 +137,7 @@ public: std::map> clientStatus; Future clientCounter; int clientCount; + AsyncVar blobGranulesEnabled; DBInfo() : clientInfo(new AsyncVar()), serverInfo(new AsyncVar()), @@ -147,7 +148,8 @@ public: EnableLocalityLoadBalance::True, TaskPriority::DefaultEndpoint, LockAware::True)), // SOMEDAY: Locality! - unfinishedRecoveries(0), logGenerations(0), cachePopulated(false), clientCount(0) { + unfinishedRecoveries(0), logGenerations(0), cachePopulated(false), clientCount(0), + blobGranulesEnabled(config.blobGranulesEnabled) { clientCounter = countClients(this); } @@ -3225,6 +3227,7 @@ public: // recruitX is used to signal when role X needs to be (re)recruited. // recruitingXID is used to track the ID of X's interface which is being recruited. // We use AsyncVars to kill (i.e. halt) singletons that have been replaced. + double lastRecruitTime = 0; AsyncVar recruitDistributor; Optional recruitingDistributorID; AsyncVar recruitRatekeeper; diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 137084c640..90254d612b 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1887,11 +1887,21 @@ ACTOR Future proxyCheckSafeExclusion(Reference cons return Void(); } try { - state Future> safeFuture = + state Future> ddSafeFuture = db->get().distributor.get().distributorExclCheckReq.tryGetReply( DistributorExclusionSafetyCheckRequest(req.exclusions)); - DistributorExclusionSafetyCheckReply _reply = wait(throwErrorOr(safeFuture)); + DistributorExclusionSafetyCheckReply _reply = wait(throwErrorOr(ddSafeFuture)); reply.safe = _reply.safe; + if (db->get().blobManager.present()) { + TraceEvent("SafetyCheckCommitProxyBM").detail("BMID", db->get().blobManager.get().id()); + state Future> bmSafeFuture = + db->get().blobManager.get().blobManagerExclCheckReq.tryGetReply( + BlobManagerExclusionSafetyCheckRequest(req.exclusions)); + BlobManagerExclusionSafetyCheckReply _reply = wait(throwErrorOr(bmSafeFuture)); + reply.safe &= _reply.safe; + } else { + TraceEvent("SafetyCheckCommitProxyNoBM"); + } } catch (Error& e) { TraceEvent("SafetyCheckCommitProxyResponseError").error(e); if (e.code() != error_code_operation_cancelled) { diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp index 8e6b847d5f..68cccaa05c 100644 --- a/fdbserver/QuietDatabase.actor.cpp +++ b/fdbserver/QuietDatabase.actor.cpp @@ -19,6 +19,8 @@ */ #include +#include + #include "fdbclient/FDBOptions.g.h" #include "fdbclient/SystemData.h" #include "flow/ActorCollection.h" diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index d55ba58c24..dc3c1595c1 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -70,6 +70,10 @@ namespace { const int MACHINE_REBOOT_TIME = 10; +// The max number of extra blob worker machines we might (i.e. randomly) add to the simulated cluster. +// Note that this is in addition to the two we always have. +const int NUM_EXTRA_BW_MACHINES = 5; + bool destructed = false; // Configuration details specified in workload test files that change the simulation @@ -262,6 +266,9 @@ class TestConfig { configDBType = configDBTypeFromString(value); } } + if (attrib == "blobGranulesEnabled") { + blobGranulesEnabled = strcmp(value.c_str(), "true") == 0; + } } ifs.close(); @@ -298,6 +305,7 @@ public: Optional generateFearless, buggify; Optional datacenters, desiredTLogCount, commitProxyCount, grvProxyCount, resolverCount, storageEngineType, stderrSeverity, machineCount, processesPerMachine, coordinators; + bool blobGranulesEnabled = false; Optional config; bool allowDefaultTenant = true; @@ -354,6 +362,7 @@ public: .add("coordinators", &coordinators) .add("configDB", &configDBType) .add("extraMachineCountDC", &extraMachineCountDC) + .add("blobGranulesEnabled", &blobGranulesEnabled) .add("allowDefaultTenant", &allowDefaultTenant) .add("allowDisablingTenants", &allowDisablingTenants); try { @@ -2085,16 +2094,20 @@ void setupSimulatedSystem(std::vector>* systemActors, coordinatorCount); ASSERT_LE(dcCoordinators, machines); - // FIXME: temporarily code to test storage cache + // FIXME: we hardcode some machines to specifically test storage cache and blob workers // TODO: caching disabled for this merge - if (dc == 0) { - machines++; + int storageCacheMachines = dc == 0 ? 1 : 0; + int blobWorkerMachines = 0; + if (testConfig.blobGranulesEnabled) { + int blobWorkerProcesses = 1 + deterministicRandom()->randomInt(0, NUM_EXTRA_BW_MACHINES + 1); + blobWorkerMachines = std::max(1, blobWorkerProcesses / processesPerMachine); } - int useSeedForMachine = deterministicRandom()->randomInt(0, machines); + int totalMachines = machines + storageCacheMachines + blobWorkerMachines; + int useSeedForMachine = deterministicRandom()->randomInt(0, totalMachines); Standalone zoneId; Standalone newZoneId; - for (int machine = 0; machine < machines; machine++) { + for (int machine = 0; machine < totalMachines; machine++) { Standalone machineId(deterministicRandom()->randomUniqueID().toString()); if (machine == 0 || machineCount - dataCenters <= 4 || assignedMachines != 4 || simconfig.db.regions.size() || deterministicRandom()->random01() < 0.5) { @@ -2124,11 +2137,19 @@ void setupSimulatedSystem(std::vector>* systemActors, } } - // FIXME: temporarily code to test storage cache + // FIXME: hack to add machines specifically to test storage cache and blob workers // TODO: caching disabled for this merge - if (machine == machines - 1 && dc == 0) { - processClass = ProcessClass(ProcessClass::StorageCacheClass, ProcessClass::CommandLineSource); - nonVersatileMachines++; + // `machines` here is the normal (non-temporary) machines that totalMachines comprises of + if (machine >= machines) { + if (storageCacheMachines > 0 && dc == 0) { + processClass = ProcessClass(ProcessClass::StorageCacheClass, ProcessClass::CommandLineSource); + nonVersatileMachines++; + storageCacheMachines--; + } else if (blobWorkerMachines > 0) { // add blob workers to every DC + processClass = ProcessClass(ProcessClass::BlobWorkerClass, ProcessClass::CommandLineSource); + nonVersatileMachines++; + blobWorkerMachines--; + } } std::vector ips; @@ -2303,8 +2324,9 @@ ACTOR void setupAndRun(std::string dataFolder, // Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate // the tenant map and related state. - // TODO: reenable when backup/DR supports tenants. - if (std::string_view(testFile).find("Backup") != std::string_view::npos || testConfig.extraDB != 0) { + // TODO: reenable when backup/DR or BlobGranule supports tenants. + if (std::string_view(testFile).find("Backup") != std::string_view::npos || + std::string_view(testFile).find("BlobGranule") != std::string_view::npos || testConfig.extraDB != 0) { allowDefaultTenant = false; } diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 427c57a65d..5549083b8e 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -804,7 +804,7 @@ ACTOR static Future processStatusFetcher( roles.addRole("ratekeeper", db->get().ratekeeper.get()); } - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES && db->get().blobManager.present()) { + if (configuration.present() && configuration.get().blobGranulesEnabled && db->get().blobManager.present()) { roles.addRole("blob_manager", db->get().blobManager.get()); } @@ -875,7 +875,7 @@ ACTOR static Future processStatusFetcher( wait(yield()); } - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (configuration.present() && configuration.get().blobGranulesEnabled) { for (auto blobWorker : blobWorkers) { roles.addRole("blob_worker", blobWorker); wait(yield()); @@ -2983,7 +2983,7 @@ ACTOR Future clusterGetStatus( errorOr(getGrvProxiesAndMetrics(db, address_workers)); state Future>> blobWorkersFuture; - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (configuration.present() && configuration.get().blobGranulesEnabled) { blobWorkersFuture = errorOr(timeoutError(getBlobWorkers(cx, true), 5.0)); } @@ -3121,7 +3121,7 @@ ACTOR Future clusterGetStatus( } // ...also blob workers - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (configuration.present() && configuration.get().blobGranulesEnabled) { ErrorOr> _blobWorkers = wait(blobWorkersFuture); if (_blobWorkers.present()) { blobWorkers = _blobWorkers.get(); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 561b6747f6..c90332a019 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -23,6 +23,7 @@ #include #include +#include "contrib/fmt-8.1.1/include/fmt/format.h" #include "fdbrpc/fdbrpc.h" #include "fdbrpc/LoadBalance.h" #include "flow/ActorCollection.h" @@ -87,6 +88,7 @@ bool canReplyWith(Error e) { case error_code_watch_cancelled: case error_code_unknown_change_feed: case error_code_server_overloaded: + case error_code_change_feed_popped: case error_code_tenant_name_required: case error_code_unknown_tenant: // getMappedRange related exceptions that are not retriable: @@ -132,7 +134,7 @@ static const KeyRangeRef persistByteSampleSampleKeys = static const KeyRef persistLogProtocol = LiteralStringRef(PERSIST_PREFIX "LogProtocol"); static const KeyRef persistPrimaryLocality = LiteralStringRef(PERSIST_PREFIX "PrimaryLocality"); static const KeyRangeRef persistChangeFeedKeys = - KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "RF/"), LiteralStringRef(PERSIST_PREFIX "RF0")); + KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "CF/"), LiteralStringRef(PERSIST_PREFIX "CF0")); static const KeyRangeRef persistTenantMapKeys = KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "TM/"), LiteralStringRef(PERSIST_PREFIX "TM0")); // data keys are unmangled (but never start with PERSIST_PREFIX because they are always in allKeys) @@ -168,6 +170,9 @@ struct AddingShard : NonCopyable { // During Fetching phase, it fetches data before fetchVersion and write it to storage, then let updater know it // is ready to update the deferred updates` (see the comment of member variable `updates` above). Fetching, + // During the FetchingCF phase, the shard data is transferred but the remaining change feed data is still being + // transferred. This is equivalent to the waiting phase for non-changefeed data. + FetchingCF, // During Waiting phase, it sends updater the deferred updates, and wait until they are durable. Waiting // The shard's state is changed from adding to readWrite then. @@ -190,7 +195,8 @@ struct AddingShard : NonCopyable { void addMutation(Version version, bool fromFetch, MutationRef const& mutation); - bool isTransferred() const { return phase == Waiting; } + bool isDataTransferred() const { return phase >= FetchingCF; } + bool isDataAndCFTransferred() const { return phase >= Waiting; } }; class ShardInfo : public ReferenceCounted, NonCopyable { @@ -216,14 +222,17 @@ public: bool isReadable() const { return readWrite != nullptr; } bool notAssigned() const { return !readWrite && !adding; } bool assigned() const { return readWrite || adding; } - bool isInVersionedData() const { return readWrite || (adding && adding->isTransferred()); } + bool isInVersionedData() const { return readWrite || (adding && adding->isDataTransferred()); } + bool isCFInVersionedData() const { return readWrite || (adding && adding->isDataAndCFTransferred()); } void addMutation(Version version, bool fromFetch, MutationRef const& mutation); bool isFetched() const { return readWrite || (adding && adding->fetchComplete.isSet()); } const char* debugDescribeState() const { if (notAssigned()) return "NotAssigned"; - else if (adding && !adding->isTransferred()) + else if (adding && !adding->isDataAndCFTransferred()) + return "AddingFetchingCF"; + else if (adding && !adding->isDataTransferred()) return "AddingFetching"; else if (adding) return "AddingTransferred"; @@ -407,15 +416,57 @@ struct FetchInjectionInfo { struct ChangeFeedInfo : ReferenceCounted { std::deque> mutations; + Version fetchVersion = invalidVersion; // The version that commits from a fetch have been written to storage, but + // have not yet been committed as part of updateStorage. Version storageVersion = invalidVersion; // The version between the storage version and the durable version are - // currently being written to disk + // being written to disk as part of the current commit in updateStorage. Version durableVersion = invalidVersion; // All versions before the durable version are durable on disk Version emptyVersion = 0; // The change feed does not have any mutations before emptyVersion KeyRange range; Key id; AsyncTrigger newMutations; - bool stopped = false; // A stopped change feed no longer adds new mutations, but is still queriable + NotifiedVersion durableFetchVersion; + // A stopped change feed no longer adds new mutations, but is still queriable. + // stopVersion = MAX_VERSION means the feed has not been stopped + Version stopVersion = MAX_VERSION; + + // We need to track the version the change feed metadata was created by private mutation, so that if it is rolled + // back, we can avoid notifying other SS of change feeds that don't durably exist + Version metadataCreateVersion = invalidVersion; + bool removing = false; + + KeyRangeMap>> moveTriggers; + + void triggerOnMove(KeyRange range, UID streamUID, Promise p) { + auto toInsert = moveTriggers.modify(range); + for (auto triggerRange = toInsert.begin(); triggerRange != toInsert.end(); ++triggerRange) { + triggerRange->value().insert({ streamUID, p }); + } + } + + void moved(KeyRange range) { + auto toTrigger = moveTriggers.intersectingRanges(range); + for (auto& triggerRange : toTrigger) { + for (auto& triggerStream : triggerRange.cvalue()) { + if (triggerStream.second.canBeSet()) { + triggerStream.second.send(Void()); + } + } + } + // coalesce doesn't work with promises + moveTriggers.insert(range, std::unordered_map>()); + } + + void removeOnMoveTrigger(KeyRange range, UID streamUID) { + auto toRemove = moveTriggers.modify(range); + for (auto triggerRange = toRemove.begin(); triggerRange != toRemove.end(); ++triggerRange) { + auto streamToRemove = triggerRange->value().find(streamUID); + ASSERT(streamToRemove != triggerRange->cvalue().end()); + triggerRange->value().erase(streamToRemove); + } + // TODO: may be more cleanup possible here + } }; class ServerWatchMetadata : public ReferenceCounted { @@ -698,7 +749,10 @@ public: Deque, Version>> changeFeedVersions; std::map> changeFeedRemovals; std::set currentChangeFeeds; + std::set fetchingChangeFeeds; std::unordered_map> changeFeedClientVersions; + std::unordered_map changeFeedCleanupDurable; + int64_t activeFeedQueries = 0; // newestAvailableVersion[k] // == invalidVersion -> k is unavailable at all versions @@ -722,7 +776,7 @@ public: NotifiedVersion durableVersion; // At least this version will be readable from storage after a power failure Version rebootAfterDurableVersion; int8_t primaryLocality; - Version knownCommittedVersion; + NotifiedVersion knownCommittedVersion; Deque> recoveryVersionSkips; int64_t versionLag; // An estimate for how many versions it takes for the data to move from the logs to this storage @@ -768,6 +822,7 @@ public: FlowLock durableVersionLock; FlowLock fetchKeysParallelismLock; + FlowLock fetchChangeFeedParallelismLock; int64_t fetchKeysBytesBudget; AsyncVar fetchKeysBudgetUsed; std::vector> readyFetchKeys; @@ -874,7 +929,7 @@ public: CounterCollection cc; Counter allQueries, getKeyQueries, getValueQueries, getRangeQueries, getMappedRangeQueries, getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried, bytesQueried, watchQueries, - emptyQueries; + emptyQueries, feedRowsQueried, feedBytesQueried; // Bytes of the mutations that have been added to the memory of the storage server. When the data is durable // and cleared from the memory, we do not subtract it but add it to bytesDurable. @@ -903,6 +958,9 @@ public: // and the lengths of both parameters. Counter mutationBytes; + // Bytes fetched by fetchChangeFeed for data movements. + Counter feedBytesFetched; + Counter sampledBytesCleared; // The number of key-value pairs fetched by fetchKeys() Counter kvFetched; @@ -942,23 +1000,24 @@ public: getRangeStreamQueries("GetRangeStreamQueries", cc), finishedQueries("FinishedQueries", cc), lowPriorityQueries("LowPriorityQueries", cc), rowsQueried("RowsQueried", cc), bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc), emptyQueries("EmptyQueries", cc), + feedRowsQueried("FeedRowsQueried", cc), feedBytesQueried("FeedBytesQueried", cc), bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc), logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc), kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc), kvSystemClearRanges("KVSystemClearRanges", cc), bytesDurable("BytesDurable", cc), bytesFetched("BytesFetched", cc), mutationBytes("MutationBytes", cc), - sampledBytesCleared("SampledBytesCleared", cc), kvFetched("KVFetched", cc), mutations("Mutations", cc), - setMutations("SetMutations", cc), clearRangeMutations("ClearRangeMutations", cc), - atomicMutations("AtomicMutations", cc), updateBatches("UpdateBatches", cc), - updateVersions("UpdateVersions", cc), loops("Loops", cc), fetchWaitingMS("FetchWaitingMS", cc), - fetchWaitingCount("FetchWaitingCount", cc), fetchExecutingMS("FetchExecutingMS", cc), - fetchExecutingCount("FetchExecutingCount", cc), readsRejected("ReadsRejected", cc), - wrongShardServer("WrongShardServer", cc), fetchedVersions("FetchedVersions", cc), - fetchesFromLogs("FetchesFromLogs", cc), quickGetValueHit("QuickGetValueHit", cc), - quickGetValueMiss("QuickGetValueMiss", cc), quickGetKeyValuesHit("QuickGetKeyValuesHit", cc), - quickGetKeyValuesMiss("QuickGetKeyValuesMiss", cc), kvScanBytes("KVScanBytes", cc), - kvGetBytes("KVGetBytes", cc), eagerReadsKeys("EagerReadsKeys", cc), kvGets("KVGets", cc), - kvScans("KVScans", cc), kvCommits("KVCommits", cc), + feedBytesFetched("FeedBytesFetched", cc), sampledBytesCleared("SampledBytesCleared", cc), + kvFetched("KVFetched", cc), mutations("Mutations", cc), setMutations("SetMutations", cc), + clearRangeMutations("ClearRangeMutations", cc), atomicMutations("AtomicMutations", cc), + updateBatches("UpdateBatches", cc), updateVersions("UpdateVersions", cc), loops("Loops", cc), + fetchWaitingMS("FetchWaitingMS", cc), fetchWaitingCount("FetchWaitingCount", cc), + fetchExecutingMS("FetchExecutingMS", cc), fetchExecutingCount("FetchExecutingCount", cc), + readsRejected("ReadsRejected", cc), wrongShardServer("WrongShardServer", cc), + fetchedVersions("FetchedVersions", cc), fetchesFromLogs("FetchesFromLogs", cc), + quickGetValueHit("QuickGetValueHit", cc), quickGetValueMiss("QuickGetValueMiss", cc), + quickGetKeyValuesHit("QuickGetKeyValuesHit", cc), quickGetKeyValuesMiss("QuickGetKeyValuesMiss", cc), + kvScanBytes("KVScanBytes", cc), kvGetBytes("KVGetBytes", cc), eagerReadsKeys("EagerReadsKeys", cc), + kvGets("KVGets", cc), kvScans("KVScans", cc), kvCommits("KVCommits", cc), readLatencySample("ReadLatencyMetrics", self->thisServerID, SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL, @@ -976,6 +1035,11 @@ public: specialCounter( cc, "FetchKeysFetchActive", [self]() { return self->fetchKeysParallelismLock.activePermits(); }); specialCounter(cc, "FetchKeysWaiting", [self]() { return self->fetchKeysParallelismLock.waiters(); }); + specialCounter(cc, "FetchChangeFeedFetchActive", [self]() { + return self->fetchChangeFeedParallelismLock.activePermits(); + }); + specialCounter( + cc, "FetchChangeFeedWaiting", [self]() { return self->fetchChangeFeedParallelismLock.waiters(); }); specialCounter(cc, "QueryQueueMax", [self]() { return self->getAndResetMaxQueryQueueSize(); }); specialCounter(cc, "BytesStored", [self]() { return self->metrics.byteSample.getEstimate(allKeys); }); specialCounter(cc, "ActiveWatches", [self]() { return self->numWatches; }); @@ -983,6 +1047,8 @@ public: specialCounter(cc, "KvstoreSizeTotal", [self]() { return std::get<0>(self->storage.getSize()); }); specialCounter(cc, "KvstoreNodeTotal", [self]() { return std::get<1>(self->storage.getSize()); }); specialCounter(cc, "KvstoreInlineKey", [self]() { return std::get<2>(self->storage.getSize()); }); + specialCounter(cc, "ActiveChangeFeeds", [self]() { return self->uidChangeFeed.size(); }); + specialCounter(cc, "ActiveChangeFeedQueries", [self]() { return self->activeFeedQueries; }); } } counters; @@ -1027,6 +1093,7 @@ public: numWatches(0), noRecentUpdates(false), lastUpdate(now()), readQueueSizeMetric(LiteralStringRef("StorageServer.ReadQueueSize")), updateEagerReads(nullptr), fetchKeysParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM), + fetchChangeFeedParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM), fetchKeysBytesBudget(SERVER_KNOBS->STORAGE_FETCH_BYTES), fetchKeysBudgetUsed(false), instanceID(deterministicRandom()->randomUniqueID().first()), shuttingDown(false), behind(false), versionBehind(false), debug_inApplyUpdate(false), debug_lastValidateTime(0), lastBytesInputEBrake(0), @@ -1189,6 +1256,19 @@ public: return fun(this, request); } + Version minFeedVersionForAddress(const NetworkAddress& addr) { + auto& clientVersions = changeFeedClientVersions[addr]; + Version minVersion = version.get(); + for (auto& it : clientVersions) { + /*fmt::print("SS {0} Blocked client {1} @ {2}\n", + thisServerID.toString().substr(0, 4), + it.first.toString().substr(0, 8), + it.second);*/ + minVersion = std::min(minVersion, it.second); + } + return minVersion; + } + void getSplitPoints(SplitRangeRequest const& req) { try { Optional entry = getTenantEntry(version.get(), req.tenantInfo); @@ -1328,6 +1408,8 @@ void validate(StorageServer* data, bool force = false) { } } + // FIXME: do some change feed validation? + latest.validate(); validateRange(latest, allKeys, data->version.get(), data->thisServerID, data->durableVersion.get()); @@ -1742,43 +1824,6 @@ ACTOR Future watchValueSendReply(StorageServer* data, } } -ACTOR Future changeFeedPopQ(StorageServer* self, ChangeFeedPopRequest req) { - wait(delay(0)); - - TraceEvent(SevDebug, "ChangeFeedPopQuery", self->thisServerID) - .detail("RangeID", req.rangeID.printable()) - .detail("Version", req.version) - .detail("Range", req.range.toString()); - - if (!self->isReadable(req.range)) { - req.reply.sendError(wrong_shard_server()); - return Void(); - } - auto feed = self->uidChangeFeed.find(req.rangeID); - if (feed == self->uidChangeFeed.end()) { - req.reply.sendError(unknown_change_feed()); - return Void(); - } - if (req.version - 1 > feed->second->emptyVersion) { - feed->second->emptyVersion = req.version - 1; - while (!feed->second->mutations.empty() && feed->second->mutations.front().version < req.version) { - feed->second->mutations.pop_front(); - } - if (feed->second->storageVersion != invalidVersion) { - self->storage.clearRange(KeyRangeRef(changeFeedDurableKey(feed->second->id, 0), - changeFeedDurableKey(feed->second->id, req.version))); - ++self->counters.kvSystemClearRanges; - if (req.version > feed->second->storageVersion) { - feed->second->storageVersion = invalidVersion; - feed->second->durableVersion = invalidVersion; - } - wait(self->durableVersion.whenAtLeast(self->storageVersion() + 1)); - } - } - req.reply.send(Void()); - return Void(); -} - // Finds a checkpoint. ACTOR Future getCheckpointQ(StorageServer* self, GetCheckpointRequest req) { // Wait until the desired version is durable. @@ -1898,16 +1943,38 @@ ACTOR Future overlappingChangeFeedsQ(StorageServer* data, OverlappingChang return Void(); } + Version metadataVersion = invalidVersion; + auto ranges = data->keyChangeFeed.intersectingRanges(req.range); - std::map> rangeIds; + std::map> rangeIds; for (auto r : ranges) { for (auto& it : r.value()) { - rangeIds[it->id] = std::make_pair(it->range, it->stopped); + // Can't tell other SS about a change feed create or stopVersion that may get rolled back, and we only need + // to tell it about the metadata if req.minVersion > metadataVersion, since it will get the information from + // its own private mutations if it hasn't processed up that version yet + metadataVersion = std::max(metadataVersion, it->metadataCreateVersion); + + Version stopVersion; + if (it->stopVersion != MAX_VERSION && req.minVersion > it->stopVersion) { + stopVersion = it->stopVersion; + metadataVersion = std::max(metadataVersion, stopVersion); + } else { + stopVersion = MAX_VERSION; + } + + rangeIds[it->id] = std::tuple(it->range, it->emptyVersion, stopVersion); } } - OverlappingChangeFeedsReply reply; + state OverlappingChangeFeedsReply reply; for (auto& it : rangeIds) { - reply.rangeIds.push_back(OverlappingChangeFeedEntry(it.first, it.second.first, it.second.second)); + reply.rangeIds.push_back(OverlappingChangeFeedEntry( + it.first, std::get<0>(it.second), std::get<1>(it.second), std::get<2>(it.second))); + } + + // Make sure all of the metadata we are sending won't get rolled back + if (metadataVersion != invalidVersion && metadataVersion > data->knownCommittedVersion.get()) { + TEST(true); // overlapping change feeds waiting for metadata version to be committed + wait(data->desiredOldestVersion.whenAtLeast(metadataVersion)); } req.reply.send(reply); return Void(); @@ -1927,8 +1994,7 @@ MutationsAndVersionRef filterMutationsInverted(Arena& arena, MutationsAndVersion } else { ASSERT(m.mutations[i].type == MutationRef::ClearRange); if (!modifiedMutations.present() && - ((m.mutations[i].param1 < range.begin && m.mutations[i].param2 > range.begin) || - (m.mutations[i].param2 > range.end && m.mutations[i].param1 < range.end))) { + (m.mutations[i].param2 > range.begin && m.mutations[i].param1 < range.end)) { modifiedMutations = m.mutations.slice(0, i); arena.dependsOn(range.arena()); } @@ -1999,16 +2065,46 @@ MutationsAndVersionRef filterMutations(Arena& arena, return m; } +// set this for VERY verbose logs on change feed SS reads +#define DEBUG_CF_TRACE false + +// To easily find if a change feed read missed data. Set the CF to the feedId, the key to the missing key, and the +// version to the version the mutation is missing at. +#define DO_DEBUG_CF_MISSING false +#define DEBUG_CF_MISSING_CF ""_sr +#define DEBUG_CF_MISSING_KEY ""_sr +#define DEBUG_CF_MISSING_VERSION invalidVersion +#define DEBUG_CF_MISSING(cfId, keyRange, beginVersion, lastVersion) \ + DO_DEBUG_CF_MISSING&& cfId.printable().substr(0, 6) == \ + DEBUG_CF_MISSING_CF&& keyRange.contains(DEBUG_CF_MISSING_KEY) && \ + beginVersion <= DEBUG_CF_MISSING_VERSION&& lastVersion >= DEBUG_CF_MISSING_VERSION + ACTOR Future> getChangeFeedMutations(StorageServer* data, ChangeFeedStreamRequest req, - bool inverted) { + bool inverted, + bool atLatest, + UID streamUID /* for debugging */) { state ChangeFeedStreamReply reply; state ChangeFeedStreamReply memoryReply; state int remainingLimitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT; state int remainingDurableBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT; + state Version startVersion = data->version.get(); + + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedMutationsBegin", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end); + } if (data->version.get() < req.begin) { wait(data->version.whenAtLeast(req.begin)); + // we must delay here to ensure that any up-to-date change feeds that are waiting on the + // mutation trigger run BEFORE any blocked change feeds run, in order to preserve the + // correct minStreamVersion ordering + wait(delay(0)); } state uint64_t changeCounter = data->shardChangeCounter; @@ -2021,54 +2117,166 @@ ACTOR Future> getChangeFeedMutations(Stor throw unknown_change_feed(); } + state Reference feedInfo = feed->second; + // We must copy the mutationDeque when fetching the durable bytes in case mutations are popped from memory while // waiting for the results state Version dequeVersion = data->version.get(); - state Version dequeKnownCommit = data->knownCommittedVersion; + state Version dequeKnownCommit = data->knownCommittedVersion.get(); + state Version emptyVersion = feedInfo->emptyVersion; + Version fetchStorageVersion = std::max(feedInfo->fetchVersion, feedInfo->durableFetchVersion.get()); - if (req.end > feed->second->emptyVersion + 1) { - for (auto& it : feed->second->mutations) { + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedMutationsDetails", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("AtLatest", atLatest) + .detail("DequeVersion", dequeVersion) + .detail("EmptyVersion", feedInfo->emptyVersion) + .detail("StorageVersion", feedInfo->storageVersion) + .detail("DurableVersion", feedInfo->durableVersion) + .detail("FetchStorageVersion", fetchStorageVersion) + .detail("FetchVersion", feedInfo->fetchVersion) + .detail("DurableFetchVersion", feedInfo->durableFetchVersion.get()); + } + + if (req.end > emptyVersion + 1) { + // FIXME: do exponential backwards search from end to find beginVersion if atLatest to reduce cpu + for (auto& it : feedInfo->mutations) { if (it.version >= req.end || it.version > dequeVersion || remainingLimitBytes <= 0) { break; } if (it.version >= req.begin) { - memoryReply.arena.dependsOn(it.arena()); auto m = filterMutations(memoryReply.arena, it, req.range, inverted); - memoryReply.mutations.push_back(memoryReply.arena, m); - remainingLimitBytes -= sizeof(MutationsAndVersionRef) + m.expectedSize(); + if (m.mutations.size()) { + memoryReply.arena.dependsOn(it.arena()); + memoryReply.mutations.push_back(memoryReply.arena, m); + remainingLimitBytes -= sizeof(MutationsAndVersionRef) + m.expectedSize(); + } } } } - if (req.end > feed->second->emptyVersion + 1 && feed->second->durableVersion != invalidVersion && - req.begin <= feed->second->durableVersion) { - RangeResult res = wait(data->storage.readRange( - KeyRangeRef(changeFeedDurableKey(req.rangeID, std::max(req.begin, feed->second->emptyVersion)), - changeFeedDurableKey(req.rangeID, req.end)), - 1 << 30, - remainingDurableBytes)); + state bool readDurable = feedInfo->durableVersion != invalidVersion && req.begin <= feedInfo->durableVersion; + state bool readFetched = req.begin <= fetchStorageVersion && !atLatest; + state bool waitFetched = false; + if (req.end > emptyVersion + 1 && (readDurable || readFetched)) { + if (readFetched && req.begin <= feedInfo->fetchVersion) { + waitFetched = true; + // Request needs data that has been written to storage by a change feed fetch, but not committed yet + // To not block fetchKeys making normal SS data readable on making change feed data written to storage, we + // wait in here instead for all fetched data to become readable from the storage engine. + ASSERT(req.begin <= feedInfo->fetchVersion); + TEST(true); // getChangeFeedMutations before fetched data durable + + // Wait for next commit to write pending feed data to storage + wait(feedInfo->durableFetchVersion.whenAtLeast(feedInfo->fetchVersion)); + // To let update storage finish + wait(delay(0)); + } + RangeResult res = wait( + data->storage.readRange(KeyRangeRef(changeFeedDurableKey(req.rangeID, std::max(req.begin, emptyVersion)), + changeFeedDurableKey(req.rangeID, req.end)), + 1 << 30, + remainingDurableBytes)); + data->counters.kvScanBytes += res.logicalSize(); - if (!req.range.empty()) { + if (!inverted && !req.range.empty()) { data->checkChangeCounter(changeCounter, req.range); } + // TODO eventually: only do verify in simulation? + int memoryVerifyIdx = 0; + Version lastVersion = req.begin - 1; + Version lastKnownCommitted = invalidVersion; for (auto& kv : res) { Key id; Version version, knownCommittedVersion; Standalone> mutations; std::tie(id, version) = decodeChangeFeedDurableKey(kv.key); std::tie(mutations, knownCommittedVersion) = decodeChangeFeedDurableValue(kv.value); - reply.arena.dependsOn(mutations.arena()); + + // gap validation + while (memoryVerifyIdx < memoryReply.mutations.size() && + version > memoryReply.mutations[memoryVerifyIdx].version) { + if (req.canReadPopped) { + // There are weird cases where SS fetching mixed with SS durability and popping can mean there are + // gaps before the popped version temporarily + memoryVerifyIdx++; + continue; + } + + // There is a case where this can happen - if we wait on a fetching change feed, and the feed is + // popped while we wait, we could have copied the memory mutations into memoryReply before the + // pop, but they may or may not have been skipped writing to disk + if (waitFetched && feedInfo->emptyVersion > emptyVersion && + memoryReply.mutations[memoryVerifyIdx].version <= feedInfo->emptyVersion) { + memoryVerifyIdx++; + continue; + } else { + fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but not on disk (next disk " + "is {4}) (emptyVersion={5}, emptyBefore={6})!\n", + data->thisServerID.toString().substr(0, 4), + req.rangeID.printable().substr(0, 6), + streamUID.toString().substr(0, 8), + memoryReply.mutations[memoryVerifyIdx].version, + version, + feedInfo->emptyVersion, + emptyVersion); + + fmt::print(" Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size()); + for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) { + if (it.type == MutationRef::SetValue) { + fmt::print(" {}=\n", it.param1.printable()); + } else { + fmt::print(" {} - {}\n", it.param1.printable(), it.param2.printable()); + } + } + ASSERT(false); + } + } + auto m = filterMutations( reply.arena, MutationsAndVersionRef(mutations, version, knownCommittedVersion), req.range, inverted); - reply.mutations.push_back(reply.arena, m); + if (m.mutations.size()) { + reply.arena.dependsOn(mutations.arena()); + reply.mutations.push_back(reply.arena, m); + + if (memoryVerifyIdx < memoryReply.mutations.size() && + version == memoryReply.mutations[memoryVerifyIdx].version) { + // We could do validation of mutations here too, but it's complicated because clears can get split + // and stuff + memoryVerifyIdx++; + } + } else if (memoryVerifyIdx < memoryReply.mutations.size() && + version == memoryReply.mutations[memoryVerifyIdx].version) { + fmt::print("ERROR: SS {0} CF {1} SQ {2} has mutation at {3} in memory but all filtered out on disk!\n", + data->thisServerID.toString().substr(0, 4), + req.rangeID.printable().substr(0, 6), + streamUID.toString().substr(0, 8), + version); + + fmt::print(" Memory: ({})\n", memoryReply.mutations[memoryVerifyIdx].mutations.size()); + for (auto& it : memoryReply.mutations[memoryVerifyIdx].mutations) { + if (it.type == MutationRef::SetValue) { + fmt::print(" {}=\n", it.param1.printable().c_str()); + } else { + fmt::print(" {} - {}\n", it.param1.printable().c_str(), it.param2.printable().c_str()); + } + } + ASSERT(false); + } remainingDurableBytes -= sizeof(KeyValueRef) + kv.expectedSize(); // This is tracking the size on disk rather than the reply size // because we cannot add mutations from memory if there are potentially more on disk lastVersion = version; + lastKnownCommitted = knownCommittedVersion; } if (remainingDurableBytes > 0) { reply.arena.dependsOn(memoryReply.arena); @@ -2079,23 +2287,64 @@ ACTOR Future> getChangeFeedMutations(Stor --totalCount; } reply.mutations.append(reply.arena, it, totalCount); + // If still empty, that means disk results were filtered out, but skipped all memory results. Add an empty, + // either the last version from disk + if (reply.mutations.empty() && res.size()) { + TEST(true); // Change feed adding empty version after disk + memory filtered + reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted)); + } + } else if (reply.mutations.empty() || reply.mutations.back().version < lastVersion) { + TEST(true); // Change feed adding empty version after disk filtered + reply.mutations.push_back(reply.arena, MutationsAndVersionRef(lastVersion, lastKnownCommitted)); } } else { reply = memoryReply; } + bool gotAll = remainingLimitBytes > 0 && remainingDurableBytes > 0 && data->version.get() == startVersion; Version finalVersion = std::min(req.end - 1, dequeVersion); if ((reply.mutations.empty() || reply.mutations.back().version < finalVersion) && remainingLimitBytes > 0 && remainingDurableBytes > 0) { + TEST(true); // Change feed adding empty version after empty results reply.mutations.push_back( reply.arena, MutationsAndVersionRef(finalVersion, finalVersion == dequeVersion ? dequeKnownCommit : 0)); + // if we add empty mutation after the last thing in memory, and didn't read from disk, gotAll is true + if (data->version.get() == startVersion) { + gotAll = true; + } + } + + // This check is done just before returning, after all waits in this function + // Check if pop happened concurently + if (!req.canReadPopped && req.begin <= feedInfo->emptyVersion) { + // This can happen under normal circumstances if this part of a change feed got no updates, but then the feed + // was popped. We can check by confirming that the client was sent empty versions as part of another feed's + // response's minStorageVersion, or a ChangeFeedUpdateRequest. If this was the case, we know no updates could + // have happened between req.begin and minVersion. + Version minVersion = data->minFeedVersionForAddress(req.reply.getEndpoint().getPrimaryAddress()); + bool ok = atLatest && minVersion > feedInfo->emptyVersion; + TEST(ok); // feed popped while valid read waiting + TEST(!ok); // feed popped while invalid read waiting + if (!ok) { + TraceEvent("ChangeFeedMutationsPopped", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("EmptyVersion", feedInfo->emptyVersion) + .detail("AtLatest", atLatest) + .detail("MinVersionSent", minVersion); + throw change_feed_popped(); + } } if (MUTATION_TRACKING_ENABLED) { for (auto& mutations : reply.mutations) { for (auto& m : mutations.mutations) { - DEBUG_MUTATION("ChangeFeedRead", mutations.version, m, data->thisServerID) + DEBUG_MUTATION("ChangeFeedSSRead", mutations.version, m, data->thisServerID) .detail("ChangeFeedID", req.rangeID) + .detail("StreamUID", streamUID) .detail("ReqBegin", req.begin) .detail("ReqEnd", req.end) .detail("ReqRange", req.range); @@ -2103,7 +2352,73 @@ ACTOR Future> getChangeFeedMutations(Stor } } - return std::make_pair(reply, remainingLimitBytes > 0 && remainingDurableBytes > 0); + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "ChangeFeedMutationsDone", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("FirstVersion", reply.mutations.empty() ? invalidVersion : reply.mutations.front().version) + .detail("LastVersion", reply.mutations.empty() ? invalidVersion : reply.mutations.back().version) + .detail("Count", reply.mutations.size()) + .detail("GotAll", gotAll); + } + + if (DEBUG_CF_MISSING(req.rangeID, req.range, req.begin, reply.mutations.back().version) && !req.canReadPopped) { + bool foundVersion = false; + bool foundKey = false; + for (auto& it : reply.mutations) { + if (it.version == DEBUG_CF_MISSING_VERSION) { + foundVersion = true; + for (auto& m : it.mutations) { + if (m.type == MutationRef::SetValue && m.param1 == DEBUG_CF_MISSING_KEY) { + foundKey = true; + break; + } + } + break; + } + } + if (!foundVersion || !foundKey) { + fmt::print("ERROR: SS {0} CF {1} SQ {2} missing {3} @ {4} from request for [{5} - {6}) {7} - {8}\n", + data->thisServerID.toString().substr(0, 4), + req.rangeID.printable().substr(0, 6), + streamUID.toString().substr(0, 8), + foundVersion ? "key" : "version", + DEBUG_CF_MISSING_VERSION, + req.range.begin.printable(), + req.range.end.printable(), + req.begin, + req.end); + fmt::print("ERROR: {0} versions in response {1} - {2}:\n", + reply.mutations.size(), + reply.mutations.front().version, + reply.mutations.back().version); + for (auto& it : reply.mutations) { + fmt::print("ERROR: {0} ({1}){2}\n", + it.version, + it.mutations.size(), + it.version == DEBUG_CF_MISSING_VERSION ? "<-------" : ""); + } + } else { + fmt::print("DBG: SS {0} CF {1} SQ {2} correct @ {3} from request for [{4} - {5}) {6} - {7}\n", + data->thisServerID.toString().substr(0, 4), + req.rangeID.printable().substr(0, 6), + streamUID.toString().substr(0, 8), + DEBUG_CF_MISSING_VERSION, + req.range.begin.printable(), + req.range.end.printable(), + req.begin, + req.end); + } + } + + reply.popVersion = feedInfo->emptyVersion + 1; + + // If the SS's version advanced at all during any of the waits, the read from memory may have missed some + // mutations, so gotAll can only be true if data->version didn't change over the course of this actor + return std::make_pair(reply, gotAll); } ACTOR Future localChangeFeedStream(StorageServer* data, @@ -2120,7 +2435,7 @@ ACTOR Future localChangeFeedStream(StorageServer* data, feedRequest.end = end; feedRequest.range = range; state std::pair feedReply = - wait(getChangeFeedMutations(data, feedRequest, true)); + wait(getChangeFeedMutations(data, feedRequest, true, false, UID())); begin = feedReply.first.mutations.back().version + 1; state int resultLoc = 0; while (resultLoc < feedReply.first.mutations.size()) { @@ -2137,59 +2452,164 @@ ACTOR Future localChangeFeedStream(StorageServer* data, } } } catch (Error& e) { - TraceEvent(SevError, "LocalChangeFeedError", data->thisServerID).error(e); + if (e.code() == error_code_unknown_change_feed) { + TEST(true); // CF was moved away, no more local data to merge with + // Send endVersion so local stream is effectively done. We couldn't have send that already, because that + // would mean the stream would have finished without error + results.send(MutationsAndVersionRef(end, invalidVersion)); + } else { + TraceEvent(SevError, "LocalChangeFeedError", data->thisServerID) + .error(e) + .detail("CFID", rangeID.printable()); + } throw; } } -ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamRequest req) { +// Change feed stream must be sent an error as soon as it is moved away, or change feed can get incorrect results +ACTOR Future stopChangeFeedOnMove(StorageServer* data, ChangeFeedStreamRequest req, UID streamUID) { + wait(delay(0, TaskPriority::DefaultEndpoint)); + + auto feed = data->uidChangeFeed.find(req.rangeID); + if (feed == data->uidChangeFeed.end() || feed->second->removing) { + req.reply.sendError(unknown_change_feed()); + return Void(); + } + state Promise moved; + feed->second->triggerOnMove(req.range, streamUID, moved); + try { + wait(moved.getFuture()); + } catch (Error& e) { + ASSERT(e.code() == error_code_operation_cancelled); + // remove from tracking + + auto feed = data->uidChangeFeed.find(req.rangeID); + if (feed != data->uidChangeFeed.end()) { + feed->second->removeOnMoveTrigger(req.range, streamUID); + } + return Void(); + } + TEST(true); // Change feed moved away cancelling queries + // DO NOT call req.reply.onReady before sending - we need to propagate this error through regardless of how far + // behind client is + req.reply.sendError(wrong_shard_server()); + return Void(); +} + +ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamRequest req, UID streamUID) { state Span span("SS:getChangeFeedStream"_loc, { req.spanContext }); state bool atLatest = false; - state UID streamUID = deterministicRandom()->randomUniqueID(); state bool removeUID = false; state Optional blockedVersion; - req.reply.setByteLimit(SERVER_KNOBS->RANGESTREAM_LIMIT_BYTES); + if (req.replyBufferSize <= 0) { + req.reply.setByteLimit(SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES); + } else { + req.reply.setByteLimit(std::min((int64_t)req.replyBufferSize, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES)); + } wait(delay(0, TaskPriority::DefaultEndpoint)); try { + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedStreamStart", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("CanReadPopped", req.canReadPopped); + } + data->activeFeedQueries++; + + // send an empty version at begin - 1 to establish the stream quickly + ChangeFeedStreamReply emptyInitialReply; + MutationsAndVersionRef emptyInitialVersion; + emptyInitialVersion.version = req.begin - 1; + emptyInitialReply.mutations.push_back_deep(emptyInitialReply.arena, emptyInitialVersion); + ASSERT(emptyInitialReply.atLatestVersion == false); + ASSERT(emptyInitialReply.minStreamVersion == invalidVersion); + req.reply.send(emptyInitialReply); + + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedStreamSentInitialEmpty", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("CanReadPopped", req.canReadPopped) + .detail("Version", req.begin - 1); + } + loop { Future onReady = req.reply.onReady(); - if (atLatest && !onReady.isReady()) { + if (atLatest && !onReady.isReady() && !removeUID) { data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()][streamUID] = blockedVersion.present() ? blockedVersion.get() : data->prevVersion; + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedStreamBlockedOnReady", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("CanReadPopped", req.canReadPopped) + .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion); + } removeUID = true; } wait(onReady); - state Future> feedReplyFuture = - getChangeFeedMutations(data, req, false); + // keep this as not state variable so it is freed after sending to reduce memory + Future> feedReplyFuture = + getChangeFeedMutations(data, req, false, atLatest, streamUID); if (atLatest && !removeUID && !feedReplyFuture.isReady()) { data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()][streamUID] = blockedVersion.present() ? blockedVersion.get() : data->prevVersion; removeUID = true; + if (DEBUG_CF_TRACE) { + TraceEvent(SevDebug, "TraceChangeFeedStreamBlockedMutations", data->thisServerID) + .detail("FeedID", req.rangeID) + .detail("StreamUID", streamUID) + .detail("Range", req.range) + .detail("Begin", req.begin) + .detail("End", req.end) + .detail("CanReadPopped", req.canReadPopped) + .detail("Version", blockedVersion.present() ? blockedVersion.get() : data->prevVersion); + } } std::pair _feedReply = wait(feedReplyFuture); ChangeFeedStreamReply feedReply = _feedReply.first; bool gotAll = _feedReply.second; + ASSERT(feedReply.mutations.size() > 0); req.begin = feedReply.mutations.back().version + 1; if (!atLatest && gotAll) { atLatest = true; } + auto& clientVersions = data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()]; Version minVersion = removeUID ? data->version.get() : data->prevVersion; if (removeUID) { - data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()].erase(streamUID); - removeUID = false; + if (gotAll || req.begin == req.end) { + data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()].erase(streamUID); + removeUID = false; + } else { + data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()][streamUID] = + feedReply.mutations.back().version; + } } for (auto& it : clientVersions) { minVersion = std::min(minVersion, it.second); } feedReply.atLatestVersion = atLatest; - feedReply.minStreamVersion = gotAll ? minVersion : feedReply.mutations.back().version; + feedReply.minStreamVersion = minVersion; + + data->counters.feedRowsQueried += feedReply.mutations.size(); + data->counters.feedBytesQueried += feedReply.mutations.expectedSize(); + req.reply.send(feedReply); - if (feedReply.mutations.back().version == req.end - 1) { + if (req.begin == req.end) { req.reply.sendError(end_of_stream()); return Void(); } @@ -2198,26 +2618,27 @@ ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques auto feed = data->uidChangeFeed.find(req.rangeID); if (feed == data->uidChangeFeed.end() || feed->second->removing) { req.reply.sendError(unknown_change_feed()); - return Void(); + // throw to delete from changeFeedClientVersions if present + throw unknown_change_feed(); } + state Version emptyBefore = feed->second->emptyVersion; choose { - when(wait(feed->second->newMutations.onTrigger())) { - } // FIXME: check that this is triggered when the range is moved to a different - // server, also check that the stream is closed + when(wait(feed->second->newMutations.onTrigger())) {} when(wait(req.end == std::numeric_limits::max() ? Future(Never()) : data->version.whenAtLeast(req.end))) {} - when(wait(delay(5.0))) {} // TODO REMOVE this once empty version logic is fully implemented } auto feed = data->uidChangeFeed.find(req.rangeID); if (feed == data->uidChangeFeed.end() || feed->second->removing) { req.reply.sendError(unknown_change_feed()); - return Void(); + // throw to delete from changeFeedClientVersions if present + throw unknown_change_feed(); } } else { blockedVersion = feedReply.mutations.back().version; } } } catch (Error& e) { + data->activeFeedQueries--; auto it = data->changeFeedClientVersions.find(req.reply.getEndpoint().getPrimaryAddress()); if (it != data->changeFeedClientVersions.end()) { if (removeUID) { @@ -2239,11 +2660,7 @@ ACTOR Future changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques ACTOR Future changeFeedVersionUpdateQ(StorageServer* data, ChangeFeedVersionUpdateRequest req) { wait(data->version.whenAtLeast(req.minVersion)); wait(delay(0)); - auto& clientVersions = data->changeFeedClientVersions[req.reply.getEndpoint().getPrimaryAddress()]; - Version minVersion = data->version.get(); - for (auto& it : clientVersions) { - minVersion = std::min(minVersion, it.second); - } + Version minVersion = data->minFeedVersionForAddress(req.reply.getEndpoint().getPrimaryAddress()); req.reply.send(ChangeFeedVersionUpdateReply(minVersion)); return Void(); } @@ -3924,42 +4341,9 @@ Optional clipMutation(MutationRef const& m, KeyRangeRef range) { return Optional(); } -// Return true if the mutation need to be applied, otherwise (it's a CompareAndClear mutation and failed the comparison) -// false. -bool expandMutation(MutationRef& m, - StorageServer::VersionedData const& data, - UpdateEagerReadInfo* eager, - KeyRef eagerTrustedEnd, - Arena& ar) { +bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, UpdateEagerReadInfo* eager, Arena& ar) { // After this function call, m should be copied into an arena immediately (before modifying data, shards, or eager) - if (m.type == MutationRef::ClearRange) { - // Expand the clear - const auto& d = data.atLatest(); - - // If another clear overlaps the beginning of this one, engulf it - auto i = d.lastLess(m.param1); - if (i && i->isClearTo() && i->getEndKey() >= m.param1) - m.param1 = i.key(); - - // If another clear overlaps the end of this one, engulf it; otherwise expand - i = d.lastLessOrEqual(m.param2); - if (i && i->isClearTo() && i->getEndKey() >= m.param2) { - m.param2 = i->getEndKey(); - } else if (SERVER_KNOBS->ENABLE_CLEAR_RANGE_EAGER_READS) { - // Expand to the next set or clear (from storage or latestVersion), and if it - // is a clear, engulf it as well - i = d.lower_bound(m.param2); - KeyRef endKeyAtStorageVersion = - m.param2 == eagerTrustedEnd ? eagerTrustedEnd : std::min(eager->getKeyEnd(m.param2), eagerTrustedEnd); - if (!i || endKeyAtStorageVersion < i.key()) - m.param2 = endKeyAtStorageVersion; - else if (i->isClearTo()) - m.param2 = i->getEndKey(); - else - m.param2 = i.key(); - } - } else if (m.type != MutationRef::SetValue && (m.type)) { - + if (m.type != MutationRef::ClearRange && m.type != MutationRef::SetValue) { Optional oldVal; auto it = data.atLatest().lastLessOrEqual(m.param1); if (it != data.atLatest().end() && it->isValue() && it.key() == m.param1) @@ -4010,22 +4394,53 @@ bool expandMutation(MutationRef& m, if (oldVal.present() && m.param2 == oldVal.get()) { m.type = MutationRef::ClearRange; m.param2 = keyAfter(m.param1, ar); - return expandMutation(m, data, eager, eagerTrustedEnd, ar); + return true; } return false; } m.type = MutationRef::SetValue; } - return true; } +void expandClear(MutationRef& m, + StorageServer::VersionedData const& data, + UpdateEagerReadInfo* eager, + KeyRef eagerTrustedEnd) { + // After this function call, m should be copied into an arena immediately (before modifying data, shards, or eager) + ASSERT(m.type == MutationRef::ClearRange); + // Expand the clear + const auto& d = data.atLatest(); + + // If another clear overlaps the beginning of this one, engulf it + auto i = d.lastLess(m.param1); + if (i && i->isClearTo() && i->getEndKey() >= m.param1) + m.param1 = i.key(); + + // If another clear overlaps the end of this one, engulf it; otherwise expand + i = d.lastLessOrEqual(m.param2); + if (i && i->isClearTo() && i->getEndKey() >= m.param2) { + m.param2 = i->getEndKey(); + } else if (SERVER_KNOBS->ENABLE_CLEAR_RANGE_EAGER_READS) { + // Expand to the next set or clear (from storage or latestVersion), and if it + // is a clear, engulf it as well + i = d.lower_bound(m.param2); + KeyRef endKeyAtStorageVersion = + m.param2 == eagerTrustedEnd ? eagerTrustedEnd : std::min(eager->getKeyEnd(m.param2), eagerTrustedEnd); + if (!i || endKeyAtStorageVersion < i.key()) + m.param2 = endKeyAtStorageVersion; + else if (i->isClearTo()) + m.param2 = i->getEndKey(); + else + m.param2 = i.key(); + } +} + void applyMutation(StorageServer* self, MutationRef const& m, Arena& arena, StorageServer::VersionedData& data, - Version version, - bool fromFetch) { + Version version) { // m is expected to be in arena already // Clear split keys are added to arena StorageMetrics metrics; @@ -4058,43 +4473,63 @@ void applyMutation(StorageServer* self, } data.insert(m.param1, ValueOrClearToRef::value(m.param2)); self->watches.trigger(m.param1); - - if (!fromFetch) { - for (auto& it : self->keyChangeFeed[m.param1]) { - if (!it->stopped) { - if (it->mutations.empty() || it->mutations.back().version != version) { - it->mutations.push_back(MutationsAndVersionRef(version, self->knownCommittedVersion)); - } - it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m); - self->currentChangeFeeds.insert(it->id); - - DEBUG_MUTATION("ChangeFeedWriteSet", version, m, self->thisServerID) - .detail("Range", it->range) - .detail("ChangeFeedID", it->id); - } - } - } } else if (m.type == MutationRef::ClearRange) { data.erase(m.param1, m.param2); ASSERT(m.param2 > m.param1); ASSERT(!data.isClearContaining(data.atLatest(), m.param1)); data.insert(m.param1, ValueOrClearToRef::clearTo(m.param2)); self->watches.triggerRange(m.param1, m.param2); + } +} - if (!fromFetch) { - auto ranges = self->keyChangeFeed.intersectingRanges(KeyRangeRef(m.param1, m.param2)); - for (auto& r : ranges) { - for (auto& it : r.value()) { - if (!it->stopped) { - if (it->mutations.empty() || it->mutations.back().version != version) { - it->mutations.push_back(MutationsAndVersionRef(version, self->knownCommittedVersion)); - } - it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m); - self->currentChangeFeeds.insert(it->id); - DEBUG_MUTATION("ChangeFeedWriteClear", version, m, self->thisServerID) - .detail("Range", it->range) - .detail("ChangeFeedID", it->id); +void applyChangeFeedMutation(StorageServer* self, MutationRef const& m, Version version) { + if (m.type == MutationRef::SetValue) { + for (auto& it : self->keyChangeFeed[m.param1]) { + if (version < it->stopVersion && !it->removing && version > it->emptyVersion) { + if (it->mutations.empty() || it->mutations.back().version != version) { + it->mutations.push_back(MutationsAndVersionRef(version, self->knownCommittedVersion.get())); + } + it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m); + self->currentChangeFeeds.insert(it->id); + + DEBUG_MUTATION("ChangeFeedWriteSet", version, m, self->thisServerID) + .detail("Range", it->range) + .detail("ChangeFeedID", it->id); + } else { + TEST(version <= it->emptyVersion); // Skip CF write because version <= emptyVersion + TEST(it->removing); // Skip CF write because removing + TEST(version >= it->stopVersion); // Skip CF write because stopped + DEBUG_MUTATION("ChangeFeedWriteSetIgnore", version, m, self->thisServerID) + .detail("Range", it->range) + .detail("ChangeFeedID", it->id) + .detail("StopVersion", it->stopVersion) + .detail("EmptyVersion", it->emptyVersion) + .detail("Removing", it->removing); + } + } + } else if (m.type == MutationRef::ClearRange) { + auto ranges = self->keyChangeFeed.intersectingRanges(KeyRangeRef(m.param1, m.param2)); + for (auto& r : ranges) { + for (auto& it : r.value()) { + if (version < it->stopVersion && !it->removing && version > it->emptyVersion) { + if (it->mutations.empty() || it->mutations.back().version != version) { + it->mutations.push_back(MutationsAndVersionRef(version, self->knownCommittedVersion.get())); } + it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), m); + self->currentChangeFeeds.insert(it->id); + DEBUG_MUTATION("ChangeFeedWriteClear", version, m, self->thisServerID) + .detail("Range", it->range) + .detail("ChangeFeedID", it->id); + } else { + TEST(version <= it->emptyVersion); // Skip CF clear because version <= emptyVersion + TEST(it->removing); // Skip CF clear because removing + TEST(version >= it->stopVersion); // Skip CF clear because stopped + DEBUG_MUTATION("ChangeFeedWriteClearIgnore", version, m, self->thisServerID) + .detail("Range", it->range) + .detail("ChangeFeedID", it->id) + .detail("StopVersion", it->stopVersion) + .detail("EmptyVersion", it->emptyVersion) + .detail("Removing", it->removing); } } } @@ -4309,135 +4744,359 @@ ACTOR Future tryGetRange(PromiseStream results, Transaction* } } -ACTOR Future fetchChangeFeedApplier(StorageServer* data, - Reference changeFeedInfo, - Key rangeId, - KeyRange range, - Version fetchVersion, - bool existing) { +// We have to store the version the change feed was stopped at in the SS instead of just the stopped status +// In addition to simplifying stopping logic, it enables communicating stopped status when fetching change feeds +// from other SS correctly +const Value changeFeedSSValue(KeyRangeRef const& range, Version popVersion, Version stopVersion) { + BinaryWriter wr(IncludeVersion(ProtocolVersion::withChangeFeed())); + wr << range; + wr << popVersion; + wr << stopVersion; + return wr.toValue(); +} + +std::tuple decodeChangeFeedSSValue(ValueRef const& value) { + KeyRange range; + Version popVersion, stopVersion; + BinaryReader reader(value, IncludeVersion()); + reader >> range; + reader >> popVersion; + reader >> stopVersion; + return std::make_tuple(range, popVersion, stopVersion); +} + +ACTOR Future changeFeedPopQ(StorageServer* self, ChangeFeedPopRequest req) { + // if a SS restarted and is way behind, wait for it to at least have caught up through the pop version + wait(self->version.whenAtLeast(req.version)); + wait(delay(0)); + + if (!self->isReadable(req.range)) { + req.reply.sendError(wrong_shard_server()); + return Void(); + } + auto feed = self->uidChangeFeed.find(req.rangeID); + if (feed == self->uidChangeFeed.end()) { + req.reply.sendError(unknown_change_feed()); + return Void(); + } + + TraceEvent(SevDebug, "ChangeFeedPopQuery", self->thisServerID) + .detail("RangeID", req.rangeID.printable()) + .detail("Version", req.version) + .detail("SSVersion", self->version.get()) + .detail("Range", req.range.toString()); + + if (req.version - 1 > feed->second->emptyVersion) { + feed->second->emptyVersion = req.version - 1; + while (!feed->second->mutations.empty() && feed->second->mutations.front().version < req.version) { + feed->second->mutations.pop_front(); + } + Version durableVersion = self->data().getLatestVersion(); + auto& mLV = self->addVersionToMutationLog(durableVersion); + self->addMutationToMutationLog( + mLV, + MutationRef( + MutationRef::SetValue, + persistChangeFeedKeys.begin.toString() + feed->second->id.toString(), + changeFeedSSValue(feed->second->range, feed->second->emptyVersion + 1, feed->second->stopVersion))); + if (feed->second->storageVersion != invalidVersion) { + ++self->counters.kvSystemClearRanges; + self->addMutationToMutationLog(mLV, + MutationRef(MutationRef::ClearRange, + changeFeedDurableKey(feed->second->id, 0), + changeFeedDurableKey(feed->second->id, req.version))); + if (req.version > feed->second->storageVersion) { + feed->second->storageVersion = invalidVersion; + feed->second->durableVersion = invalidVersion; + } + } + wait(self->durableVersion.whenAtLeast(durableVersion)); + } + req.reply.send(Void()); + return Void(); +} + +// FIXME: there's a decent amount of duplicated code around fetching and popping change feeds +// Returns max version fetched +ACTOR Future fetchChangeFeedApplier(StorageServer* data, + Reference changeFeedInfo, + Key rangeId, + KeyRange range, + Version emptyVersion, + Version beginVersion, + Version endVersion) { + + state Version startVersion = beginVersion; + startVersion = std::max(startVersion, emptyVersion + 1); + startVersion = std::max(startVersion, changeFeedInfo->fetchVersion + 1); + startVersion = std::max(startVersion, changeFeedInfo->durableFetchVersion.get() + 1); + + ASSERT(startVersion >= 0); + + if (startVersion >= endVersion || (changeFeedInfo->removing)) { + TEST(true); // Change Feed popped before fetch + TraceEvent(SevDebug, "FetchChangeFeedNoOp", data->thisServerID) + .detail("RangeID", rangeId.printable()) + .detail("Range", range.toString()) + .detail("StartVersion", startVersion) + .detail("EndVersion", endVersion) + .detail("Removing", changeFeedInfo->removing); + return invalidVersion; + } + state Reference feedResults = makeReference(); state Future feed = data->cx->getChangeFeedStream( - feedResults, rangeId, 0, existing ? fetchVersion + 1 : data->version.get() + 1, range); + feedResults, rangeId, startVersion, endVersion, range, SERVER_KNOBS->CHANGEFEEDSTREAM_LIMIT_BYTES, true); - if (!existing) { - try { - loop { - Standalone> res = waitNext(feedResults->mutations.getFuture()); - for (auto& it : res) { - if (it.mutations.size()) { - data->storage.writeKeyValue( - KeyValueRef(changeFeedDurableKey(rangeId, it.version), - changeFeedDurableValue(it.mutations, it.knownCommittedVersion))); - changeFeedInfo->storageVersion = std::max(changeFeedInfo->durableVersion, it.version); - changeFeedInfo->durableVersion = changeFeedInfo->storageVersion; - } - } - wait(yield()); - } - } catch (Error& e) { - if (e.code() != error_code_end_of_stream) { - throw; - } - return Void(); - } - } + state Version firstVersion = invalidVersion; + state Version lastVersion = invalidVersion; + state int64_t versionsFetched = 0; state PromiseStream> localResults; - // Add 2 to fetch version to make sure the local stream will have more versions in the stream than the remote stream + // Add 1 to fetch version to make sure the local stream will have more versions in the stream than the remote stream // to avoid edge cases in the merge logic - state Future localStream = localChangeFeedStream(data, localResults, rangeId, 0, fetchVersion + 2, range); + + state Future localStream = + localChangeFeedStream(data, localResults, rangeId, startVersion, endVersion + 1, range); state Standalone localResult; Standalone _localResult = waitNext(localResults.getFuture()); localResult = _localResult; try { loop { + while (data->fetchKeysBudgetUsed.get()) { + wait(data->fetchKeysBudgetUsed.onChange()); + } + state Standalone> remoteResult = waitNext(feedResults->mutations.getFuture()); state int remoteLoc = 0; + while (remoteLoc < remoteResult.size()) { - if (remoteResult[remoteLoc].version < localResult.version) { - if (remoteResult[remoteLoc].mutations.size()) { + if (feedResults->popVersion - 1 > changeFeedInfo->emptyVersion) { + TEST(true); // CF fetched updated popped version from src SS + changeFeedInfo->emptyVersion = feedResults->popVersion - 1; + // pop mutations + while (!changeFeedInfo->mutations.empty() && + changeFeedInfo->mutations.front().version <= changeFeedInfo->emptyVersion) { + changeFeedInfo->mutations.pop_front(); + } + auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); + data->addMutationToMutationLog( + mLV, + MutationRef(MutationRef::SetValue, + persistChangeFeedKeys.begin.toString() + changeFeedInfo->id.toString(), + changeFeedSSValue(changeFeedInfo->range, + changeFeedInfo->emptyVersion + 1, + changeFeedInfo->stopVersion))); + data->addMutationToMutationLog( + mLV, + MutationRef(MutationRef::ClearRange, + changeFeedDurableKey(changeFeedInfo->id, 0), + changeFeedDurableKey(changeFeedInfo->id, feedResults->popVersion))); + ++data->counters.kvSystemClearRanges; + } + + Version localVersion = localResult.version; + Version remoteVersion = remoteResult[remoteLoc].version; + + if (remoteVersion <= localVersion) { + if (remoteVersion > changeFeedInfo->emptyVersion) { + // merge if same version + if (remoteVersion == localVersion && remoteResult[remoteLoc].mutations.size() && + remoteResult[remoteLoc].mutations.back().param1 != lastEpochEndPrivateKey) { + int remoteSize = remoteResult[remoteLoc].mutations.size(); + ASSERT(localResult.mutations.size()); + remoteResult[remoteLoc].mutations.append( + remoteResult.arena(), localResult.mutations.begin(), localResult.mutations.size()); + if (MUTATION_TRACKING_ENABLED) { + int midx = 0; + for (auto& m : remoteResult[remoteLoc].mutations) { + DEBUG_MUTATION("ChangeFeedWriteMoveMerge", remoteVersion, m, data->thisServerID) + .detail("Range", range) + .detail("FromLocal", midx >= remoteSize) + .detail("ChangeFeedID", rangeId); + midx++; + } + } + } else { + if (MUTATION_TRACKING_ENABLED) { + for (auto& m : remoteResult[remoteLoc].mutations) { + DEBUG_MUTATION("ChangeFeedWriteMove", remoteVersion, m, data->thisServerID) + .detail("Range", range) + .detail("ChangeFeedID", rangeId); + } + } + } + data->storage.writeKeyValue( - KeyValueRef(changeFeedDurableKey(rangeId, remoteResult[remoteLoc].version), + KeyValueRef(changeFeedDurableKey(rangeId, remoteVersion), changeFeedDurableValue(remoteResult[remoteLoc].mutations, remoteResult[remoteLoc].knownCommittedVersion))); - changeFeedInfo->storageVersion = - std::max(changeFeedInfo->durableVersion, remoteResult[remoteLoc].version); - changeFeedInfo->durableVersion = changeFeedInfo->storageVersion; + ++data->counters.kvSystemClearRanges; + changeFeedInfo->fetchVersion = std::max(changeFeedInfo->fetchVersion, remoteVersion); + + if (firstVersion == invalidVersion) { + firstVersion = remoteVersion; + } + lastVersion = remoteVersion; + versionsFetched++; + } else { + TEST(true); // Change feed ignoring write on move because it was popped concurrently + if (MUTATION_TRACKING_ENABLED) { + for (auto& m : remoteResult[remoteLoc].mutations) { + DEBUG_MUTATION("ChangeFeedWriteMoveIgnore", remoteVersion, m, data->thisServerID) + .detail("Range", range) + .detail("ChangeFeedID", rangeId) + .detail("EmptyVersion", changeFeedInfo->emptyVersion); + } + } + if (versionsFetched > 0) { + ASSERT(firstVersion != invalidVersion); + ASSERT(lastVersion != invalidVersion); + data->storage.clearRange( + KeyRangeRef(changeFeedDurableKey(changeFeedInfo->id, firstVersion), + changeFeedDurableKey(changeFeedInfo->id, lastVersion + 1))); + ++data->counters.kvSystemClearRanges; + firstVersion = invalidVersion; + lastVersion = invalidVersion; + versionsFetched = 0; + } } remoteLoc++; - } else if (remoteResult[remoteLoc].version == localResult.version) { - if (remoteResult[remoteLoc].mutations.size()) { - ASSERT(localResult.mutations.size()); - remoteResult[remoteLoc].mutations.append( - remoteResult.arena(), localResult.mutations.begin(), localResult.mutations.size()); - data->storage.writeKeyValue( - KeyValueRef(changeFeedDurableKey(rangeId, remoteResult[remoteLoc].version), - changeFeedDurableValue(remoteResult[remoteLoc].mutations, - remoteResult[remoteLoc].knownCommittedVersion))); - changeFeedInfo->storageVersion = - std::max(changeFeedInfo->durableVersion, remoteResult[remoteLoc].version); - changeFeedInfo->durableVersion = changeFeedInfo->storageVersion; - } - remoteLoc++; - Standalone _localResult = waitNext(localResults.getFuture()); - localResult = _localResult; - } else { + } + if (localVersion <= remoteVersion) { + // Do this once per wait instead of once per version for efficiency + data->fetchingChangeFeeds.insert(changeFeedInfo->id); Standalone _localResult = waitNext(localResults.getFuture()); localResult = _localResult; } } + // Do this once per wait instead of once per version for efficiency + data->fetchingChangeFeeds.insert(changeFeedInfo->id); + + data->counters.feedBytesFetched += remoteResult.expectedSize(); + data->fetchKeysBytesBudget -= remoteResult.expectedSize(); + if (data->fetchKeysBytesBudget <= 0) { + data->fetchKeysBudgetUsed.set(true); + } wait(yield()); } } catch (Error& e) { if (e.code() != error_code_end_of_stream) { + TraceEvent(SevDebug, "FetchChangeFeedError", data->thisServerID) + .errorUnsuppressed(e) + .detail("RangeID", rangeId.printable()) + .detail("Range", range.toString()) + .detail("EndVersion", endVersion); throw; } } - return Void(); -} -ACTOR Future fetchChangeFeed(StorageServer* data, - Key rangeId, - KeyRange range, - bool stopped, - Version fetchVersion) { - state Reference changeFeedInfo; - wait(delay(0)); // allow this actor to be cancelled by removals - state bool existing = data->uidChangeFeed.count(rangeId); - - TraceEvent(SevDebug, "FetchChangeFeed", data->thisServerID) - .detail("RangeID", rangeId.printable()) - .detail("Range", range.toString()) - .detail("Existing", existing); - - if (!existing) { - changeFeedInfo = Reference(new ChangeFeedInfo()); - changeFeedInfo->range = range; - changeFeedInfo->id = rangeId; - changeFeedInfo->stopped = stopped; - data->uidChangeFeed[rangeId] = changeFeedInfo; - auto rs = data->keyChangeFeed.modify(range); - for (auto r = rs.begin(); r != rs.end(); ++r) { - r->value().push_back(changeFeedInfo); + if (feedResults->popVersion - 1 > changeFeedInfo->emptyVersion) { + TEST(true); // CF fetched updated popped version from src SS at end + changeFeedInfo->emptyVersion = feedResults->popVersion - 1; + while (!changeFeedInfo->mutations.empty() && + changeFeedInfo->mutations.front().version <= changeFeedInfo->emptyVersion) { + changeFeedInfo->mutations.pop_front(); } - data->keyChangeFeed.coalesce(range.contents()); auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); data->addMutationToMutationLog( mLV, MutationRef(MutationRef::SetValue, - persistChangeFeedKeys.begin.toString() + rangeId.toString(), - changeFeedValue(range, invalidVersion, ChangeFeedStatus::CHANGE_FEED_CREATE))); - } else { - changeFeedInfo = data->uidChangeFeed[rangeId]; + persistChangeFeedKeys.begin.toString() + changeFeedInfo->id.toString(), + changeFeedSSValue( + changeFeedInfo->range, changeFeedInfo->emptyVersion + 1, changeFeedInfo->stopVersion))); + data->addMutationToMutationLog(mLV, + MutationRef(MutationRef::ClearRange, + changeFeedDurableKey(changeFeedInfo->id, 0), + changeFeedDurableKey(changeFeedInfo->id, feedResults->popVersion))); + ++data->counters.kvSystemClearRanges; + } + + // if we were popped or removed while fetching but it didn't pass the fetch version while writing, clean up here + if (versionsFetched > 0 && startVersion < changeFeedInfo->emptyVersion) { + TEST(true); // Change feed cleaning up popped data after move + ASSERT(firstVersion != invalidVersion); + ASSERT(lastVersion != invalidVersion); + Version endClear = std::min(lastVersion + 1, changeFeedInfo->emptyVersion); + if (endClear > firstVersion) { + auto& mLV2 = data->addVersionToMutationLog(data->data().getLatestVersion()); + data->addMutationToMutationLog(mLV2, + MutationRef(MutationRef::ClearRange, + changeFeedDurableKey(changeFeedInfo->id, firstVersion), + changeFeedDurableKey(changeFeedInfo->id, endClear))); + ++data->counters.kvSystemClearRanges; + } + } + + TraceEvent(SevDebug, "FetchChangeFeedDone", data->thisServerID) + .detail("RangeID", rangeId.printable()) + .detail("Range", range.toString()) + .detail("StartVersion", startVersion) + .detail("EndVersion", endVersion) + .detail("EmptyVersion", changeFeedInfo->emptyVersion) + .detail("FirstFetchedVersion", firstVersion) + .detail("LastFetchedVersion", lastVersion) + .detail("VersionsFetched", versionsFetched) + .detail("Removed", changeFeedInfo->removing); + return lastVersion; +} + +// returns largest version fetched +ACTOR Future fetchChangeFeed(StorageServer* data, + Reference changeFeedInfo, + Version beginVersion, + Version endVersion) { + wait(delay(0)); // allow this actor to be cancelled by removals + + // bound active change feed fetches + wait(data->fetchChangeFeedParallelismLock.take(TaskPriority::DefaultYield)); + state FlowLock::Releaser holdingFCFPL(data->fetchChangeFeedParallelismLock); + + TraceEvent(SevDebug, "FetchChangeFeed", data->thisServerID) + .detail("RangeID", changeFeedInfo->id.printable()) + .detail("Range", changeFeedInfo->range.toString()) + .detail("BeginVersion", beginVersion) + .detail("EndVersion", endVersion); + + auto cleanupPending = data->changeFeedCleanupDurable.find(changeFeedInfo->id); + if (cleanupPending != data->changeFeedCleanupDurable.end()) { + TEST(true); // Change feed waiting for dirty previous move to finish + TraceEvent(SevDebug, "FetchChangeFeedWaitCleanup", data->thisServerID) + .detail("RangeID", changeFeedInfo->id.printable()) + .detail("Range", changeFeedInfo->range.toString()) + .detail("CleanupVersion", cleanupPending->second) + .detail("EmptyVersion", changeFeedInfo->emptyVersion) + .detail("BeginVersion", beginVersion) + .detail("EndVersion", endVersion); + wait(data->durableVersion.whenAtLeast(cleanupPending->second + 1)); + wait(delay(0)); + // shard might have gotten moved away (again) while we were waiting + auto cleanupPendingAfter = data->changeFeedCleanupDurable.find(changeFeedInfo->id); + if (cleanupPendingAfter != data->changeFeedCleanupDurable.end()) { + ASSERT(cleanupPendingAfter->second >= endVersion); + TraceEvent(SevDebug, "FetchChangeFeedCancelledByCleanup", data->thisServerID) + .detail("RangeID", changeFeedInfo->id.printable()) + .detail("Range", changeFeedInfo->range.toString()) + .detail("BeginVersion", beginVersion) + .detail("EndVersion", endVersion); + return invalidVersion; + } } loop { try { - wait(fetchChangeFeedApplier(data, changeFeedInfo, rangeId, range, fetchVersion, existing)); - return Void(); + Version maxFetched = wait(fetchChangeFeedApplier(data, + changeFeedInfo, + changeFeedInfo->id, + changeFeedInfo->range, + changeFeedInfo->emptyVersion, + beginVersion, + endVersion)); + data->fetchingChangeFeeds.insert(changeFeedInfo->id); + return maxFetched; } catch (Error& e) { if (e.code() != error_code_change_feed_not_registered) { throw; @@ -4447,24 +5106,143 @@ ACTOR Future fetchChangeFeed(StorageServer* data, } } -ACTOR Future dispatchChangeFeeds(StorageServer* data, UID fetchKeysID, KeyRange keys, Version fetchVersion) { +ACTOR Future> fetchChangeFeedMetadata(StorageServer* data, KeyRange keys, Version fetchVersion) { + TraceEvent(SevDebug, "FetchChangeFeedMetadata", data->thisServerID) + .detail("Range", keys.toString()) + .detail("FetchVersion", fetchVersion); + std::vector feeds = wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion + 1)); + std::vector feedIds; + feedIds.reserve(feeds.size()); + // create change feed metadata if it does not exist + for (auto& cfEntry : feeds) { + auto cleanupEntry = data->changeFeedCleanupDurable.find(cfEntry.rangeId); + bool cleanupPending = cleanupEntry != data->changeFeedCleanupDurable.end(); + feedIds.push_back(cfEntry.rangeId); + auto existingEntry = data->uidChangeFeed.find(cfEntry.rangeId); + bool existing = existingEntry != data->uidChangeFeed.end(); + + TraceEvent(SevDebug, "FetchedChangeFeedInfo", data->thisServerID) + .detail("RangeID", cfEntry.rangeId.printable()) + .detail("Range", cfEntry.range.toString()) + .detail("FetchVersion", fetchVersion) + .detail("EmptyVersion", cfEntry.emptyVersion) + .detail("StopVersion", cfEntry.stopVersion) + .detail("Existing", existing) + .detail("CleanupPendingVersion", cleanupPending ? cleanupEntry->second : invalidVersion); + + bool addMutationToLog = false; + Reference changeFeedInfo; + + if (!existing) { + TEST(cleanupPending); // Fetch change feed which is cleanup pending. This means there was a move away and a + // move back, this will remake the metadata + + changeFeedInfo = Reference(new ChangeFeedInfo()); + changeFeedInfo->range = cfEntry.range; + changeFeedInfo->id = cfEntry.rangeId; + + changeFeedInfo->emptyVersion = cfEntry.emptyVersion; + changeFeedInfo->stopVersion = cfEntry.stopVersion; + data->uidChangeFeed[cfEntry.rangeId] = changeFeedInfo; + auto rs = data->keyChangeFeed.modify(cfEntry.range); + for (auto r = rs.begin(); r != rs.end(); ++r) { + r->value().push_back(changeFeedInfo); + } + data->keyChangeFeed.coalesce(cfEntry.range.contents()); + + addMutationToLog = true; + } else { + changeFeedInfo = existingEntry->second; + auto feedCleanup = data->changeFeedCleanupDurable.find(cfEntry.rangeId); + + if (cfEntry.stopVersion < changeFeedInfo->stopVersion) { + TEST(true); // Change feed updated stop version from fetch metadata + changeFeedInfo->stopVersion = cfEntry.stopVersion; + addMutationToLog = true; + } + + if (feedCleanup != data->changeFeedCleanupDurable.end() && changeFeedInfo->removing) { + TEST(true); // re-fetching feed scheduled for deletion! Un-mark it as removing + if (cfEntry.emptyVersion < data->version.get()) { + changeFeedInfo->emptyVersion = cfEntry.emptyVersion; + } + + changeFeedInfo->removing = false; + // reset fetch versions because everything previously fetched was cleaned up + changeFeedInfo->fetchVersion = invalidVersion; + changeFeedInfo->durableFetchVersion = NotifiedVersion(); + + // Since cleanup put a mutation in the log to delete the change feed data, put one in the log to restore + // it + // We may just want to refactor this so updateStorage does explicit deletes based on + // changeFeedCleanupDurable and not use the mutation log at all for the change feed metadata cleanup. + // Then we wouldn't have to reset anything here + addMutationToLog = true; + } + } + if (addMutationToLog) { + ASSERT(changeFeedInfo.isValid()); + auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); + data->addMutationToMutationLog( + mLV, + MutationRef( + MutationRef::SetValue, + persistChangeFeedKeys.begin.toString() + cfEntry.rangeId.toString(), + changeFeedSSValue(cfEntry.range, changeFeedInfo->emptyVersion + 1, changeFeedInfo->stopVersion))); + // if we updated pop version, remove mutations + while (!changeFeedInfo->mutations.empty() && + changeFeedInfo->mutations.front().version <= changeFeedInfo->emptyVersion) { + changeFeedInfo->mutations.pop_front(); + } + } + } + return feedIds; +} + +// returns max version fetched for each feed +// newFeedIds is used for the second fetch to get data for new feeds that weren't there for the first fetch +ACTOR Future> dispatchChangeFeeds(StorageServer* data, + UID fetchKeysID, + KeyRange keys, + Version beginVersion, + Version endVersion, + std::vector feedIds, + std::unordered_set newFeedIds) { + state std::unordered_map feedMaxFetched; + if (feedIds.empty() && newFeedIds.empty()) { + return feedMaxFetched; + } + // find overlapping range feeds - state std::map> feedFetches; + state std::map> feedFetches; state PromiseStream removals; data->changeFeedRemovals[fetchKeysID] = removals; try { - state std::vector feeds = - wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion + 1)); - for (auto& feed : feeds) { - feedFetches[feed.rangeId] = fetchChangeFeed(data, feed.rangeId, feed.range, feed.stopped, fetchVersion); + for (auto& feedId : feedIds) { + auto feedIt = data->uidChangeFeed.find(feedId); + // feed may have been moved away or deleted after move was scheduled, do nothing in that case + if (feedIt != data->uidChangeFeed.end() && !feedIt->second->removing) { + feedFetches[feedIt->second->id] = fetchChangeFeed(data, feedIt->second, beginVersion, endVersion); + } + } + for (auto& feedId : newFeedIds) { + auto feedIt = data->uidChangeFeed.find(feedId); + // we just read the change feed data map earlier in fetchKeys without yielding, so these feeds must exist + ASSERT(feedIt != data->uidChangeFeed.end()); + ASSERT(!feedIt->second->removing); + feedFetches[feedIt->second->id] = fetchChangeFeed(data, feedIt->second, 0, endVersion); } loop { - Future nextFeed = Never(); + Future nextFeed = Never(); if (!removals.getFuture().isReady()) { bool done = true; while (!feedFetches.empty()) { if (feedFetches.begin()->second.isReady()) { + Version maxFetched = feedFetches.begin()->second.get(); + if (maxFetched != invalidVersion) { + feedFetches[feedFetches.begin()->first] = maxFetched; + } feedFetches.erase(feedFetches.begin()); } else { nextFeed = feedFetches.begin()->second; @@ -4474,12 +5252,12 @@ ACTOR Future dispatchChangeFeeds(StorageServer* data, UID fetchKeysID, Key } if (done) { data->changeFeedRemovals.erase(fetchKeysID); - return Void(); + return feedMaxFetched; } } choose { when(Key remove = waitNext(removals.getFuture())) { feedFetches.erase(remove); } - when(wait(nextFeed)) {} + when(wait(success(nextFeed))) {} } } @@ -4510,12 +5288,24 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { // This allows adding->start() to be called inline with CSK. wait(data->coreStarted.getFuture() && delay(0)); + // On SS Reboot, durableVersion == latestVersion, so any mutations we add to the mutation log would be skipped if + // added before latest version advances. + // To ensure this doesn't happen, we wait for version to increase by one if this fetchKeys was initiated by a + // changeServerKeys from restoreDurableState + if (data->version.get() == data->durableVersion.get()) { + wait(data->version.whenAtLeast(data->version.get() + 1)); + wait(delay(0)); + } + try { DEBUG_KEY_RANGE("fetchKeysBegin", data->version.get(), shard->keys, data->thisServerID); TraceEvent(SevDebug, interval.begin(), data->thisServerID) .detail("KeyBegin", shard->keys.begin) - .detail("KeyEnd", shard->keys.end); + .detail("KeyEnd", shard->keys.end) + .detail("Version", data->version.get()); + + state Future> fetchCFMetadata = fetchChangeFeedMetadata(data, keys, data->version.get()); validate(data); @@ -4548,6 +5338,12 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { // Fetch keys gets called while the update actor is processing mutations. data->version will not be updated // until all mutations for a version have been processed. We need to take the durableVersionLock to ensure // data->version is greater than the version of the mutation which caused the fetch to be initiated. + + // We must also ensure we have fetched all change feed metadata BEFORE changing the phase to fetching to ensure + // change feed mutations get applied correctly + state std::vector changeFeedsToFetch; + std::vector _cfToFetch = wait(fetchCFMetadata); + changeFeedsToFetch = _cfToFetch; wait(data->durableVersionLock.take()); shard->phase = AddingShard::Fetching; @@ -4736,13 +5532,17 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { // being recovered. Instead we wait for the updateStorage loop to commit something (and consequently also what // we have written) + state Future> feedFetchMain = dispatchChangeFeeds( + data, fetchKeysID, keys, 0, fetchVersion + 1, changeFeedsToFetch, std::unordered_set()); + state Future fetchDurable = data->durableVersion.whenAtLeast(data->storageVersion() + 1); state Future dataArrive = data->version.whenAtLeast(fetchVersion); - wait(dispatchChangeFeeds(data, fetchKeysID, keys, fetchVersion)); holdingFKPL.release(); wait(dataArrive && fetchDurable); + state std::unordered_map feedFetchedVersions = wait(feedFetchMain); + TraceEvent(SevDebug, "FKAfterFinalCommit", data->thisServerID) .detail("FKID", interval.pairID) .detail("SV", data->storageVersion()) @@ -4758,7 +5558,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { FetchInjectionInfo* batch = wait(p.getFuture()); TraceEvent(SevDebug, "FKUpdateBatch", data->thisServerID).detail("FKID", interval.pairID); - shard->phase = AddingShard::Waiting; + shard->phase = AddingShard::FetchingCF; ASSERT(data->version.get() >= fetchVersion); // Choose a transferredVersion. This choice and timing ensure that // * The transferredVersion can be mutated in versionedData @@ -4773,6 +5573,24 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { ASSERT(shard->transferredVersion > data->storageVersion()); ASSERT(shard->transferredVersion == data->data().getLatestVersion()); + // find new change feeds for this range that didn't exist when we started the fetch + auto ranges = data->keyChangeFeed.intersectingRanges(keys); + std::unordered_set newChangeFeeds; + for (auto& r : ranges) { + for (auto& cfInfo : r.value()) { + TEST(true); // SS fetching new change feed that didn't exist when fetch started + newChangeFeeds.insert(cfInfo->id); + } + } + for (auto& cfId : changeFeedsToFetch) { + newChangeFeeds.erase(cfId); + } + // This is split into two fetches to reduce tail. Fetch [0 - fetchVersion+1) + // once fetchVersion is finalized, and [fetchVersion+1, transferredVersion) here once transferredVersion is + // finalized. Also fetch new change feeds alongside it + state Future> feedFetchTransferred = dispatchChangeFeeds( + data, fetchKeysID, keys, fetchVersion + 1, shard->transferredVersion, changeFeedsToFetch, newChangeFeeds); + TraceEvent(SevDebug, "FetchKeysHaveData", data->thisServerID) .detail("FKID", interval.pairID) .detail("Version", shard->transferredVersion) @@ -4812,6 +5630,27 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { shard->updates.clear(); + // wait on change feed fetch to complete writing to storage before marking data as available + std::unordered_map feedFetchedVersions2 = wait(feedFetchTransferred); + for (auto& newFetch : feedFetchedVersions2) { + auto prevFetch = feedFetchedVersions.find(newFetch.first); + if (prevFetch != feedFetchedVersions.end()) { + prevFetch->second = std::max(prevFetch->second, newFetch.second); + } else { + feedFetchedVersions[newFetch.first] = newFetch.second; + } + } + + shard->phase = AddingShard::Waiting; + + // Similar to transferred version, but wait for all feed data and + Version feedTransferredVersion = data->version.get() + 1; + + TraceEvent(SevDebug, "FetchKeysHaveFeedData", data->thisServerID) + .detail("FKID", interval.pairID) + .detail("Version", feedTransferredVersion) + .detail("StorageVersion", data->storageVersion()); + setAvailableStatus(data, keys, true); // keys will be available when getLatestVersion()==transferredVersion is durable @@ -4819,8 +5658,8 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { // Note that since it receives a pointer to FetchInjectionInfo, the thread does not leave this actor until this // point. - // Wait for the transferredVersion (and therefore the shard data) to be committed and durable. - wait(data->durableVersion.whenAtLeast(shard->transferredVersion)); + // Wait for the transferred version (and therefore the shard data) to be committed and durable. + wait(data->durableVersion.whenAtLeast(feedTransferredVersion)); ASSERT(data->shards[shard->keys.begin]->assigned() && data->shards[shard->keys.begin]->keys == @@ -4842,7 +5681,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { .detail("Version", data->version.get()); if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) { - if (shard->phase < AddingShard::Waiting) { + if (shard->phase < AddingShard::FetchingCF) { data->storage.clearRange(keys); ++data->counters.kvSystemClearRanges; data->byteSampleApplyClear(keys, invalidVersion); @@ -4903,33 +5742,7 @@ void AddingShard::addMutation(Version version, bool fromFetch, MutationRef const } // Add the mutation to the version. updates.back().mutations.push_back_deep(updates.back().arena(), mutation); - if (!fromFetch) { - if (mutation.type == MutationRef::SetValue) { - for (auto& it : server->keyChangeFeed[mutation.param1]) { - if (!it->stopped) { - if (it->mutations.empty() || it->mutations.back().version != version) { - it->mutations.push_back(MutationsAndVersionRef(version, server->knownCommittedVersion)); - } - it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), mutation); - server->currentChangeFeeds.insert(it->id); - } - } - } else if (mutation.type == MutationRef::ClearRange) { - auto ranges = server->keyChangeFeed.intersectingRanges(KeyRangeRef(mutation.param1, mutation.param2)); - for (auto& r : ranges) { - for (auto& it : r.value()) { - if (!it->stopped) { - if (it->mutations.empty() || it->mutations.back().version != version) { - it->mutations.push_back(MutationsAndVersionRef(version, server->knownCommittedVersion)); - } - it->mutations.back().mutations.push_back_deep(it->mutations.back().arena(), mutation); - server->currentChangeFeeds.insert(it->id); - } - } - } - } - } - } else if (phase == Waiting) { + } else if (phase == FetchingCF || phase == Waiting) { server->addMutation(version, fromFetch, mutation, keys, server->updateEagerReads); } else ASSERT(false); @@ -5092,6 +5905,7 @@ void changeServerKeys(StorageServer* data, } validate(data); + // find any change feeds that no longer have shards on this server, and clean them up if (!nowAssigned) { std::map candidateFeeds; auto ranges = data->keyChangeFeed.intersectingRanges(keys); @@ -5109,9 +5923,18 @@ void changeServerKeys(StorageServer* data, break; } } + if (!foundAssigned) { + Version durableVersion = data->data().getLatestVersion(); + TraceEvent(SevDebug, "ChangeFeedCleanup", data->thisServerID) + .detail("FeedID", f.first) + .detail("Version", version) + .detail("DurableVersion", durableVersion); + + data->changeFeedCleanupDurable[f.first] = durableVersion; + Key beginClearKey = f.first.withPrefix(persistChangeFeedKeys.begin); - auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); + auto& mLV = data->addVersionToMutationLog(durableVersion); data->addMutationToMutationLog( mLV, MutationRef(MutationRef::ClearRange, beginClearKey, keyAfter(beginClearKey))); ++data->counters.kvSystemClearRanges; @@ -5119,21 +5942,24 @@ void changeServerKeys(StorageServer* data, MutationRef(MutationRef::ClearRange, changeFeedDurableKey(f.first, 0), changeFeedDurableKey(f.first, version))); - ++data->counters.kvSystemClearRanges; - auto rs = data->keyChangeFeed.modify(f.second); - for (auto r = rs.begin(); r != rs.end(); ++r) { - auto& feedList = r->value(); - for (int i = 0; i < feedList.size(); i++) { - if (feedList[i]->id == f.first) { - swapAndPop(&feedList, i--); - } - } - } + + // We can't actually remove this change feed fully until the mutations clearing its data become durable. + // If the SS restarted at version R before the clearing mutations became durable at version D (R < D), + // then the restarted SS would restore the change feed clients would be able to read data and would miss + // mutations from versions [R, D), up until we got the private mutation triggering the cleanup again. + auto feed = data->uidChangeFeed.find(f.first); if (feed != data->uidChangeFeed.end()) { + feed->second->emptyVersion = version - 1; feed->second->removing = true; + feed->second->moved(feed->second->range); feed->second->newMutations.trigger(); - data->uidChangeFeed.erase(feed); + } + } else { + // if just part of feed's range is moved away + auto feed = data->uidChangeFeed.find(f.first); + if (feed != data->uidChangeFeed.end()) { + feed->second->moved(keys); } } } @@ -5162,16 +5988,29 @@ void StorageServer::addMutation(Version version, KeyRangeRef const& shard, UpdateEagerReadInfo* eagerReads) { MutationRef expanded = mutation; + MutationRef + nonExpanded; // need to keep non-expanded but atomic converted version of clear mutations for change feeds auto& mLog = addVersionToMutationLog(version); - if (!expandMutation(expanded, data(), eagerReads, shard.end, mLog.arena())) { + if (!convertAtomicOp(expanded, data(), eagerReads, mLog.arena())) { return; } + if (expanded.type == MutationRef::ClearRange) { + nonExpanded = expanded; + expandClear(expanded, data(), eagerReads, shard.end); + } expanded = addMutationToMutationLog(mLog, expanded); DEBUG_MUTATION("applyMutation", version, expanded, thisServerID) .detail("ShardBegin", shard.begin) .detail("ShardEnd", shard.end); - applyMutation(this, expanded, mLog.arena(), mutableData(), version, fromFetch); + + if (!fromFetch) { + // have to do change feed before applyMutation because nonExpanded wasn't copied into the mutation log arena, + // and thus would go out of scope if it wasn't copied into the change feed arena + applyChangeFeedMutation(this, expanded.type == MutationRef::ClearRange ? nonExpanded : expanded, version); + } + applyMutation(this, expanded, mLog.arena(), mutableData(), version); + // printf("\nSSUpdate: Printing versioned tree after applying mutation\n"); // mutableData().printTree(version); } @@ -5213,10 +6052,9 @@ public: applyPrivateData(data, m); } } else { - // FIXME: enable when DEBUG_MUTATION is active - // for(auto m = changes[c].mutations.begin(); m; ++m) { - // DEBUG_MUTATION("SSUpdateMutation", changes[c].version, *m, data->thisServerID); - //} + if (MUTATION_TRACKING_ENABLED) { + DEBUG_MUTATION("SSUpdateMutation", ver, m, data->thisServerID).detail("FromFetch", fromFetch); + } splitMutation(data, data->shards, m, ver, fromFetch); } @@ -5286,14 +6124,24 @@ private: .detail("FromVersion", fromVersion) .detail("ToVersion", rollbackVersion) .detail("AtVersion", currentVersion) + .detail("RestoredVersion", restoredVersion) .detail("StorageVersion", data->storageVersion()); ASSERT(rollbackVersion >= data->storageVersion()); rollback(data, rollbackVersion, currentVersion); + } else { + TraceEvent(SevDebug, "RollbackSkip", data->thisServerID) + .detail("FromVersion", fromVersion) + .detail("ToVersion", rollbackVersion) + .detail("AtVersion", currentVersion) + .detail("RestoredVersion", restoredVersion) + .detail("StorageVersion", data->storageVersion()); } for (auto& it : data->uidChangeFeed) { - it.second->mutations.push_back(MutationsAndVersionRef(currentVersion, rollbackVersion)); - it.second->mutations.back().mutations.push_back_deep(it.second->mutations.back().arena(), m); - data->currentChangeFeeds.insert(it.first); + if (!it.second->removing && currentVersion < it.second->stopVersion) { + it.second->mutations.push_back(MutationsAndVersionRef(currentVersion, rollbackVersion)); + it.second->mutations.back().mutations.push_back_deep(it.second->mutations.back().arena(), m); + data->currentChangeFeeds.insert(it.first); + } } data->recoveryVersionSkips.emplace_back(rollbackVersion, currentVersion - rollbackVersion); @@ -5343,76 +6191,126 @@ private: ChangeFeedStatus status; std::tie(changeFeedRange, popVersion, status) = decodeChangeFeedValue(m.param2); auto feed = data->uidChangeFeed.find(changeFeedId); - if (feed == data->uidChangeFeed.end()) { - if (status == ChangeFeedStatus::CHANGE_FEED_CREATE) { - TraceEvent(SevDebug, "AddingChangeFeed", data->thisServerID) - .detail("RangeID", changeFeedId.printable()) - .detail("Range", changeFeedRange.toString()) - .detail("Version", currentVersion); - Reference changeFeedInfo(new ChangeFeedInfo()); - changeFeedInfo->range = changeFeedRange; - changeFeedInfo->id = changeFeedId; - changeFeedInfo->emptyVersion = currentVersion - 1; - data->uidChangeFeed[changeFeedId] = changeFeedInfo; - auto rs = data->keyChangeFeed.modify(changeFeedRange); - for (auto r = rs.begin(); r != rs.end(); ++r) { - r->value().push_back(changeFeedInfo); - } - data->keyChangeFeed.coalesce(changeFeedRange.contents()); - auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); - data->addMutationToMutationLog( - mLV, - MutationRef(MutationRef::SetValue, - persistChangeFeedKeys.begin.toString() + changeFeedId.toString(), - m.param2)); + TraceEvent(SevDebug, "ChangeFeedPrivateMutation", data->thisServerID) + .detail("RangeID", changeFeedId.printable()) + .detail("Range", changeFeedRange.toString()) + .detail("Version", currentVersion) + .detail("PopVersion", popVersion) + .detail("Status", status); + + // Because of data moves, we can get mutations operating on a change feed we don't yet know about, because + // the fetch hasn't started yet + bool createdFeed = false; + if (feed == data->uidChangeFeed.end() && status != ChangeFeedStatus::CHANGE_FEED_DESTROY) { + createdFeed = true; + + Reference changeFeedInfo(new ChangeFeedInfo()); + changeFeedInfo->range = changeFeedRange; + changeFeedInfo->id = changeFeedId; + if (status == ChangeFeedStatus::CHANGE_FEED_CREATE && popVersion == invalidVersion) { + // for a create, the empty version should be now, otherwise it will be set in a later pop + changeFeedInfo->emptyVersion = currentVersion - 1; + } else { + TEST(true); // SS got non-create change feed private mutation before move created its metadata + changeFeedInfo->emptyVersion = invalidVersion; } - } else { - if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY) { - Key beginClearKey = changeFeedId.withPrefix(persistChangeFeedKeys.begin); - auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); - data->addMutationToMutationLog( - mLV, MutationRef(MutationRef::ClearRange, beginClearKey, keyAfter(beginClearKey))); + changeFeedInfo->metadataCreateVersion = currentVersion; + data->uidChangeFeed[changeFeedId] = changeFeedInfo; + + feed = data->uidChangeFeed.find(changeFeedId); + ASSERT(feed != data->uidChangeFeed.end()); + + TraceEvent(SevDebug, "AddingChangeFeed", data->thisServerID) + .detail("RangeID", changeFeedId.printable()) + .detail("Range", changeFeedRange.toString()) + .detail("EmptyVersion", feed->second->emptyVersion); + + auto rs = data->keyChangeFeed.modify(changeFeedRange); + for (auto r = rs.begin(); r != rs.end(); ++r) { + r->value().push_back(changeFeedInfo); + } + data->keyChangeFeed.coalesce(changeFeedRange.contents()); + } + + bool popMutationLog = false; + bool addMutationToLog = false; + if (popVersion != invalidVersion && status != ChangeFeedStatus::CHANGE_FEED_DESTROY) { + // pop the change feed at pop version, no matter what state it is in + if (popVersion - 1 > feed->second->emptyVersion) { + feed->second->emptyVersion = popVersion - 1; + while (!feed->second->mutations.empty() && feed->second->mutations.front().version < popVersion) { + feed->second->mutations.pop_front(); + } + if (feed->second->storageVersion != invalidVersion) { + ++data->counters.kvSystemClearRanges; + // do this clear in the mutation log, as we want it to be committed consistently with the + // popVersion update + popMutationLog = true; + if (popVersion > feed->second->storageVersion) { + feed->second->storageVersion = invalidVersion; + feed->second->durableVersion = invalidVersion; + } + } + addMutationToLog = true; + } + + } else if (status == ChangeFeedStatus::CHANGE_FEED_CREATE && createdFeed) { + TraceEvent(SevDebug, "CreatingChangeFeed", data->thisServerID) + .detail("RangeID", changeFeedId.printable()) + .detail("Range", changeFeedRange.toString()) + .detail("Version", currentVersion); + // no-op, already created metadata + addMutationToLog = true; + } + if (status == ChangeFeedStatus::CHANGE_FEED_STOP && currentVersion < feed->second->stopVersion) { + TraceEvent(SevDebug, "StoppingChangeFeed", data->thisServerID) + .detail("RangeID", changeFeedId.printable()) + .detail("Range", changeFeedRange.toString()) + .detail("Version", currentVersion); + feed->second->stopVersion = currentVersion; + addMutationToLog = true; + } + if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY && !createdFeed) { + TraceEvent(SevDebug, "DestroyingChangeFeed", data->thisServerID) + .detail("RangeID", changeFeedId.printable()) + .detail("Range", changeFeedRange.toString()) + .detail("Version", currentVersion); + Key beginClearKey = changeFeedId.withPrefix(persistChangeFeedKeys.begin); + Version cleanupVersion = data->data().getLatestVersion(); + auto& mLV = data->addVersionToMutationLog(cleanupVersion); + data->addMutationToMutationLog( + mLV, MutationRef(MutationRef::ClearRange, beginClearKey, keyAfter(beginClearKey))); + ++data->counters.kvSystemClearRanges; + data->addMutationToMutationLog(mLV, + MutationRef(MutationRef::ClearRange, + changeFeedDurableKey(feed->second->id, 0), + changeFeedDurableKey(feed->second->id, currentVersion))); + ++data->counters.kvSystemClearRanges; + + feed->second->emptyVersion = currentVersion - 1; + feed->second->stopVersion = currentVersion; + feed->second->removing = true; + feed->second->moved(feed->second->range); + feed->second->newMutations.trigger(); + + data->changeFeedCleanupDurable[feed->first] = cleanupVersion; + } + + if (addMutationToLog) { + auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); + data->addMutationToMutationLog( + mLV, + MutationRef(MutationRef::SetValue, + persistChangeFeedKeys.begin.toString() + changeFeedId.toString(), + changeFeedSSValue( + feed->second->range, feed->second->emptyVersion + 1, feed->second->stopVersion))); + if (popMutationLog) { ++data->counters.kvSystemClearRanges; data->addMutationToMutationLog(mLV, MutationRef(MutationRef::ClearRange, changeFeedDurableKey(feed->second->id, 0), - changeFeedDurableKey(feed->second->id, currentVersion))); - ++data->counters.kvSystemClearRanges; - auto rs = data->keyChangeFeed.modify(feed->second->range); - for (auto r = rs.begin(); r != rs.end(); ++r) { - auto& feedList = r->value(); - for (int i = 0; i < feedList.size(); i++) { - if (feedList[i] == feed->second) { - swapAndPop(&feedList, i--); - } - } - } - data->uidChangeFeed.erase(feed); - } else { - if (popVersion != invalidVersion && popVersion - 1 > feed->second->emptyVersion) { - feed->second->emptyVersion = popVersion - 1; - while (!feed->second->mutations.empty() && - feed->second->mutations.front().version < popVersion) { - feed->second->mutations.pop_front(); - } - if (feed->second->storageVersion != invalidVersion) { - data->storage.clearRange(KeyRangeRef(changeFeedDurableKey(feed->second->id, 0), - changeFeedDurableKey(feed->second->id, popVersion))); - ++data->counters.kvSystemClearRanges; - if (popVersion > feed->second->storageVersion) { - feed->second->storageVersion = invalidVersion; - feed->second->durableVersion = invalidVersion; - } - } - } - feed->second->stopped = (status == ChangeFeedStatus::CHANGE_FEED_STOP); - auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion()); - data->addMutationToMutationLog( - mLV, - MutationRef(MutationRef::SetValue, - persistChangeFeedKeys.begin.toString() + changeFeedId.toString(), - m.param2)); + changeFeedDurableKey(feed->second->id, popVersion))); } } } else if ((m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) && @@ -5645,7 +6543,9 @@ ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { ++data->counters.updateBatches; data->lastTLogVersion = cursor->getMaxKnownVersion(); - data->knownCommittedVersion = cursor->getMinKnownCommittedVersion(); + if (cursor->getMinKnownCommittedVersion() > data->knownCommittedVersion.get()) { + data->knownCommittedVersion.set(cursor->getMinKnownCommittedVersion()); + } data->versionLag = std::max(0, data->lastTLogVersion - data->version.get()); ASSERT(*pReceivedUpdate == false); @@ -6105,27 +7005,51 @@ ACTOR Future updateStorage(StorageServer* data) { break; } - std::set modifiedChangeFeeds; + std::set modifiedChangeFeeds = data->fetchingChangeFeeds; + data->fetchingChangeFeeds.clear(); while (!data->changeFeedVersions.empty() && data->changeFeedVersions.front().second <= newOldestVersion) { modifiedChangeFeeds.insert(data->changeFeedVersions.front().first.begin(), data->changeFeedVersions.front().first.end()); data->changeFeedVersions.pop_front(); } + state std::vector> feedFetchVersions; + state std::vector updatedChangeFeeds(modifiedChangeFeeds.begin(), modifiedChangeFeeds.end()); state int curFeed = 0; while (curFeed < updatedChangeFeeds.size()) { auto info = data->uidChangeFeed.find(updatedChangeFeeds[curFeed]); if (info != data->uidChangeFeed.end()) { + // Cannot yield in mutation updating loop because of race with fetchVersion + Version alreadyFetched = std::max(info->second->fetchVersion, info->second->durableFetchVersion.get()); for (auto& it : info->second->mutations) { - if (it.version > newOldestVersion) { + if (it.version <= alreadyFetched) { + continue; + } else if (it.version > newOldestVersion) { break; } data->storage.writeKeyValue( KeyValueRef(changeFeedDurableKey(info->second->id, it.version), changeFeedDurableValue(it.mutations, it.knownCommittedVersion))); + // FIXME: there appears to be a bug somewhere where the exact same mutation appears twice in a row + // in the stream. We should fix this assert to be strictly > and re-enable it + ASSERT(it.version >= info->second->storageVersion); info->second->storageVersion = it.version; } + + if (info->second->fetchVersion != invalidVersion && !info->second->removing) { + feedFetchVersions.push_back(std::pair(info->second->id, info->second->fetchVersion)); + } + // handle case where fetch had version ahead of last in-memory mutation + if (alreadyFetched > info->second->storageVersion) { + info->second->storageVersion = std::min(alreadyFetched, newOldestVersion); + if (alreadyFetched > info->second->storageVersion) { + // This change feed still has pending mutations fetched and written to storage that are higher + // than the new durableVersion. To ensure its storage and durable version get updated, we need + // to add it back to fetchingChangeFeeds + data->fetchingChangeFeeds.insert(info->first); + } + } wait(yield(TaskPriority::UpdateStorage)); } curFeed++; @@ -6184,12 +7108,60 @@ ACTOR Future updateStorage(StorageServer* data) { while (!info->second->mutations.empty() && info->second->mutations.front().version < newOldestVersion) { info->second->mutations.pop_front(); } + ASSERT(info->second->storageVersion >= info->second->durableVersion); info->second->durableVersion = info->second->storageVersion; wait(yield(TaskPriority::UpdateStorage)); } curFeed++; } + // if commit included fetched data from this change feed, update the fetched durable version + curFeed = 0; + while (curFeed < feedFetchVersions.size()) { + auto info = data->uidChangeFeed.find(feedFetchVersions[curFeed].first); + // Don't update if the feed is pending cleanup. Either it will get cleaned up and destroyed, or it will get + // fetched again, where the fetch version will get reset. + if (info != data->uidChangeFeed.end() && !data->changeFeedCleanupDurable.count(info->second->id)) { + if (feedFetchVersions[curFeed].second > info->second->durableFetchVersion.get()) { + info->second->durableFetchVersion.set(feedFetchVersions[curFeed].second); + } + if (feedFetchVersions[curFeed].second == info->second->fetchVersion) { + // haven't fetched anything else since commit started, reset fetch version + info->second->fetchVersion = invalidVersion; + } + } + curFeed++; + } + + // remove any entries from changeFeedCleanupPending that were persisted + auto cfCleanup = data->changeFeedCleanupDurable.begin(); + while (cfCleanup != data->changeFeedCleanupDurable.end()) { + if (cfCleanup->second <= newOldestVersion) { + // remove from the data structure here, if it wasn't added back by another fetch or something + auto feed = data->uidChangeFeed.find(cfCleanup->first); + ASSERT(feed != data->uidChangeFeed.end()); + if (feed->second->removing) { + auto rs = data->keyChangeFeed.modify(feed->second->range); + for (auto r = rs.begin(); r != rs.end(); ++r) { + auto& feedList = r->value(); + for (int i = 0; i < feedList.size(); i++) { + if (feedList[i]->id == cfCleanup->first) { + swapAndPop(&feedList, i--); + } + } + } + data->keyChangeFeed.coalesce(feed->second->range.contents()); + + data->uidChangeFeed.erase(feed); + } else { + TEST(true); // Feed re-fetched after remove + } + cfCleanup = data->changeFeedCleanupDurable.erase(cfCleanup); + } else { + cfCleanup++; + } + } + durableInProgress.send(Void()); wait(delay(0, TaskPriority::UpdateStorage)); // Setting durableInProgess could cause the storage server to // shut down, so delay to check for cancellation @@ -6267,7 +7239,7 @@ void setAvailableStatus(StorageServer* self, KeyRangeRef keys, bool available) { availableKeys.begin, available ? LiteralStringRef("1") : LiteralStringRef("0"))); if (keys.end != allKeys.end) { - bool endAvailable = self->shards.rangeContaining(keys.end)->value()->isInVersionedData(); + bool endAvailable = self->shards.rangeContaining(keys.end)->value()->isCFInVersionedData(); self->addMutationToMutationLog(mLV, MutationRef(MutationRef::SetValue, availableKeys.end, @@ -6698,13 +7670,12 @@ ACTOR Future restoreDurableState(StorageServer* data, IKeyValueStore* stor for (feedLoc = 0; feedLoc < changeFeeds.size(); feedLoc++) { Key changeFeedId = changeFeeds[feedLoc].key.removePrefix(persistChangeFeedKeys.begin); KeyRange changeFeedRange; - Version popVersion; - ChangeFeedStatus status; - std::tie(changeFeedRange, popVersion, status) = decodeChangeFeedValue(changeFeeds[feedLoc].value); + Version popVersion, stopVersion; + std::tie(changeFeedRange, popVersion, stopVersion) = decodeChangeFeedSSValue(changeFeeds[feedLoc].value); TraceEvent(SevDebug, "RestoringChangeFeed", data->thisServerID) .detail("RangeID", changeFeedId.printable()) .detail("Range", changeFeedRange.toString()) - .detail("Status", status) + .detail("StopVersion", stopVersion) .detail("PopVer", popVersion); Reference changeFeedInfo(new ChangeFeedInfo()); changeFeedInfo->range = changeFeedRange; @@ -6712,7 +7683,7 @@ ACTOR Future restoreDurableState(StorageServer* data, IKeyValueStore* stor changeFeedInfo->durableVersion = version; changeFeedInfo->storageVersion = version; changeFeedInfo->emptyVersion = popVersion - 1; - changeFeedInfo->stopped = status == ChangeFeedStatus::CHANGE_FEED_STOP; + changeFeedInfo->stopVersion = stopVersion; data->uidChangeFeed[changeFeedId] = changeFeedInfo; auto rs = data->keyChangeFeed.modify(changeFeedRange); for (auto r = rs.begin(); r != rs.end(); ++r) { @@ -7290,7 +8261,8 @@ ACTOR Future serveChangeFeedStreamRequests(StorageServer* self, FutureStream changeFeedStream) { loop { ChangeFeedStreamRequest req = waitNext(changeFeedStream); - self->actors.add(changeFeedStreamQ(self, req)); + // must notify change feed that its shard is moved away ASAP + self->actors.add(changeFeedStreamQ(self, req, req.debugUID) || stopChangeFeedOnMove(self, req, req.debugUID)); } } diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index d4ae7de299..6863ec39c6 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -524,20 +524,21 @@ std::vector getDiskStores(std::string folder) { // Register the worker interf to cluster controller (cc) and // re-register the worker when key roles interface, e.g., cc, dd, ratekeeper, change. -ACTOR Future registrationClient(Reference> const> ccInterface, - WorkerInterface interf, - Reference> asyncPriorityInfo, - ProcessClass initialClass, - Reference> const> ddInterf, - Reference> const> rkInterf, - Reference> const> bmInterf, - Reference> const> ekpInterf, - Reference const> degraded, - Reference connRecord, - Reference> const> issues, - Reference configNode, - Reference localConfig, - Reference> dbInfo) { +ACTOR Future registrationClient( + Reference> const> ccInterface, + WorkerInterface interf, + Reference> asyncPriorityInfo, + ProcessClass initialClass, + Reference> const> ddInterf, + Reference> const> rkInterf, + Reference>> const> bmInterf, + Reference> const> ekpInterf, + Reference const> degraded, + Reference connRecord, + Reference> const> issues, + Reference configNode, + Reference localConfig, + Reference> dbInfo) { // Keeps the cluster controller (as it may be re-elected) informed that this worker exists // The cluster controller uses waitFailureClient to find out if we die, and returns from registrationReply // (requiring us to re-register) The registration request piggybacks optional distributor interface if it exists. @@ -567,7 +568,8 @@ ACTOR Future registrationClient(Referenceget(), rkInterf->get(), - bmInterf->get(), + bmInterf->get().present() ? bmInterf->get().get().second + : Optional(), ekpInterf->get(), degraded->get(), localConfig->lastSeenVersion(), @@ -1138,7 +1140,9 @@ ACTOR Future storageServerRollbackRebooter(std::set(), Reference(nullptr)); @@ -1375,6 +1379,24 @@ ACTOR Future chaosMetricsLogger() { } } +// like genericactors setWhenDoneOrError, but we need to take into account the bm epoch. We don't want to reset it if +// this manager was replaced by a later manager (with a higher epoch) on this worker +ACTOR Future resetBlobManagerWhenDoneOrError( + Future blobManagerProcess, + Reference>>> var, + int64_t epoch) { + try { + wait(blobManagerProcess); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + } + if (var->get().present() && var->get().get().first == epoch) { + var->set(Optional>()); + } + return Void(); +} + ACTOR Future workerServer(Reference connRecord, Reference> const> ccInterface, LocalityData locality, @@ -1396,7 +1418,8 @@ ACTOR Future workerServer(Reference connRecord, state Reference>> ddInterf( new AsyncVar>()); state Reference>> rkInterf(new AsyncVar>()); - state Reference>> bmInterf(new AsyncVar>()); + state Reference>>> bmEpochAndInterf( + new AsyncVar>>()); state Reference>> ekpInterf( new AsyncVar>()); state Future handleErrors = workerHandleErrors(errors.getFuture()); // Needs to be stopped last @@ -1417,6 +1440,7 @@ ACTOR Future workerServer(Reference connRecord, state std::map sharedLogs; state Reference> activeSharedTLog(new AsyncVar()); state WorkerCache backupWorkerCache; + state WorkerCache blobWorkerCache; state std::string coordFolder = abspath(_coordFolder); @@ -1529,6 +1553,7 @@ ACTOR Future workerServer(Reference connRecord, DUMPTOKEN(recruited.getValue); DUMPTOKEN(recruited.getKey); DUMPTOKEN(recruited.getKeyValues); + DUMPTOKEN(recruited.getMappedKeyValues); DUMPTOKEN(recruited.getShardState); DUMPTOKEN(recruited.waitMetrics); DUMPTOKEN(recruited.splitMetrics); @@ -1540,7 +1565,9 @@ ACTOR Future workerServer(Reference connRecord, DUMPTOKEN(recruited.getKeyValueStoreType); DUMPTOKEN(recruited.watchValue); DUMPTOKEN(recruited.getKeyValuesStream); - DUMPTOKEN(recruited.getMappedKeyValues); + DUMPTOKEN(recruited.changeFeedStream); + DUMPTOKEN(recruited.changeFeedPop); + DUMPTOKEN(recruited.changeFeedVersionUpdate); Promise recovery; Future f = storageServer(kv, recruited, dbInfo, folder, recovery, connRecord); @@ -1668,7 +1695,7 @@ ACTOR Future workerServer(Reference connRecord, initialClass, ddInterf, rkInterf, - bmInterf, + bmEpochAndInterf, ekpInterf, degraded, connRecord, @@ -1870,21 +1897,30 @@ ACTOR Future workerServer(Reference connRecord, BlobManagerInterface recruited(locality, req.reqId); recruited.initEndpoints(); - if (bmInterf->get().present()) { - recruited = bmInterf->get().get(); + if (bmEpochAndInterf->get().present() && bmEpochAndInterf->get().get().first == req.epoch) { + recruited = bmEpochAndInterf->get().get().second; + TEST(true); // Recruited while already a blob manager. } else { + // TODO: it'd be more optimal to halt the last manager if present here, but it will figure it out + // via the epoch check + // Also, not halting lets us handle the case here where the last BM had a higher + // epoch and somehow the epochs got out of order by a delayed initialize request. The one we start + // here will just halt on the lock check. startRole(Role::BLOB_MANAGER, recruited.id(), interf.id()); DUMPTOKEN(recruited.waitFailure); DUMPTOKEN(recruited.haltBlobManager); + DUMPTOKEN(recruited.haltBlobGranules); + DUMPTOKEN(recruited.blobManagerExclCheckReq); Future blobManagerProcess = blobManager(recruited, dbInfo, req.epoch); - errorForwarders.add(forwardError( - errors, - Role::BLOB_MANAGER, - recruited.id(), - setWhenDoneOrError(blobManagerProcess, bmInterf, Optional()))); - bmInterf->set(Optional(recruited)); + errorForwarders.add( + forwardError(errors, + Role::BLOB_MANAGER, + recruited.id(), + resetBlobManagerWhenDoneOrError(blobManagerProcess, bmEpochAndInterf, req.epoch))); + bmEpochAndInterf->set( + Optional>(std::pair(req.epoch, recruited))); } TraceEvent("BlobManagerReceived", req.reqId).detail("BlobManagerId", recruited.id()); req.reply.send(recruited); @@ -2028,6 +2064,7 @@ ACTOR Future workerServer(Reference connRecord, DUMPTOKEN(recruited.getValue); DUMPTOKEN(recruited.getKey); DUMPTOKEN(recruited.getKeyValues); + DUMPTOKEN(recruited.getMappedKeyValues); DUMPTOKEN(recruited.getShardState); DUMPTOKEN(recruited.waitMetrics); DUMPTOKEN(recruited.splitMetrics); @@ -2039,7 +2076,9 @@ ACTOR Future workerServer(Reference connRecord, DUMPTOKEN(recruited.getKeyValueStoreType); DUMPTOKEN(recruited.watchValue); DUMPTOKEN(recruited.getKeyValuesStream); - DUMPTOKEN(recruited.getMappedKeyValues); + DUMPTOKEN(recruited.changeFeedStream); + DUMPTOKEN(recruited.changeFeedPop); + DUMPTOKEN(recruited.changeFeedVersionUpdate); // printf("Recruited as storageServer\n"); std::string filename = @@ -2086,13 +2125,26 @@ ACTOR Future workerServer(Reference connRecord, } } when(InitializeBlobWorkerRequest req = waitNext(interf.blobWorker.getFuture())) { - BlobWorkerInterface recruited(locality, req.interfaceId); - recruited.initEndpoints(); - startRole(Role::BLOB_WORKER, recruited.id(), interf.id()); + if (!blobWorkerCache.exists(req.reqId)) { + BlobWorkerInterface recruited(locality, req.interfaceId); + recruited.initEndpoints(); + startRole(Role::BLOB_WORKER, recruited.id(), interf.id()); - ReplyPromise blobWorkerReady = req.reply; - Future bw = blobWorker(recruited, blobWorkerReady, dbInfo); - errorForwarders.add(forwardError(errors, Role::BLOB_WORKER, recruited.id(), bw)); + DUMPTOKEN(recruited.waitFailure); + DUMPTOKEN(recruited.blobGranuleFileRequest); + DUMPTOKEN(recruited.assignBlobRangeRequest); + DUMPTOKEN(recruited.revokeBlobRangeRequest); + DUMPTOKEN(recruited.granuleAssignmentsRequest); + DUMPTOKEN(recruited.granuleStatusStreamRequest); + DUMPTOKEN(recruited.haltBlobWorker); + + ReplyPromise blobWorkerReady = req.reply; + Future bw = blobWorker(recruited, blobWorkerReady, dbInfo); + errorForwarders.add(forwardError(errors, Role::BLOB_WORKER, recruited.id(), bw)); + + } else { + forwardPromise(req.reply, blobWorkerCache.get(req.reqId)); + } } when(InitializeCommitProxyRequest req = waitNext(interf.commitProxy.getFuture())) { LocalLineage _; diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp new file mode 100644 index 0000000000..ea43cebbb7 --- /dev/null +++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp @@ -0,0 +1,884 @@ +/* + * BlobGranuleCorrectnessWorkload.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "contrib/fmt-8.1.1/include/fmt/format.h" +#include "fdbclient/BlobGranuleReader.actor.h" +#include "fdbclient/ManagementAPI.actor.h" +#include "fdbclient/NativeAPI.actor.h" +#include "fdbclient/ReadYourWrites.h" +#include "fdbclient/SystemData.h" +#include "fdbserver/Knobs.h" +#include "fdbserver/TesterInterface.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "flow/Arena.h" +#include "flow/IRandom.h" +#include "flow/genericactors.actor.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +#define BGW_DEBUG true + +struct WriteData { + Version writeVersion; + Version clearVersion; + int32_t val; + int16_t valLength; + + // start as MAX_VERSION while uncommitted/uncleared so that they're ignored by concurrent readers + explicit WriteData(int32_t val, int16_t valLength) + : writeVersion(MAX_VERSION), clearVersion(MAX_VERSION), val(val), valLength(valLength) {} +}; + +struct KeyData { + int nextClearIdx; + std::vector writes; +}; + +static std::vector targetValSizes = { 40, 100, 500 }; + +struct ThreadData : ReferenceCounted, NonCopyable { + // directory info + int32_t directoryID; + KeyRange directoryRange; + + // key + value gen data + // in vector for efficient random selection + std::vector usedKeys; + // by key for tracking data + std::map keyData; + + std::deque writeVersions; + + // randomized parameters that can be different per directory + int targetByteRate; + bool nextKeySequential; + int16_t targetValLength; + double reuseKeyProb; + int targetIDsPerKey; + + // communication between workers + Promise firstWriteSuccessful; + Version minSuccessfulReadVersion = MAX_VERSION; + + // stats + int64_t errors = 0; + int64_t mismatches = 0; + int64_t reads = 0; + int64_t timeTravelReads = 0; + int64_t timeTravelTooOld = 0; + int64_t rowsRead = 0; + int64_t bytesRead = 0; + int64_t rowsWritten = 0; + int64_t bytesWritten = 0; + + ThreadData(uint32_t directoryID, int64_t targetByteRate) + : directoryID(directoryID), targetByteRate(targetByteRate) { + directoryRange = + KeyRangeRef(StringRef(format("%08x", directoryID)), StringRef(format("%08x", directoryID + 1))); + + targetByteRate *= (0.5 + deterministicRandom()->random01()); + + targetValLength = deterministicRandom()->randomChoice(targetValSizes); + targetValLength *= (0.5 + deterministicRandom()->random01()); + + nextKeySequential = deterministicRandom()->random01() < 0.5; + reuseKeyProb = 0.1 + (deterministicRandom()->random01() * 0.8); + targetIDsPerKey = 1 + deterministicRandom()->randomInt(1, 10); + + if (BGW_DEBUG) { + fmt::print("Directory {0} initialized with the following parameters:\n", directoryID); + fmt::print(" targetByteRate={0}\n", targetByteRate); + fmt::print(" targetValLength={0}\n", targetValLength); + fmt::print(" nextKeySequential={0}\n", nextKeySequential); + fmt::print(" reuseKeyProb={0}\n", reuseKeyProb); + fmt::print(" targetIDsPerKey={0}\n", targetIDsPerKey); + } + } + + // TODO could make keys variable length? + Key getKey(uint32_t key, uint32_t id) { return StringRef(format("%08x/%08x/%08x", directoryID, key, id)); } +}; + +// For debugging mismatches on what data should be and why +// set mismatch to true, dir id and key id to the directory and key id that are wrong, and rv to read version that read +// the wrong value +#define DEBUG_MISMATCH false +#define DEBUG_DIR_ID 0 +#define DEBUG_KEY_ID 0 +#define DEBUG_RV invalidVersion + +#define DEBUG_KEY_OP(dirId, keyId) BGW_DEBUG&& DEBUG_MISMATCH&& dirId == DEBUG_DIR_ID&& DEBUG_KEY_ID == keyId +#define DEBUG_READ_OP(dirId, rv) BGW_DEBUG&& DEBUG_MISMATCH&& dirId == DEBUG_DIR_ID&& rv == DEBUG_RV + +/* + * This is a stand-alone workload designed to validate blob granule correctness. + * By enabling distinct ranges and writing to those parts of the key space, we can control what parts of the key space + * are written to blob, and can validate that the granule data is correct at any desired version. + */ +struct BlobGranuleCorrectnessWorkload : TestWorkload { + bool doSetup; + double testDuration; + + // parameters global across all clients + int64_t targetByteRate; + + std::vector> directories; + std::vector> clients; + DatabaseConfiguration config; + Reference bstore; + + BlobGranuleCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + doSetup = !clientId; // only do this on the "first" client + testDuration = getOption(options, LiteralStringRef("testDuration"), 120.0); + + // randomize global test settings based on shared parameter to get similar workload across tests, but then vary + // different parameters within those constraints + int64_t randomness = sharedRandomNumber; + + // randomize between low and high directory count + int64_t targetDirectories = 1 + (randomness % 8); + randomness /= 8; + + int64_t targetMyDirectories = + (targetDirectories / clientCount) + ((targetDirectories % clientCount > clientId) ? 1 : 0); + + if (targetMyDirectories > 0) { + int myDirectories = 1; + if (targetMyDirectories > 1) { + myDirectories = deterministicRandom()->randomInt(1, 2 * targetMyDirectories + 1); + } + + // anywhere from 2 delta files per second to 1 delta file every 2 seconds, spread across all directories + int denom = std::min(clientCount, (int)targetDirectories); + targetByteRate = 2 * SERVER_KNOBS->BG_DELTA_FILE_TARGET_BYTES / (1 + (randomness % 4)) / denom; + randomness /= 4; + + // either do equal across all of my directories, or skewed + bool skewed = myDirectories > 1 && deterministicRandom()->random01() < 0.4; + int skewMultiplier; + if (skewed) { + // first directory has 1/2, second has 1/4, third has 1/8, etc... + skewMultiplier = 2; + targetByteRate /= 2; + } else { + skewMultiplier = 1; + targetByteRate /= myDirectories; + } + for (int i = 0; i < myDirectories; i++) { + // set up directory with its own randomness + uint32_t dirId = i * clientCount + clientId; + if (BGW_DEBUG) { + fmt::print("Client {0}/{1} creating directory {2}\n", clientId, clientCount, dirId); + } + directories.push_back(makeReference(dirId, targetByteRate)); + targetByteRate /= skewMultiplier; + } + } + } + + ACTOR Future setUpBlobRange(Database cx, KeyRange range) { + if (BGW_DEBUG) { + fmt::print( + "Setting up blob granule range for [{0} - {1})\n", range.begin.printable(), range.end.printable()); + } + state Reference tr = makeReference(cx); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString()); + wait(krmSetRange(tr, blobRangeKeys.begin, range, LiteralStringRef("1"))); + wait(tr->commit()); + if (BGW_DEBUG) { + fmt::print("Successfully set up blob granule range for [{0} - {1})\n", + range.begin.printable(), + range.end.printable()); + } + return Void(); + } catch (Error& e) { + wait(tr->onError(e)); + } + } + } + + std::string description() const override { return "BlobGranuleCorrectnessWorkload"; } + Future setup(Database const& cx) override { return _setup(cx, this); } + + ACTOR Future _setup(Database cx, BlobGranuleCorrectnessWorkload* self) { + if (self->doSetup) { + // FIXME: run the actual FDBCLI command instead of copy/pasting its implementation + wait(success(ManagementAPI::changeConfig(cx.getReference(), "blob_granules_enabled=1", true))); + } + + if (self->directories.empty()) { + return Void(); + } + + state int directoryIdx = 0; + for (; directoryIdx < self->directories.size(); directoryIdx++) { + // Set up the blob range first + wait(self->setUpBlobRange(cx, self->directories[directoryIdx]->directoryRange)); + } + + if (BGW_DEBUG) { + printf("Initializing Blob Granule Correctness s3 stuff\n"); + } + try { + if (g_network->isSimulated()) { + if (BGW_DEBUG) { + printf("Blob Granule Correctness constructing simulated backup container\n"); + } + self->bstore = BackupContainerFileSystem::openContainerFS("file://fdbblob/"); + } else { + if (BGW_DEBUG) { + printf("Blob Granule Correctness constructing backup container from %s\n", + SERVER_KNOBS->BG_URL.c_str()); + } + self->bstore = BackupContainerFileSystem::openContainerFS(SERVER_KNOBS->BG_URL); + if (BGW_DEBUG) { + printf("Blob Granule Correctness constructed backup container\n"); + } + } + } catch (Error& e) { + if (BGW_DEBUG) { + printf("Blob Granule Correctness got backup container init error %s\n", e.name()); + } + throw e; + } + + return Void(); + } + + // FIXME: typedef this pair type and/or chunk list + ACTOR Future>>> + readFromBlob(Database cx, BlobGranuleCorrectnessWorkload* self, KeyRange range, Version version) { + state RangeResult out; + state Standalone> chunks; + state Transaction tr(cx); + + loop { + try { + Standalone> chunks_ = wait(tr.readBlobGranules(range, 0, version)); + chunks = chunks_; + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + for (const BlobGranuleChunkRef& chunk : chunks) { + RangeResult chunkRows = wait(readBlobGranule(chunk, range, version, self->bstore)); + out.arena().dependsOn(chunkRows.arena()); + out.append(out.arena(), chunkRows.begin(), chunkRows.size()); + } + return std::pair(out, chunks); + } + + // handle retries + errors + // It's ok to reset the transaction here because its read version is only used for reading the granule mapping from + // the system keyspace + ACTOR Future doGrv(Transaction* tr) { + loop { + try { + Version readVersion = wait(tr->getReadVersion()); + return readVersion; + } catch (Error& e) { + wait(tr->onError(e)); + } + } + } + + ACTOR Future waitFirstSnapshot(BlobGranuleCorrectnessWorkload* self, + Database cx, + Reference threadData, + bool doSetup) { + // read entire keyspace at the start until granules for the entire thing are available + loop { + state Transaction tr(cx); + try { + Version rv = wait(self->doGrv(&tr)); + state Version readVersion = rv; + std::pair>> blob = + wait(self->readFromBlob(cx, self, threadData->directoryRange, readVersion)); + fmt::print("Directory {0} got {1} RV {2}\n", + threadData->directoryID, + doSetup ? "initial" : "final", + readVersion); + threadData->minSuccessfulReadVersion = readVersion; + return Void(); + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw e; + } + if (e.code() != error_code_blob_granule_transaction_too_old) { + wait(tr.onError(e)); + } else { + wait(delay(1.0)); + } + } + } + } + + void logMismatch(Reference threadData, + const Optional& lastMatching, + const Optional& expectedKey, + const Optional& blobKey, + const Optional& expectedValue, + const Optional& blobValue, + uint32_t startKey, + uint32_t endKey, + Version readVersion, + const std::pair>>& blob) { + threadData->mismatches++; + if (!BGW_DEBUG) { + return; + } + + TraceEvent ev(SevError, "BGMismatch"); + ev.detail("DirectoryID", format("%08x", threadData->directoryID)) + .detail("RangeStart", format("%08x", startKey)) + .detail("RangeEnd", format("%08x", endKey)) + .detail("Version", readVersion); + fmt::print("Found mismatch! Request for dir {0} [{1} - {2}) @ {3}\n", + format("%08x", threadData->directoryID), + format("%08x", startKey), + format("%08x", endKey), + readVersion); + if (lastMatching.present()) { + fmt::print(" last correct: {}\n", lastMatching.get().printable()); + } + if (expectedValue.present() || blobValue.present()) { + // value mismatch + ASSERT(blobKey.present()); + ASSERT(blobKey == expectedKey); + fmt::print(" Value mismatch for {0}.\n Expected={1}\n Actual={2}\n", + blobKey.get().printable(), + expectedValue.get().printable(), + blobValue.get().printable()); + } else { + // key mismatch + fmt::print(" Expected Key: {0}\n", expectedKey.present() ? expectedKey.get().printable() : ""); + fmt::print(" Actual Key: {0}\n", blobKey.present() ? blobKey.get().printable() : ""); + } + + fmt::print("Chunks:\n"); + for (auto& chunk : blob.second) { + fmt::print("[{0} - {1})\n", chunk.keyRange.begin.printable(), chunk.keyRange.end.printable()); + + fmt::print(" SnapshotFile:\n {}\n", + chunk.snapshotFile.present() ? chunk.snapshotFile.get().toString().c_str() : ""); + fmt::print(" DeltaFiles:\n"); + for (auto& df : chunk.deltaFiles) { + fmt::print(" {}\n", df.toString()); + } + fmt::print(" Deltas: ({})", chunk.newDeltas.size()); + if (chunk.newDeltas.size() > 0) { + fmt::print(" with version [{0} - {1}]", + chunk.newDeltas[0].version, + chunk.newDeltas[chunk.newDeltas.size() - 1].version); + } + fmt::print(" IncludedVersion: {}\n", chunk.includedVersion); + } + printf("\n"); + } + + Value genVal(uint32_t val, uint16_t valLen) { + std::string v(valLen, 'x'); + auto valFormatted = format("%08x", val); + ASSERT(valFormatted.size() <= v.size()); + + for (int i = 0; i < valFormatted.size(); i++) { + v[i] = valFormatted[i]; + } + // copy into an arena + // TODO do this in original arena? a bit more efficient that way + Arena a; + return Standalone(StringRef(a, v), a); + } + + bool validateValue(const Value& v, uint32_t val, uint16_t valLen) { + if (v.size() != valLen) { + return false; + } + // check for correct value portion + auto valFormatted = format("%08x", val); + ASSERT(valFormatted.size() <= v.size()); + if (v.substr(0, valFormatted.size()) != valFormatted) { + return false; + } + // check for corruption + for (int i = valFormatted.size(); i < v.size(); i++) { + if (v[i] != 'x') { + return false; + } + } + return true; + } + + bool validateResult(Reference threadData, + std::pair>> blob, + int startKeyInclusive, + int endKeyExclusive, + Version beginVersion, + Version readVersion) { + auto checkIt = threadData->keyData.lower_bound(startKeyInclusive); + if (checkIt != threadData->keyData.end() && checkIt->first < startKeyInclusive) { + checkIt++; + } + int resultIdx = 0; + Optional lastMatching; + if (DEBUG_READ_OP(threadData->directoryID, readVersion)) { + fmt::print("DBG READ: [{0} - {1}) @ {2}\n", + format("%08x", startKeyInclusive), + format("%08x", endKeyExclusive), + readVersion); + } + + while (checkIt != threadData->keyData.end() && checkIt->first < endKeyExclusive) { + uint32_t key = checkIt->first; + if (DEBUG_READ_OP(threadData->directoryID, readVersion)) { + fmt::print("DBG READ: Key {0}\n", format("%08x", key)); + } + + // TODO could binary search this to find clearVersion if it gets long + int idIdx = 0; + for (; idIdx < checkIt->second.writes.size() && checkIt->second.writes[idIdx].clearVersion <= readVersion; + idIdx++) { + // iterate until we find the oldest tag that should have not been cleared + /*if (DEBUG_READ_OP(threadData->directoryID, readVersion)) { + fmt::print( + "DBG READ: Skip ID {0} cleared @ {1}\n", idIdx, checkIt->second.writes[idIdx].clearVersion); + }*/ + } + for (; idIdx < checkIt->second.writes.size() && checkIt->second.writes[idIdx].writeVersion <= readVersion; + idIdx++) { + Key nextKeyShouldBe = threadData->getKey(key, idIdx); + if (DEBUG_READ_OP(threadData->directoryID, readVersion)) { + fmt::print("DBG READ: Checking ID {0} ({1}) written @ {2}\n", + format("%08x", idIdx), + idIdx, + checkIt->second.writes[idIdx].writeVersion); + } + if (resultIdx >= blob.first.size()) { + // missing at end!! + logMismatch(threadData, + lastMatching, + nextKeyShouldBe, + Optional(), + Optional(), + Optional(), + startKeyInclusive, + endKeyExclusive, + readVersion, + blob); + return false; + } + + if (nextKeyShouldBe != blob.first[resultIdx].key) { + // key mismatch! + if (DEBUG_READ_OP(threadData->directoryID, readVersion)) { + printf("key mismatch!\n"); + } + logMismatch(threadData, + lastMatching, + nextKeyShouldBe, + blob.first[resultIdx].key, + Optional(), + Optional(), + startKeyInclusive, + endKeyExclusive, + readVersion, + blob); + return false; + } else if (!validateValue(blob.first[resultIdx].value, + checkIt->second.writes[idIdx].val, + checkIt->second.writes[idIdx].valLength)) { + logMismatch(threadData, + lastMatching, + nextKeyShouldBe, + blob.first[resultIdx].key, + genVal(checkIt->second.writes[idIdx].val, checkIt->second.writes[idIdx].valLength), + blob.first[resultIdx].value, + startKeyInclusive, + endKeyExclusive, + readVersion, + blob); + return false; + // value mismatch for same key + } else { + lastMatching = nextKeyShouldBe; + } + resultIdx++; + } + checkIt++; + } + + if (resultIdx < blob.first.size()) { + // blob has extra stuff!! + logMismatch(threadData, + lastMatching, + Optional(), + blob.first[resultIdx].key, + Optional(), + Optional(), + startKeyInclusive, + endKeyExclusive, + readVersion, + blob); + return false; + } + + return true; + } + + ACTOR Future readWorker(BlobGranuleCorrectnessWorkload* self, + Future firstSnapshot, + Database cx, + Reference threadData) { + state double last = now(); + state double targetBytesReadPerQuery = + SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES * 2.0 / deterministicRandom()->randomInt(1, 11); + + // read at higher read rate than write rate to validate data + state double targetReadBytesPerSec = threadData->targetByteRate * 4; + ASSERT(targetReadBytesPerSec > 0); + + state Version readVersion; + + TraceEvent("BlobGranuleCorrectnessReaderStart").log(); + if (BGW_DEBUG) { + printf("BGW read thread starting\n"); + } + + // wait for data to read + wait(firstSnapshot); + wait(threadData->firstWriteSuccessful.getFuture()); + + TraceEvent("BlobGranuleCorrectnessReaderReady").log(); + if (BGW_DEBUG) { + printf("BGW read thread ready\n"); + } + + loop { + try { + // Do 1 read + + // pick key range by doing random start key, and then picking the end key based on that + int startKeyIdx = deterministicRandom()->randomInt(0, threadData->usedKeys.size()); + state uint32_t startKey = threadData->usedKeys[startKeyIdx]; + auto endKeyIt = threadData->keyData.find(startKey); + ASSERT(endKeyIt != threadData->keyData.end()); + + int targetQueryBytes = (deterministicRandom()->randomInt(1, 20) * targetBytesReadPerQuery) / 10; + int estimatedQueryBytes = 0; + for (int i = 0; estimatedQueryBytes < targetQueryBytes && endKeyIt != threadData->keyData.end(); + i++, endKeyIt++) { + // iterate forward until end or target keys have passed + estimatedQueryBytes += (1 + endKeyIt->second.writes.size() - endKeyIt->second.nextClearIdx) * + threadData->targetValLength; + } + + state uint32_t endKey; + if (endKeyIt == threadData->keyData.end()) { + endKey = std::numeric_limits::max(); + } else { + endKey = endKeyIt->first; + } + + state KeyRange range = KeyRangeRef(threadData->getKey(startKey, 0), threadData->getKey(endKey, 0)); + + // pick read version + // TODO could also pick begin version here + ASSERT(threadData->writeVersions.back() >= threadData->minSuccessfulReadVersion); + // randomly choose up to date vs time travel read + if (deterministicRandom()->random01() < 0.5) { + threadData->reads++; + readVersion = threadData->writeVersions.back(); + } else { + threadData->timeTravelReads++; + loop { + int readVersionIdx = deterministicRandom()->randomInt(0, threadData->writeVersions.size()); + readVersion = threadData->writeVersions[readVersionIdx]; + if (readVersion >= threadData->minSuccessfulReadVersion) { + break; + } + } + } + + std::pair>> blob = + wait(self->readFromBlob(cx, self, range, readVersion)); + self->validateResult(threadData, blob, startKey, endKey, 0, readVersion); + + int resultBytes = blob.first.expectedSize(); + threadData->rowsRead += blob.first.size(); + threadData->bytesRead += resultBytes; + + wait(poisson(&last, (resultBytes + 1) / targetReadBytesPerSec)); + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { + throw; + } + if (e.code() == error_code_blob_granule_transaction_too_old) { + threadData->timeTravelTooOld++; + } else { + threadData->errors++; + if (BGW_DEBUG) { + printf("BGWorkload got unexpected error %s\n", e.name()); + } + } + } + } + } + + ACTOR Future writeWorker(BlobGranuleCorrectnessWorkload* self, + Future firstSnapshot, + Database cx, + Reference threadData) { + + state double last = now(); + state int keysPerQuery = 100; + // state int targetBytesPerQuery = threadData->targetValLength * keysPerQuery; + // state double targetTps = (1.0 * threadData->targetByteRate) / targetBytesPerQuery; + state uint32_t nextVal = 0; + + TraceEvent("BlobGranuleCorrectnessWriterStart").log(); + + wait(firstSnapshot); + + TraceEvent("BlobGranuleCorrectnessWriterReady").log(); + + loop { + state Transaction tr(cx); + + // pick rows to write and clear, generate values for writes + state std::vector> keyAndIdToWrite; + state std::vector> keyAndIdToClear; + + state int queryKeys = + keysPerQuery * (0.1 + deterministicRandom()->random01() * 1.8); // 10% to 190% of target keys per query + for (int i = 0; i < queryKeys; i++) { + uint32_t key; + if (threadData->keyData.empty() || deterministicRandom()->random01() > threadData->reuseKeyProb) { + // new key + if (threadData->nextKeySequential) { + key = threadData->usedKeys.size(); + } else { + key = std::numeric_limits::max(); + while (key == std::numeric_limits::max() || + threadData->keyData.find(key) != threadData->keyData.end()) { + key = deterministicRandom()->randomUInt32(); + } + } + + // add new key to data structures + threadData->usedKeys.push_back(key); + threadData->keyData.insert({ key, KeyData() }); + } else { + int keyIdx = deterministicRandom()->randomInt(0, threadData->usedKeys.size()); + key = threadData->usedKeys[keyIdx]; + } + + auto keyIt = threadData->keyData.find(key); + ASSERT(keyIt != threadData->keyData.end()); + + int unclearedIds = keyIt->second.writes.size() - keyIt->second.nextClearIdx; + // if we are at targetIDs, 50% chance of adding one or clearing. If we are closer to 0, higher chance of + // adding one, if we are closer to 2x target IDs, higher chance of clearing one + double probAddId = (threadData->targetIDsPerKey * 2.0 - unclearedIds) / threadData->targetIDsPerKey; + if (deterministicRandom()->random01() < probAddId || + keyIt->second.nextClearIdx == keyIt->second.writes.size()) { + int32_t val = nextVal++; + int16_t valLen = (0.5 + deterministicRandom()->random01()) * threadData->targetValLength; + if (valLen < 10) { + valLen = 10; + } + + uint32_t nextId = keyIt->second.writes.size(); + keyIt->second.writes.push_back(WriteData(val, valLen)); + + keyAndIdToWrite.push_back(std::tuple(key, nextId, val, valLen)); + } else { + uint32_t idToClear = keyIt->second.nextClearIdx++; + keyAndIdToClear.push_back(std::pair(key, idToClear)); + } + } + + state int64_t txnBytes; + loop { + try { + // write rows in txn + for (auto& it : keyAndIdToWrite) { + Value v = self->genVal(std::get<2>(it), std::get<3>(it)); + tr.set(threadData->getKey(std::get<0>(it), std::get<1>(it)), v); + } + for (auto& it : keyAndIdToClear) { + tr.clear(singleKeyRange(threadData->getKey(it.first, it.second))); + } + txnBytes = tr.getSize(); + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + + Version commitVersion = tr.getCommittedVersion(); + + // once txn is committed, update write map + + for (auto& it : keyAndIdToWrite) { + uint32_t key = std::get<0>(it); + uint32_t id = std::get<1>(it); + auto keyIt = threadData->keyData.find(key); + ASSERT(keyIt != threadData->keyData.end()); + + keyIt->second.writes[id].writeVersion = commitVersion; + if (DEBUG_KEY_OP(threadData->directoryID, key)) { + fmt::print("DBG: {0} WRITE {1} = {2}:{3}\n", + commitVersion, + format("%08x/%08x/%08x", threadData->directoryID, key, id), + std::get<2>(it), + std::get<3>(it)); + } + } + + for (auto& it : keyAndIdToClear) { + auto keyIt = threadData->keyData.find(it.first); + ASSERT(keyIt != threadData->keyData.end()); + keyIt->second.writes[it.second].clearVersion = commitVersion; + if (DEBUG_KEY_OP(threadData->directoryID, it.first)) { + fmt::print("DBG: {0} CLEAR {1}\n", + commitVersion, + format("%08x/%08x/%08x", threadData->directoryID, it.first, it.second)); + } + } + + threadData->writeVersions.push_back(commitVersion); + + if (threadData->firstWriteSuccessful.canBeSet()) { + threadData->firstWriteSuccessful.send(Void()); + } + + threadData->rowsWritten += queryKeys; + threadData->bytesWritten += txnBytes; + + // wait + wait(poisson(&last, (txnBytes + 1.0) / threadData->targetByteRate)); + } + } + + Future start(Database const& cx) override { + clients.reserve(3 * directories.size()); + for (auto& it : directories) { + // Wait for blob worker to initialize snapshot before starting test for that range + Future start = waitFirstSnapshot(this, cx, it, true); + clients.push_back(timeout(writeWorker(this, start, cx, it), testDuration, Void())); + clients.push_back(timeout(readWorker(this, start, cx, it), testDuration, Void())); + } + return delay(testDuration); + } + + ACTOR Future checkDirectory(Database cx, + BlobGranuleCorrectnessWorkload* self, + Reference threadData) { + + state bool result = true; + state int finalRowsValidated; + if (threadData->writeVersions.empty()) { + // never had a successful write during the test, likely due to many chaos events. Just wait for granules to + // become available and call that a pass, since writer is stopped and will never guarantee anything is + // written + if (BGW_DEBUG) { + fmt::print("Directory {0} doing final availability check\n", threadData->directoryID); + } + wait(self->waitFirstSnapshot(self, cx, threadData, false)); + } else { + // otherwise, read at last write version and ensure everything becomes available and matches + // it's possible that waitFirstSnapshot finished but then writer never wrote anything before test timed out + state Version readVersion = threadData->writeVersions.back(); + if (BGW_DEBUG) { + fmt::print("Directory {0} doing final data check @ {1}\n", threadData->directoryID, readVersion); + } + std::pair>> blob = + wait(self->readFromBlob(cx, self, threadData->directoryRange, readVersion)); + result = self->validateResult(threadData, blob, 0, std::numeric_limits::max(), 0, readVersion); + finalRowsValidated = blob.first.size(); + + // then if we are still good, do another check at a higher version (not checking data) to ensure availabiity + // of empty versions + if (result) { + if (BGW_DEBUG) { + fmt::print("Directory {0} doing final availability check after data check\n", + threadData->directoryID); + } + wait(self->waitFirstSnapshot(self, cx, threadData, false)); + } + } + + bool initialCheck = result; + result &= threadData->mismatches == 0 && (threadData->timeTravelTooOld == 0); + + fmt::print("Blob Granule Workload Directory {0} {1}:\n", threadData->directoryID, result ? "passed" : "failed"); + fmt::print(" Final granule check {0}successful\n", initialCheck ? "" : "un"); + fmt::print(" {} Rows read in final check\n", finalRowsValidated); + fmt::print(" {} mismatches\n", threadData->mismatches); + fmt::print(" {} time travel too old\n", threadData->timeTravelTooOld); + fmt::print(" {} errors\n", threadData->errors); + fmt::print(" {} rows written\n", threadData->rowsWritten); + fmt::print(" {} bytes written\n", threadData->bytesWritten); + fmt::print(" {} unique keys\n", threadData->usedKeys.size()); + fmt::print(" {} real-time reads\n", threadData->reads); + fmt::print(" {} time travel reads\n", threadData->timeTravelReads); + fmt::print(" {} rows read\n", threadData->rowsRead); + fmt::print(" {} bytes read\n", threadData->bytesRead); + // FIXME: add above as details to trace event + + TraceEvent("BlobGranuleWorkloadChecked").detail("Directory", threadData->directoryID).detail("Result", result); + + // For some reason simulation is still passing when this fails?.. so assert for now + ASSERT(result); + + return result; + } + + ACTOR Future _check(Database cx, BlobGranuleCorrectnessWorkload* self) { + // check error counts, and do an availability check at the end + state std::vector> results; + for (auto& it : self->directories) { + results.push_back(self->checkDirectory(cx, self, it)); + } + state bool allSuccessful = true; + for (auto& f : results) { + bool dirSuccess = wait(f); + allSuccessful &= dirSuccess; + } + return allSuccessful; + } + + Future check(Database const& cx) override { return _check(cx, this); } + void getMetrics(std::vector& m) override {} +}; + +WorkloadFactory BlobGranuleCorrectnessWorkloadFactory("BlobGranuleCorrectnessWorkload"); diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index 1af8c5e9d4..d4264058ca 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -24,17 +24,20 @@ #include "contrib/fmt-8.1.1/include/fmt/format.h" #include "fdbclient/BlobGranuleReader.actor.h" +#include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" +#include "fdbclient/SystemData.h" #include "fdbserver/Knobs.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" +#include "flow/Error.h" #include "flow/IRandom.h" #include "flow/genericactors.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. -#define BGV_DEBUG false +#define BGV_DEBUG true /* * This workload is designed to verify the correctness of the blob data produced by the blob workers. @@ -59,6 +62,9 @@ struct BlobGranuleVerifierWorkload : TestWorkload { int64_t rowsRead = 0; int64_t bytesRead = 0; std::vector> clients; + bool enablePruning; + + DatabaseConfiguration config; Reference bstore; AsyncVar>> granuleRanges; @@ -72,6 +78,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { timeTravelLimit = getOption(options, LiteralStringRef("timeTravelLimit"), testDuration); timeTravelBufferSize = getOption(options, LiteralStringRef("timeTravelBufferSize"), 100000000); threads = getOption(options, LiteralStringRef("threads"), 1); + enablePruning = getOption(options, LiteralStringRef("enablePruning"), false /*sharedRandomNumber % 2 == 0*/); ASSERT(threads >= 1); if (BGV_DEBUG) { @@ -126,19 +133,22 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } std::string description() const override { return "BlobGranuleVerifier"; } - Future setup(Database const& cx) override { - if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + Future setup(Database const& cx) override { return _setup(cx, this); } + + ACTOR Future _setup(Database cx, BlobGranuleVerifierWorkload* self) { + if (!self->doSetup) { + wait(delay(0)); return Void(); } - if (doSetup) { - double initialDelay = deterministicRandom()->random01() * (maxDelay - minDelay) + minDelay; - if (BGV_DEBUG) { - printf("BGW setup initial delay of %.3f\n", initialDelay); - } - return setUpBlobRange(cx, delay(initialDelay)); + wait(success(ManagementAPI::changeConfig(cx.getReference(), "blob_granules_enabled=1", true))); + + double initialDelay = deterministicRandom()->random01() * (self->maxDelay - self->minDelay) + self->minDelay; + if (BGV_DEBUG) { + printf("BGW setup initial delay of %.3f\n", initialDelay); } - return delay(0); + wait(self->setUpBlobRange(cx, delay(initialDelay))); + return Void(); } ACTOR Future findGranules(Database cx, BlobGranuleVerifierWorkload* self) { @@ -159,20 +169,35 @@ struct BlobGranuleVerifierWorkload : TestWorkload { // assumes we can read the whole range in one transaction at a single version ACTOR Future> readFromFDB(Database cx, KeyRange range) { + state bool first = true; state Version v; state RangeResult out; state Transaction tr(cx); state KeyRange currentRange = range; loop { try { - RangeResult r = wait(tr.getRange(currentRange, CLIENT_KNOBS->TOO_MANY)); + state RangeResult r = wait(tr.getRange(currentRange, CLIENT_KNOBS->TOO_MANY)); + Version grv = wait(tr.getReadVersion()); + // need consistent version snapshot of range + if (first) { + v = grv; + first = false; + } else if (v != grv) { + // reset the range and restart the read at a higher version + TraceEvent(SevDebug, "BGVFDBReadReset").detail("ReadVersion", v); + TEST(true); // BGV transaction reset + fmt::print("Resetting BGV GRV {0} -> {1}\n", v, grv); + first = true; + out = RangeResult(); + currentRange = range; + tr.reset(); + continue; + } out.arena().dependsOn(r.arena()); out.append(out.arena(), r.begin(), r.size()); if (r.more) { currentRange = KeyRangeRef(keyAfter(r.back().key), currentRange.end); } else { - Version _v = wait(tr.getReadVersion()); - v = _v; break; } } catch (Error& e) { @@ -299,11 +324,108 @@ struct BlobGranuleVerifierWorkload : TestWorkload { OldRead(KeyRange range, Version v, RangeResult oldResult) : range(range), v(v), oldResult(oldResult) {} }; - ACTOR Future verifyGranules(Database cx, BlobGranuleVerifierWorkload* self) { + // utility to prune at pruneVersion= with the flag + ACTOR Future pruneAtVersion(Database cx, KeyRange range, Version version, bool force) { + state Reference tr = makeReference(cx); + state Key pruneKey; + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + Value pruneValue = blobGranulePruneValueFor(version, range, force); + tr->atomicOp( + addVersionStampAtEnd(blobGranulePruneKeys.begin), pruneValue, MutationRef::SetVersionstampedKey); + tr->set(blobGranulePruneChangeKey, deterministicRandom()->randomUniqueID().toString()); + state Future> fTrVs = tr->getVersionstamp(); + wait(tr->commit()); + Standalone vs = wait(fTrVs); + pruneKey = blobGranulePruneKeys.begin.withSuffix(vs); + if (BGV_DEBUG) { + fmt::print("pruneAtVersion for range [{0} - {1}) at version {2} succeeded\n", + range.begin.printable(), + range.end.printable(), + version); + } + break; + } catch (Error& e) { + if (BGV_DEBUG) { + fmt::print("pruneAtVersion for range [{0} - {1}) at version {2} encountered error {3}\n", + range.begin.printable(), + range.end.printable(), + version, + e.name()); + } + wait(tr->onError(e)); + } + } + tr->reset(); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + Optional pruneVal = wait(tr->get(pruneKey)); + if (!pruneVal.present()) { + return Void(); + } + state Future watchFuture = tr->watch(pruneKey); + wait(tr->commit()); + wait(watchFuture); + } catch (Error& e) { + wait(tr->onError(e)); + } + } + } + + ACTOR Future killBlobWorkers(Database cx, BlobGranuleVerifierWorkload* self) { + state Transaction tr(cx); + state std::set knownWorkers; + state bool first = true; + loop { + try { + RangeResult r = wait(tr.getRange(blobWorkerListKeys, CLIENT_KNOBS->TOO_MANY)); + + state std::vector haltIds; + state std::vector>> haltRequests; + for (auto& it : r) { + BlobWorkerInterface interf = decodeBlobWorkerListValue(it.value); + if (first) { + knownWorkers.insert(interf.id()); + } + if (knownWorkers.count(interf.id())) { + haltIds.push_back(interf.id()); + haltRequests.push_back(interf.haltBlobWorker.tryGetReply(HaltBlobWorkerRequest(1e6, UID()))); + } + } + first = false; + wait(waitForAll(haltRequests)); + bool allPresent = true; + for (int i = 0; i < haltRequests.size(); i++) { + if (haltRequests[i].get().present()) { + knownWorkers.erase(haltIds[i]); + } else { + allPresent = false; + } + } + if (allPresent) { + return Void(); + } else { + wait(delay(1.0)); + } + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + + ACTOR Future verifyGranules(Database cx, BlobGranuleVerifierWorkload* self, bool allowPruning) { state double last = now(); state double endTime = last + self->testDuration; state std::map timeTravelChecks; state int64_t timeTravelChecksMemory = 0; + state Version prevPruneVersion = -1; + state UID dbgId = debugRandom()->randomUniqueID(); TraceEvent("BlobGranuleVerifierStart"); if (BGV_DEBUG) { @@ -325,15 +447,53 @@ struct BlobGranuleVerifierWorkload : TestWorkload { state OldRead oldRead = timeTravelIt->second; timeTravelChecksMemory -= oldRead.oldResult.expectedSize(); timeTravelIt = timeTravelChecks.erase(timeTravelIt); + if (prevPruneVersion == -1) { + prevPruneVersion = oldRead.v; + } // advance iterator before doing read, so if it gets error we don't retry it try { + state Version newPruneVersion = 0; + state bool doPruning = allowPruning && deterministicRandom()->random01() < 0.5; + if (doPruning) { + Version maxPruneVersion = oldRead.v; + for (auto& it : timeTravelChecks) { + maxPruneVersion = std::min(it.second.v, maxPruneVersion); + } + if (prevPruneVersion < maxPruneVersion) { + newPruneVersion = deterministicRandom()->randomInt64(prevPruneVersion, maxPruneVersion); + prevPruneVersion = std::max(prevPruneVersion, newPruneVersion); + wait(self->pruneAtVersion(cx, normalKeys, newPruneVersion, false)); + } else { + doPruning = false; + } + } std::pair>> reReadResult = wait(self->readFromBlob(cx, self, oldRead.range, oldRead.v)); self->compareResult(oldRead.oldResult, reReadResult, oldRead.range, oldRead.v, false); self->timeTravelReads++; + + if (doPruning) { + wait(self->killBlobWorkers(cx, self)); + std::pair>> versionRead = + wait(self->readFromBlob(cx, self, oldRead.range, prevPruneVersion)); + try { + Version minSnapshotVersion = newPruneVersion; + for (auto& it : versionRead.second) { + minSnapshotVersion = std::min(minSnapshotVersion, it.snapshotVersion); + } + std::pair>> versionRead = + wait(self->readFromBlob(cx, self, oldRead.range, minSnapshotVersion - 1)); + ASSERT(false); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + throw; + } + ASSERT(e.code() == error_code_blob_granule_transaction_too_old); + } + } } catch (Error& e) { - if (e.code() == error_code_transaction_too_old) { + if (e.code() == error_code_blob_granule_transaction_too_old) { self->timeTravelTooOld++; // TODO: add debugging info for when this is a failure } @@ -365,8 +525,7 @@ struct BlobGranuleVerifierWorkload : TestWorkload { if (e.code() == error_code_operation_cancelled) { throw; } - if (e.code() != error_code_transaction_too_old && e.code() != error_code_wrong_shard_server && - BGV_DEBUG) { + if (e.code() != error_code_blob_granule_transaction_too_old && BGV_DEBUG) { printf("BGVerifier got unexpected error %s\n", e.name()); } self->errors++; @@ -377,32 +536,65 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } Future start(Database const& cx) override { - if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { - return Void(); - } - clients.reserve(threads + 1); clients.push_back(timeout(findGranules(cx, this), testDuration, Void())); - for (int i = 0; i < threads; i++) { + if (enablePruning && clientId == 0) { clients.push_back( - timeout(reportErrors(verifyGranules(cx, this), "BlobGranuleVerifier"), testDuration, Void())); + timeout(reportErrors(verifyGranules(cx, this, true), "BlobGranuleVerifier"), testDuration, Void())); + } else if (!enablePruning) { + for (int i = 0; i < threads; i++) { + clients.push_back(timeout( + reportErrors(verifyGranules(cx, this, false), "BlobGranuleVerifier"), testDuration, Void())); + } } return delay(testDuration); } + // handle retries + errors + // It's ok to reset the transaction here because its read version is only used for reading the granule mapping from + // the system keyspace + ACTOR Future doGrv(Transaction* tr) { + loop { + try { + Version readVersion = wait(tr->getReadVersion()); + return readVersion; + } catch (Error& e) { + wait(tr->onError(e)); + } + } + } + ACTOR Future _check(Database cx, BlobGranuleVerifierWorkload* self) { // check error counts, and do an availability check at the end state Transaction tr(cx); - state Version readVersion = wait(tr.getReadVersion()); + state Version readVersion = wait(self->doGrv(&tr)); + state Version startReadVersion = readVersion; state int checks = 0; state KeyRange last; state bool availabilityPassed = true; - state Standalone> allRanges = self->granuleRanges.get(); + + state Standalone> allRanges; + if (self->granuleRanges.get().empty()) { + if (BGV_DEBUG) { + fmt::print("Waiting to get granule ranges for check\n"); + } + state Future rangeFetcher = self->findGranules(cx, self); + loop { + wait(self->granuleRanges.onChange()); + if (!self->granuleRanges.get().empty()) { + break; + } + } + rangeFetcher.cancel(); + if (BGV_DEBUG) { + fmt::print("Got granule ranges for check\n"); + } + } + allRanges = self->granuleRanges.get(); for (auto& range : allRanges) { state KeyRange r = range; - state PromiseStream> chunkStream; if (BGV_DEBUG) { fmt::print("Final availability check [{0} - {1}) @ {2}\n", r.begin.printable(), @@ -412,18 +604,32 @@ struct BlobGranuleVerifierWorkload : TestWorkload { try { loop { - tr.reset(); try { Standalone> chunks = wait(tr.readBlobGranules(r, 0, readVersion)); ASSERT(chunks.size() > 0); last = chunks.back().keyRange; checks += chunks.size(); + + break; } catch (Error& e) { - wait(tr.onError(e)); + // it's possible for blob granules to never get opened for the entire test due to fault + // injection. If we get blob_granule_transaction_too_old, for the latest read version, the + // granule still needs to open. Wait for that to happen at a higher read version. + if (e.code() == error_code_blob_granule_transaction_too_old) { + wait(delay(1.0)); + tr.reset(); + Version rv = wait(self->doGrv(&tr)); + readVersion = rv; + } else { + wait(tr.onError(e)); + } } } } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + throw; + } if (e.code() == error_code_end_of_stream) { break; } @@ -441,7 +647,11 @@ struct BlobGranuleVerifierWorkload : TestWorkload { break; } } - fmt::print("Blob Granule Verifier finished with:\n"); + if (BGV_DEBUG && startReadVersion != readVersion) { + fmt::print("Availability check updated read version from {0} to {1}\n", startReadVersion, readVersion); + } + bool result = availabilityPassed && self->mismatches == 0 && (checks > 0) && (self->timeTravelTooOld == 0); + fmt::print("Blob Granule Verifier {0} {1}:\n", self->clientId, result ? "passed" : "failed"); fmt::print(" {} successful final granule checks\n", checks); fmt::print(" {} failed final granule checks\n", availabilityPassed ? 0 : 1); fmt::print(" {} mismatches\n", self->mismatches); @@ -451,18 +661,17 @@ struct BlobGranuleVerifierWorkload : TestWorkload { fmt::print(" {} time travel reads\n", self->timeTravelReads); fmt::print(" {} rows\n", self->rowsRead); fmt::print(" {} bytes\n", self->bytesRead); - // FIXME: add above as details - TraceEvent("BlobGranuleVerifierChecked"); - return availabilityPassed && self->mismatches == 0 && checks > 0 && self->timeTravelTooOld == 0; + // FIXME: add above as details to trace event + + TraceEvent("BlobGranuleVerifierChecked").detail("Result", result); + + // For some reason simulation is still passing when this fails?.. so assert for now + ASSERT(result); + + return result; } - Future check(Database const& cx) override { - if (!CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { - return true; - } - - return _check(cx, this); - } + Future check(Database const& cx) override { return _check(cx, this); } void getMetrics(std::vector& m) override {} }; diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp index 431de4de5c..27a8e6f4cf 100644 --- a/fdbserver/workloads/ConsistencyCheck.actor.cpp +++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp @@ -297,7 +297,7 @@ struct ConsistencyCheckWorkload : TestWorkload { wait(::success(self->checkForExtraDataStores(cx, self))); // Check blob workers are operating as expected - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES) { + if (configuration.blobGranulesEnabled) { bool blobWorkersCorrect = wait(self->checkBlobWorkers(cx, configuration, self)); if (!blobWorkersCorrect) self->testFailure("Blob workers incorrect"); @@ -2003,25 +2003,24 @@ struct ConsistencyCheckWorkload : TestWorkload { int numBlobWorkerProcesses = 0; for (const auto& worker : workers) { NetworkAddress addr = worker.interf.stableAddress(); + bool inCCDc = worker.interf.locality.dcId() == ccDcId; if (!configuration.isExcludedServer(worker.interf.addresses())) { if (worker.processClass == ProcessClass::BlobWorkerClass) { numBlobWorkerProcesses++; - // this is a worker with processClass == BWClass, so should have exactly one blob worker - if (blobWorkersByAddr[addr] == 0) { - TraceEvent("ConsistencyCheck_NoBWsOnBWClass") + // this is a worker with processClass == BWClass, so should have exactly one blob worker if it's in + // the same DC + int desiredBlobWorkersOnAddr = inCCDc ? 1 : 0; + + if (blobWorkersByAddr[addr] != desiredBlobWorkersOnAddr) { + TraceEvent("ConsistencyCheck_WrongBWCountOnBWClass") .detail("Address", addr) - .detail("NumBlobWorkersOnAddr", blobWorkersByAddr[addr]); + .detail("NumBlobWorkersOnAddr", blobWorkersByAddr[addr]) + .detail("DesiredBlobWorkersOnAddr", desiredBlobWorkersOnAddr) + .detail("BwDcId", worker.interf.locality.dcId()) + .detail("CcDcId", ccDcId); return false; } - /* TODO: replace above code with this once blob manager recovery is handled - if (blobWorkersByAddr[addr] != 1) { - TraceEvent("ConsistencyCheck_NoBWOrManyBWsOnBWClass") - .detail("Address", addr) - .detail("NumBlobWorkersOnAddr", blobWorkersByAddr[addr]); - return false; - } - */ } else { // this is a worker with processClass != BWClass, so there should be no BWs on it if (blobWorkersByAddr[addr] > 0) { @@ -2359,7 +2358,7 @@ struct ConsistencyCheckWorkload : TestWorkload { } // Check BlobManager - if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES && db.blobManager.present() && + if (config.blobGranulesEnabled && db.blobManager.present() && (!nonExcludedWorkerProcessMap.count(db.blobManager.get().address()) || nonExcludedWorkerProcessMap[db.blobManager.get().address()].processClass.machineClassFitness( ProcessClass::BlobManager) > fitnessLowerBound)) { diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp index 9bcbf22898..8516f5f3ed 100644 --- a/fdbserver/workloads/WriteDuringRead.actor.cpp +++ b/fdbserver/workloads/WriteDuringRead.actor.cpp @@ -45,6 +45,9 @@ struct WriteDuringReadWorkload : TestWorkload { bool useSystemKeys; std::string keyPrefix; int64_t maximumTotalData; + int64_t maximumDataWritten; + + int64_t dataWritten = 0; bool success; Database extraDB; @@ -57,6 +60,8 @@ struct WriteDuringReadWorkload : TestWorkload { numOps = getOption(options, LiteralStringRef("numOps"), 21); rarelyCommit = getOption(options, LiteralStringRef("rarelyCommit"), false); maximumTotalData = getOption(options, LiteralStringRef("maximumTotalData"), 3e6); + maximumDataWritten = + getOption(options, LiteralStringRef("maximumDataWritten"), std::numeric_limits::max()); minNode = getOption(options, LiteralStringRef("minNode"), 0); useSystemKeys = getOption(options, LiteralStringRef("useSystemKeys"), deterministicRandom()->random01() < 0.5); adjacentKeys = deterministicRandom()->random01() < 0.5; @@ -73,6 +78,7 @@ struct WriteDuringReadWorkload : TestWorkload { nodes = deterministicRandom()->randomInt(1, 4 << deterministicRandom()->randomInt(0, 20)); } + dataWritten = 0; int newNodes = std::min(nodes, maximumTotalData / (getKeyForIndex(nodes).size() + valueSizeRange.second)); minNode = std::max(minNode, nodes - newNodes); nodes = newNodes; @@ -539,11 +545,13 @@ struct WriteDuringReadWorkload : TestWorkload { } } + state int64_t txnSize = tr->getApproximateSize(); state std::map committedDB = self->memoryDatabase; *doingCommit = true; wait(tr->commit()); *doingCommit = false; self->finished.trigger(); + self->dataWritten += txnSize; if (readYourWritesDisabled) tr->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE); @@ -616,11 +624,12 @@ struct WriteDuringReadWorkload : TestWorkload { state Transaction tr(cx); loop { try { - if (now() - startTime > self->testDuration) + if (now() - startTime > self->testDuration || self->dataWritten >= self->maximumDataWritten) return Void(); if (self->useSystemKeys) tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + state int64_t txnSize = 0; if (i == 0) { tr.clear(normalKeys); } @@ -641,10 +650,13 @@ struct WriteDuringReadWorkload : TestWorkload { value.substr(0, std::min(value.size(), CLIENT_KNOBS->VALUE_SIZE_LIMIT)); self->memoryDatabase[key] = value; tr.set(key, value); + int64_t rowSize = key.expectedSize() + value.expectedSize(); + txnSize += rowSize; } } } wait(tr.commit()); + self->dataWritten += txnSize; //TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr.getCommittedVersion()); break; } catch (Error& e) { @@ -671,7 +683,7 @@ struct WriteDuringReadWorkload : TestWorkload { throw; break; } - if (now() - startTime > self->testDuration) + if (now() - startTime > self->testDuration || self->dataWritten >= self->maximumDataWritten) return Void(); } } diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 7214df4c3d..1f82b0be46 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -84,6 +84,9 @@ ERROR( change_feed_not_registered, 1060, "Change feed not registered" ) ERROR( granule_assignment_conflict, 1061, "Conflicting attempts to assign blob granules" ) ERROR( change_feed_cancelled, 1062, "Change feed was cancelled" ) ERROR( blob_granule_file_load_error, 1063, "Error loading a blob file during granule materialization" ) +ERROR( blob_granule_transaction_too_old, 1064, "Read version is older than blob granule history supports" ) +ERROR( blob_manager_replaced, 1065, "This blob manager has been replaced." ) +ERROR( change_feed_popped, 1066, "Tried to read a version older than what has been popped from the change feed" ) ERROR( broken_promise, 1100, "Broken promise" ) ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" ) @@ -171,7 +174,7 @@ ERROR( quick_get_key_values_has_more, 2033, "One of the mapped range queries is ERROR( quick_get_value_miss, 2034, "Found a mapped key that is not served in the same SS" ) ERROR( quick_get_key_values_miss, 2035, "Found a mapped range that is not served in the same SS" ) ERROR( blob_granule_no_ryw, 2036, "Blob Granule Read Transactions must be specified as ryw-disabled" ) -ERROR( blob_granule_not_materialized, 2037, "Blob Granule Read Transactions must be specified as ryw-disabled" ) +ERROR( blob_granule_not_materialized, 2037, "Blob Granule Read was not materialized" ) ERROR( get_mapped_key_values_has_more, 2038, "getMappedRange does not support continuation for now" ) ERROR( get_mapped_range_reads_your_writes, 2039, "getMappedRange tries to read data that were previously written in the transaction" ) ERROR( checkpoint_not_found, 2040, "Checkpoint not found" ) diff --git a/flow/flow.h b/flow/flow.h index 23fced7bf4..a3c156c9d1 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -970,8 +970,10 @@ struct NotifiedQueue : private SingleCallback, FastAllocated std::queue> queue; Promise onEmpty; Error error; + Promise onError; - NotifiedQueue(int futures, int promises) : promises(promises), futures(futures), onEmpty(nullptr) { + NotifiedQueue(int futures, int promises) + : promises(promises), futures(futures), onEmpty(nullptr), onError(nullptr) { SingleCallback::next = this; } @@ -979,6 +981,7 @@ struct NotifiedQueue : private SingleCallback, FastAllocated bool isReady() const { return !queue.empty() || error.isValid(); } bool isError() const { return queue.empty() && error.isValid(); } // the *next* thing queued is an error + bool hasError() const { return error.isValid(); } // there is an error queued uint32_t size() const { return queue.size(); } virtual T pop() { @@ -1013,7 +1016,17 @@ struct NotifiedQueue : private SingleCallback, FastAllocated if (error.isValid()) return; + ASSERT(this->error.code() != error_code_success); this->error = err; + + // end_of_stream error is "expected", don't terminate reading stream early for this + // onError must be triggered before callback, otherwise callback could cause delPromiseRef/delFutureRef. This + // could destroy *this* in the callback, causing onError to be referenced after this object is destroyed. + if (err.code() != error_code_end_of_stream && err.code() != error_code_broken_promise && onError.isValid()) { + ASSERT(onError.canBeSet()); + onError.sendError(err); + } + if (shouldFireImmediately()) { SingleCallback::next->error(err); } @@ -1062,11 +1075,14 @@ protected: } auto copy = std::move(queue.front()); queue.pop(); + if (onEmpty.isValid() && queue.empty()) { + Promise hold = onEmpty; + onEmpty = Promise(nullptr); + hold.send(Void()); + } return copy; } - bool hasError() { return error.isValid(); } - bool shouldFireImmediately() { return SingleCallback::next != this; } }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 775f46c294..42713ad6cb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,8 +50,8 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES s3VersionHeaders.txt IGNORE) add_fdb_test(TEST_FILES BandwidthThrottle.txt IGNORE) add_fdb_test(TEST_FILES BigInsert.txt IGNORE) - add_fdb_test(TEST_FILES BlobGranuleFileUnit.txt IGNORE) - add_fdb_test(TEST_FILES BlobManagerUnit.txt IGNORE) + add_fdb_test(TEST_FILES BlobGranuleFileUnit.txt) + add_fdb_test(TEST_FILES BlobManagerUnit.txt) add_fdb_test(TEST_FILES ConsistencyCheck.txt IGNORE) add_fdb_test(TEST_FILES DDMetricsExclude.txt IGNORE) add_fdb_test(TEST_FILES DataDistributionMetrics.txt IGNORE) @@ -129,8 +129,10 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/BackupCorrectnessClean.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectness.toml) add_fdb_test(TEST_FILES fast/BackupToDBCorrectnessClean.toml) - add_fdb_test(TEST_FILES fast/BlobGranuleCorrectness.toml IGNORE) - add_fdb_test(TEST_FILES fast/BlobGranuleCorrectnessClean.toml IGNORE) + add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmall.toml) + add_fdb_test(TEST_FILES fast/BlobGranuleVerifySmallClean.toml) + add_fdb_test(TEST_FILES fast/BlobGranuleVerifyAtomicOps.toml) + add_fdb_test(TEST_FILES fast/BlobGranuleVerifyCycle.toml) add_fdb_test(TEST_FILES fast/CacheTest.toml) add_fdb_test(TEST_FILES fast/CloggedSideband.toml) add_fdb_test(TEST_FILES fast/ConfigureLocked.toml) @@ -273,8 +275,12 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES slow/ApiCorrectness.toml) add_fdb_test(TEST_FILES slow/ApiCorrectnessAtomicRestore.toml) add_fdb_test(TEST_FILES slow/ApiCorrectnessSwitchover.toml) - add_fdb_test(TEST_FILES slow/BlobGranuleCorrectnessLarge.toml IGNORE) - add_fdb_test(TEST_FILES slow/BlobGranuleCorrectnessLargeClean.toml IGNORE) + add_fdb_test(TEST_FILES slow/BlobGranuleVerifyLarge.toml) + add_fdb_test(TEST_FILES slow/BlobGranuleVerifyLargeClean.toml) + add_fdb_test(TEST_FILES slow/BlobGranuleVerifyBalance.toml) + add_fdb_test(TEST_FILES slow/BlobGranuleVerifyBalanceClean.toml) + add_fdb_test(TEST_FILES slow/BlobGranuleCorrectnessClean.toml) + add_fdb_test(TEST_FILES slow/BlobGranuleCorrectness.toml) add_fdb_test(TEST_FILES slow/ClogWithRollbacks.toml) add_fdb_test(TEST_FILES slow/CloggedCycleTest.toml) add_fdb_test(TEST_FILES slow/CloggedStorefront.toml) diff --git a/tests/fast/BlobGranuleCorrectnessClean.toml b/tests/fast/BlobGranuleCorrectnessClean.toml deleted file mode 100644 index 168790dd9d..0000000000 --- a/tests/fast/BlobGranuleCorrectnessClean.toml +++ /dev/null @@ -1,10 +0,0 @@ -[[test]] -testTitle = 'BlobGranuleCorrectnessCleanTest' - - [[test.workload]] - testName = 'WriteDuringRead' - testDuration = 120.0 - - [[test.workload]] - testName = 'BlobGranuleVerifier' - testDuration = 120.0 diff --git a/tests/fast/BlobGranuleVerifyAtomicOps.toml b/tests/fast/BlobGranuleVerifyAtomicOps.toml new file mode 100644 index 0000000000..4831d8b985 --- /dev/null +++ b/tests/fast/BlobGranuleVerifyAtomicOps.toml @@ -0,0 +1,37 @@ +[configuration] +blobGranulesEnabled = true + +[[test]] +testTitle = 'BlobGranuleVerifyAtomicOps' + + [[test.workload]] + testName = 'AtomicOps' + transactionsPerSecond = 2500.0 + testDuration = 30.0 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 30.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 30.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 30.0 + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 diff --git a/tests/fast/BlobGranuleVerifyCycle.toml b/tests/fast/BlobGranuleVerifyCycle.toml new file mode 100644 index 0000000000..b15bc34a85 --- /dev/null +++ b/tests/fast/BlobGranuleVerifyCycle.toml @@ -0,0 +1,38 @@ +[configuration] +blobGranulesEnabled = true + +[[test]] +testTitle = 'BlobGranuleVerifyCycle' + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 250.0 + testDuration = 60.0 + expectedRate = 0 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 60.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 60.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 diff --git a/tests/fast/BlobGranuleVerifySmall.toml b/tests/fast/BlobGranuleVerifySmall.toml new file mode 100644 index 0000000000..22a4b15ae6 --- /dev/null +++ b/tests/fast/BlobGranuleVerifySmall.toml @@ -0,0 +1,40 @@ +[configuration] +blobGranulesEnabled = true +storageEngineExcludeTypes = [3] # FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet + +[[test]] +testTitle = 'BlobGranuleVerifySmall' + + [[test.workload]] + testName = 'WriteDuringRead' + testDuration = 60.0 + useSystemKeys = false + maximumDataWritten=50000000 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 60.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 30.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + diff --git a/tests/fast/BlobGranuleVerifySmallClean.toml b/tests/fast/BlobGranuleVerifySmallClean.toml new file mode 100644 index 0000000000..0a7d2a95d6 --- /dev/null +++ b/tests/fast/BlobGranuleVerifySmallClean.toml @@ -0,0 +1,16 @@ +[configuration] +blobGranulesEnabled = true +storageEngineExcludeTypes = [3] # FIXME: exclude redwood because WriteDuringRead can write massive KV pairs and we don't chunk change feed data on disk yet + +[[test]] +testTitle = 'BlobGranuleVerifySmallClean' + + [[test.workload]] + testName = 'WriteDuringRead' + testDuration = 60.0 + useSystemKeys = false + maximumDataWritten=50000000 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 60.0 diff --git a/tests/fast/BlobGranuleCorrectness.toml b/tests/slow/BlobGranuleCorrectness.toml similarity index 64% rename from tests/fast/BlobGranuleCorrectness.toml rename to tests/slow/BlobGranuleCorrectness.toml index 20446ec66c..10c03ab63b 100644 --- a/tests/fast/BlobGranuleCorrectness.toml +++ b/tests/slow/BlobGranuleCorrectness.toml @@ -1,34 +1,32 @@ +[configuration] +blobGranulesEnabled = true + [[test]] -testTitle = 'BlobGranuleCorrectnessTest' +testTitle = 'BlobGranuleCorrectness' [[test.workload]] - testName = 'WriteDuringRead' - testDuration = 120.0 - - [[test.workload]] - testName = 'BlobGranuleVerifier' + testName = 'BlobGranuleCorrectnessWorkload' testDuration = 120.0 [[test.workload]] testName = 'RandomClogging' - testDuration = 120.0 + testDuration = 120.0 [[test.workload]] testName = 'Rollback' meanDelay = 30.0 - testDuration = 120.0 + testDuration = 120.0 [[test.workload]] testName = 'Attrition' machinesToKill = 10 machinesToLeave = 3 reboot = true - testDuration = 120.0 + testDuration = 120.0 [[test.workload]] testName = 'Attrition' machinesToKill = 10 machinesToLeave = 3 reboot = true - testDuration = 120.0 - + testDuration = 120.0 \ No newline at end of file diff --git a/tests/slow/BlobGranuleCorrectnessClean.toml b/tests/slow/BlobGranuleCorrectnessClean.toml new file mode 100644 index 0000000000..a538e7203b --- /dev/null +++ b/tests/slow/BlobGranuleCorrectnessClean.toml @@ -0,0 +1,9 @@ +[configuration] +blobGranulesEnabled = true + +[[test]] +testTitle = 'BlobGranuleCorrectness' + + [[test.workload]] + testName = 'BlobGranuleCorrectnessWorkload' + testDuration = 120.0 \ No newline at end of file diff --git a/tests/slow/BlobGranuleVerifyBalance.toml b/tests/slow/BlobGranuleVerifyBalance.toml new file mode 100644 index 0000000000..385b88ff69 --- /dev/null +++ b/tests/slow/BlobGranuleVerifyBalance.toml @@ -0,0 +1,51 @@ +[configuration] +blobGranulesEnabled = true + +[[test]] +testTitle = 'BlobGranuleVerifyBalance' + + [[test.workload]] + testName = 'DDBalance' + testDuration = 120.0 + transactionsPerSecond = 250.0 + binCount = 1000 + writesPerTransaction = 5 + keySpaceDriftFactor = 10 + moversPerClient = 10 + actorsPerClient = 100 + nodes = 100000 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 120.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 120.0 + swizzle = 1 + + [[test.workload]] + testName = 'Rollback' + testDuration = 120.0 + meanDelay = 10.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 120.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 120.0 + + [[test.workload]] + testName = 'RemoveServersSafely' + minDelay = 0 + maxDelay = 100 + kill1Timeout = 30 + kill2Timeout = 6000 diff --git a/tests/slow/BlobGranuleVerifyBalanceClean.toml b/tests/slow/BlobGranuleVerifyBalanceClean.toml new file mode 100644 index 0000000000..65bb8ad15c --- /dev/null +++ b/tests/slow/BlobGranuleVerifyBalanceClean.toml @@ -0,0 +1,20 @@ +[configuration] +blobGranulesEnabled = true + +[[test]] +testTitle = 'BlobGranuleVerifyBalanceClean' + + [[test.workload]] + testName = 'DDBalance' + testDuration = 120.0 + transactionsPerSecond = 250.0 + binCount = 1000 + writesPerTransaction = 5 + keySpaceDriftFactor = 10 + moversPerClient = 10 + actorsPerClient = 100 + nodes = 100000 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 120.0 diff --git a/tests/slow/BlobGranuleCorrectnessLarge.toml b/tests/slow/BlobGranuleVerifyLarge.toml similarity index 70% rename from tests/slow/BlobGranuleCorrectnessLarge.toml rename to tests/slow/BlobGranuleVerifyLarge.toml index edf400b6d9..de55422d89 100644 --- a/tests/slow/BlobGranuleCorrectnessLarge.toml +++ b/tests/slow/BlobGranuleVerifyLarge.toml @@ -1,12 +1,15 @@ +[configuration] +blobGranulesEnabled = true + [[test]] -testTitle = 'BlobGranuleCorrectnessLargeTest' +testTitle = 'BlobGranuleVerifyLarge' [[test.workload]] testName = 'ReadWrite' - testDuration = 200.0 + testDuration = 120.0 transactionsPerSecond = 200 - writesPerTransactionA = 0 - readsPerTransactionA = 10 + writesPerTransactionA = 5 + readsPerTransactionA = 1 writesPerTransactionB = 10 readsPerTransactionB = 1 alpha = 0.5 @@ -18,28 +21,28 @@ testTitle = 'BlobGranuleCorrectnessLargeTest' [[test.workload]] testName = 'BlobGranuleVerifier' - testDuration = 200.0 + testDuration = 120.0 [[test.workload]] testName = 'RandomClogging' - testDuration = 200.0 + testDuration = 120.0 [[test.workload]] testName = 'Rollback' meanDelay = 30.0 - testDuration = 200.0 + testDuration = 120.0 [[test.workload]] testName = 'Attrition' machinesToKill = 10 machinesToLeave = 3 reboot = true - testDuration = 200.0 + testDuration = 120.0 [[test.workload]] testName = 'Attrition' machinesToKill = 10 machinesToLeave = 3 reboot = true - testDuration = 200.0 + testDuration = 120.0 diff --git a/tests/slow/BlobGranuleCorrectnessLargeClean.toml b/tests/slow/BlobGranuleVerifyLargeClean.toml similarity index 64% rename from tests/slow/BlobGranuleCorrectnessLargeClean.toml rename to tests/slow/BlobGranuleVerifyLargeClean.toml index 1a12e6f47f..782935a68b 100644 --- a/tests/slow/BlobGranuleCorrectnessLargeClean.toml +++ b/tests/slow/BlobGranuleVerifyLargeClean.toml @@ -1,12 +1,15 @@ +[configuration] +blobGranulesEnabled = true + [[test]] -testTitle = 'BlobGranuleCorrectnessLargeCleanTest' +testTitle = 'BlobGranuleVerifyLargeClean' [[test.workload]] testName = 'ReadWrite' - testDuration = 200.0 + testDuration = 120.0 transactionsPerSecond = 200 - writesPerTransactionA = 0 - readsPerTransactionA = 10 + writesPerTransactionA = 5 + readsPerTransactionA = 1 writesPerTransactionB = 10 readsPerTransactionB = 1 alpha = 0.5 @@ -18,4 +21,4 @@ testTitle = 'BlobGranuleCorrectnessLargeCleanTest' [[test.workload]] testName = 'BlobGranuleVerifier' - testDuration = 200.0 + testDuration = 120.0