From 72e589105881dd562478cec679d84897afddf072 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 27 Mar 2020 01:35:26 -0700 Subject: [PATCH 01/89] Clean up and rework the debugMutation API. As a relatively unknown debugging tool for simulation tests, one could have simulation print when a particular key is handled in various stages of the commit process. This functionality was enabled by changing a 0 to a 1 in an #if, and changing a constant to the key in question. As a proxy and storage server handle mutations, they call debugMutation or debugKeyRange, which then checks against the mutation against the key in question, and logs if they match. A mixture of printfs and TraceEvents would then be emitted, and for this to actually be usable, one also needs to comment out some particularly spammy debugKeyRange() calls. This PR reworks the API of debugMutation/debugKeyRange, pulls it out into its own file, and trims what is logged by default into something useful and understandable: * debugMutation() now returns a TraceEvent, that one can add more details to before it is logged. * Data distribution and storage server cleanup operations are no longer logged by default --- fdbclient/CommitTransaction.h | 10 +++- fdbclient/FDBTypes.h | 14 +++++ fdbserver/MasterProxyServer.actor.cpp | 10 ++-- fdbserver/MutationTracking.cpp | 61 ++++++++++++++++++++ fdbserver/MutationTracking.h | 33 +++++++++++ fdbserver/StorageCache.actor.cpp | 15 +---- fdbserver/WorkerInterface.actor.h | 20 ------- fdbserver/fdbserver.actor.cpp | 57 ------------------ fdbserver/fdbserver.vcxproj | 2 + fdbserver/storageserver.actor.cpp | 31 +++++----- fdbserver/workloads/ApiCorrectness.actor.cpp | 1 + flow/Trace.cpp | 2 + 12 files changed, 139 insertions(+), 117 deletions(-) create mode 100644 fdbserver/MutationTracking.cpp create mode 100644 fdbserver/MutationTracking.h diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index be5821cfa4..ca6c525ef1 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -124,6 +124,13 @@ struct MutationRef { }; }; +template<> +struct Traceable : std::true_type { + static std::string toString(MutationRef const& value) { + return value.toString(); + } +}; + // A 'single key mutation' is one which affects exactly the value of the key specified by its param1 static inline bool isSingleKeyMutation(MutationRef::Type type) { return (MutationRef::SINGLE_KEY_MASK & (1< const& items, int max_items = -1 ) { return describeList(items, max_items); } +template +struct Traceable> : std::true_type { + static std::string toString(const std::vector& value) { + return describe(value); + } +}; + template std::string describe( std::set const& items, int max_items = -1 ) { return describeList(items, max_items); } +template +struct Traceable> : std::true_type { + static std::string toString(const std::set& value) { + return describe(value); + } +}; + std::string printable( const StringRef& val ); std::string printable( const std::string& val ); std::string printable( const KeyRangeRef& range ); diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 6e5b03a96e..afdd1317b5 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -38,6 +38,7 @@ #include "fdbserver/LogSystem.h" #include "fdbserver/LogSystemDiskQueueAdapter.h" #include "fdbserver/MasterInterface.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RecoveryState.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/WaitFailure.h" @@ -938,8 +939,7 @@ ACTOR Future commitBatch( self->singleKeyMutationEvent->log(); } - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("Mutation", m.toString()).detail("Version", commitVersion); + debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", tags).detail("Mutation", m); toCommit.addTags(tags); if(self->cacheInfo[m.param1]) { @@ -954,8 +954,7 @@ ACTOR Future commitBatch( ++firstRange; if (firstRange == ranges.end()) { // Fast path - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(ranges.begin().value().tags)).detail("Mutation", m.toString()).detail("Version", commitVersion); + debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", ranges.begin().value().tags).detail("Mutation", m); ranges.begin().value().populateTags(); toCommit.addTags(ranges.begin().value().tags); @@ -967,8 +966,7 @@ ACTOR Future commitBatch( r.value().populateTags(); allSources.insert(r.value().tags.begin(), r.value().tags.end()); } - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(allSources)).detail("Mutation", m.toString()).detail("Version", commitVersion); + debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", allSources).detail("Mutation", m); toCommit.addTags(allSources); } diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp new file mode 100644 index 0000000000..2adef04b9c --- /dev/null +++ b/fdbserver/MutationTracking.cpp @@ -0,0 +1,61 @@ +/* + * MutationTracking.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "fdbserver/MutationTracking.h" +#include "fdbserver/LogProtocolMessage.h" + +#if defined(FDB_CLEAN_BUILD) && MUTATION_TRACKING_ENABLED +#error "You cannot use mutation tracking in a clean/release build." +#endif + +StringRef debugKey = LiteralStringRef( "\xff/globals/lastEpochEnd" ); +StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" ); + +TraceEvent debugMutationEnabled( const char* context, Version version, MutationRef const& mutation ) { + if ((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) && + ((mutation.param1<=debugKey && mutation.param2>debugKey) || (mutation.param1<=debugKey2 && mutation.param2>debugKey2))) { + return std::move(TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", typeString[mutation.type]).detail("KeyBegin", mutation.param1).detail("KeyEnd", mutation.param2)); + } else if (mutation.param1 == debugKey || mutation.param1 == debugKey2) { + return std::move(TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", typeString[mutation.type]).detail("Key", mutation.param1).detail("Value", mutation.param2)); + } else { + return std::move(TraceEvent()); + } +} + +TraceEvent debugKeyRangeEnabled( const char* context, Version version, KeyRangeRef const& keys ) { + if (keys.contains(debugKey) || keys.contains(debugKey2)) { + return std::move(debugMutation(context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) )); + } else { + return std::move(TraceEvent()); + } +} + +#if MUTATION_TRACKING_ENABLED +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { + return debugMutationEnabled( context, version, mutation ); +} +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { + return debugKeyRangeEnabled( context, version, keys ); +} +#else +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { return std::move(TraceEvent()); } +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return std::move(TraceEvent()); } +#endif diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h new file mode 100644 index 0000000000..5fa97ef50b --- /dev/null +++ b/fdbserver/MutationTracking.h @@ -0,0 +1,33 @@ +/* + * MutationTracking.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _FDBSERVER_MUTATIONTRACKING_H_ +#define _FDBSERVER_MUTATIONTRACKING_H_ +#pragma once + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/CommitTransaction.h" + +#define MUTATION_TRACKING_ENABLED 0 + +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); + +#endif diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index 67516e291e..2192ca527c 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -26,6 +26,7 @@ #include "fdbclient/Atomic.h" #include "fdbclient/Notified.h" #include "fdbserver/LogSystem.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -710,22 +711,10 @@ void StorageCacheData::addMutation(KeyRangeRef const& cachedKeyRange, Version ve return; } expanded = addMutationToMutationLog(mLog, expanded); - if (debugMutation("expandedMutation", version, expanded)) { - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%s\t%s\t%s\n", - now(), g_network->getLocalAddress().toString().c_str(), "originalMutation", - type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - printf(" Cached Key-range: %s - %s\n", printable(cachedKeyRange.begin).c_str(), printable(cachedKeyRange.end).c_str()); - } + debugMutation("expandedMutation", version, expanded).detail("Begin", cachedKeyRange.begin).detail("End", cachedKeyRange.end); applyMutation( this, expanded, mLog.arena(), mutableData() ); printf("\nSCUpdate: Printing versioned tree after applying mutation\n"); mutableData().printTree(version); - } // Helper class for updating the storage cache (i.e. applying mutations) diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index ee613912a1..8b44507b79 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -388,26 +388,6 @@ struct EventLogRequest { } }; -struct DebugEntryRef { - double time; - NetworkAddress address; - StringRef context; - Version version; - MutationRef mutation; - DebugEntryRef() {} - DebugEntryRef( const char* c, Version v, MutationRef const& m ) : context((const uint8_t*)c,strlen(c)), version(v), mutation(m), time(now()), address( g_network->getLocalAddress() ) {} - DebugEntryRef( Arena& a, DebugEntryRef const& d ) : time(d.time), address(d.address), context(d.context), version(d.version), mutation(a, d.mutation) {} - - size_t expectedSize() const { - return context.expectedSize() + mutation.expectedSize(); - } - - template - void serialize(Ar& ar) { - serializer(ar, time, address, context, version, mutation); - } -}; - struct DiskStoreRequest { constexpr static FileIdentifier file_identifier = 1986262; bool includePartialStores; diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 437ec78f25..e9dd653bc6 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -198,63 +198,6 @@ bool enableFailures = true; #define test_assert(x) if (!(x)) { cout << "Test failed: " #x << endl; return false; } -vector< Standalone> > debugEntries; -int64_t totalDebugEntriesSize = 0; - -#if CENABLED(0, NOT_IN_CLEAN) -StringRef debugKey = LiteralStringRef( "" ); -StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" ); - -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { - if ((mutation.type == mutation.SetValue || mutation.type == mutation.AddValue || mutation.type==mutation.DebugKey) && (mutation.param1 == debugKey || mutation.param1 == debugKey2)) - ;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "SetValue").detail("Key", mutation.param1).detail("Value", mutation.param2); - else if ((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) && ((mutation.param1<=debugKey && mutation.param2>debugKey) || (mutation.param1<=debugKey2 && mutation.param2>debugKey2))) - ;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "ClearRange").detail("KeyBegin", mutation.param1).detail("KeyEnd", mutation.param2); - else - return false; - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::AddValue ? "AddValue" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%lld\t%s\t%s\t%s\n", now(), g_network->getLocalAddress().toString().c_str(), context, version, type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - - return true; -} - -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { - if (keys.contains(debugKey) || keys.contains(debugKey2)) { - debugMutation(context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) ); - //TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("KeyBegin", keys.begin).detail("KeyEnd", keys.end); - return true; - } else - return false; -} - -#elif CENABLED(0, NOT_IN_CLEAN) -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { - if (!debugEntries.size() || debugEntries.back().size() >= 1000) { - if (debugEntries.size()) totalDebugEntriesSize += debugEntries.back().arena().getSize() + sizeof(debugEntries.back()); - debugEntries.push_back(Standalone>()); - TraceEvent("DebugMutationBuffer").detail("Bytes", totalDebugEntriesSize); - } - auto& v = debugEntries.back(); - v.push_back_deep( v.arena(), DebugEntryRef(context, version, mutation) ); - - return false; // No auxiliary logging -} - -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { - return debugMutation( context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) ); -} - -#else // Default implementation. -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { return false; } -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return false; } -#endif - #ifdef _WIN32 #include diff --git a/fdbserver/fdbserver.vcxproj b/fdbserver/fdbserver.vcxproj index 5957516a3b..52e9755d78 100644 --- a/fdbserver/fdbserver.vcxproj +++ b/fdbserver/fdbserver.vcxproj @@ -41,6 +41,7 @@ + @@ -202,6 +203,7 @@ + false false diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index e8a9bbca76..c673e03f25 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -42,6 +42,7 @@ #include "fdbserver/LogProtocolMessage.h" #include "fdbserver/LogSystem.h" #include "fdbserver/MoveKeys.actor.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RecoveryState.h" #include "fdbserver/StorageMetrics.h" #include "fdbserver/ServerDBInfo.h" @@ -2288,7 +2289,8 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss // .detail("Context", changeServerKeysContextName[(int)context]); validate(data); - debugKeyRange( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); + // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. + //debugKeyRange( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); bool isDifferent = false; auto existingShards = data->shards.intersectingRanges(keys); @@ -2414,18 +2416,7 @@ void StorageServer::addMutation(Version version, MutationRef const& mutation, Ke return; } expanded = addMutationToMutationLog(mLog, expanded); - if (debugMutation("expandedMutation", version, expanded)) { - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%" PRId64 "\t%s\t%s\t%s\n", now(), g_network->getLocalAddress().toString().c_str(), "originalMutation", version, type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - printf(" shard: %s - %s\n", printable(shard.begin).c_str(), printable(shard.end).c_str()); - if (mutation.type == MutationRef::ClearRange && mutation.param2 != shard.end) - printf(" eager: %s\n", printable( eagerReads->getKeyEnd( mutation.param2 ) ).c_str() ); - } + debugMutation("applyMutation", version, expanded).detail("UID", thisServerID).detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end); applyMutation( this, expanded, mLog.arena(), mutableData() ); //printf("\nSSUpdate: Printing versioned tree after applying mutation\n"); //mutableData().printTree(version); @@ -2766,7 +2757,8 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) rd >> msg; if (ver != invalidVersion) { // This change belongs to a version < minVersion - if (debugMutation("SSPeek", ver, msg) || ver == 1) { + debugMutation("SSPeek", ver, msg).detail("ServerID", data->thisServerID); + if (ver == 1) { TraceEvent("SSPeekMutation", data->thisServerID); // The following trace event may produce a value with special characters //TraceEvent("SSPeekMutation", data->thisServerID).detail("Mutation", msg.toString()).detail("Version", cloneCursor2->version().toString()); @@ -2819,7 +2811,8 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) } if(ver != invalidVersion && ver > data->version.get()) { - debugKeyRange("SSUpdate", ver, allKeys); + // TODO(alexmiller): Update to version tracking. + debugKeyRange("SSUpdate", ver, KeyRangeRef()); data->mutableData().createNewVersion(ver); if (data->otherError.getFuture().isReady()) data->otherError.getFuture().get(); @@ -3033,7 +3026,7 @@ void StorageServerDisk::writeMutation( MutationRef mutation ) { void StorageServerDisk::writeMutations( MutationListRef mutations, Version debugVersion, const char* debugContext ) { for(auto m = mutations.begin(); m; ++m) { - debugMutation(debugContext, debugVersion, *m); + debugMutation(debugContext, debugVersion, *m).detail("UID", data->thisServerID); if (m->type == MutationRef::SetValue) { storage->set( KeyValueRef(m->param1, m->param2) ); } else if (m->type == MutationRef::ClearRange) { @@ -3050,7 +3043,8 @@ bool StorageServerDisk::makeVersionMutationsDurable( Version& prevStorageVersion if (u != data->getMutationLog().end() && u->first <= newStorageVersion) { VersionUpdateRef const& v = u->second; ASSERT( v.version > prevStorageVersion && v.version <= newStorageVersion ); - debugKeyRange("makeVersionMutationsDurable", v.version, allKeys); + // TODO(alexmiller): Update to version tracking. + debugKeyRange("makeVersionMutationsDurable", v.version, KeyRangeRef()); writeMutations(v.mutations, v.version, "makeVersionDurable"); for(auto m=v.mutations.begin(); m; ++m) bytesLeft -= mvccStorageBytes(*m); @@ -3236,7 +3230,8 @@ ACTOR Future restoreDurableState( StorageServer* data, IKeyValueStore* sto for(auto it = data->newestAvailableVersion.ranges().begin(); it != data->newestAvailableVersion.ranges().end(); ++it) { if (it->value() == invalidVersion) { KeyRangeRef clearRange(it->begin(), it->end()); - debugKeyRange("clearInvalidVersion", invalidVersion, clearRange); + // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. + //debugKeyRange("clearInvalidVersion", invalidVersion, clearRange); storage->clear( clearRange ); data->byteSampleApplyClear( clearRange, invalidVersion ); } diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.actor.cpp index c122c7c550..03651f19f2 100644 --- a/fdbserver/workloads/ApiCorrectness.actor.cpp +++ b/fdbserver/workloads/ApiCorrectness.actor.cpp @@ -20,6 +20,7 @@ #include "fdbserver/QuietDatabase.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/ApiWorkload.h" #include "fdbserver/workloads/MemoryKeyValueStore.h" diff --git a/flow/Trace.cpp b/flow/Trace.cpp index 4ce0b96e8a..3c42395eac 100644 --- a/flow/Trace.cpp +++ b/flow/Trace.cpp @@ -765,6 +765,7 @@ TraceEvent::TraceEvent(TraceEvent &&ev) { tmpEventMetric = ev.tmpEventMetric; trackingKey = ev.trackingKey; type = ev.type; + timeIndex = ev.timeIndex; ev.initialized = true; ev.enabled = false; @@ -785,6 +786,7 @@ TraceEvent& TraceEvent::operator=(TraceEvent &&ev) { tmpEventMetric = ev.tmpEventMetric; trackingKey = ev.trackingKey; type = ev.type; + timeIndex = ev.timeIndex; ev.initialized = true; ev.enabled = false; From 122762cce1d11e802b2f6199de2dea5713b03d4e Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 27 Mar 2020 03:31:04 -0700 Subject: [PATCH 02/89] Add debugMessagesAndTags, and track mutations in more places. Like: * Leaving the proxy * Entering the TLog * Leaving the TLog * Being read on a cursor All of this brought to you by TagsAndMessage! This also slides in a minor optimization as to how mutations are serialized per target log. --- fdbclient/FDBTypes.h | 7 +++++ fdbserver/LogSystem.h | 28 ++++++++++++++------ fdbserver/LogSystemPeekCursor.actor.cpp | 2 ++ fdbserver/MutationTracking.cpp | 35 +++++++++++++++++++++++++ fdbserver/MutationTracking.h | 1 + fdbserver/TLogServer.actor.cpp | 8 ++++++ 6 files changed, 73 insertions(+), 8 deletions(-) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index aec96cd715..82617b6a3e 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -108,6 +108,13 @@ struct struct_like_traits : std::true_type { } }; +template<> +struct Traceable : std::true_type { + static std::string toString(const Tag& value) { + return value.toString(); + } +}; + static const Tag invalidTag {tagLocalitySpecial, 0}; static const Tag txsTag {tagLocalitySpecial, 1}; static const Tag cacheTag {tagLocalitySpecial, 2}; diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index dad55b047f..814bc27e0d 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -27,6 +27,7 @@ #include "fdbserver/TLogInterface.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbclient/DatabaseConfiguration.h" +#include "fdbserver/MutationTracking.h" #include "flow/IndexedSet.h" #include "fdbrpc/ReplicationPolicy.h" #include "fdbrpc/Locality.h" @@ -875,16 +876,27 @@ struct LogPushData : NonCopyable { msg_locations.clear(); logSystem->getPushLocations(prev_tags, msg_locations, allLocations); + BinaryWriter bw(AssumeVersion(currentProtocolVersion)); uint32_t subseq = this->subsequence++; + bool first = true; + int firstOffset=-1, firstLength=-1; for(int loc : msg_locations) { - // FIXME: memcpy after the first time - BinaryWriter& wr = messagesWriter[loc]; - int offset = wr.getLength(); - wr << uint32_t(0) << subseq << uint16_t(prev_tags.size()); - for(auto& tag : prev_tags) - wr << tag; - wr << item; - *(uint32_t*)((uint8_t*)wr.getData() + offset) = wr.getLength() - offset - sizeof(uint32_t); + if (first) { + BinaryWriter& wr = messagesWriter[loc]; + firstOffset = wr.getLength(); + wr << uint32_t(0) << subseq << uint16_t(prev_tags.size()); + for(auto& tag : prev_tags) + wr << tag; + wr << item; + firstLength = wr.getLength() - firstOffset; + *(uint32_t*)((uint8_t*)wr.getData() + firstOffset) = firstLength - sizeof(uint32_t); + debugMessagesAndTags("ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength)).detail("PushLocations", msg_locations); + first = false; + } else { + BinaryWriter& wr = messagesWriter[loc]; + BinaryWriter& from = messagesWriter[msg_locations[0]]; + wr.serializeBytes( (uint8_t*)from.getData() + firstOffset, firstLength ); + } } next_message_tags.clear(); } diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp index 51880f5064..5e72092d3d 100644 --- a/fdbserver/LogSystemPeekCursor.actor.cpp +++ b/fdbserver/LogSystemPeekCursor.actor.cpp @@ -21,6 +21,7 @@ #include "fdbserver/LogSystem.h" #include "fdbrpc/FailureMonitor.h" #include "fdbserver/Knobs.h" +#include "fdbserver/MutationTracking.h" #include "fdbrpc/ReplicationUtils.h" #include "flow/actorcompiler.h" // has to be last include @@ -90,6 +91,7 @@ void ILogSystem::ServerPeekCursor::nextMessage() { } messageAndTags.loadFromArena(&rd, &messageVersion.sub); + debugMessagesAndTags("ServerPeekCursor", messageVersion.version, messageAndTags.getRawMessage()).detail("CursorID", this->randomID); // Rewind and consume the header so that reader() starts from the message. rd.rewind(); rd.readBytes(messageAndTags.getHeaderSize()); diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index 2adef04b9c..64ec3f4872 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -48,6 +48,41 @@ TraceEvent debugKeyRangeEnabled( const char* context, Version version, KeyRangeR } } +TraceEvent debugMessagesAndTagsEnabled( const char* context, Version version, StringRef commitBlob ) { + BinaryReader rdr(commitBlob, AssumeVersion(currentProtocolVersion)); + while (!rdr.empty()) { + if (*(int32_t*)rdr.peekBytes(4) == VERSION_HEADER) { + int32_t dummy; + rdr >> dummy >> version; + continue; + } + TagsAndMessage msg; + msg.loadFromArena(&rdr, nullptr); + bool logAdapterMessage = std::any_of( + msg.tags.begin(), msg.tags.end(), [](const Tag& t) { return t == txsTag || t.locality == tagLocalityTxs; }); + StringRef mutationData = msg.getMessageWithoutTags(); + uint8_t mutationType = *mutationData.begin(); + if (logAdapterMessage) { + // Skip the message, as there will always be an idential non-logAdapterMessage mutation + // that we can match against in the same commit. + } else if (LogProtocolMessage::startsLogProtocolMessage(mutationType)) { + BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); + LogProtocolMessage lpm; + br >> lpm; + rdr.setProtocolVersion(br.protocolVersion()); + } else { + MutationRef m; + BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); + br >> m; + TraceEvent&& event = debugMutation(context, version, m); + if (event.isEnabled()) { + return std::move(event.detail("MessageTags", msg.tags)); + } + } + } + return std::move(TraceEvent()); +} + #if MUTATION_TRACKING_ENABLED TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { return debugMutationEnabled( context, version, mutation ); diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h index 5fa97ef50b..db42579286 100644 --- a/fdbserver/MutationTracking.h +++ b/fdbserver/MutationTracking.h @@ -29,5 +29,6 @@ TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); +TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ); #endif diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 7ce0bb5d79..c6d9ecf2e0 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -31,6 +31,7 @@ #include "fdbserver/TLogInterface.h" #include "fdbserver/Knobs.h" #include "fdbserver/IKeyValueStore.h" +#include "fdbserver/MutationTracking.h" #include "flow/ActorCollection.h" #include "fdbrpc/FailureMonitor.h" #include "fdbserver/IDiskQueue.h" @@ -1221,6 +1222,7 @@ void commitMessages( TLogData* self, Reference logData, Version version block.reserve(block.arena(), std::max(SERVER_KNOBS->TLOG_MESSAGE_BLOCK_BYTES, msgSize)); } + debugMessagesAndTags("TLogCommitMessages", version, msg.getRawMessage()).detail("UID", self->dbgid).detail("LogId", logData->logId); block.append(block.arena(), msg.message.begin(), msg.message.size()); for(auto tag : msg.tags) { if(logData->locality == tagLocalitySatellite) { @@ -1335,7 +1337,12 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req messages << VERSION_HEADER << currentVersion; } + // We need the 4 byte length prefix to be a TagsAndMessage format, but that prefix is added as part of StringRef serialization. + int offset = messages.getLength(); messages << it->second.toStringRef(); + void* data = messages.getData(); + debugMessagesAndTags("TLogPeek", currentVersion, StringRef((uint8_t*)data+offset, messages.getLength()-offset)) + .detail("LogId", self->logId).detail("PeekTag", req.tag); } } @@ -1580,6 +1587,7 @@ ACTOR Future tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere wait(parseMessagesForTag(entry.messages, req.tag, logData->logRouterTags)); for (const StringRef& msg : rawMessages) { messages.serializeBytes(msg); + debugMessagesAndTags("TLogPeekFromDisk", entry.version, msg).detail("UID", self->dbgid).detail("LogId", logData->logId).detail("PeekTag", req.tag); } lastRefMessageVersion = entry.version; From 40d10aa9903af400f89aacfa3748e9e84bf08390 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 27 Mar 2020 04:01:18 -0700 Subject: [PATCH 03/89] Fix debugMutation uses that were concurrently added in new backup code --- fdbserver/BackupWorker.actor.cpp | 6 ++---- fdbserver/MutationTracking.h | 2 +- fdbserver/RestoreApplier.actor.h | 15 +++++++------ fdbserver/RestoreLoader.actor.cpp | 35 +++++++++++++++---------------- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index 3cfc1662d3..d5a0426c43 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -652,13 +652,11 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int MutationRef m; if (!message.isBackupMessage(&m)) continue; - if (debugMutation("addMutation", message.version.version, m)) { - TraceEvent("BackupWorkerDebug", self->myId) + debugMutation("addMutation", message.version.version, m) .detail("Version", message.version.toString()) - .detail("Mutation", m.toString()) + .detail("Mutation", m) .detail("KCV", self->minKnownCommittedVersion) .detail("SavedVersion", self->savedVersion); - } std::vector> adds; if (m.type != MutationRef::Type::ClearRange) { diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h index db42579286..d0f6bacb39 100644 --- a/fdbserver/MutationTracking.h +++ b/fdbserver/MutationTracking.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbclient/CommitTransaction.h" -#define MUTATION_TRACKING_ENABLED 0 +#define MUTATION_TRACKING_ENABLED 1 TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h index 2d2cf69d1c..dbe959fa5d 100644 --- a/fdbserver/RestoreApplier.actor.h +++ b/fdbserver/RestoreApplier.actor.h @@ -36,6 +36,7 @@ #include "fdbrpc/Locality.h" #include "fdbserver/CoordinationInterface.h" #include "fdbclient/RestoreWorkerInterface.actor.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RestoreUtil.h" #include "fdbserver/RestoreRoleCommon.actor.h" @@ -60,19 +61,17 @@ struct StagingKey { // Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set void add(const MutationRef& m, LogMessageVersion newVersion) { ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue); - if (debugMutation("StagingKeyAdd", newVersion.version, m)) { - TraceEvent("StagingKeyAdd") - .detail("Version", version.toString()) - .detail("NewVersion", newVersion.toString()) - .detail("Mutation", m.toString()); - } + debugMutation("StagingKeyAdd", newVersion.version, m) + .detail("Version", version.toString()) + .detail("NewVersion", newVersion.toString()) + .detail("Mutation", m); if (version == newVersion) { // This could happen because the same mutation can be present in // overlapping mutation logs, because new TLogs can copy mutations // from old generation TLogs (or backup worker is recruited without // knowning previously saved progress). ASSERT(type == m.type && key == m.param1 && val == m.param2); - TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString()); + TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m); return; } @@ -93,7 +92,7 @@ struct StagingKey { // Duplicated mutation ignored. TraceEvent("SameVersion") .detail("Version", version.toString()) - .detail("Mutation", m.toString()) + .detail("Mutation", m) .detail("NewVersion", newVersion.toString()); ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2); } diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index fadda9e28c..d7a76b3288 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -24,6 +24,7 @@ #include "fdbclient/BackupContainer.h" #include "fdbserver/RestoreLoader.actor.h" #include "fdbserver/RestoreRoleCommon.actor.h" +#include "fdbserver/MutationTracking.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -463,13 +464,15 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat nodeIDs.contents()); ASSERT(mvector.size() == nodeIDs.size()); - if (debugMutation("RestoreLoader", commitVersion.version, kvm)) { - TraceEvent e("DebugSplit"); - int i = 0; - for (auto& [key, uid] : *pRangeToApplier) { - e.detail(format("Range%d", i).c_str(), printable(key)) - .detail(format("UID%d", i).c_str(), uid.toString()); - i++; + { + TraceEvent&& e = debugMutation("RestoreLoaderDebugSplit", commitVersion.version, kvm); + if (e.isEnabled()) { + int i = 0; + for (auto& [key, uid] : *pRangeToApplier) { + e.detail(format("Range%d", i).c_str(), printable(key)) + .detail(format("UID%d", i).c_str(), uid.toString()); + i++; + } } } for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) { @@ -477,11 +480,9 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat UID applierID = nodeIDs[splitMutationIndex]; // printf("SPLITTED MUTATION: %d: mutation:%s applierID:%s\n", splitMutationIndex, // mutation.toString().c_str(), applierID.toString().c_str()); - if (debugMutation("RestoreLoader", commitVersion.version, mutation)) { - TraceEvent("SplittedMutation") - .detail("Version", commitVersion.toString()) - .detail("Mutation", mutation.toString()); - } + debugMutation("RestoreLoaderSplittedMutation", commitVersion.version, mutation) + .detail("Version", commitVersion.toString()) + .detail("Mutation", mutation); applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), mutation); applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub); applierMutationsSize[applierID] += mutation.expectedSize(); @@ -495,12 +496,10 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat UID applierID = itlow->second; kvCount++; - if (debugMutation("RestoreLoader", commitVersion.version, kvm)) { - TraceEvent("SendMutation") - .detail("Applier", applierID) - .detail("Version", commitVersion.toString()) - .detail("Mutation", kvm.toString()); - } + debugMutation("RestoreLoaderSendMutation", commitVersion.version, kvm) + .detail("Applier", applierID) + .detail("Version", commitVersion.toString()) + .detail("Mutation", kvm); applierMutationsBuffer[applierID].push_back_deep(applierMutationsBuffer[applierID].arena(), kvm); applierSubsBuffer[applierID].push_back(applierSubsBuffer[applierID].arena(), commitVersion.sub); applierMutationsSize[applierID] += kvm.expectedSize(); From 32519e92b2431fb111b3a54c99fc593678a3b042 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 27 Mar 2020 04:04:13 -0700 Subject: [PATCH 04/89] And fix a couple more issues from me doing manual things wrong. --- fdbserver/MutationTracking.cpp | 4 ++++ fdbserver/MutationTracking.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index 64ec3f4872..f776bf95b4 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -90,7 +90,11 @@ TraceEvent debugMutation( const char* context, Version version, MutationRef cons TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return debugKeyRangeEnabled( context, version, keys ); } +TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ) { + return debugMessagesAndTagsEnabled( context, version, commitBlob ); +} #else TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { return std::move(TraceEvent()); } TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return std::move(TraceEvent()); } +TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ) { return std::move(TraceEvent()); } #endif diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h index d0f6bacb39..db42579286 100644 --- a/fdbserver/MutationTracking.h +++ b/fdbserver/MutationTracking.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbclient/CommitTransaction.h" -#define MUTATION_TRACKING_ENABLED 1 +#define MUTATION_TRACKING_ENABLED 0 TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); From 146787e8d6f79b5a3323d61950d4c468ba7cd63d Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 27 Mar 2020 04:16:15 -0700 Subject: [PATCH 05/89] Add MutationTracking to CMake. --- fdbserver/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index fedb6be995..7d632243b2 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -47,8 +47,10 @@ set(FDBSERVER_SRCS MasterInterface.h MasterProxyServer.actor.cpp masterserver.actor.cpp - MoveKeys.actor.cpp + MutationTracking.h + MutationTracking.cpp MoveKeys.actor.h + MoveKeys.actor.cpp networktest.actor.cpp NetworkTest.h OldTLogServer_4_6.actor.cpp From 10795cdc1a894a67277561fd37f3d91dd4b5783e Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 30 Mar 2020 13:17:29 -0700 Subject: [PATCH 06/89] Some performance improvements to the transaction profiling analyzer: * Fetch the boundary keys once and cache the results * Try to get addresses for shards in the same transaction and in parallel * Simplify the range count logic (this seems not to be a clear win performance-wise, I've seen this go twice as fast or take 50% longer) Also added a flag to enable ports on addresses. --- .../transaction_profiling_analyzer.py | 167 +++++++++++------- 1 file changed, 102 insertions(+), 65 deletions(-) diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index 15fa19d166..2d95ddc220 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -39,7 +39,9 @@ from json import JSONEncoder import logging import struct from bisect import bisect_left +from bisect import bisect_right import time +import datetime PROTOCOL_VERSION_5_2 = 0x0FDB00A552000001 PROTOCOL_VERSION_6_0 = 0x0FDB00A570010001 @@ -514,6 +516,7 @@ class RangeCounter(object): self.k = k from sortedcontainers import SortedDict self.ranges = SortedDict() + self.ranges[b''] = 0 def process(self, transaction_info): for get_range in transaction_info.get_ranges: @@ -521,52 +524,18 @@ class RangeCounter(object): def _insert_range(self, start_key, end_key): keys = self.ranges.keys() - if len(keys) == 0: - self.ranges[start_key] = end_key, 1 - return + start_pos = bisect_right(keys, start_key)-1 + end_pos = bisect_right(keys, end_key)-1 + + start_count = self.ranges[keys[start_pos]] + end_count = self.ranges[keys[end_pos]] - start_pos = bisect_left(keys, start_key) end_pos = bisect_left(keys, end_key) - #print("start_pos=%d, end_pos=%d" % (start_pos, end_pos)) + for k in self.ranges.islice(start_pos+1, end_pos): + self.ranges[k] += 1 - possible_intersection_keys = keys[max(0, start_pos - 1):min(len(keys), end_pos+1)] - - start_range_left = start_key - - for key in possible_intersection_keys: - cur_end_key, cur_count = self.ranges[key] - #logger.debug("key=%s, cur_end_key=%s, cur_count=%d, start_range_left=%s" % (key, cur_end_key, cur_count, start_range_left)) - if start_range_left < key: - if end_key <= key: - self.ranges[start_range_left] = end_key, 1 - return - self.ranges[start_range_left] = key, 1 - start_range_left = key - assert start_range_left >= key - if start_range_left >= cur_end_key: - continue - - # [key, start_range_left) = cur_count - # if key == start_range_left this will get overwritten below - self.ranges[key] = start_range_left, cur_count - - if end_key <= cur_end_key: - # [start_range_left, end_key) = cur_count+1 - # [end_key, cur_end_key) = cur_count - self.ranges[start_range_left] = end_key, cur_count + 1 - if end_key != cur_end_key: - self.ranges[end_key] = cur_end_key, cur_count - start_range_left = end_key - break - else: - # [start_range_left, cur_end_key) = cur_count+1 - self.ranges[start_range_left] = cur_end_key, cur_count+1 - start_range_left = cur_end_key - assert start_range_left <= end_key - - # there may be some range left - if start_range_left < end_key: - self.ranges[start_range_left] = end_key, 1 + self.ranges[start_key] = start_count+1 + self.ranges[end_key] = end_count def get_count_for_key(self, key): if key in self.ranges: @@ -574,16 +543,12 @@ class RangeCounter(object): keys = self.ranges.keys() index = bisect_left(keys, key) - if index == 0: - return 0 index_key = keys[index-1] - if index_key <= key < self.ranges[index_key][0]: - return self.ranges[index_key][1] - return 0 + return self.ranges[keys[index_key]] def get_range_boundaries(self, shard_finder=None): - total = sum([count for _, (_, count) in self.ranges.items()]) + total = sum([count for count in self.ranges.values()]) range_size = total // self.k output_range_counts = [] @@ -599,42 +564,102 @@ class RangeCounter(object): output_range_counts.append((start, end, count, None, None)) this_range_start_key = None + this_range_end_key = None count_this_range = 0 - for (start_key, (end_key, count)) in self.ranges.items(): - if not this_range_start_key: - this_range_start_key = start_key - count_this_range += count + prev_count = 0 + for (start_key, count) in self.ranges.items(): + if prev_count > 0: + this_range_end_key = start_key + if count_this_range >= range_size: - add_boundary(this_range_start_key, end_key, count_this_range) + add_boundary(this_range_start_key, this_range_end_key, count_this_range) count_this_range = 0 this_range_start_key = None + + if count != 0 and not this_range_start_key: + this_range_start_key = start_key + + prev_count = count + count_this_range += count + + if this_range_end_key is None: + this_range_end_key = b'\xff' if count_this_range > 0: - add_boundary(this_range_start_key, end_key, count_this_range) + add_boundary(this_range_start_key, this_range_end_key, count_this_range) + + for index in range(len(output_range_counts)): + item = output_range_counts[index] + if item[4] is not None: + while True: + try: + output_range_counts[index] = item[0:4] + ([a.decode('ascii') for a in item[4].wait()],) + break + except fdb.FDBError as e: + output_range_counts[index] = item[0:4] + (shard_finder.get_addresses_for_key(item[0]),) return output_range_counts class ShardFinder(object): - def __init__(self, db): + def __init__(self, db, include_ports): self.db = db + self.include_ports = include_ports - @staticmethod - @fdb.transactional - def _get_boundary_keys(tr, begin, end): - tr.options.set_read_lock_aware() - return fdb.locality.get_boundary_keys(tr, begin, end) + self.tr = db.create_transaction() + self.refresh_tr() + + self.outstanding = [] + self.boundary_keys = list(fdb.locality.get_boundary_keys(db, b'', b'\xff\xff')) + self.shard_cache = {} + + def _get_boundary_keys(self, begin, end): + start_pos = max(0, bisect_right(self.boundary_keys, begin)-1) + end_pos = max(0, bisect_right(self.boundary_keys, end)-1) + + return self.boundary_keys[start_pos:end_pos] + + def refresh_tr(self): + self.tr.options.set_read_lock_aware() + if self.include_ports: + self.tr.options.set_include_port_in_address() @staticmethod @fdb.transactional def _get_addresses_for_key(tr, key): - tr.options.set_read_lock_aware() return fdb.locality.get_addresses_for_key(tr, key) def get_shard_count(self, start_key, end_key): - return len(list(self._get_boundary_keys(self.db, start_key, end_key))) + 1 + return len(self._get_boundary_keys(start_key, end_key)) + 1 def get_addresses_for_key(self, key): - return [a.decode('ascii') for a in self._get_addresses_for_key(self.db, key).wait()] + shard = self.boundary_keys[max(0, bisect_right(self.boundary_keys, key)-1)] + do_load = False + if not shard in self.shard_cache: + do_load = True + elif self.shard_cache[shard].is_ready(): + try: + self.shard_cache[shard].wait() + except fdb.FDBError as e: + self.tr.on_error(e).wait() + self.refresh_tr() + do_load = True + + if do_load: + if len(self.outstanding) > 1000: + for f in self.outstanding: + try: + f.wait() + except fdb.FDBError as e: + pass + + self.outstanding = [] + self.tr.reset() + self.refresh_tr() + + self.outstanding.append(self._get_addresses_for_key(self.tr, shard)) + self.shard_cache[shard] = self.outstanding[-1] + + return self.shard_cache[shard] class TopKeysCounter(object): @@ -683,6 +708,16 @@ class TopKeysCounter(object): if count_this_range > 0: add_boundary(start_key, k, count_this_range) + for index in range(len(output_range_counts)): + item = output_range_counts[index] + if item[4] is not None: + while True: + try: + output_range_counts[index] = item[0:4] + ([a.decode('ascii') for a in item[4].wait()],) + break + except fdb.FDBError as e: + output_range_counts[index] = item[0:4] + (shard_finder.get_addresses_for_key(item[0]),) + return output_range_counts def _get_top_k(self, counts): @@ -733,6 +768,7 @@ def main(): end_time_group.add_argument("--max-timestamp", type=int, help="Don't return events newer than this epoch time") end_time_group.add_argument("-e", "--end-time", type=str, help="Don't return events older than this parsed time") parser.add_argument("--top-keys", type=int, help="If specified will output this many top keys for reads or writes", default=0) + parser.add_argument("--include-ports", type=bool, help="Print addresses with the port number. 6.2 clusters only", default=False) args = parser.parse_args() type_filter = set() @@ -802,7 +838,8 @@ def main(): addresses_string = "addresses=%s" % ','.join(addresses) if addresses else '' print("[%s, %s] %d shards=%d %s" % (start, end, count, shard_count, addresses_string)) - shard_finder = ShardFinder(db) + shard_finder = ShardFinder(db, args.include_ports) + top_reads = key_counter.get_top_k_reads() if top_reads: print("Top %d reads:" % min(top_keys, len(top_reads))) From a45533b32692243b60e573352de916211126a1b6 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 30 Mar 2020 14:43:59 -0700 Subject: [PATCH 07/89] Performance improvements to range insertion --- .../transaction_profiling_analyzer.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index 2d95ddc220..b564f01134 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -524,18 +524,25 @@ class RangeCounter(object): def _insert_range(self, start_key, end_key): keys = self.ranges.keys() - start_pos = bisect_right(keys, start_key)-1 - end_pos = bisect_right(keys, end_key)-1 + start_pos = self.ranges.bisect_right(start_key)-1 + end_pos = self.ranges.bisect_right(end_key)-1 - start_count = self.ranges[keys[start_pos]] - end_count = self.ranges[keys[end_pos]] + exact_end = end_key == keys[end_pos] + if not exact_end: + end_count = self.ranges[keys[end_pos]] + end_pos += 1 - end_pos = bisect_left(keys, end_key) for k in self.ranges.islice(start_pos+1, end_pos): self.ranges[k] += 1 - self.ranges[start_key] = start_count+1 - self.ranges[end_key] = end_count + if keys[start_pos] == start_key: + self.ranges[start_key] += 1 + else: + self.ranges[start_key] = self.ranges[keys[start_pos]] + 1 + + if not exact_end: + self.ranges[end_key] = end_count + def get_count_for_key(self, key): if key in self.ranges: From 60484af8e0a0f79016d089a38248c4f8e1c49ad5 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 2 Apr 2020 13:50:27 -0700 Subject: [PATCH 08/89] More performance improvements to loading ranges. Fix range accounting to avoid double counting ranges. Add some new features related to filtering by address, listing top read requests, querying for all reads. --- .../transaction_profiling_analyzer.py | 394 +++++++++++------- 1 file changed, 251 insertions(+), 143 deletions(-) diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index b564f01134..71bf833712 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -411,7 +411,7 @@ class TransactionInfoLoader(object): else: end_key = self.client_latency_end_key_selector - valid_transaction_infos = 0 + transaction_infos = 0 invalid_transaction_infos = 0 def build_client_transaction_info(v): @@ -443,11 +443,12 @@ class TransactionInfoLoader(object): info = build_client_transaction_info(v) if info.has_types(): buffer.append(info) - valid_transaction_infos += 1 except UnsupportedProtocolVersionError as e: invalid_transaction_infos += 1 except ValueError: invalid_transaction_infos += 1 + + transaction_infos += 1 else: if chunk_num == 1: # first chunk @@ -473,14 +474,15 @@ class TransactionInfoLoader(object): info = build_client_transaction_info(b''.join([chunk.value for chunk in c_list])) if info.has_types(): buffer.append(info) - valid_transaction_infos += 1 except UnsupportedProtocolVersionError as e: invalid_transaction_infos += 1 except ValueError: invalid_transaction_infos += 1 + + transaction_infos += 1 self._check_and_adjust_chunk_cache_size() - if (valid_transaction_infos + invalid_transaction_infos) % 1000 == 0: - print("Processed valid: %d, invalid: %d" % (valid_transaction_infos, invalid_transaction_infos)) + if transaction_infos % 1000 == 0: + print("Processed %d transactions, %d invalid" % (transaction_infos, invalid_transaction_infos)) if found == 0: more = False except fdb.FDBError as e: @@ -492,13 +494,15 @@ class TransactionInfoLoader(object): for item in buffer: yield item + print("Processed %d transactions, %d invalid\n" % (transaction_infos, invalid_transaction_infos)) + def has_sortedcontainers(): try: import sortedcontainers return True except ImportError: - logger.warn("Can't find sortedcontainers so disabling RangeCounter") + logger.warn("Can't find sortedcontainers so disabling ReadCounter") return False @@ -510,107 +514,125 @@ def has_dateparser(): logger.warn("Can't find dateparser so disabling human date parsing") return False +def wait_for_shard_addresses(ranges, shard_finder, key_idx, addr_idx): + for index in range(len(ranges)): + item = ranges[index] + if item[addr_idx] is not None: + while True: + try: + ranges[index] = item[0:addr_idx] + ([a.decode('ascii') for a in item[addr_idx].wait()],) + item[addr_idx+1:] + break + except fdb.FDBError as e: + ranges[index] = item[0:addr_idx] + (shard_finder.get_addresses_for_key(item[key_idx]),) + item[addr_idx+1:] -class RangeCounter(object): - def __init__(self, k): - self.k = k +class ReadCounter(object): + def __init__(self): from sortedcontainers import SortedDict - self.ranges = SortedDict() - self.ranges[b''] = 0 + self.reads = SortedDict() + self.reads[b''] = [0, 0] + + self.read_counts = {} + self.hit_count=0 def process(self, transaction_info): + for get in transaction_info.gets: + self._insert_read(get.key, None) for get_range in transaction_info.get_ranges: - self._insert_range(get_range.key_range.start_key, get_range.key_range.end_key) + self._insert_read(get_range.key_range.start_key, get_range.key_range.end_key) - def _insert_range(self, start_key, end_key): - keys = self.ranges.keys() - start_pos = self.ranges.bisect_right(start_key)-1 - end_pos = self.ranges.bisect_right(end_key)-1 + def _insert_read(self, start_key, end_key): + self.read_counts.setdefault((start_key, end_key), 0) + self.read_counts[(start_key, end_key)] += 1 - exact_end = end_key == keys[end_pos] - if not exact_end: - end_count = self.ranges[keys[end_pos]] - end_pos += 1 - - for k in self.ranges.islice(start_pos+1, end_pos): - self.ranges[k] += 1 - - if keys[start_pos] == start_key: - self.ranges[start_key] += 1 + self.reads.setdefault(start_key, [0, 0])[0] += 1 + if end_key is not None: + self.reads.setdefault(end_key, [0, 0])[1] += 1 else: - self.ranges[start_key] = self.ranges[keys[start_pos]] + 1 + self.reads.setdefault(start_key+b'\x00', [0, 0])[1] += 1 - if not exact_end: - self.ranges[end_key] = end_count + def get_total_reads(self): + return sum([v for v in self.read_counts.values()]) + + def matches_filter(addresses, required_addresses): + for addr in required_addresses: + if addr not in addresses: + return False + return True + def get_top_k_reads(self, num, filter_addresses, shard_finder=None): + count_pairs = sorted([(v, k) for (k, v) in self.read_counts.items()], reverse=True) + if not filter_addresses: + count_pairs = count_pairs[0:num] - def get_count_for_key(self, key): - if key in self.ranges: - return self.ranges[key][1] + if shard_finder: + results = [] + for (count, (start, end)) in count_pairs: + results.append((start, end, count, shard_finder.get_addresses_for_key(start))) - keys = self.ranges.keys() - index = bisect_left(keys, key) + wait_for_shard_addresses(results, shard_finder, 0, 3) - index_key = keys[index-1] - return self.ranges[keys[index_key]] + if filter_addresses: + filter_addresses = set(filter_addresses) + results = [r for r in results if filter_addresses.issubset(set(r[3]))][0:num] + else: + results = [(start, end, count) for (count, (start, end)) in count_pairs[0:num]] - def get_range_boundaries(self, shard_finder=None): - total = sum([count for count in self.ranges.values()]) - range_size = total // self.k + return results + + def get_range_boundaries(self, num_buckets, shard_finder=None): + total = sum([start_count for (start_count, end_count) in self.reads.values()]) + range_size = total // num_buckets output_range_counts = [] - def add_boundary(start, end, count): + def add_boundary(start, end, started_count, total_count): if shard_finder: shard_count = shard_finder.get_shard_count(start, end) if shard_count == 1: addresses = shard_finder.get_addresses_for_key(start) else: addresses = None - output_range_counts.append((start, end, count, shard_count, addresses)) + output_range_counts.append((start, end, started_count, total_count, shard_count, addresses)) else: - output_range_counts.append((start, end, count, None, None)) + output_range_counts.append((start, end, started_count, total_count, None, None)) this_range_start_key = None - this_range_end_key = None + last_end = None + open_count = 0 + opened_this_range = 0 count_this_range = 0 - prev_count = 0 - for (start_key, count) in self.ranges.items(): - if prev_count > 0: - this_range_end_key = start_key - if count_this_range >= range_size: - add_boundary(this_range_start_key, this_range_end_key, count_this_range) - count_this_range = 0 + for (start_key, (start_count, end_count)) in self.reads.items(): + open_count -= end_count + + if opened_this_range >= range_size: + add_boundary(this_range_start_key, start_key, opened_this_range, count_this_range) + count_this_range = open_count + opened_this_range = 0 this_range_start_key = None - if count != 0 and not this_range_start_key: + count_this_range += start_count + opened_this_range += start_count + open_count += start_count + + if count_this_range > 0 and this_range_start_key is None: this_range_start_key = start_key - prev_count = count - count_this_range += count + if end_count > 0: + last_end = start_key - if this_range_end_key is None: - this_range_end_key = b'\xff' + if last_end is None: + last_end = b'\xff' if count_this_range > 0: - add_boundary(this_range_start_key, this_range_end_key, count_this_range) - - for index in range(len(output_range_counts)): - item = output_range_counts[index] - if item[4] is not None: - while True: - try: - output_range_counts[index] = item[0:4] + ([a.decode('ascii') for a in item[4].wait()],) - break - except fdb.FDBError as e: - output_range_counts[index] = item[0:4] + (shard_finder.get_addresses_for_key(item[0]),) + add_boundary(this_range_start_key, last_end, opened_this_range, count_this_range) + wait_for_shard_addresses(output_range_counts, shard_finder, 0, 5) return output_range_counts class ShardFinder(object): - def __init__(self, db, include_ports): + def __init__(self, db, exclude_ports): self.db = db - self.include_ports = include_ports + self.exclude_ports = exclude_ports self.tr = db.create_transaction() self.refresh_tr() @@ -627,7 +649,7 @@ class ShardFinder(object): def refresh_tr(self): self.tr.options.set_read_lock_aware() - if self.include_ports: + if not self.exclude_ports: self.tr.options.set_include_port_in_address() @staticmethod @@ -669,26 +691,22 @@ class ShardFinder(object): return self.shard_cache[shard] -class TopKeysCounter(object): +class WriteCounter(object): mutation_types_to_consider = frozenset([MutationType.SET_VALUE, MutationType.ADD_VALUE]) - def __init__(self, k): - self.k = k - self.reads = defaultdict(lambda: 0) + def __init__(self): self.writes = defaultdict(lambda: 0) def process(self, transaction_info): - for get in transaction_info.gets: - self.reads[get.key] += 1 if transaction_info.commit: for mutation in transaction_info.commit.mutations: if mutation.code in self.mutation_types_to_consider: self.writes[mutation.param_one] += 1 - def _get_range_boundaries(self, counts, shard_finder=None): - total = sum([v for (k, v) in counts.items()]) - range_size = total // self.k - key_counts_sorted = sorted(counts.items()) + def get_range_boundaries(self, num_buckets, shard_finder=None): + total = sum([v for (k, v) in self.writes.items()]) + range_size = total // num_buckets + key_counts_sorted = sorted(self.writes.items()) output_range_counts = [] def add_boundary(start, end, count): @@ -698,9 +716,9 @@ class TopKeysCounter(object): addresses = shard_finder.get_addresses_for_key(start) else: addresses = None - output_range_counts.append((start, end, count, shard_count, addresses)) + output_range_counts.append((start, end, count, None, shard_count, addresses)) else: - output_range_counts.append((start, end, count, None, None)) + output_range_counts.append((start, end, count, None, None, None)) start_key = None count_this_range = 0 @@ -715,34 +733,31 @@ class TopKeysCounter(object): if count_this_range > 0: add_boundary(start_key, k, count_this_range) - for index in range(len(output_range_counts)): - item = output_range_counts[index] - if item[4] is not None: - while True: - try: - output_range_counts[index] = item[0:4] + ([a.decode('ascii') for a in item[4].wait()],) - break - except fdb.FDBError as e: - output_range_counts[index] = item[0:4] + (shard_finder.get_addresses_for_key(item[0]),) - + wait_for_shard_addresses(output_range_counts, shard_finder, 0, 5) return output_range_counts - def _get_top_k(self, counts): - count_key_pairs = sorted([(v, k) for (k, v) in counts.items()], reverse=True) - return count_key_pairs[0:self.k] + def get_total_writes(self): + return sum([v for v in self.writes.values()]) - def get_top_k_reads(self): - return self._get_top_k(self.reads) + def get_top_k_writes(self, num, filter_addresses, shard_finder=None): + count_pairs = sorted([(v, k) for (k, v) in self.writes.items()], reverse=True) + if not filter_addresses: + count_pairs = count_pairs[0:num] - def get_top_k_writes(self): - return self._get_top_k(self.writes) + if shard_finder: + results = [] + for (count, key) in count_pairs: + results.append((key, None, count, shard_finder.get_addresses_for_key(key))) - def get_k_read_range_boundaries(self, shard_finder=None): - return self._get_range_boundaries(self.reads, shard_finder) + wait_for_shard_addresses(results, shard_finder, 0, 3) - def get_k_write_range_boundaries(self, shard_finder=None): - return self._get_range_boundaries(self.writes, shard_finder) + if filter_addresses: + filter_addresses = set(filter_addresses) + results = [r for r in results if filter_addresses.issubset(set(r[3]))][0:num] + else: + results = [(key, end, count) for (count, key) in count_pairs[0:num]] + return results def connect(cluster_file=None): db = fdb.open(cluster_file=cluster_file) @@ -759,6 +774,8 @@ def main(): help="Include get type. If no filter args are given all will be returned.") parser.add_argument("--filter-get-range", action="store_true", help="Include get_range type. If no filter args are given all will be returned.") + parser.add_argument("--filter-reads", action="store_true", + help="Include get and get_range type. If no filter args are given all will be returned.") parser.add_argument("--filter-commit", action="store_true", help="Include commit type. If no filter args are given all will be returned.") parser.add_argument("--filter-error-get", action="store_true", @@ -774,22 +791,34 @@ def main(): end_time_group = parser.add_mutually_exclusive_group() end_time_group.add_argument("--max-timestamp", type=int, help="Don't return events newer than this epoch time") end_time_group.add_argument("-e", "--end-time", type=str, help="Don't return events older than this parsed time") - parser.add_argument("--top-keys", type=int, help="If specified will output this many top keys for reads or writes", default=0) - parser.add_argument("--include-ports", type=bool, help="Print addresses with the port number. 6.2 clusters only", default=False) + parser.add_argument("--num-buckets", type=int, help="The number of buckets to partition the key-space into for operation counts", default=100) + parser.add_argument("--top-requests", type=int, help="If specified will output this many top keys for reads or writes", default=0) + parser.add_argument("--exclude-ports", action="store_true", help="Print addresses without the port number. Only works in versions older than 6.3, and is required in versions older than 6.2.") + parser.add_argument("--single-shard-ranges-only", action="store_true", help="Only print range boundaries that exist in a single shard") + parser.add_argument("-a", "--filter-address", action="append", help="Only print range boundaries that include the given address. This option can used multiple times to include more than one address in the filter, in which case all addresses must match.") + args = parser.parse_args() type_filter = set() if args.filter_get_version: type_filter.add("get_version") - if args.filter_get: type_filter.add("get") - if args.filter_get_range: type_filter.add("get_range") + if args.filter_get or args.filter_reads: type_filter.add("get") + if args.filter_get_range or args.filter_reads: type_filter.add("get_range") if args.filter_commit: type_filter.add("commit") if args.filter_error_get: type_filter.add("error_get") if args.filter_error_get_range: type_filter.add("error_get_range") if args.filter_error_commit: type_filter.add("error_commit") - top_keys = args.top_keys - key_counter = TopKeysCounter(top_keys) if top_keys else None - range_counter = RangeCounter(top_keys) if (has_sortedcontainers() and top_keys) else None - full_output = args.full_output or (top_keys is not None) + + if (not type_filter or "commit" in type_filter): + write_counter = WriteCounter() if args.num_buckets else None + else: + write_counter = None + + if (not type_filter or "get" in type_filter or "get_range" in type_filter): + read_counter = ReadCounter() if (has_sortedcontainers() and args.num_buckets) else None + else: + read_counter = None + + full_output = args.full_output or (args.num_buckets is not None) if args.min_timestamp: min_timestamp = args.min_timestamp @@ -822,49 +851,128 @@ def main(): db = connect(cluster_file=args.cluster_file) loader = TransactionInfoLoader(db, full_output=full_output, type_filter=type_filter, min_timestamp=min_timestamp, max_timestamp=max_timestamp) + for info in loader.fetch_transaction_info(): if info.has_types(): - if not key_counter and not range_counter: + if not write_counter and not read_counter: print(info.to_json()) else: - if key_counter: - key_counter.process(info) - if range_counter: - range_counter.process(info) + if write_counter: + write_counter.process(info) + if read_counter: + read_counter.process(info) - if key_counter: - def print_top(top): - for (count, key) in top: - print("%s %d" % (key, count)) - - def print_range_boundaries(range_boundaries): - for (start, end, count, shard_count, addresses) in range_boundaries: - if not shard_count: - print("[%s, %s] %d" % (start, end, count)) + def print_top(top, total, context): + if top: + running_count = 0 + for (idx, (start, end, count, addresses)) in enumerate(top): + running_count += count + if end is not None: + op_str = 'Range %r - %r' % (start, end) else: - addresses_string = "addresses=%s" % ','.join(addresses) if addresses else '' - print("[%s, %s] %d shards=%d %s" % (start, end, count, shard_count, addresses_string)) + op_str = 'Key %r' % start - shard_finder = ShardFinder(db, args.include_ports) + print(" %d. %s\n %d sampled %s (%.2f%%, %.2f%% cumulative)" % (idx+1, op_str, count, context, 100*count/total, 100*running_count/total)) + print(" shard addresses: %s\n" % ", ".join(addresses)) + + else: + print(" No %s found" % context) + + def print_range_boundaries(range_boundaries, context): + omit_start = None + for (idx, (start, end, start_count, total_count, shard_count, addresses)) in enumerate(range_boundaries): + omit = args.single_shard_ranges_only and shard_count is not None and shard_count > 1 + if args.filter_address: + if not addresses: + omit = True + else: + for addr in args.filter_address: + if addr not in addresses: + omit = True + break + + if not omit: + if omit_start is not None: + if omit_start == idx-1: + print(" %d. Omitted\n" % (idx)) + else: + print(" %d - %d. Omitted\n" % (omit_start+1, idx)) + omit_start = None + + if total_count is None: + count_str = '%d sampled %s' % (start_count, context) + else: + count_str = '%d sampled %s (%d intersecting)' % (start_count, context, total_count) + if not shard_count: + print(" %d. [%s, %s]\n %d sampled %s\n" % (idx+1, start, end, count, context)) + else: + addresses_string = "; addresses=%s" % ', '.join(addresses) if addresses else '' + print(" %d. [%s, %s]\n %s spanning %d shard(s)%s\n" % (idx+1, start, end, count_str, shard_count, addresses_string)) + elif omit_start is None: + omit_start = idx + + if omit_start is not None: + if omit_start == len(range_boundaries)-1: + print(" %d. Omitted\n" % len(range_boundaries)) + else: + print(" %d - %d. Omitted\n" % (omit_start+1, len(range_boundaries))) + + shard_finder = ShardFinder(db, args.exclude_ports) + + print("NOTE: shard locations are current and may not reflect where an operation was performed in the past\n") + + if write_counter: + if args.top_requests: + top_writes = write_counter.get_top_k_writes(args.top_requests, args.filter_address, shard_finder=shard_finder) + + range_boundaries = write_counter.get_range_boundaries(args.num_buckets, shard_finder=shard_finder) + num_writes = write_counter.get_total_writes() + + if args.top_requests or range_boundaries: + print("WRITES") + print("------\n") + print("Processed %d total writes\n" % num_writes) + + if args.top_requests: + suffix = "" + if args.filter_address: + suffix = " (%s)" % ", ".join(args.filter_address) + print("Top %d writes%s:\n" % (args.top_requests, suffix)) + + print_top(top_writes, write_counter.get_total_writes(), "writes") + print("") - top_reads = key_counter.get_top_k_reads() - if top_reads: - print("Top %d reads:" % min(top_keys, len(top_reads))) - print_top(top_reads) - print("Approx equal sized gets range boundaries:") - print_range_boundaries(key_counter.get_k_read_range_boundaries(shard_finder=shard_finder)) - top_writes = key_counter.get_top_k_writes() - if top_writes: - print("Top %d writes:" % min(top_keys, len(top_writes))) - print_top(top_writes) - print("Approx equal sized commits range boundaries:") - print_range_boundaries(key_counter.get_k_write_range_boundaries(shard_finder=shard_finder)) - if range_counter: - range_boundaries = range_counter.get_range_boundaries(shard_finder=shard_finder) if range_boundaries: - print("Approx equal sized get_ranges boundaries:") - print_range_boundaries(range_boundaries) + print("Key-space boundaries with approximately equal mutation counts:\n") + print_range_boundaries(range_boundaries, "writes") + if args.top_requests or range_boundaries: + print("") + + if read_counter: + if args.top_requests: + top_reads = read_counter.get_top_k_reads(args.top_requests, args.filter_address, shard_finder=shard_finder) + + range_boundaries = read_counter.get_range_boundaries(args.num_buckets, shard_finder=shard_finder) + num_reads = read_counter.get_total_reads() + + if args.top_requests or range_boundaries: + print("READS") + print("-----\n") + print("Processed %d total reads\n" % num_reads) + + if args.top_requests: + suffix = "" + if args.filter_address: + suffix = " (%s)" % ", ".join(args.filter_address) + print("Top %d reads%s:\n" % (args.top_requests, suffix)) + + print_top(top_reads, num_reads, "reads") + print("") + + if range_boundaries: + print("Key-space boundaries with approximately equal read counts:\n") + print_range_boundaries(range_boundaries, "reads") if __name__ == "__main__": main() From 902c08715e56c573addf8eb73f0190c6521f9a83 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 16 Apr 2020 11:52:58 -0700 Subject: [PATCH 09/89] Fix sorting bug that can occur when ordering point and range reads --- .../transaction_profiling_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index 71bf833712..fca842161b 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -560,7 +560,7 @@ class ReadCounter(object): return True def get_top_k_reads(self, num, filter_addresses, shard_finder=None): - count_pairs = sorted([(v, k) for (k, v) in self.read_counts.items()], reverse=True) + count_pairs = sorted([(v, k) for (k, v) in self.read_counts.items()], reverse=True, key=lambda item: item[0]) if not filter_addresses: count_pairs = count_pairs[0:num] From 88fa9bdb63633ba231b090e30b9497edc4aa96a2 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 2 May 2020 20:43:50 -0700 Subject: [PATCH 10/89] Unrevert "Enable -Wclass-memaccess and fix warnings" --- cmake/ConfigureCompiler.cmake | 1 + fdbclient/FDBTypes.h | 5 +++++ fdbserver/DiskQueue.actor.cpp | 2 +- fdbserver/VFSAsync.cpp | 2 +- flow/Arena.h | 35 +++++++++++++++++++---------------- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 20960a61fa..265246de1b 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -243,6 +243,7 @@ else() -fvisibility=hidden -Wreturn-type -fPIC) + add_compile_options($<$:-Wclass-memaccess>) if (GPERFTOOLS_FOUND AND GCC) add_compile_options( -fno-builtin-malloc diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 21b5d00dc5..26934c77ca 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -26,6 +26,7 @@ #include #include +#include "flow/Arena.h" #include "flow/flow.h" #include "fdbclient/Knobs.h" @@ -77,6 +78,10 @@ struct Tag { serializer(ar, locality, id); } }; + +template <> +struct non_flow_ref : std::integral_constant {}; + #pragma pack(pop) template void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); } diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp index 1f38dfb8ee..9ec422ad7c 100644 --- a/fdbserver/DiskQueue.actor.cpp +++ b/fdbserver/DiskQueue.actor.cpp @@ -1013,7 +1013,7 @@ private: ASSERT( nextPageSeq%sizeof(Page)==0 ); auto& p = backPage(); - memset(&p, 0, sizeof(Page)); // FIXME: unnecessary? + memset(static_cast(&p), 0, sizeof(Page)); // FIXME: unnecessary? p.magic = 0xFDB; switch (diskQueueVersion) { case DiskQueueVersion::V0: diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/VFSAsync.cpp index 3d53aaccfb..0a1feff976 100644 --- a/fdbserver/VFSAsync.cpp +++ b/fdbserver/VFSAsync.cpp @@ -531,7 +531,7 @@ static int asyncOpen( if (flags & SQLITE_OPEN_WAL) oflags |= IAsyncFile::OPEN_LARGE_PAGES; oflags |= IAsyncFile::OPEN_LOCK; - memset(p, 0, sizeof(VFSAsyncFile)); + memset(static_cast(p), 0, sizeof(VFSAsyncFile)); new (p) VFSAsyncFile(zName, flags); try { // Note that SQLiteDB::open also opens the db file, so its flags and modes are important, too diff --git a/flow/Arena.h b/flow/Arena.h index cfc756506d..cad0b13083 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -699,15 +699,18 @@ inline bool operator != (const StringRef& lhs, const StringRef& rhs ) { return ! inline bool operator <= ( const StringRef& lhs, const StringRef& rhs ) { return !(lhs>rhs); } inline bool operator >= ( const StringRef& lhs, const StringRef& rhs ) { return !(lhs -struct memcpy_able : std::is_trivial {}; +struct non_flow_ref : std::is_fundamental {}; template <> -struct memcpy_able : std::integral_constant {}; +struct non_flow_ref : std::integral_constant {}; + +template +struct non_flow_ref> : std::integral_constant {}; template struct string_serialized_traits : std::false_type { @@ -783,7 +786,7 @@ public: using value_type = T; static_assert(SerStrategy == VecSerStrategy::FlatBuffers || string_serialized_traits::value); - // T must be trivially destructible (and copyable)! + // T must be trivially destructible! VectorRef() : data(0), m_size(0), m_capacity(0) {} template @@ -798,19 +801,19 @@ public: return *this; } - // Arena constructor for non-Ref types, identified by memcpy_able + // Arena constructor for non-Ref types, identified by non_flow_ref template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(toCopy), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { if (m_size > 0) { - memcpy(data, toCopy.data, m_size * sizeof(T)); + std::copy(toCopy.data, toCopy.data + m_size, data); } } // Arena constructor for Ref types, which must have an Arena constructor template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { for (int i = 0; i < m_size; i++) { auto ptr = new (&data[i]) T(p, toCopy[i]); @@ -900,7 +903,7 @@ public: if (m_size + count > m_capacity) reallocate(p, m_size + count); VPS::invalidate(); if (count > 0) { - memcpy(data + m_size, begin, sizeof(T) * count); + std::copy(begin, begin + count, data + m_size); } m_size += count; } @@ -940,15 +943,15 @@ public: if (size > m_capacity) reallocate(p, size); } - // expectedSize() for non-Ref types, identified by memcpy_able + // expectedSize() for non-Ref types, identified by non_flow_ref template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { return sizeof(T) * m_size; } // expectedSize() for Ref types, which must in turn have expectedSize() implemented. template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { size_t t = sizeof(T) * m_size; for (int i = 0; i < m_size; i++) t += data[i].expectedSize(); return t; @@ -967,7 +970,7 @@ private: // SOMEDAY: Maybe we are right at the end of the arena and can expand cheaply T* newData = (T*)new (p) uint8_t[requiredCapacity * sizeof(T)]; if (m_size > 0) { - memcpy(newData, data, m_size * sizeof(T)); + std::move(data, data + m_size, newData); } data = newData; m_capacity = requiredCapacity; From d0d859c313ee98927fbc39e9c0eddfb19ea6479f Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 2 May 2020 11:12:53 -0700 Subject: [PATCH 11/89] Fix segmentation fault in VectorRef::reallocate --- flow/Arena.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/flow/Arena.h b/flow/Arena.h index cad0b13083..159a3d55ac 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -380,12 +380,6 @@ public: } #else Standalone( const T& t, const Arena& arena ) : Arena( arena ), T( t ) {} - Standalone( const Standalone & t ) : Arena((Arena const&)t), T((T const&)t) {} - Standalone& operator=( const Standalone & t ) { - *(Arena*)this = (Arena const&)t; - *(T*)this = (T const&)t; - return *this; - } #endif template Standalone castTo() const { @@ -968,7 +962,7 @@ private: void reallocate(Arena& p, int requiredCapacity) { requiredCapacity = std::max(m_capacity * 2, requiredCapacity); // SOMEDAY: Maybe we are right at the end of the arena and can expand cheaply - T* newData = (T*)new (p) uint8_t[requiredCapacity * sizeof(T)]; + T* newData = new (p) T[requiredCapacity]; if (m_size > 0) { std::move(data, data + m_size, newData); } From 19a9ae14c2f6dd91a371c11dc6101b6cf8d15f4a Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Sun, 10 May 2020 23:51:15 -0700 Subject: [PATCH 12/89] update master version to 7.0.0 --- CMakeLists.txt | 2 +- flow/ProtocolVersion.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 00bdde8e1e..fa762949a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.13) project(foundationdb - VERSION 6.3.0 + VERSION 7.0.0 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index a7ddcb21dd..fa288b56fe 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -101,7 +101,7 @@ public: // introduced features // // xyzdev // vvvv -constexpr ProtocolVersion currentProtocolVersion(0x0FDB00B063010001LL); +constexpr ProtocolVersion currentProtocolVersion(0x0FDB00B070010001LL); // This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to // change when we reach version 10. static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); From 81184e79c5f28a4bfbe471c9eb9a3e5678524bd4 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Mon, 11 May 2020 15:56:51 -0700 Subject: [PATCH 13/89] Fixed a bug where the loop might get stuck. --- fdbserver/StorageMetrics.actor.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h index a04007b6f9..41f51ac227 100644 --- a/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/StorageMetrics.actor.h @@ -436,7 +436,13 @@ struct StorageServerMetrics { IndexedSet::iterator endKey = byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + baseChunkSize); while (endKey != byteSample.sample.end()) { - if (*endKey > shard.end) endKey = byteSample.sample.lower_bound(shard.end); + if (*endKey > shard.end) { + endKey = byteSample.sample.lower_bound(shard.end); + if (*endKey == beginKey) { + // No need to increment endKey since otherwise it would stuck here forever. + break; + } + } if (*endKey == beginKey) { ++endKey; continue; From ccaac162e2843df9b1f5bf8745b567bc9a849780 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 13 May 2020 14:28:04 -0700 Subject: [PATCH 14/89] Resolve performance concerns of nearly-no-op debugMutation being frequently called This introduces unhygenic macro variants that inline a `ENABLED &&` before the TraceEvent. This way, they get entirely compiled out unless enabled. Then rewrite all debugMutation uses via sed. --- fdbserver/BackupWorker.actor.cpp | 2 +- fdbserver/LogSystem.h | 2 +- fdbserver/LogSystemPeekCursor.actor.cpp | 2 +- fdbserver/MasterProxyServer.actor.cpp | 8 ++--- fdbserver/MutationTracking.cpp | 8 ++--- fdbserver/MutationTracking.h | 8 ++++- fdbserver/RestoreApplier.actor.h | 4 +-- fdbserver/RestoreLoader.actor.cpp | 6 ++-- fdbserver/RestoreUtil.actor.cpp | 2 +- fdbserver/StorageCache.actor.cpp | 16 ++++----- fdbserver/TLogServer.actor.cpp | 6 ++-- fdbserver/storageserver.actor.cpp | 36 ++++++++++---------- fdbserver/workloads/ApiCorrectness.actor.cpp | 6 ++-- flow/Trace.h | 4 +++ 14 files changed, 58 insertions(+), 52 deletions(-) diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index 48d8d92356..b1d666e0f4 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -689,7 +689,7 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int MutationRef m; if (!message.isBackupMessage(&m)) continue; - debugMutation("addMutation", message.version.version, m) + DEBUG_MUTATION("addMutation", message.version.version, m) .detail("Version", message.version.toString()) .detail("Mutation", m) .detail("KCV", self->minKnownCommittedVersion) diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 63d6fa26ae..f2ee81bc5f 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -892,7 +892,7 @@ struct LogPushData : NonCopyable { wr << item; firstLength = wr.getLength() - firstOffset; *(uint32_t*)((uint8_t*)wr.getData() + firstOffset) = firstLength - sizeof(uint32_t); - debugMessagesAndTags("ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength)).detail("PushLocations", msg_locations); + DEBUG_TAGS_AND_MESSAGE("ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength)).detail("PushLocations", msg_locations); first = false; } else { BinaryWriter& wr = messagesWriter[loc]; diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp index 5e72092d3d..6f2d15cedd 100644 --- a/fdbserver/LogSystemPeekCursor.actor.cpp +++ b/fdbserver/LogSystemPeekCursor.actor.cpp @@ -91,7 +91,7 @@ void ILogSystem::ServerPeekCursor::nextMessage() { } messageAndTags.loadFromArena(&rd, &messageVersion.sub); - debugMessagesAndTags("ServerPeekCursor", messageVersion.version, messageAndTags.getRawMessage()).detail("CursorID", this->randomID); + DEBUG_TAGS_AND_MESSAGE("ServerPeekCursor", messageVersion.version, messageAndTags.getRawMessage()).detail("CursorID", this->randomID); // Rewind and consume the header so that reader() starts from the message. rd.rewind(); rd.readBytes(messageAndTags.getHeaderSize()); diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index ff35f74759..3ea5d5c0d1 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -749,7 +749,7 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::mapaddTags(tags); toCommit->addTypedMessage(backupMutation); -// if (debugMutation("BackupProxyCommit", commitVersion, backupMutation)) { +// if (DEBUG_MUTATION("BackupProxyCommit", commitVersion, backupMutation)) { // TraceEvent("BackupProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("BackupMutation", backupMutation.toString()) // .detail("BackupMutationSize", val.size()).detail("Version", commitVersion).detail("DestPath", logRangeMutation.first) // .detail("PartIndex", part).detail("PartIndexEndian", bigEndian32(part)).detail("PartData", backupMutation.param1); @@ -1067,7 +1067,7 @@ ACTOR Future commitBatch( self->singleKeyMutationEvent->log(); } - debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", tags).detail("Mutation", m); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", tags).detail("Mutation", m); toCommit.addTags(tags); if(self->cacheInfo[m.param1]) { @@ -1082,7 +1082,7 @@ ACTOR Future commitBatch( ++firstRange; if (firstRange == ranges.end()) { // Fast path - debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", ranges.begin().value().tags).detail("Mutation", m); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", ranges.begin().value().tags).detail("Mutation", m); ranges.begin().value().populateTags(); toCommit.addTags(ranges.begin().value().tags); @@ -1094,7 +1094,7 @@ ACTOR Future commitBatch( r.value().populateTags(); allSources.insert(r.value().tags.begin(), r.value().tags.end()); } - debugMutation("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", allSources).detail("Mutation", m); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", allSources).detail("Mutation", m); toCommit.addTags(allSources); } diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index f776bf95b4..867ce6482f 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -48,7 +48,7 @@ TraceEvent debugKeyRangeEnabled( const char* context, Version version, KeyRangeR } } -TraceEvent debugMessagesAndTagsEnabled( const char* context, Version version, StringRef commitBlob ) { +TraceEvent debugTagsAndMessageEnabled( const char* context, Version version, StringRef commitBlob ) { BinaryReader rdr(commitBlob, AssumeVersion(currentProtocolVersion)); while (!rdr.empty()) { if (*(int32_t*)rdr.peekBytes(4) == VERSION_HEADER) { @@ -90,11 +90,11 @@ TraceEvent debugMutation( const char* context, Version version, MutationRef cons TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return debugKeyRangeEnabled( context, version, keys ); } -TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ) { - return debugMessagesAndTagsEnabled( context, version, commitBlob ); +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ) { + return debugTagsAndMessageEnabled( context, version, commitBlob ); } #else TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { return std::move(TraceEvent()); } TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return std::move(TraceEvent()); } -TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ) { return std::move(TraceEvent()); } +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ) { return std::move(TraceEvent()); } #endif diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h index db42579286..99594ffe70 100644 --- a/fdbserver/MutationTracking.h +++ b/fdbserver/MutationTracking.h @@ -27,8 +27,14 @@ #define MUTATION_TRACKING_ENABLED 0 + +#define DEBUG_MUTATION(context, version, mutation) MUTATION_TRACKING_ENABLED && debugMutation(context, version, mutation) TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); + +#define DEBUG_KEY_RANGE(context, version, keys) MUTATION_TRACKING_ENABLED && debugKeyRange(context, version, keys) TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); -TraceEvent debugMessagesAndTags( const char* context, Version version, StringRef commitBlob ); + +#define DEBUG_TAGS_AND_MESSAGE(context, version, commitBlob) MUTATION_TRACKING_ENABLED && debugTagsAndMessage(context, version, commitBlob) +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ); #endif diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h index 422d79686f..7aaf44263a 100644 --- a/fdbserver/RestoreApplier.actor.h +++ b/fdbserver/RestoreApplier.actor.h @@ -61,7 +61,7 @@ struct StagingKey { // Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set void add(const MutationRef& m, LogMessageVersion newVersion) { ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue); - debugMutation("StagingKeyAdd", newVersion.version, m) + DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m) .detail("Version", version.toString()) .detail("NewVersion", newVersion.toString()) .detail("Mutation", m); @@ -83,7 +83,7 @@ struct StagingKey { ASSERT(m.param1 == m.param2); } if (version < newVersion) { - debugMutation("StagingKeyAdd", newVersion.version, m) + DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m) .detail("Version", version.toString()) .detail("NewVersion", newVersion.toString()) .detail("MType", getTypeString(type)) diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index e64ba4ed61..570d46d4bd 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -506,7 +506,7 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat nodeIDs.contents()); ASSERT(mvector.size() == nodeIDs.size()); - { + if (MUTATION_TRACKING_ENABLED) { TraceEvent&& e = debugMutation("RestoreLoaderDebugSplit", commitVersion.version, kvm); if (e.isEnabled()) { int i = 0; @@ -520,7 +520,7 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) { MutationRef mutation = mvector[splitMutationIndex]; UID applierID = nodeIDs[splitMutationIndex]; - debugMutation("RestoreLoaderSplittedMutation", commitVersion.version, mutation) + DEBUG_MUTATION("RestoreLoaderSplittedMutation", commitVersion.version, mutation) .detail("Version", commitVersion.toString()) .detail("Mutation", mutation); // CAREFUL: The splitted mutations' lifetime is shorter than the for-loop @@ -538,7 +538,7 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat UID applierID = itlow->second; kvCount++; - debugMutation("RestoreLoaderSendMutation", commitVersion.version, kvm) + DEBUG_MUTATION("RestoreLoaderSendMutation", commitVersion.version, kvm) .detail("Applier", applierID) .detail("Version", commitVersion.toString()) .detail("Mutation", kvm); diff --git a/fdbserver/RestoreUtil.actor.cpp b/fdbserver/RestoreUtil.actor.cpp index 7965ab60e4..7451f16570 100644 --- a/fdbserver/RestoreUtil.actor.cpp +++ b/fdbserver/RestoreUtil.actor.cpp @@ -76,4 +76,4 @@ bool isRangeMutation(MutationRef m) { ASSERT(m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type)m.type)); return false; } -} \ No newline at end of file +} diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index 2192ca527c..dad354713f 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -268,8 +268,8 @@ ACTOR Future getValueQ( StorageCacheData* data, GetValueRequest req ) { path = 1; } - //debugMutation("CacheGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); - //debugMutation("CacheGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); + //DEBUG_MUTATION("CacheGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); + //DEBUG_MUTATION("CacheGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); if (v.present()) { ++data->counters.rowsQueried; @@ -711,7 +711,7 @@ void StorageCacheData::addMutation(KeyRangeRef const& cachedKeyRange, Version ve return; } expanded = addMutationToMutationLog(mLog, expanded); - debugMutation("expandedMutation", version, expanded).detail("Begin", cachedKeyRange.begin).detail("End", cachedKeyRange.end); + DEBUG_MUTATION("expandedMutation", version, expanded).detail("Begin", cachedKeyRange.begin).detail("End", cachedKeyRange.end); applyMutation( this, expanded, mLog.arena(), mutableData() ); printf("\nSCUpdate: Printing versioned tree after applying mutation\n"); mutableData().printTree(version); @@ -731,15 +731,11 @@ public: data->mutableData().createNewVersion(ver); } + DEBUG_MUTATION("SCUpdateMutation", ver, m); if (m.param1.startsWith( systemKeys.end )) { //TraceEvent("PrivateData", data->thisServerID).detail("Mutation", m.toString()).detail("Version", ver); applyPrivateCacheData( data, m ); } else { - // FIXME: enable when debugMutation is active - //for(auto m = changes[c].mutations.begin(); m; ++m) { - // debugMutation("SCUpdateMutation", changes[c].version, *m); - //} - splitMutation(data, data->cachedRangeMap, m, ver); } @@ -757,7 +753,7 @@ private: //that this cache server is responsible for // TODO Revisit during failure handling. Might we loose some private mutations? void applyPrivateCacheData( StorageCacheData* data, MutationRef const& m ) { - TraceEvent(SevDebug, "SCPrivateCacheMutation", data->thisServerID).detail("Mutation", m.toString()); + TraceEvent(SevDebug, "SCPrivateCacheMutation", data->thisServerID).detail("Mutation", m); if (processedCacheStartKey) { // we expect changes in pairs, [begin,end). This mutation is for end key of the range @@ -903,7 +899,7 @@ ACTOR Future pullAsyncData( StorageCacheData *data ) { } if(ver != invalidVersion && ver > data->version.get()) { - debugKeyRange("SCUpdate", ver, allKeys); + DEBUG_KEY_RANGE("SCUpdate", ver, allKeys); data->mutableData().createNewVersion(ver); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index d0cfc32e03..bbcd9f7606 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -1259,7 +1259,7 @@ void commitMessages( TLogData* self, Reference logData, Version version block.reserve(block.arena(), std::max(SERVER_KNOBS->TLOG_MESSAGE_BLOCK_BYTES, msgSize)); } - debugMessagesAndTags("TLogCommitMessages", version, msg.getRawMessage()).detail("UID", self->dbgid).detail("LogId", logData->logId); + DEBUG_TAGS_AND_MESSAGE("TLogCommitMessages", version, msg.getRawMessage()).detail("UID", self->dbgid).detail("LogId", logData->logId); block.append(block.arena(), msg.message.begin(), msg.message.size()); for(auto tag : msg.tags) { if(logData->locality == tagLocalitySatellite) { @@ -1378,7 +1378,7 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req int offset = messages.getLength(); messages << it->second.toStringRef(); void* data = messages.getData(); - debugMessagesAndTags("TLogPeek", currentVersion, StringRef((uint8_t*)data+offset, messages.getLength()-offset)) + DEBUG_TAGS_AND_MESSAGE("TLogPeek", currentVersion, StringRef((uint8_t*)data+offset, messages.getLength()-offset)) .detail("LogId", self->logId).detail("PeekTag", req.tag); } } @@ -1641,7 +1641,7 @@ ACTOR Future tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere wait(parseMessagesForTag(entry.messages, req.tag, logData->logRouterTags)); for (const StringRef& msg : rawMessages) { messages.serializeBytes(msg); - debugMessagesAndTags("TLogPeekFromDisk", entry.version, msg).detail("UID", self->dbgid).detail("LogId", logData->logId).detail("PeekTag", req.tag); + DEBUG_TAGS_AND_MESSAGE("TLogPeekFromDisk", entry.version, msg).detail("UID", self->dbgid).detail("LogId", logData->logId).detail("PeekTag", req.tag); } lastRefMessageVersion = entry.version; diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index c02ba4fb56..85b8f0dec7 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -961,8 +961,8 @@ ACTOR Future getValueQ( StorageServer* data, GetValueRequest req ) { v = vv; } - debugMutation("ShardGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); - debugMutation("ShardGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); + DEBUG_MUTATION("ShardGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); + DEBUG_MUTATION("ShardGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); /* StorageMetrics m; @@ -1036,7 +1036,7 @@ ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) throw reply.error.get(); } - debugMutation("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("") ) ); + DEBUG_MUTATION("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("") ) ); if( req.debugID.present() ) g_traceBatch.addEvent("WatchValueDebug", req.debugID.get().first(), "watchValueQ.AfterRead"); //.detail("TaskID", g_network->getCurrentTask()); @@ -2092,7 +2092,7 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { wait( data->coreStarted.getFuture() && delay( 0 ) ); try { - debugKeyRange("fetchKeysBegin", data->version.get(), shard->keys); + DEBUG_KEY_RANGE("fetchKeysBegin", data->version.get(), shard->keys); TraceEvent(SevDebug, interval.begin(), data->thisServerID) .detail("KeyBegin", shard->keys.begin) @@ -2160,8 +2160,8 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { .detail("KeyBegin", keys.begin).detail("KeyEnd", keys.end) .detail("Last", this_block.size() ? this_block.end()[-1].key : std::string()) .detail("Version", fetchVersion).detail("More", this_block.more); - debugKeyRange("fetchRange", fetchVersion, keys); - for(auto k = this_block.begin(); k != this_block.end(); ++k) debugMutation("fetch", fetchVersion, MutationRef(MutationRef::SetValue, k->key, k->value)); + DEBUG_KEY_RANGE("fetchRange", fetchVersion, keys); + for(auto k = this_block.begin(); k != this_block.end(); ++k) DEBUG_MUTATION("fetch", fetchVersion, MutationRef(MutationRef::SetValue, k->key, k->value)); data->counters.bytesFetched += expectedSize; if( fetchBlockBytes > expectedSize ) { @@ -2308,7 +2308,7 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { ASSERT( b->version >= checkv ); checkv = b->version; for(auto& m : b->mutations) - debugMutation("fetchKeysFinalCommitInject", batch->changes[0].version, m); + DEBUG_MUTATION("fetchKeysFinalCommitInject", batch->changes[0].version, m); } shard->updates.clear(); @@ -2418,7 +2418,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss validate(data); // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. - //debugKeyRange( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); + //DEBUG_KEY_RANGE( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); bool isDifferent = false; auto existingShards = data->shards.intersectingRanges(keys); @@ -2523,7 +2523,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss void rollback( StorageServer* data, Version rollbackVersion, Version nextVersion ) { TEST(true); // call to shard rollback - debugKeyRange("Rollback", rollbackVersion, allKeys); + DEBUG_KEY_RANGE("Rollback", rollbackVersion, allKeys); // We used to do a complicated dance to roll back in MVCC history. It's much simpler, and more testable, // to simply restart the storage server actor and restore from the persistent disk state, and then roll @@ -2544,7 +2544,7 @@ void StorageServer::addMutation(Version version, MutationRef const& mutation, Ke return; } expanded = addMutationToMutationLog(mLog, expanded); - debugMutation("applyMutation", version, expanded).detail("UID", thisServerID).detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end); + DEBUG_MUTATION("applyMutation", version, expanded).detail("UID", thisServerID).detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end); applyMutation( this, expanded, mLog.arena(), mutableData() ); //printf("\nSSUpdate: Printing versioned tree after applying mutation\n"); //mutableData().printTree(version); @@ -2597,9 +2597,9 @@ public: applyPrivateData( data, m ); } } else { - // FIXME: enable when debugMutation is active + // FIXME: enable when DEBUG_MUTATION is active //for(auto m = changes[c].mutations.begin(); m; ++m) { - // debugMutation("SSUpdateMutation", changes[c].version, *m); + // DEBUG_MUTATION("SSUpdateMutation", changes[c].version, *m); //} splitMutation(data, data->shards, m, ver); @@ -2885,7 +2885,7 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) rd >> msg; if (ver != invalidVersion) { // This change belongs to a version < minVersion - debugMutation("SSPeek", ver, msg).detail("ServerID", data->thisServerID); + DEBUG_MUTATION("SSPeek", ver, msg).detail("ServerID", data->thisServerID); if (ver == 1) { TraceEvent("SSPeekMutation", data->thisServerID); // The following trace event may produce a value with special characters @@ -2940,7 +2940,7 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) if(ver != invalidVersion && ver > data->version.get()) { // TODO(alexmiller): Update to version tracking. - debugKeyRange("SSUpdate", ver, KeyRangeRef()); + DEBUG_KEY_RANGE("SSUpdate", ver, KeyRangeRef()); data->mutableData().createNewVersion(ver); if (data->otherError.getFuture().isReady()) data->otherError.getFuture().get(); @@ -3143,7 +3143,7 @@ void StorageServerDisk::writeKeyValue( KeyValueRef kv ) { } void StorageServerDisk::writeMutation( MutationRef mutation ) { - // FIXME: debugMutation(debugContext, debugVersion, *m); + // FIXME: DEBUG_MUTATION(debugContext, debugVersion, *m); if (mutation.type == MutationRef::SetValue) { storage->set( KeyValueRef(mutation.param1, mutation.param2) ); } else if (mutation.type == MutationRef::ClearRange) { @@ -3154,7 +3154,7 @@ void StorageServerDisk::writeMutation( MutationRef mutation ) { void StorageServerDisk::writeMutations( MutationListRef mutations, Version debugVersion, const char* debugContext ) { for(auto m = mutations.begin(); m; ++m) { - debugMutation(debugContext, debugVersion, *m).detail("UID", data->thisServerID); + DEBUG_MUTATION(debugContext, debugVersion, *m).detail("UID", data->thisServerID); if (m->type == MutationRef::SetValue) { storage->set( KeyValueRef(m->param1, m->param2) ); } else if (m->type == MutationRef::ClearRange) { @@ -3172,7 +3172,7 @@ bool StorageServerDisk::makeVersionMutationsDurable( Version& prevStorageVersion VersionUpdateRef const& v = u->second; ASSERT( v.version > prevStorageVersion && v.version <= newStorageVersion ); // TODO(alexmiller): Update to version tracking. - debugKeyRange("makeVersionMutationsDurable", v.version, KeyRangeRef()); + DEBUG_KEY_RANGE("makeVersionMutationsDurable", v.version, KeyRangeRef()); writeMutations(v.mutations, v.version, "makeVersionDurable"); for(auto m=v.mutations.begin(); m; ++m) bytesLeft -= mvccStorageBytes(*m); @@ -3359,7 +3359,7 @@ ACTOR Future restoreDurableState( StorageServer* data, IKeyValueStore* sto if (it->value() == invalidVersion) { KeyRangeRef clearRange(it->begin(), it->end()); // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. - //debugKeyRange("clearInvalidVersion", invalidVersion, clearRange); + //DEBUG_KEY_RANGE("clearInvalidVersion", invalidVersion, clearRange); storage->clear( clearRange ); data->byteSampleApplyClear( clearRange, invalidVersion ); } diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.actor.cpp index 03651f19f2..afd04d6f97 100644 --- a/fdbserver/workloads/ApiCorrectness.actor.cpp +++ b/fdbserver/workloads/ApiCorrectness.actor.cpp @@ -329,7 +329,7 @@ public: wait(transaction->commit()); for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size()); i++) - debugMutation("ApiCorrectnessSet", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, data[i].key, data[i].value)); + DEBUG_MUTATION("ApiCorrectnessSet", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, data[i].key, data[i].value)); currentIndex += self->maxKeysPerTransaction; break; @@ -661,7 +661,7 @@ public: wait(transaction->commit()); for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size()); i++) - debugMutation("ApiCorrectnessClear", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, keys[i], StringRef())); + DEBUG_MUTATION("ApiCorrectnessClear", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, keys[i], StringRef())); currentIndex += self->maxKeysPerTransaction; break; @@ -712,7 +712,7 @@ public: } transaction->clear(range); wait(transaction->commit()); - debugKeyRange("ApiCorrectnessClear", transaction->getCommittedVersion(), range); + DEBUG_KEY_RANGE("ApiCorrectnessClear", transaction->getCommittedVersion(), range); break; } catch(Error &e) { diff --git a/flow/Trace.h b/flow/Trace.h index 3aeb5a9a8d..ae32302a74 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -479,6 +479,10 @@ public: return enabled; } + explicit operator bool() const { + return enabled; + } + void log(); ~TraceEvent(); // Actually logs the event From 342eebebdb8b09b33a299cf576b3b9ae062d26fb Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 13 May 2020 18:44:22 -0700 Subject: [PATCH 15/89] Add a bit of documentation, and the TODO trail for future work. --- fdbserver/MutationTracking.cpp | 2 +- fdbserver/MutationTracking.h | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index 867ce6482f..f49febf05d 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -26,7 +26,7 @@ #error "You cannot use mutation tracking in a clean/release build." #endif -StringRef debugKey = LiteralStringRef( "\xff/globals/lastEpochEnd" ); +StringRef debugKey = LiteralStringRef( "" ); StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" ); TraceEvent debugMutationEnabled( const char* context, Version version, MutationRef const& mutation ) { diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h index 99594ffe70..978ddca6a3 100644 --- a/fdbserver/MutationTracking.h +++ b/fdbserver/MutationTracking.h @@ -26,15 +26,24 @@ #include "fdbclient/CommitTransaction.h" #define MUTATION_TRACKING_ENABLED 0 +// The keys to track are defined in the .cpp file to limit recompilation. #define DEBUG_MUTATION(context, version, mutation) MUTATION_TRACKING_ENABLED && debugMutation(context, version, mutation) TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); +// debugKeyRange and debugTagsAndMessage only log the *first* occurrence of a key in their range/commit. +// TODO: Create a TraceEventGroup that forwards all calls to each element of a vector, +// to allow "multiple" TraceEvents to be returned. + #define DEBUG_KEY_RANGE(context, version, keys) MUTATION_TRACKING_ENABLED && debugKeyRange(context, version, keys) TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); #define DEBUG_TAGS_AND_MESSAGE(context, version, commitBlob) MUTATION_TRACKING_ENABLED && debugTagsAndMessage(context, version, commitBlob) TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ); + +// TODO: Version Tracking. If the bug is in handling a version rather than a key, then it'd be good to be able to log each time +// that version is handled within simulation. A similar set of functions should be implemented. + #endif From bf6d056095935da801792f7ac2533a619609a78d Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 13 May 2020 18:48:43 -0700 Subject: [PATCH 16/89] Changing the last suggestions from review. --- fdbserver/MutationTracking.cpp | 1 + fdbserver/TLogServer.actor.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index f49febf05d..ddd17437a3 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -26,6 +26,7 @@ #error "You cannot use mutation tracking in a clean/release build." #endif +// Track up to 2 keys in simulation via enabling MUTATION_TRACKING_ENABLED and setting the keys here. StringRef debugKey = LiteralStringRef( "" ); StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" ); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index bbcd9f7606..50c4ea4038 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -1375,7 +1375,7 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req } // We need the 4 byte length prefix to be a TagsAndMessage format, but that prefix is added as part of StringRef serialization. - int offset = messages.getLength(); + int offset = messages.getLength(); messages << it->second.toStringRef(); void* data = messages.getData(); DEBUG_TAGS_AND_MESSAGE("TLogPeek", currentVersion, StringRef((uint8_t*)data+offset, messages.getLength()-offset)) From 25390f0968cf3a9b5649e977d74554ac72359e5b Mon Sep 17 00:00:00 2001 From: tclinken Date: Wed, 13 May 2020 22:46:07 -0700 Subject: [PATCH 17/89] Remove SimpleExternalTest.txt from fdb_test_files even if UBSAN is enabled --- tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7dc5af5a75..26632451cf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -79,6 +79,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES SimpleExternalTest.txt) else() message(WARNING "Python not found, won't configure ctest") + add_fdb_test(TEST_FILES SimpleExternalTest.txt IGNORE) endif() add_fdb_test(TEST_FILES SlowTask.txt IGNORE) add_fdb_test(TEST_FILES SpecificUnitTest.txt IGNORE) From ae2b2700fc0abd64be30291541feb097c4856e8f Mon Sep 17 00:00:00 2001 From: tclinken Date: Wed, 13 May 2020 22:49:41 -0700 Subject: [PATCH 18/89] Remove duplicate code from RequestStream::send --- fdbrpc/fdbrpc.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h index 655bf25a14..4989bf28cc 100644 --- a/fdbrpc/fdbrpc.h +++ b/fdbrpc/fdbrpc.h @@ -246,20 +246,13 @@ public: // stream.send( request ) // Unreliable at most once delivery: Delivers request unless there is a connection failure (zero or one times) - void send(const T& value) const { + template + void send(U && value) const { if (queue->isRemoteEndpoint()) { - FlowTransport::transport().sendUnreliable(SerializeSource(value), getEndpoint(), true); + FlowTransport::transport().sendUnreliable(SerializeSource(std::forward(value)), getEndpoint(), true); } else - queue->send(value); - } - - void send(T&& value) const { - if (queue->isRemoteEndpoint()) { - FlowTransport::transport().sendUnreliable(SerializeSource(std::move(value)), getEndpoint(), true); - } - else - queue->send(std::move(value)); + queue->send(std::forward(value)); } /*void sendError(const Error& error) const { From 3ff2fa9c2a82595a64eaf87c9e217ff70b27d620 Mon Sep 17 00:00:00 2001 From: tclinken Date: Wed, 13 May 2020 22:57:03 -0700 Subject: [PATCH 19/89] Removed uses of outdated cpuTicks and errorCounts --- fdbbackup/backup.actor.cpp | 10 ---------- fdbrpc/simulator.h | 12 +++++------- fdbserver/fdbserver.actor.cpp | 14 -------------- flow/Error.cpp | 5 ----- 4 files changed, 5 insertions(+), 36 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index c712bb9090..54ef9fbb06 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -3467,16 +3467,6 @@ int main(int argc, char* argv[]) { std::set_new_handler( &platform::outOfMemory ); setMemoryQuota( memLimit ); - int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - total += i->second; - if (total) - printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - if (i->second > 0) - printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - - Reference ccf; Database db; Reference sourceCcf; diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index d81e5763a1..6f8164a0f4 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -58,7 +58,6 @@ public: bool failed; bool excluded; bool cleared; - int64_t cpuTicks; bool rebooting; std::vector globals; @@ -68,12 +67,11 @@ public: double fault_injection_p1, fault_injection_p2; ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddressList addresses, - INetworkConnections *net, const char* dataFolder, const char* coordinationFolder ) - : name(name), locality(locality), startingClass(startingClass), - addresses(addresses), address(addresses.address), dataFolder(dataFolder), - network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0), - rebooting(false), fault_injection_p1(0), fault_injection_p2(0), - fault_injection_r(0), machine(0), cleared(false) {} + INetworkConnections* net, const char* dataFolder, const char* coordinationFolder) + : name(name), locality(locality), startingClass(startingClass), addresses(addresses), + address(addresses.address), dataFolder(dataFolder), network(net), coordinationFolder(coordinationFolder), + failed(false), excluded(false), rebooting(false), fault_injection_p1(0), fault_injection_p2(0), + fault_injection_r(0), machine(0), cleared(false) {} Future onShutdown() { return shutdownSignal.getFuture(); } diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 5016f5e34a..a23bfcb5a6 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -1921,20 +1921,6 @@ int main(int argc, char* argv[]) { cout << " " << i->second << " " << i->first << endl;*/ // cout << " " << Actor::allActors[i]->getName() << endl; - int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - total += i->second; - if (total) - printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - if (i->second > 0) - printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - - if (&g_simulator == g_network) { - auto processes = g_simulator.getAllProcesses(); - for(auto i = processes.begin(); i != processes.end(); ++i) - printf("%s %s: %0.3f Mclocks\n", (*i)->name, (*i)->address.toString().c_str(), (*i)->cpuTicks / 1e6); - } if (role == Simulation) { unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError); if (sevErrorEventsLogged > 0) { diff --git a/flow/Error.cpp b/flow/Error.cpp index 3edb81adf9..a4eefdfc66 100644 --- a/flow/Error.cpp +++ b/flow/Error.cpp @@ -28,11 +28,6 @@ using std::make_pair; bool g_crashOnError = false; -std::map& Error::errorCounts() { - static std::map counts; - return counts; -} - #include Error Error::fromUnvalidatedCode(int code) { From 7003a68ba182ddf8068b4d0ebd096053888d3c36 Mon Sep 17 00:00:00 2001 From: tclinken Date: Fri, 15 May 2020 11:04:26 -0700 Subject: [PATCH 20/89] Removed outdated comments and errorCounts() declaration --- fdbrpc/sim2.actor.cpp | 7 ------- flow/Error.cpp | 2 -- flow/Error.h | 7 +++++-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 973da3ad64..ddfc66c42c 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -1661,15 +1661,8 @@ public: this->currentProcess = t.machine; try { - //auto before = getCPUTicks(); t.action.send(Void()); ASSERT( this->currentProcess == t.machine ); - /*auto elapsed = getCPUTicks() - before; - currentProcess->cpuTicks += elapsed; - if (deterministicRandom()->random01() < 0.01){ - TraceEvent("TaskDuration").detail("CpuTicks", currentProcess->cpuTicks); - currentProcess->cpuTicks = 0; - }*/ } catch (Error& e) { TraceEvent(SevError, "UnhandledSimulationEventError").error(e, true); killProcess(t.machine, KillInstantly); diff --git a/flow/Error.cpp b/flow/Error.cpp index a4eefdfc66..cf2fa34b63 100644 --- a/flow/Error.cpp +++ b/flow/Error.cpp @@ -65,8 +65,6 @@ Error::Error(int error_code) crashAndDie(); } } - /*if (error_code) - errorCounts()[error_code]++;*/ } ErrorCodeTable& Error::errorCodeTable() { diff --git a/flow/Error.h b/flow/Error.h index 0afe4d0d99..98cb35c576 100644 --- a/flow/Error.h +++ b/flow/Error.h @@ -58,9 +58,12 @@ public: explicit Error(int error_code); static void init(); - static std::map& errorCounts(); static ErrorCodeTable& errorCodeTable(); - static Error fromCode(int error_code) { Error e; e.error_code = error_code; return e; } // Doesn't change errorCounts + static Error fromCode(int error_code) { + Error e; + e.error_code = error_code; + return e; + } static Error fromUnvalidatedCode(int error_code); // Converts codes that are outside the legal range (but not necessarily individually unknown error codes) to unknown_error() Error asInjectedFault() const; // Returns an error with the same code() as this but isInjectedFault() is true From 84cfafe1641acc7c11653ec4eedff8c93f76a0d4 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 10:34:36 -0700 Subject: [PATCH 21/89] Added IndexedSet::const_iterator --- flow/IKeyValueContainer.h | 2 +- flow/IndexedSet.cpp | 12 +- flow/IndexedSet.h | 373 ++++++++++++++++++++++++++++---------- 3 files changed, 288 insertions(+), 99 deletions(-) diff --git a/flow/IKeyValueContainer.h b/flow/IKeyValueContainer.h index 64a5752c2e..7b32d6e61e 100644 --- a/flow/IKeyValueContainer.h +++ b/flow/IKeyValueContainer.h @@ -85,7 +85,7 @@ public: iterator lower_bound(const StringRef& key) { return data.lower_bound(key); } iterator upper_bound(const StringRef& key) { return data.upper_bound(key); } - iterator previous(iterator i) const { return data.previous(i); } + iterator previous(iterator i) { return data.previous(i); } void erase(iterator begin, iterator end) { data.erase(begin, end); } iterator insert(const StringRef& key, const StringRef& val, bool replaceExisting = true) { diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index 2065fabac0..38eb24a5ec 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -210,12 +210,12 @@ struct IndexedSetHarness { map s; void insert(K const& k) { s.insert(K(k), 1); } - result find(K const& k) const { return s.find(k); } - result not_found() const { return s.end(); } - result begin() const { return s.begin(); } - result end() const { return s.end(); } - result lower_bound(K const& k) const { return s.lower_bound(k); } - result upper_bound(K const& k) const { return s.upper_bound(k); } + result find(K const& k) { return s.find(k); } + result not_found() { return s.end(); } + result begin() { return s.begin(); } + result end() { return s.end(); } + result lower_bound(K const& k) { return s.lower_bound(k); } + result upper_bound(K const& k) { return s.upper_bound(k); } void erase(K const& k) { s.erase(k); } }; diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 2e22e71e64..e4b3f1f18b 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -29,6 +29,7 @@ #include "flow/Error.h" #include +#include #include // IndexedSet is similar to a std::set, with the following additional features: @@ -39,7 +40,6 @@ // - Search functions (find(), lower_bound(), etc) can accept a type comparable to T instead of T // (e.g. StringRef when T is std::string or Standalone). This can save a lot of needless // copying at query time for read-mostly sets with string keys. -// - iterators are not const; the responsibility of not changing the order lies with the caller // - the size() function is missing; if the metric being used is a count sumTo(end()) will do instead // A number of STL compatibility features are missing and should be added as needed. // T must define operator <, which must define a total order. Unlike std::set, @@ -70,8 +70,10 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou // combinations, but still take advantage of move constructors when available (or required). template Node(T_&& data, Metric_&& m, Node* parent=0) : data(std::forward(data)), total(std::forward(m)), parent(parent), balance(0) { - child[0] = child[1] = NULL; + child[0] = child[1] = nullptr; } + Node(Node const&) = delete; + Node& operator=(Node const&) = delete; ~Node(){ delete child[0]; delete child[1]; @@ -84,32 +86,100 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou Node *parent; }; -public: - struct iterator{ - typename IndexedSet::Node *i; - iterator() : i(0) {}; - iterator(typename IndexedSet::Node *n) : i(n) {}; - T& operator*() { return i->data; }; - T* operator->() { return &i->data; } + template + struct IteratorImpl { + typename std::conditional_t* i; + + template > + IteratorImpl(const IteratorImpl& nonConstIter) : i(nonConstIter.i) {} + template > + IteratorImpl& operator=(const IteratorImpl& nonConstIter) { + i = nonConstIter.i; + return *this; + } + + IteratorImpl(decltype(i) n = nullptr) : i(n){}; + + const T& operator*() const { return i->data; } + template > + T& operator*() { + return i->data; + } + + const T* operator->() const { return &i->data; } + template > + T* operator->() { + return &i->data; + } + void operator++(); void decrementNonEnd(); - bool operator == ( const iterator& r ) const { return i == r.i; } - bool operator != ( const iterator& r ) const { return i != r.i; } + template + bool operator==(const IteratorImpl& r) const { + return i == r.i; + } + template + bool operator!=(const IteratorImpl& r) const { + return i != r.i; + } // following two methods are for memory storage engine(KeyValueStoreMemory class) use only // in order to have same interface as radixtree StringRef& getKey(uint8_t* dummyContent) const { return i->data.key; } StringRef& getValue() const { return i->data.value; } }; + template + struct Impl { + using IteratorT = IteratorImpl; + using SetT = std::conditional_t, IndexedSet>; + + static IteratorT begin(SetT&); + + static IteratorT end(SetT&); + + template + static IteratorImpl previous(SetT&, IteratorImpl); + + template + static IteratorT index(SetT&, const M&); + + template + static IteratorT find(SetT&, const Key&); + + template + static IteratorT upper_bound(SetT&, const Key&); + + template + static IteratorT lower_bound(SetT&, const Key&); + + template + static IteratorT lastLessOrEqual(SetT&, const Key&); + + static IteratorT lastItem(SetT&); + }; + +public: + using iterator = IteratorImpl; + using const_iterator = IteratorImpl; + IndexedSet() : root(NULL) {}; ~IndexedSet() { delete root; } IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = NULL; } IndexedSet& operator=(IndexedSet&& r) BOOST_NOEXCEPT { delete root; root = r.root; r.root = 0; return *this; } - iterator begin() const; - iterator end() const { return iterator(); } - iterator previous(iterator i) const; - iterator lastItem() const; + const_iterator begin() const { return Impl::begin(*this); }; + iterator begin() { return Impl::begin(*this); }; + const_iterator end() const { return Impl::end(*this); } + iterator end() { return Impl::end(*this); } + + const_iterator previous(const_iterator i) const { return Impl::previous(*this, i); } + template + IteratorImpl previous(IteratorImpl i) { + return Impl::previous(*this, i); + }; + + const_iterator lastItem() const { return Impl::lastItem(*this); } + iterator lastItem() { return Impl::lastItem(*this); } bool empty() const { return !root; } void clear() { delete root; root = NULL; } @@ -159,36 +229,74 @@ public: // Returns x such that key==*x, or end() template - iterator find(const Key &key) const; + const_iterator find(const Key& key) const { + return Impl::find(*this, key); + } + + template + iterator find(const Key& key) { + return Impl::find(*this, key); + } // Returns the smallest x such that *x>=key, or end() template - iterator lower_bound(const Key &key) const; + const_iterator lower_bound(const Key& key) const { + return Impl::lower_bound(*this, key); + } + + template + iterator lower_bound(const Key& key) { + return Impl::lower_bound(*this, key); + }; // Returns the smallest x such that *x>key, or end() template - iterator upper_bound(const Key &key) const; + const_iterator upper_bound(const Key& key) const { + return Impl::upper_bound(*this, key); + } + + template + iterator upper_bound(const Key& key) { + return Impl::upper_bound(*this, key); + }; // Returns the largest x such that *x<=key, or end() template - iterator lastLessOrEqual( const Key &key ) const; + const_iterator lastLessOrEqual(const Key& key) const { + return Impl::lastLessOrEqual(*this, key); + }; + + template + iterator lastLessOrEqual(const Key& key) { + return Impl::lastLessOrEqual(*this, key); + } // Returns smallest x such that sumTo(x+1) > metric, or end() template - iterator index( M const& metric ) const; + const_iterator index(M const& metric) const { + return Impl::index(*this, metric); + }; + + template + iterator index(M const& metric) { + return Impl::index(*this, metric); + } // Return the metric inserted with item x - Metric getMetric(iterator x) const; + Metric getMetric(const_iterator x) const; // Return the sum of getMetric(x) for begin()<=x - Metric sumRange(const Key& begin, const Key& end) const { return sumRange(lower_bound(begin), lower_bound(end)); } + template + Metric sumRange(const Key& begin, const Key& end) const { + return sumRange(lower_bound(begin), lower_bound(end)); + } // Return the amount of memory used by an entry in the IndexedSet static int getElementBytes() { return sizeof(Node); } @@ -215,13 +323,22 @@ private: // direction 0 = left, 1 = right template static void moveIterator(Node* &i){ - if (i->child[0^direction]) { - i = i->child[0^direction]; - while (i->child[1^direction]) - i = i->child[1^direction]; + if (i->child[0 ^ direction]) { + i = i->child[0 ^ direction]; + while (i->child[1 ^ direction]) i = i->child[1 ^ direction]; } else { - while (i->parent && i->parent->child[0^direction] == i) - i = i->parent; + while (i->parent && i->parent->child[0 ^ direction] == i) i = i->parent; + i = i->parent; + } + } + + template + static void moveIterator(const Node*& i) { + if (i->child[0 ^ direction]) { + i = i->child[0 ^ direction]; + while (i->child[1 ^ direction]) i = i->child[1 ^ direction]; + } else { + while (i->parent && i->parent->child[0 ^ direction] == i) i = i->parent; i = i->parent; } } @@ -284,12 +401,17 @@ template , class Metric= class Map { public: typedef typename IndexedSet::iterator iterator; + typedef typename IndexedSet::const_iterator const_iterator; Map() {} - iterator begin() const { return set.begin(); } - iterator end() const { return set.end(); } - iterator lastItem() const { return set.lastItem(); } - iterator previous(iterator i) const { return set.previous(i); } + const_iterator begin() const { return set.begin(); } + iterator begin() { return set.begin(); } + const_iterator end() const { return set.end(); } + iterator end() { return set.end(); } + const_iterator lastItem() const { return set.lastItem(); } + iterator lastItem() { return set.lastItem(); } + const_iterator previous(const_iterator i) const { return set.previous(i); } + iterator previous(iterator i) { return set.previous(i); } bool empty() const { return set.empty(); } Value& operator[]( const Key& key ) { @@ -317,18 +439,58 @@ public: } template - iterator find( KeyCompatible const& k ) const { return set.find(k); } + const_iterator find(KeyCompatible const& k) const { + return set.find(k); + } template - iterator lower_bound( KeyCompatible const& k ) const { return set.lower_bound(k); } + iterator find(KeyCompatible const& k) { + return set.find(k); + } + template - iterator upper_bound( KeyCompatible const& k ) const { return set.upper_bound(k); } + const_iterator lower_bound(KeyCompatible const& k) const { + return set.lower_bound(k); + } template - iterator lastLessOrEqual( KeyCompatible const& k ) const { return set.lastLessOrEqual(k); } - template - iterator index( M const& metric ) const { return set.index(metric); } - Metric getMetric(iterator x) const { return set.getMetric(x); } - Metric sumTo(iterator to) const { return set.sumTo(to); } - Metric sumRange(iterator begin, iterator end) const { return set.sumRange(begin,end); } + iterator lower_bound(KeyCompatible const& k) { + return set.lower_bound(k); + } + + template + const_iterator upper_bound(KeyCompatible const& k) const { + return set.upper_bound(k); + } + template + iterator upper_bound(KeyCompatible const& k) { + return set.upper_bound(k); + } + + template + const_iterator lastLessOrEqual(KeyCompatible const& k) const { + return set.lastLessOrEqual(k); + } + template + iterator lastLessOrEqual(KeyCompatible const& k) { + return set.lastLessOrEqual(k); + } + + template + const_iterator index(M const& metric) const { + return set.index(metric); + } + template + iterator index(M const& metric) { + return set.index(metric); + } + + Metric getMetric(const_iterator x) const { return set.getMetric(x); } + Metric getMetric(iterator x) const { return getMetric(const_iterator{ x }); } + + Metric sumTo(const_iterator to) const { return set.sumTo(to); } + Metric sumTo(iterator to) const { return sumTo(const_iterator{ to }); } + + Metric sumRange(const_iterator begin, const_iterator end) const { return set.sumRange(begin, end); } + Metric sumRange(iterator begin, iterator end) const { return set.sumRange(begin, end); } template Metric sumRange(const KeyCompatible& begin, const KeyCompatible& end) const { return set.sumRange(begin,end); } @@ -347,12 +509,14 @@ private: /////////////////////// implementation ////////////////////////// template -void IndexedSet::iterator::operator++(){ +template +void IndexedSet::IteratorImpl::operator++() { moveIterator<1>(i); } template -void IndexedSet::iterator::decrementNonEnd(){ +template +void IndexedSet::IteratorImpl::decrementNonEnd() { moveIterator<0>(i); } @@ -578,28 +742,42 @@ Node* ISCommonSubtreeRoot(Node* first, Node* last) { } template -typename IndexedSet::iterator IndexedSet::begin() const { - Node *x = root; - while (x && x->child[0]) - x = x->child[0]; - return x; +template +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::begin( + IndexedSet::Impl::SetT& self) { + using NodeT = std::conditional_t; + NodeT* x = self.root; + while (x && x->child[0]) x = x->child[0]; + return { x }; } template -typename IndexedSet::iterator IndexedSet::previous(typename IndexedSet::iterator i) const { - if (i==end()) - return lastItem(); - - moveIterator<0>(i.i); - return i; +template +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::end( + IndexedSet::Impl::SetT& self) { + return {}; } template -typename IndexedSet::iterator IndexedSet::lastItem() const { - Node *x = root; - while (x && x->child[1]) - x = x->child[1]; - return x; +template +template +typename IndexedSet::template IteratorImpl +IndexedSet::Impl::previous(IndexedSet::Impl::SetT& self, + IndexedSet::IteratorImpl iter) { + if (iter == self.end()) return self.lastItem(); + + moveIterator<0>(iter.i); + return iter; +} + +template +template +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lastItem( + IndexedSet::Impl::SetT& self) { + using NodeT = std::conditional_t; + NodeT* x = self.root; + while (x && x->child[1]) x = x->child[1]; + return { x }; } template template @@ -842,8 +1020,8 @@ Metric IndexedSet::eraseHalf(Node* start, Node* end, int eraseDir, in metricDelta = metricDelta - n->total; n->parent = start->parent; } - - start->child[fromDir] = NULL; + + start->child[fromDir] = nullptr; toFree.push_back( start ); } @@ -1005,87 +1183,98 @@ void IndexedSet::erase(iterator toErase) { // Returns x such that key==*x, or end() template +template template -typename IndexedSet::iterator IndexedSet::find(const Key &key) const { - Node* t = root; +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::find( + IndexedSet::Impl::SetT& self, const Key& key) { + using NodeT = std::conditional_t; + NodeT* t = self.root; while (t){ int cmp = compare(key, t->data); - if (cmp == 0) return iterator(t); + if (cmp == 0) return { t }; t = t->child[cmp > 0]; } - return end(); + return self.end(); } // Returns the smallest x such that *x>=key, or end() template +template template -typename IndexedSet::iterator IndexedSet::lower_bound(const Key &key) const { - Node* t = root; - if (!t) return iterator(); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lower_bound( + IndexedSet::Impl::SetT& self, const Key& key) { + using NodeT = std::conditional_t; + NodeT* t = self.root; + if (!t) return self.end(); bool less; while (true) { less = t->data < key; - Node* n = t->child[less]; + NodeT* n = t->child[less]; if (!n) break; t = n; } if (less) moveIterator<1>(t); - return iterator(t); + return { t }; } // Returns the smallest x such that *x>key, or end() template +template template -typename IndexedSet::iterator IndexedSet::upper_bound(const Key &key) const { - Node* t = root; - if (!t) return iterator(); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::upper_bound( + IndexedSet::Impl::SetT& self, const Key& key) { + using NodeT = std::conditional_t; + NodeT* t = self.root; + if (!t) return {}; bool not_less; while (true) { not_less = !(key < t->data); - Node* n = t->child[not_less]; + NodeT* n = t->child[not_less]; if (!n) break; t = n; } if (not_less) moveIterator<1>(t); - return iterator(t); + return { t }; } template +template template -typename IndexedSet::iterator IndexedSet::lastLessOrEqual(const Key &key) const { - iterator i = upper_bound(key); - if (i == begin()) return end(); - return previous(i); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lastLessOrEqual( + IndexedSet::Impl::SetT& self, const Key& key) { + auto i = self.upper_bound(key); + if (i == self.begin()) return self.end(); + return self.previous(i); } // Returns first x such that metric < sum(begin(), x+1), or end() template +template template -typename IndexedSet::iterator IndexedSet::index( M const& metric ) const -{ +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::index( + IndexedSet::Impl::SetT& self, const M& metric) { + using NodeT = std::conditional_t; M m = metric; - Node* t = root; + NodeT* t = self.root; while (t) { if (t->child[0] && m < t->child[0]->total) t = t->child[0]; else { m = m - t->total; - if (t->child[1]) - m = m + t->child[1]->total; - if (m < M()) - return iterator(t); + if (t->child[1]) m = m + t->child[1]->total; + if (m < M()) return { t }; t = t->child[1]; } } - return end(); + return self.end(); } template -Metric IndexedSet::getMetric(typename IndexedSet::iterator x) const { +Metric IndexedSet::getMetric(typename IndexedSet::const_iterator x) const { Metric m = x.i->total; for(int i=0; i<2; i++) if (x.i->child[i]) @@ -1094,12 +1283,12 @@ Metric IndexedSet::getMetric(typename IndexedSet::iterator x } template -Metric IndexedSet::sumTo(typename IndexedSet::iterator end) const { +Metric IndexedSet::sumTo(typename IndexedSet::const_iterator end) const { if (!end.i) return root ? root->total : Metric(); Metric m = end.i->child[0] ? end.i->child[0]->total : Metric(); - for(Node* p = end.i; p->parent; p=p->parent) { + for (const Node* p = end.i; p->parent; p = p->parent) { if (p->parent->child[1] == p) { m = m - p->total; m = m + p->parent->total; From e11ba12da70b587df58db157df43804a3d6cb465 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 10:38:42 -0700 Subject: [PATCH 22/89] Added IndexedSet::ConstImpl and IndexedSet::NonConstImpl --- flow/IndexedSet.h | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index e4b3f1f18b..01c10bb772 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -158,6 +158,9 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou static IteratorT lastItem(SetT&); }; + using ConstImpl = Impl; + using NonConstImpl = Impl; + public: using iterator = IteratorImpl; using const_iterator = IteratorImpl; @@ -167,19 +170,19 @@ public: IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = NULL; } IndexedSet& operator=(IndexedSet&& r) BOOST_NOEXCEPT { delete root; root = r.root; r.root = 0; return *this; } - const_iterator begin() const { return Impl::begin(*this); }; - iterator begin() { return Impl::begin(*this); }; - const_iterator end() const { return Impl::end(*this); } - iterator end() { return Impl::end(*this); } + const_iterator begin() const { return ConstImpl::begin(*this); }; + iterator begin() { return NonConstImpl::begin(*this); }; + const_iterator end() const { return ConstImpl::end(*this); } + iterator end() { return NonConstImpl::end(*this); } - const_iterator previous(const_iterator i) const { return Impl::previous(*this, i); } + const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } template IteratorImpl previous(IteratorImpl i) { - return Impl::previous(*this, i); + return NonConstImpl::previous(*this, i); }; - const_iterator lastItem() const { return Impl::lastItem(*this); } - iterator lastItem() { return Impl::lastItem(*this); } + const_iterator lastItem() const { return ConstImpl::lastItem(*this); } + iterator lastItem() { return NonConstImpl::lastItem(*this); } bool empty() const { return !root; } void clear() { delete root; root = NULL; } @@ -230,56 +233,56 @@ public: // Returns x such that key==*x, or end() template const_iterator find(const Key& key) const { - return Impl::find(*this, key); + return ConstImpl::find(*this, key); } template iterator find(const Key& key) { - return Impl::find(*this, key); + return NonConstImpl::find(*this, key); } // Returns the smallest x such that *x>=key, or end() template const_iterator lower_bound(const Key& key) const { - return Impl::lower_bound(*this, key); + return ConstImpl::lower_bound(*this, key); } template iterator lower_bound(const Key& key) { - return Impl::lower_bound(*this, key); + return NonConstImpl::lower_bound(*this, key); }; // Returns the smallest x such that *x>key, or end() template const_iterator upper_bound(const Key& key) const { - return Impl::upper_bound(*this, key); + return ConstImpl::upper_bound(*this, key); } template iterator upper_bound(const Key& key) { - return Impl::upper_bound(*this, key); + return NonConstImpl::upper_bound(*this, key); }; // Returns the largest x such that *x<=key, or end() template const_iterator lastLessOrEqual(const Key& key) const { - return Impl::lastLessOrEqual(*this, key); + return ConstImpl::lastLessOrEqual(*this, key); }; template iterator lastLessOrEqual(const Key& key) { - return Impl::lastLessOrEqual(*this, key); + return NonConstImpl::lastLessOrEqual(*this, key); } // Returns smallest x such that sumTo(x+1) > metric, or end() template const_iterator index(M const& metric) const { - return Impl::index(*this, metric); + return ConstImpl::index(*this, metric); }; template iterator index(M const& metric) { - return Impl::index(*this, metric); + return NonConstImpl::index(*this, metric); } // Return the metric inserted with item x From 864ddd316f781e06f32867046aada9909b29cd42 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 11:31:15 -0700 Subject: [PATCH 23/89] Removed duplicate IndexedSet::moveIterator code --- flow/IndexedSet.h | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 01c10bb772..815b297f74 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -323,27 +323,25 @@ private: newNode->parent = oldNode->parent; } - // direction 0 = left, 1 = right - template - static void moveIterator(Node* &i){ - if (i->child[0 ^ direction]) { - i = i->child[0 ^ direction]; - while (i->child[1 ^ direction]) i = i->child[1 ^ direction]; + template + static void _moveIterator(std::conditional_t*& node) { + if (node->child[0 ^ direction]) { + node = node->child[0 ^ direction]; + while (node->child[1 ^ direction]) node = node->child[1 ^ direction]; } else { - while (i->parent && i->parent->child[0 ^ direction] == i) i = i->parent; - i = i->parent; + while (node->parent && node->parent->child[0 ^ direction] == node) node = node->parent; + node = node->parent; } } + // direction 0 = left, 1 = right template - static void moveIterator(const Node*& i) { - if (i->child[0 ^ direction]) { - i = i->child[0 ^ direction]; - while (i->child[1 ^ direction]) i = i->child[1 ^ direction]; - } else { - while (i->parent && i->parent->child[0 ^ direction] == i) i = i->parent; - i = i->parent; - } + static void moveIterator(Node const*& node) { + _moveIterator(node); + } + template + static void moveIterator(Node*& node) { + _moveIterator(node); } public: // but testonly From 9c9b56b8b148b5f8cb35efd2e4c448abd1f4768d Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 12:53:36 -0700 Subject: [PATCH 24/89] Avoid implicit conversion from Node* to iterator --- flow/IndexedSet.h | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 815b297f74..c5d177ce0f 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -98,7 +98,7 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou return *this; } - IteratorImpl(decltype(i) n = nullptr) : i(n){}; + explicit IteratorImpl(decltype(i) n = nullptr) : i(n){}; const T& operator*() const { return i->data; } template > @@ -130,6 +130,7 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou template struct Impl { + using NodeT = std::conditional_t; using IteratorT = IteratorImpl; using SetT = std::conditional_t, IndexedSet>; @@ -176,10 +177,7 @@ public: iterator end() { return NonConstImpl::end(*this); } const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } - template - IteratorImpl previous(IteratorImpl i) { - return NonConstImpl::previous(*this, i); - }; + iterator previous(iterator i) { return NonConstImpl::previous(*this, i); } const_iterator lastItem() const { return ConstImpl::lastItem(*this); } iterator lastItem() { return NonConstImpl::lastItem(*this); } @@ -746,17 +744,16 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::begin( IndexedSet::Impl::SetT& self) { - using NodeT = std::conditional_t; NodeT* x = self.root; while (x && x->child[0]) x = x->child[0]; - return { x }; + return IteratorT{ x }; } template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::end( IndexedSet::Impl::SetT& self) { - return {}; + return IteratorT{}; } template @@ -775,10 +772,9 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lastItem( IndexedSet::Impl::SetT& self) { - using NodeT = std::conditional_t; NodeT* x = self.root; while (x && x->child[1]) x = x->child[1]; - return { x }; + return IteratorT{ x }; } template template @@ -796,9 +792,9 @@ Metric IndexedSet::addMetric(T_&& data, Metric_&& metric){ template template typename IndexedSet::iterator IndexedSet::insert(T_&& data, Metric_&& metric, bool replaceExisting){ - if (root == NULL){ + if (root == nullptr) { root = new Node(std::forward(data), std::forward(metric)); - return root; + return iterator{ root }; } Node *t = root; int d; // direction @@ -821,7 +817,7 @@ typename IndexedSet::iterator IndexedSet::insert(T_&& data, } } - return returnNode; + return iterator{ returnNode }; } d = cmp > 0; Node *nextT = t->child[d]; @@ -864,7 +860,7 @@ typename IndexedSet::iterator IndexedSet::insert(T_&& data, t->total = t->total + metric; } - return newNode; + return iterator{ newNode }; } template @@ -1188,11 +1184,10 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::find( IndexedSet::Impl::SetT& self, const Key& key) { - using NodeT = std::conditional_t; NodeT* t = self.root; while (t){ int cmp = compare(key, t->data); - if (cmp == 0) return { t }; + if (cmp == 0) return IteratorT{ t }; t = t->child[cmp > 0]; } return self.end(); @@ -1204,7 +1199,6 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lower_bound( IndexedSet::Impl::SetT& self, const Key& key) { - using NodeT = std::conditional_t; NodeT* t = self.root; if (!t) return self.end(); bool less; @@ -1217,7 +1211,7 @@ typename IndexedSet::template Impl::IteratorT IndexedSet(t); - return { t }; + return IteratorT{ t }; } // Returns the smallest x such that *x>key, or end() @@ -1226,9 +1220,8 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::upper_bound( IndexedSet::Impl::SetT& self, const Key& key) { - using NodeT = std::conditional_t; NodeT* t = self.root; - if (!t) return {}; + if (!t) return IteratorT{}; bool not_less; while (true) { not_less = !(key < t->data); @@ -1239,7 +1232,7 @@ typename IndexedSet::template Impl::IteratorT IndexedSet(t); - return { t }; + return IteratorT{ t }; } template @@ -1258,7 +1251,6 @@ template template typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::index( IndexedSet::Impl::SetT& self, const M& metric) { - using NodeT = std::conditional_t; M m = metric; NodeT* t = self.root; while (t) { @@ -1267,7 +1259,7 @@ typename IndexedSet::template Impl::IteratorT IndexedSettotal; if (t->child[1]) m = m + t->child[1]->total; - if (m < M()) return { t }; + if (m < M()) return IteratorT{ t }; t = t->child[1]; } } From d9124abb02264d264617e0c398eb953f6c461bb5 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 13:01:57 -0700 Subject: [PATCH 25/89] Avoid implicit conversion from iterator to const_iterator --- flow/IndexedSet.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index c5d177ce0f..f2254f79fe 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -91,12 +91,7 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou typename std::conditional_t* i; template > - IteratorImpl(const IteratorImpl& nonConstIter) : i(nonConstIter.i) {} - template > - IteratorImpl& operator=(const IteratorImpl& nonConstIter) { - i = nonConstIter.i; - return *this; - } + explicit IteratorImpl(const IteratorImpl& nonConstIter) : i(nonConstIter.i) {} explicit IteratorImpl(decltype(i) n = nullptr) : i(n){}; @@ -285,13 +280,17 @@ public: // Return the metric inserted with item x Metric getMetric(const_iterator x) const; + Metric getMetric(iterator x) { return sumTo(const_iterator{ x }); } // Return the sum of getMetric(x) for begin()<=x From 52cb766b17522b7bac5ec9c00eb0c1901822e4f5 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 14:25:13 -0700 Subject: [PATCH 26/89] Renamed IndexedSet::IteratorImpl::i to IndexedSet::IteratorImpl::node --- flow/IndexedSet.h | 50 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index f2254f79fe..366a70c44a 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -88,39 +88,39 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou template struct IteratorImpl { - typename std::conditional_t* i; + typename std::conditional_t* node; template > - explicit IteratorImpl(const IteratorImpl& nonConstIter) : i(nonConstIter.i) {} + explicit IteratorImpl(const IteratorImpl& nonConstIter) : node(nonConstIter.node) {} - explicit IteratorImpl(decltype(i) n = nullptr) : i(n){}; + explicit IteratorImpl(decltype(node) n = nullptr) : node(n){}; - const T& operator*() const { return i->data; } + const T& operator*() const { return node->data; } template > T& operator*() { - return i->data; + return node->data; } - const T* operator->() const { return &i->data; } + const T* operator->() const { return &node->data; } template > T* operator->() { - return &i->data; + return &node->data; } void operator++(); void decrementNonEnd(); template bool operator==(const IteratorImpl& r) const { - return i == r.i; + return node == r.node; } template bool operator!=(const IteratorImpl& r) const { - return i != r.i; + return node != r.node; } // following two methods are for memory storage engine(KeyValueStoreMemory class) use only // in order to have same interface as radixtree - StringRef& getKey(uint8_t* dummyContent) const { return i->data.key; } - StringRef& getValue() const { return i->data.value; } + StringRef& getKey(uint8_t* dummyContent) const { return node->data.key; } + StringRef& getValue() const { return node->data.value; } }; template @@ -509,13 +509,13 @@ private: template template void IndexedSet::IteratorImpl::operator++() { - moveIterator<1>(i); + moveIterator<1>(node); } template template void IndexedSet::IteratorImpl::decrementNonEnd() { - moveIterator<0>(i); + moveIterator<0>(node); } template @@ -763,7 +763,7 @@ IndexedSet::Impl::previous(IndexedSet::Impl::IteratorImpl iter) { if (iter == self.end()) return self.lastItem(); - moveIterator<0>(iter.i); + moveIterator<0>(iter.node); return iter; } @@ -1048,13 +1048,13 @@ void IndexedSet::erase( typename IndexedSet::iterator begin, // Removes all nodes in the set between first and last, inclusive. // toFree is extended with the roots of completely removed subtrees. - ASSERT(!end.i || (begin.i && (::compare(*begin, *end) <= 0))); + ASSERT(!end.node || (begin.node && (::compare(*begin, *end) <= 0))); if(begin == end) return; - - IndexedSet::Node* first = begin.i; - IndexedSet::Node* last = previous(end).i; + + IndexedSet::Node* first = begin.node; + IndexedSet::Node* last = previous(end).node; IndexedSet::Node* subRoot = ISCommonSubtreeRoot(first, last); @@ -1099,7 +1099,7 @@ void IndexedSet::erase(iterator toErase) { { // Find the node to erase - Node* t = toErase.i; + Node* t = toErase.node; if (!t) return; if (!t->child[0] || !t->child[1]) { @@ -1267,20 +1267,18 @@ typename IndexedSet::template Impl::IteratorT IndexedSet Metric IndexedSet::getMetric(typename IndexedSet::const_iterator x) const { - Metric m = x.i->total; + Metric m = x.node->total; for(int i=0; i<2; i++) - if (x.i->child[i]) - m = m - x.i->child[i]->total; + if (x.i->child[i]) m = m - x.node->child[i]->total; return m; } template Metric IndexedSet::sumTo(typename IndexedSet::const_iterator end) const { - if (!end.i) - return root ? root->total : Metric(); + if (!end.node) return root ? root->total : Metric(); - Metric m = end.i->child[0] ? end.i->child[0]->total : Metric(); - for (const Node* p = end.i; p->parent; p = p->parent) { + Metric m = end.node->child[0] ? end.node->child[0]->total : Metric(); + for (const Node* p = end.node; p->parent; p = p->parent) { if (p->parent->child[1] == p) { m = m - p->total; m = m + p->parent->total; From dc11fb6b012967a5fecbb45f28e5ade260d6c21f Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 14:29:38 -0700 Subject: [PATCH 27/89] Removed IndexedSet::Impl::end --- flow/IndexedSet.h | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 366a70c44a..fcad834974 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -131,8 +131,6 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou static IteratorT begin(SetT&); - static IteratorT end(SetT&); - template static IteratorImpl previous(SetT&, IteratorImpl); @@ -168,8 +166,8 @@ public: const_iterator begin() const { return ConstImpl::begin(*this); }; iterator begin() { return NonConstImpl::begin(*this); }; - const_iterator end() const { return ConstImpl::end(*this); } - iterator end() { return NonConstImpl::end(*this); } + const_iterator end() const { return const_iterator{}; } + iterator end() { return iterator{}; } const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } iterator previous(iterator i) { return NonConstImpl::previous(*this, i); } @@ -748,13 +746,6 @@ typename IndexedSet::template Impl::IteratorT IndexedSet -template -typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::end( - IndexedSet::Impl::SetT& self) { - return IteratorT{}; -} - template template template From 8f883591d45c012cfc2f194f6097bd8378cba048 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 14:58:09 -0700 Subject: [PATCH 28/89] Renamed _moveIterator to moveIteratorImpl --- flow/IndexedSet.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index fcad834974..f9f6c67da9 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -319,7 +319,7 @@ private: } template - static void _moveIterator(std::conditional_t*& node) { + static void moveIteratorImpl(std::conditional_t*& node) { if (node->child[0 ^ direction]) { node = node->child[0 ^ direction]; while (node->child[1 ^ direction]) node = node->child[1 ^ direction]; @@ -332,11 +332,11 @@ private: // direction 0 = left, 1 = right template static void moveIterator(Node const*& node) { - _moveIterator(node); + moveIteratorImpl(node); } template static void moveIterator(Node*& node) { - _moveIterator(node); + moveIteratorImpl(node); } public: // but testonly From cbfe380b1d54aea9f45e618e99bcdcbe82e41a98 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 15:00:15 -0700 Subject: [PATCH 29/89] s/NULL/nullptr in IndexedSet.h --- flow/IndexedSet.h | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index f9f6c67da9..6d83d19801 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -159,9 +159,9 @@ public: using iterator = IteratorImpl; using const_iterator = IteratorImpl; - IndexedSet() : root(NULL) {}; + IndexedSet() : root(nullptr){}; ~IndexedSet() { delete root; } - IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = NULL; } + IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = nullptr; } IndexedSet& operator=(IndexedSet&& r) BOOST_NOEXCEPT { delete root; root = r.root; r.root = 0; return *this; } const_iterator begin() const { return ConstImpl::begin(*this); }; @@ -176,7 +176,10 @@ public: iterator lastItem() { return NonConstImpl::lastItem(*this); } bool empty() const { return !root; } - void clear() { delete root; root = NULL; } + void clear() { + delete root; + root = nullptr; + } void swap( IndexedSet& r ) { std::swap( root, r.root ); } // Place data in the set with the given metric. If an item equal to data is already in the set and, @@ -856,17 +859,17 @@ typename IndexedSet::iterator IndexedSet::insert(T_&& data, template int IndexedSet::insert(const std::vector>& dataVector, bool replaceExisting) { int num_inserted = 0; - Node *blockStart = NULL; - Node *blockEnd = NULL; + Node* blockStart = nullptr; + Node* blockEnd = nullptr; for(int i = 0; i < dataVector.size(); ++i) { Metric metric = dataVector[i].second; T data = std::move(dataVector[i].first); int d = 1; // direction - if(blockStart == NULL || (blockEnd != NULL && data >= blockEnd->data)) { - blockEnd = NULL; - if (root == NULL) { + if (blockStart == nullptr || (blockEnd != nullptr && data >= blockEnd->data)) { + blockEnd = nullptr; + if (root == nullptr) { root = new Node(std::move(data), metric); num_inserted++; blockStart = root; @@ -1062,7 +1065,7 @@ void IndexedSet::erase( typename IndexedSet::iterator begin, int heightDelta = leftHeightDelta + rightHeightDelta; // Rebalance and update metrics for all nodes from subRoot up to the root - for(auto p = subRoot; p != NULL; p = p->parent) { + for (auto p = subRoot; p != nullptr; p = p->parent) { p->total = p->total - metricDelta; auto& pc = p->parent ? p->parent->child[p->parent->child[1]==p] : root; From 79b46c124e8db006fe69edd6107ecdbc2fc81116 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 16:35:39 -0700 Subject: [PATCH 30/89] Added /flow/IndexedSet/const_iterator unit test --- flow/IndexedSet.cpp | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index 38eb24a5ec..71f01918c4 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include "flow/TreeBenchmark.h" #include "flow/UnitTest.h" template @@ -204,17 +205,24 @@ TEST_CASE("/flow/IndexedSet/strings") { template struct IndexedSetHarness { using map = IndexedSet; + using const_result = typename map::const_iterator; using result = typename map::iterator; using key_type = K; map s; void insert(K const& k) { s.insert(K(k), 1); } + const_result find(K const& k) const { return s.find(k); } result find(K const& k) { return s.find(k); } + const_result not_found() const { return s.end(); } result not_found() { return s.end(); } + const_result begin() const { return s.begin(); } result begin() { return s.begin(); } + const_result end() const { return s.end(); } result end() { return s.end(); } + const_result lower_bound(K const& k) const { return s.lower_bound(k); } result lower_bound(K const& k) { return s.lower_bound(k); } + result upper_bound(K const& k) const { return s.upper_bound(k); } result upper_bound(K const& k) { return s.upper_bound(k); } void erase(K const& k) { s.erase(k); } }; @@ -494,4 +502,49 @@ TEST_CASE("/flow/IndexedSet/all numbers") { return Void(); } +TEST_CASE("/flow/IndexedSet/const_iterator") { + struct Key { + int key; + explicit Key(int key) : key(key) {} + }; + struct Metric { + int metric; + explicit Metric(int metric) : metric(metric) {} + }; + IndexedSet is; + for (int i = 0; i < 10; ++i) is.insert(i, 1); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + static_assert(!std::is_const_v); + + const IndexedSet& cis = is; + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + static_assert(std::is_const_v>); + + for (auto& val : is) { + static_assert(!std::is_const_v>); + } + for (const auto& val : is) { + static_assert(std::is_const_v>); + } + for (auto& val : cis) { + static_assert(std::is_const_v>); + } + + return Void(); +} + void forceLinkIndexedSetTests() {} From 70bd53570733c36ba045d5278d12fac8d01acbfd Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 16:36:21 -0700 Subject: [PATCH 31/89] Added IKeyValueContainer::const_iterator --- flow/IKeyValueContainer.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/flow/IKeyValueContainer.h b/flow/IKeyValueContainer.h index 7b32d6e61e..6b42872e96 100644 --- a/flow/IKeyValueContainer.h +++ b/flow/IKeyValueContainer.h @@ -69,22 +69,30 @@ bool operator<(CompatibleWithKey const& l, KeyValueMapPair const& r) { class IKeyValueContainer { public: - typedef typename IndexedSet::iterator iterator; + using const_iterator = IndexedSet::const_iterator; + using iterator = IndexedSet::iterator; IKeyValueContainer() = default; ~IKeyValueContainer() = default; - bool empty() { return data.empty(); } + bool empty() const { return data.empty(); } void clear() { return data.clear(); } - std::tuple size() { return std::make_tuple(0, 0, 0); } + std::tuple size() const { return std::make_tuple(0, 0, 0); } + const_iterator find(const StringRef& key) const { return data.find(key); } iterator find(const StringRef& key) { return data.find(key); } + const_iterator begin() const { return data.begin(); } iterator begin() { return data.begin(); } + const_iterator end() const { return data.end(); } iterator end() { return data.end(); } + const_iterator lower_bound(const StringRef& key) const { return data.lower_bound(key); } iterator lower_bound(const StringRef& key) { return data.lower_bound(key); } + const_iterator upper_bound(const StringRef& key) const { return data.upper_bound(key); } iterator upper_bound(const StringRef& key) { return data.upper_bound(key); } + const_iterator previous(const_iterator i) const { return data.previous(i); } + const_iterator previous(iterator i) const { return data.previous(const_iterator{ i }); } iterator previous(iterator i) { return data.previous(i); } void erase(iterator begin, iterator end) { data.erase(begin, end); } @@ -96,7 +104,8 @@ public: return data.insert(pairs, replaceExisting); } - uint64_t sumTo(iterator to) { return data.sumTo(to); } + uint64_t sumTo(const_iterator to) const { return data.sumTo(to); } + uint64_t sumTo(iterator to) const { return data.sumTo(const_iterator{ to }); } static int getElementBytes() { return IndexedSet::getElementBytes(); } From e58698d6f76b6acadcb025ecb506916c70c8714b Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 17:01:26 -0700 Subject: [PATCH 32/89] Added IndexedSet::previous(iterator) const --- flow/IndexedSet.cpp | 1 + flow/IndexedSet.h | 1 + 2 files changed, 2 insertions(+) diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index 71f01918c4..dd83d63ac3 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -527,6 +527,7 @@ TEST_CASE("/flow/IndexedSet/const_iterator") { static_assert(std::is_const_v>); static_assert(std::is_const_v>); static_assert(std::is_const_v>); + static_assert(std::is_const_v>); static_assert(std::is_const_v>); static_assert(std::is_const_v>); static_assert(std::is_const_v>); diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 6d83d19801..9539ad5fb8 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -170,6 +170,7 @@ public: iterator end() { return iterator{}; } const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } + const_iterator previous(iterator i) const { return ConstImpl::previous(*this, const_iterator{ i }); } iterator previous(iterator i) { return NonConstImpl::previous(*this, i); } const_iterator lastItem() const { return ConstImpl::lastItem(*this); } From e16ec8beca2618e2cda91a4f663f9b43099d8171 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 18:01:07 -0700 Subject: [PATCH 33/89] Added cbegin and cend to IndexedSet Also fixed/improved /flow/IndexedSet/const_iterator test --- flow/IKeyValueContainer.h | 2 ++ flow/IndexedSet.cpp | 66 ++++++++++++++++++++++++--------------- flow/IndexedSet.h | 5 +++ 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/flow/IKeyValueContainer.h b/flow/IKeyValueContainer.h index 6b42872e96..2167f32801 100644 --- a/flow/IKeyValueContainer.h +++ b/flow/IKeyValueContainer.h @@ -84,8 +84,10 @@ public: iterator find(const StringRef& key) { return data.find(key); } const_iterator begin() const { return data.begin(); } iterator begin() { return data.begin(); } + const_iterator cbegin() const { return begin(); } const_iterator end() const { return data.end(); } iterator end() { return data.end(); } + const_iterator cend() const { return end(); } const_iterator lower_bound(const StringRef& key) const { return data.lower_bound(key); } iterator lower_bound(const StringRef& key) { return data.lower_bound(key); } diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index dd83d63ac3..fe950f8b39 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -222,7 +222,7 @@ struct IndexedSetHarness { result end() { return s.end(); } const_result lower_bound(K const& k) const { return s.lower_bound(k); } result lower_bound(K const& k) { return s.lower_bound(k); } - result upper_bound(K const& k) const { return s.upper_bound(k); } + const_result upper_bound(K const& k) const { return s.upper_bound(k); } result upper_bound(K const& k) { return s.upper_bound(k); } void erase(K const& k) { s.erase(k); } }; @@ -502,6 +502,13 @@ TEST_CASE("/flow/IndexedSet/all numbers") { return Void(); } +template +struct is_const_ref { + static constexpr bool value = std::is_const_v>; +}; +template +static constexpr bool is_const_ref_v = is_const_ref::value; + TEST_CASE("/flow/IndexedSet/const_iterator") { struct Key { int key; @@ -511,41 +518,48 @@ TEST_CASE("/flow/IndexedSet/const_iterator") { int metric; explicit Metric(int metric) : metric(metric) {} }; + IndexedSet is; for (int i = 0; i < 10; ++i) is.insert(i, 1); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); - static_assert(!std::is_const_v); + + IndexedSet& ncis = is; + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); const IndexedSet& cis = is; - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); - static_assert(std::is_const_v>); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); - for (auto& val : is) { - static_assert(!std::is_const_v>); + for (auto& val : ncis) { + static_assert(!is_const_ref_v); } - for (const auto& val : is) { - static_assert(std::is_const_v>); + for (const auto& val : ncis) { + static_assert(is_const_ref_v); } for (auto& val : cis) { - static_assert(std::is_const_v>); + static_assert(is_const_ref_v); } return Void(); } - void forceLinkIndexedSetTests() {} diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 9539ad5fb8..2d8a697b1c 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -166,8 +166,11 @@ public: const_iterator begin() const { return ConstImpl::begin(*this); }; iterator begin() { return NonConstImpl::begin(*this); }; + const_iterator cbegin() const { return begin(); } + const_iterator end() const { return const_iterator{}; } iterator end() { return iterator{}; } + const_iterator cend() const { return end(); } const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } const_iterator previous(iterator i) const { return ConstImpl::previous(*this, const_iterator{ i }); } @@ -406,8 +409,10 @@ public: Map() {} const_iterator begin() const { return set.begin(); } iterator begin() { return set.begin(); } + const_iterator cbegin() const { return begin(); } const_iterator end() const { return set.end(); } iterator end() { return set.end(); } + const_iterator cend() const { return end(); } const_iterator lastItem() const { return set.lastItem(); } iterator lastItem() { return set.lastItem(); } const_iterator previous(const_iterator i) const { return set.previous(i); } From 5632af3b4d60bb298caabd876f38029fe81c344d Mon Sep 17 00:00:00 2001 From: tclinken Date: Sat, 16 May 2020 20:20:11 -0700 Subject: [PATCH 34/89] Removed inaccurate warning --- tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 26632451cf..d60859be79 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -78,7 +78,6 @@ if(WITH_PYTHON) if (NOT USE_UBSAN) # TODO re-enable in UBSAN after https://github.com/apple/foundationdb/issues/2410 is resolved add_fdb_test(TEST_FILES SimpleExternalTest.txt) else() - message(WARNING "Python not found, won't configure ctest") add_fdb_test(TEST_FILES SimpleExternalTest.txt IGNORE) endif() add_fdb_test(TEST_FILES SlowTask.txt IGNORE) From c054b47150e3c21f75d26c8a7d2d32a73577dadd Mon Sep 17 00:00:00 2001 From: tclinken Date: Sun, 17 May 2020 11:23:10 -0700 Subject: [PATCH 35/89] Fixed IndexedSet::getMetric --- flow/IndexedSet.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 2d8a697b1c..ad996b1672 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -285,11 +285,11 @@ public: // Return the metric inserted with item x Metric getMetric(const_iterator x) const; - Metric getMetric(iterator x) { return sumTo(const_iterator{ x }); } + Metric getMetric(iterator x) const { return getMetric(const_iterator{ x }); } // Return the sum of getMetric(x) for begin()<=x typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::upper_bound( IndexedSet::Impl::SetT& self, const Key& key) { NodeT* t = self.root; - if (!t) return IteratorT{}; + if (!t) return self.end(); bool not_less; while (true) { not_less = !(key < t->data); @@ -1269,7 +1269,7 @@ template Metric IndexedSet::getMetric(typename IndexedSet::const_iterator x) const { Metric m = x.node->total; for(int i=0; i<2; i++) - if (x.i->child[i]) m = m - x.node->child[i]->total; + if (x.node->child[i]) m = m - x.node->child[i]->total; return m; } From 3399f566db3981bbdaafdd01fd638d2facdefc3e Mon Sep 17 00:00:00 2001 From: tclinken Date: Sun, 17 May 2020 12:47:32 -0700 Subject: [PATCH 36/89] Fixed MacOS build --- flow/IndexedSet.h | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index ad996b1672..bb39f29ee5 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -90,33 +90,20 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou struct IteratorImpl { typename std::conditional_t* node; - template > - explicit IteratorImpl(const IteratorImpl& nonConstIter) : node(nonConstIter.node) {} + explicit IteratorImpl(const IteratorImpl& nonConstIter) : node(nonConstIter.node) { + static_assert(isConst); + } explicit IteratorImpl(decltype(node) n = nullptr) : node(n){}; - const T& operator*() const { return node->data; } - template > - T& operator*() { - return node->data; - } + typename std::conditional_t& operator*() const { return node->data; } - const T* operator->() const { return &node->data; } - template > - T* operator->() { - return &node->data; - } + typename std::conditional_t* operator->() const { return &node->data; } void operator++(); void decrementNonEnd(); - template - bool operator==(const IteratorImpl& r) const { - return node == r.node; - } - template - bool operator!=(const IteratorImpl& r) const { - return node != r.node; - } + bool operator==(const IteratorImpl& r) const { return node == r.node; } + bool operator!=(const IteratorImpl& r) const { return node != r.node; } // following two methods are for memory storage engine(KeyValueStoreMemory class) use only // in order to have same interface as radixtree StringRef& getKey(uint8_t* dummyContent) const { return node->data.key; } From 9d1d83640cb131788065adf943474d671f70e600 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sun, 17 May 2020 13:29:22 -0700 Subject: [PATCH 37/89] Update return type for const_iterator::getKey and const_iterator::getValue --- flow/IndexedSet.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index bb39f29ee5..44c6d87be3 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -106,8 +106,10 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou bool operator!=(const IteratorImpl& r) const { return node != r.node; } // following two methods are for memory storage engine(KeyValueStoreMemory class) use only // in order to have same interface as radixtree - StringRef& getKey(uint8_t* dummyContent) const { return node->data.key; } - StringRef& getValue() const { return node->data.value; } + typename std::conditional_t& getKey(uint8_t* dummyContent) const { + return node->data.key; + } + typename std::conditional_t& getValue() const { return node->data.value; } }; template From c8d3b56ff0a888324144d47a85d93ea6c40c625c Mon Sep 17 00:00:00 2001 From: tclinken Date: Mon, 18 May 2020 09:02:08 -0700 Subject: [PATCH 38/89] Removed unnecessary is_const_ref struct --- flow/IndexedSet.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index fe950f8b39..16e9ce12ca 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -503,11 +503,7 @@ TEST_CASE("/flow/IndexedSet/all numbers") { } template -struct is_const_ref { - static constexpr bool value = std::is_const_v>; -}; -template -static constexpr bool is_const_ref_v = is_const_ref::value; +static constexpr bool is_const_ref_v = std::is_const_v>; TEST_CASE("/flow/IndexedSet/const_iterator") { struct Key { From 9a64ec934393dd3e05d079b408affb3a16907b44 Mon Sep 17 00:00:00 2001 From: tclinken Date: Sun, 17 May 2020 17:49:06 -0700 Subject: [PATCH 39/89] Use std::copy in Deque copy ctor --- flow/Deque.h | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/flow/Deque.h b/flow/Deque.h index c5c05fb895..6148e00d0c 100644 --- a/flow/Deque.h +++ b/flow/Deque.h @@ -41,21 +41,25 @@ public: Deque() : arr(0), begin(0), end(0), mask(-1) {} // TODO: iterator construction, other constructors - Deque(Deque const& r) : arr(0), begin(0), end(r.size()), mask(r.mask) { + Deque(Deque const& r) : arr(nullptr), begin(0), end(r.size()), mask(r.mask) { if (r.capacity() > 0) { arr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)), capacity() * sizeof(T)); ASSERT(arr != nullptr); } ASSERT(capacity() >= end || end == 0); - for (uint32_t i=0; i= r.begin) { + std::copy(r.arr + r.begin, r.arr + r.begin + r.size(), arr); + } else { + auto partOneSize = r.capacity() - r.begin; + std::copy(r.arr + r.begin, r.arr + r.begin + partOneSize, arr); + std::copy(r.arr, r.arr + r.end, arr + partOneSize); + } } void operator=(Deque const& r) { cleanup(); - arr = 0; + arr = nullptr; begin = 0; end = r.size(); mask = r.mask; @@ -64,13 +68,17 @@ public: ASSERT(arr != nullptr); } ASSERT(capacity() >= end || end == 0); - for (uint32_t i=0; i= r.begin) { + std::copy(r.arr + r.begin, r.arr + r.begin + r.size(), arr); + } else { + auto partOneSize = r.capacity() - r.begin; + std::copy(r.arr + r.begin, r.arr + r.begin + partOneSize, arr); + std::copy(r.arr, r.arr + r.end, arr + partOneSize); + } } Deque(Deque&& r) BOOST_NOEXCEPT : begin(r.begin), end(r.end), mask(r.mask), arr(r.arr) { - r.arr = 0; + r.arr = nullptr; r.begin = r.end = 0; r.mask = -1; } @@ -82,8 +90,8 @@ public: end = r.end; mask = r.mask; arr = r.arr; - - r.arr = 0; + + r.arr = nullptr; r.begin = r.end = 0; r.mask = -1; } From 0da199b201736dc2da3fb8088cf632f99ed37c9e Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Tue, 19 May 2020 08:28:16 -0700 Subject: [PATCH 40/89] Move wait_for_shard_addresses into ShardFinder. Remove unneeded transactional decorator. --- .../transaction_profiling_analyzer.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index fca842161b..8cc52b0cf5 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -514,17 +514,6 @@ def has_dateparser(): logger.warn("Can't find dateparser so disabling human date parsing") return False -def wait_for_shard_addresses(ranges, shard_finder, key_idx, addr_idx): - for index in range(len(ranges)): - item = ranges[index] - if item[addr_idx] is not None: - while True: - try: - ranges[index] = item[0:addr_idx] + ([a.decode('ascii') for a in item[addr_idx].wait()],) + item[addr_idx+1:] - break - except fdb.FDBError as e: - ranges[index] = item[0:addr_idx] + (shard_finder.get_addresses_for_key(item[key_idx]),) + item[addr_idx+1:] - class ReadCounter(object): def __init__(self): from sortedcontainers import SortedDict @@ -569,7 +558,7 @@ class ReadCounter(object): for (count, (start, end)) in count_pairs: results.append((start, end, count, shard_finder.get_addresses_for_key(start))) - wait_for_shard_addresses(results, shard_finder, 0, 3) + shard_finder.wait_for_shard_addresses(results, 0, 3) if filter_addresses: filter_addresses = set(filter_addresses) @@ -625,7 +614,7 @@ class ReadCounter(object): if count_this_range > 0: add_boundary(this_range_start_key, last_end, opened_this_range, count_this_range) - wait_for_shard_addresses(output_range_counts, shard_finder, 0, 5) + shard_finder.wait_for_shard_addresses(output_range_counts, 0, 5) return output_range_counts @@ -653,7 +642,6 @@ class ShardFinder(object): self.tr.options.set_include_port_in_address() @staticmethod - @fdb.transactional def _get_addresses_for_key(tr, key): return fdb.locality.get_addresses_for_key(tr, key) @@ -690,6 +678,16 @@ class ShardFinder(object): return self.shard_cache[shard] + def wait_for_shard_addresses(self, ranges, key_idx, addr_idx): + for index in range(len(ranges)): + item = ranges[index] + if item[addr_idx] is not None: + while True: + try: + ranges[index] = item[0:addr_idx] + ([a.decode('ascii') for a in item[addr_idx].wait()],) + item[addr_idx+1:] + break + except fdb.FDBError as e: + ranges[index] = item[0:addr_idx] + (self.get_addresses_for_key(item[key_idx]),) + item[addr_idx+1:] class WriteCounter(object): mutation_types_to_consider = frozenset([MutationType.SET_VALUE, MutationType.ADD_VALUE]) @@ -733,7 +731,7 @@ class WriteCounter(object): if count_this_range > 0: add_boundary(start_key, k, count_this_range) - wait_for_shard_addresses(output_range_counts, shard_finder, 0, 5) + shard_finder.wait_for_shard_addresses(output_range_counts, 0, 5) return output_range_counts def get_total_writes(self): @@ -749,7 +747,7 @@ class WriteCounter(object): for (count, key) in count_pairs: results.append((key, None, count, shard_finder.get_addresses_for_key(key))) - wait_for_shard_addresses(results, shard_finder, 0, 3) + shard_finder.wait_for_shard_addresses(results, 0, 3) if filter_addresses: filter_addresses = set(filter_addresses) From 131534e4a1a10953cdfd6aedb0f07e9172676718 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 20 May 2020 10:19:10 -0700 Subject: [PATCH 41/89] Added =default for Standalone constructors for clarity --- flow/Arena.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flow/Arena.h b/flow/Arena.h index 159a3d55ac..a8412bf6c5 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -380,6 +380,11 @@ public: } #else Standalone( const T& t, const Arena& arena ) : Arena( arena ), T( t ) {} + Standalone(const Standalone&) = default; + Standalone& operator=(const Standalone&) = default; + Standalone(Standalone&&) = default; + Standalone& operator=(Standalone&&) = default; + ~Standalone() = default; #endif template Standalone castTo() const { From 9d4d11485c3df71135342f76acbe254c0d97ba31 Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Wed, 20 May 2020 14:32:54 -0700 Subject: [PATCH 42/89] Fix include paths --- fdbservice/FDBService.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbservice/FDBService.cpp b/fdbservice/FDBService.cpp index 4571599b07..59ef5c8045 100644 --- a/fdbservice/FDBService.cpp +++ b/fdbservice/FDBService.cpp @@ -28,8 +28,8 @@ #include #include -#include "..\flow\SimpleOpt.h" -#include "..\fdbmonitor\SimpleIni.h" +#include "flow/SimpleOpt.h" +#include "fdbmonitor/SimpleIni.h" #include "fdbclient/IncludeVersions.h" // For PathFileExists From d128252e904f9cc0ee83c6218610d13daa10a399 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Fri, 22 May 2020 09:25:32 -0700 Subject: [PATCH 43/89] Merge release-6.3 into master --- README.md | 88 +- .../tuple/FastByteComparisons.java | 2 +- cmake/ConfigureCompiler.cmake | 10 + .../transaction_profiling_analyzer.py | 52 +- design/special-key-space.md | 21 +- .../source/mr-status-json-schemas.rst.inc | 2 - fdbbackup/FileDecoder.actor.cpp | 11 +- fdbbackup/backup.actor.cpp | 18 +- fdbcli/fdbcli.actor.cpp | 367 ++++++-- fdbclient/ClientLogEvents.h | 48 +- fdbclient/FDBTypes.h | 15 + fdbclient/IncludeVersions.h | 28 - fdbclient/Knobs.cpp | 1 + fdbclient/Knobs.h | 1 + fdbclient/MasterProxyInterface.h | 55 +- fdbclient/NativeAPI.actor.cpp | 41 +- fdbclient/NativeAPI.actor.h | 2 + fdbclient/Schemas.cpp | 2 - fdbclient/SpecialKeySpace.actor.cpp | 36 +- fdbclient/SpecialKeySpace.actor.h | 20 +- fdbclient/StorageServerInterface.h | 31 +- fdbclient/SystemData.cpp | 8 + fdbclient/SystemData.h | 1 + fdbclient/TagThrottle.actor.cpp | 98 +- fdbclient/TagThrottle.h | 29 +- fdbclient/ThreadSafeTransaction.actor.cpp | 2 +- fdbmonitor/fdbmonitor.cpp | 2 +- fdbrpc/ActorFuzz.actor.cpp | 60 +- fdbrpc/ActorFuzz.h | 8 - fdbrpc/AsyncFileCached.actor.cpp | 20 +- fdbrpc/AsyncFileCached.actor.h | 10 +- fdbrpc/FailureMonitor.actor.cpp | 14 +- fdbrpc/FailureMonitor.h | 2 + fdbrpc/FlowTransport.actor.cpp | 11 +- fdbrpc/FlowTransport.h | 26 +- fdbrpc/actorFuzz.py | 2 +- fdbrpc/sim2.actor.cpp | 11 - fdbserver/BackupProgress.actor.cpp | 13 +- fdbserver/BackupWorker.actor.cpp | 81 +- fdbserver/CMakeLists.txt | 1 + fdbserver/DataDistribution.actor.cpp | 18 +- fdbserver/DataDistribution.actor.h | 10 + fdbserver/DataDistributionTracker.actor.cpp | 51 ++ fdbserver/DataDistributorInterface.h | 30 +- fdbserver/FDBExecHelper.actor.cpp | 2 +- fdbserver/Knobs.cpp | 10 +- fdbserver/Knobs.h | 10 +- fdbserver/MasterInterface.h | 14 +- fdbserver/MasterProxyServer.actor.cpp | 20 + fdbserver/Ratekeeper.actor.cpp | 4 +- fdbserver/RestoreLoader.actor.cpp | 3 + fdbserver/SimulatedCluster.actor.cpp | 2 +- fdbserver/SkipList.cpp | 2 +- fdbserver/Status.actor.cpp | 33 +- fdbserver/TLogInterface.h | 26 +- fdbserver/VersionedBTree.actor.cpp | 841 ++++++++++++------ fdbserver/fdbserver.actor.cpp | 2 +- fdbserver/storageserver.actor.cpp | 1 + ...kupAndParallelRestoreCorrectness.actor.cpp | 4 +- ...entTransactionProfileCorrectness.actor.cpp | 8 +- .../workloads/ConfigureDatabase.actor.cpp | 7 +- .../DataDistributionMetrics.actor.cpp | 108 +++ fdbserver/workloads/TagThrottleApi.actor.cpp | 94 +- fdbservice/FDBService.cpp | 2 +- flow/IThreadPool.cpp | 2 +- flow/TLSConfig.actor.cpp | 2 +- flow/network.h | 20 +- tests/CMakeLists.txt | 1 + tests/DataDistributionMetrics.txt | 21 + 69 files changed, 1762 insertions(+), 836 deletions(-) delete mode 100644 fdbclient/IncludeVersions.h mode change 100644 => 100755 fdbrpc/actorFuzz.py create mode 100644 fdbserver/workloads/DataDistributionMetrics.actor.cpp create mode 100644 tests/DataDistributionMetrics.txt diff --git a/README.md b/README.md index e27dca73fc..e42dfccd63 100755 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Contributing to FoundationDB can be in contributions to the code base, sharing y ### Binary downloads -Developers interested in using the FoundationDB store for an application can get started easily by downloading and installing a binary package. Please see the [downloads page](https://www.foundationdb.org/download/) for a list of available packages. +Developers interested in using FoundationDB can get started by downloading and installing a binary package. Please see the [downloads page](https://www.foundationdb.org/download/) for a list of available packages. ### Compiling from source @@ -28,44 +28,24 @@ Developers interested in using the FoundationDB store for an application can get Developers on an OS for which there is no binary package, or who would like to start hacking on the code, can get started by compiling from source. -Currently there are two build systems: a collection of Makefiles and a -CMake-based build system. Both of them should currently work for most users, -and CMake should be the preferred choice as it will eventually become the only -build system available. +The official docker image for building is `foundationdb/foundationdb-build`. It has all dependencies installed. To build outside the official docker image you'll need at least these dependencies: + +1. Install cmake Version 3.13 or higher [CMake](https://cmake.org/) +1. Install [Mono](http://www.mono-project.com/download/stable/) +1. Install [Ninja](https://ninja-build.org/) (optional, but recommended) If compiling for local development, please set `-DUSE_WERROR=ON` in cmake. Our CI compiles with `-Werror` on, so this way you'll find out about compiler warnings that break the build earlier. -## CMake - -To build with CMake, generally the following is required (works on Linux and -Mac OS - for Windows see below): +Once you have your dependencies, you can run cmake and then build: 1. Check out this repository. -1. Install cmake Version 3.13 or higher [CMake](https://cmake.org/) -1. Download version 1.67 of [Boost](https://sourceforge.net/projects/boost/files/boost/1.67.0/). -1. Unpack boost (you don't need to compile it) -1. Install [Mono](http://www.mono-project.com/download/stable/). -1. Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8. 1. Create a build directory (you can have the build directory anywhere you - like): `mkdir build` -1. `cd build` -1. `cmake -GNinja -DBOOST_ROOT= ` -1. `ninja` - -CMake will try to find its dependencies. However, for LibreSSL this can be often -problematic (especially if OpenSSL is installed as well). For that we recommend -passing the argument `-DLibreSSL_ROOT` to cmake. So, for example, if you -LibreSSL is installed under `/usr/local/libressl-2.8.3`, you should call cmake like -this: - -``` -cmake -GNinja -DLibreSSL_ROOT=/usr/local/libressl-2.8.3/ ../foundationdb -``` - -FoundationDB will build just fine without LibreSSL, however, the resulting -binaries won't support TLS connections. + like). There is currently a directory in the source tree called build, but you should not use it. See [#3098](https://github.com/apple/foundationdb/issues/3098) +1. `cd ` +1. `cmake -G Ninja ` +1. `ninja # If this crashes it probably ran out of memory. Try ninja -j1` ### Language Bindings @@ -120,8 +100,7 @@ create a XCode-project with the following command: cmake -G Xcode -DOPEN_FOR_IDE=ON ``` -You should create a second build-directory which you will use for building -(probably with make or ninja) and debugging. +You should create a second build-directory which you will use for building and debugging. #### FreeBSD @@ -160,11 +139,8 @@ There are no special requirements for Linux. A docker image can be pulled from `foundationdb/foundationdb-build` that has all of FoundationDB's dependencies pre-installed, and is what the CI uses to build and test PRs. -If you want to create a package you have to tell cmake what platform it is for. -And then you can build by simply calling `cpack`. So for debian, call: - ``` -cmake -GNinja +cmake -G Ninja ninja cpack -G DEB ``` @@ -173,20 +149,15 @@ For RPM simply replace `DEB` with `RPM`. ### MacOS -The build under MacOS will work the same way as on Linux. To get LibreSSL, -boost, and ninja you can use [Homebrew](https://brew.sh/). LibreSSL will not be -installed in `/usr/local` instead it will stay in `/usr/local/Cellar`. So the -cmake command will look something like this: +The build under MacOS will work the same way as on Linux. To get boost and ninja you can use [Homebrew](https://brew.sh/). ```sh -cmake -GNinja -DLibreSSL_ROOT=/usr/local/Cellar/libressl/2.8.3 +cmake -G Ninja ``` -To generate a installable package, you have to call CMake with the corresponding -arguments and then use cpack to generate the package: +To generate a installable package, you can use cpack: ```sh -cmake -GNinja ninja cpack -G productbuild ``` @@ -198,15 +169,15 @@ that Visual Studio is used to compile. 1. Install Visual Studio 2017 (Community Edition is tested) 1. Install cmake Version 3.12 or higher [CMake](https://cmake.org/) -1. Download version 1.67 of [Boost](https://sourceforge.net/projects/boost/files/boost/1.67.0/). +1. Download version 1.72 of [Boost](https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2) 1. Unpack boost (you don't need to compile it) -1. Install [Mono](http://www.mono-project.com/download/stable/). -1. Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8. +1. Install [Mono](http://www.mono-project.com/download/stable/) +1. (Optional) Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8 1. Set `JAVA_HOME` to the unpacked location and JAVA_COMPILE to `$JAVA_HOME/bin/javac`. -1. Install [Python](https://www.python.org/downloads/) if it is not already installed by Visual Studio. +1. Install [Python](https://www.python.org/downloads/) if it is not already installed by Visual Studio 1. (Optional) Install [WIX](http://wixtoolset.org/). Without it Visual Studio - won't build the Windows installer. + won't build the Windows installer 1. Create a build directory (you can have the build directory anywhere you like): `mkdir build` 1. `cd build` @@ -218,22 +189,7 @@ that Visual Studio is used to compile. Studio will only know about the generated files. `msbuild` is located at `c:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe` for Visual Studio 15. -If you want TLS support to be enabled under Windows you currently have to build -and install LibreSSL yourself as the newer LibreSSL versions are not provided -for download from the LibreSSL homepage. To build LibreSSL: - -1. Download and unpack libressl (>= 2.8.2) -2. `cd libressl-2.8.2` -3. `mkdir build` -4. `cd build` -5. `cmake -G "Visual Studio 15 2017 Win64" ..` -6. Open the generated `LibreSSL.sln` in Visual Studio as administrator (this is - necessary for the install) -7. Build the `INSTALL` project in `Release` mode - -This will install LibreSSL under `C:\Program Files\LibreSSL`. After that `cmake` -will automatically find it and build with TLS support. - If you installed WIX before running `cmake` you should find the `FDBInstaller.msi` in your build directory under `packaging/msi`. +TODO: Re-add instructions for TLS support [#3022](https://github.com/apple/foundationdb/issues/3022) diff --git a/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java index 77add1db7f..83f5f399de 100644 --- a/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java +++ b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java @@ -1,5 +1,5 @@ /* - * ByteArrayUtil.java + * FastByteComparisons.java * * This source file is part of the FoundationDB open source project * diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 1a45498b37..ddb2f38792 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -85,7 +85,17 @@ include(CheckFunctionExists) set(CMAKE_REQUIRED_INCLUDES stdlib.h malloc.h) set(CMAKE_REQUIRED_LIBRARIES c) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +if(NOT WIN32) + include(CheckIncludeFile) + CHECK_INCLUDE_FILE("stdatomic.h" HAS_C11_ATOMICS) + if (NOT HAS_C11_ATOMICS) + message(FATAL_ERROR "C compiler does not support c11 atomics") + endif() +endif() if(WIN32) # see: https://docs.microsoft.com/en-us/windows/desktop/WinProg/using-the-windows-headers diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index b57d3b506c..58e2cf2548 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -173,44 +173,45 @@ class Mutation(object): class BaseInfo(object): - def __init__(self, start_timestamp): - self.start_timestamp = start_timestamp - + def __init__(self, bb, protocol_version): + self.start_timestamp = bb.get_double() + if protocol_version >= PROTOCOL_VERSION_6_3: + self.dc_id = bb.get_bytes_with_length() class GetVersionInfo(BaseInfo): def __init__(self, bb, protocol_version): - super().__init__(bb.get_double()) + super().__init__(bb, protocol_version) self.latency = bb.get_double() if protocol_version >= PROTOCOL_VERSION_6_2: self.transaction_priority_type = bb.get_int() if protocol_version >= PROTOCOL_VERSION_6_3: - self.read_version = bb.get_long() + self.read_version = bb.get_long() class GetInfo(BaseInfo): - def __init__(self, bb): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version): + super().__init__(bb, protocol_version) self.latency = bb.get_double() self.value_size = bb.get_int() self.key = bb.get_bytes_with_length() class GetRangeInfo(BaseInfo): - def __init__(self, bb): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version): + super().__init__(bb, protocol_version) self.latency = bb.get_double() self.range_size = bb.get_int() self.key_range = bb.get_key_range() class CommitInfo(BaseInfo): - def __init__(self, bb, full_output=True): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version, full_output=True): + super().__init__(bb, protocol_version) self.latency = bb.get_double() self.num_mutations = bb.get_int() self.commit_bytes = bb.get_int() - + if protocol_version >= PROTOCOL_VERSION_6_3: - self.commit_version = bb.get_long() + self.commit_version = bb.get_long() read_conflict_range = bb.get_key_range_list() if full_output: self.read_conflict_range = read_conflict_range @@ -225,22 +226,22 @@ class CommitInfo(BaseInfo): class ErrorGetInfo(BaseInfo): - def __init__(self, bb): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version): + super().__init__(bb, protocol_version) self.error_code = bb.get_int() self.key = bb.get_bytes_with_length() class ErrorGetRangeInfo(BaseInfo): - def __init__(self, bb): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version): + super().__init__(bb, protocol_version) self.error_code = bb.get_int() self.key_range = bb.get_key_range() class ErrorCommitInfo(BaseInfo): - def __init__(self, bb, full_output=True): - super().__init__(bb.get_double()) + def __init__(self, bb, protocol_version, full_output=True): + super().__init__(bb, protocol_version) self.error_code = bb.get_int() read_conflict_range = bb.get_key_range_list() @@ -282,33 +283,33 @@ class ClientTransactionInfo: if (not type_filter or "get_version" in type_filter): self.get_version = get_version elif event == 1: - get = GetInfo(bb) + get = GetInfo(bb, protocol_version) if (not type_filter or "get" in type_filter): # because of the crappy json serializtion using __dict__ we have to set the list here otherwise # it doesn't print if not self.gets: self.gets = [] self.gets.append(get) elif event == 2: - get_range = GetRangeInfo(bb) + get_range = GetRangeInfo(bb, protocol_version) if (not type_filter or "get_range" in type_filter): if not self.get_ranges: self.get_ranges = [] self.get_ranges.append(get_range) elif event == 3: - commit = CommitInfo(bb, full_output=full_output) + commit = CommitInfo(bb, protocol_version, full_output=full_output) if (not type_filter or "commit" in type_filter): self.commit = commit elif event == 4: - error_get = ErrorGetInfo(bb) + error_get = ErrorGetInfo(bb, protocol_version) if (not type_filter or "error_gets" in type_filter): if not self.error_gets: self.error_gets = [] self.error_gets.append(error_get) elif event == 5: - error_get_range = ErrorGetRangeInfo(bb) + error_get_range = ErrorGetRangeInfo(bb, protocol_version) if (not type_filter or "error_get_range" in type_filter): if not self.error_get_ranges: self.error_get_ranges = [] self.error_get_ranges.append(error_get_range) elif event == 6: - error_commit = ErrorCommitInfo(bb, full_output=full_output) + error_commit = ErrorCommitInfo(bb, protocol_version, full_output=full_output) if (not type_filter or "error_commit" in type_filter): if not self.error_commits: self.error_commits = [] self.error_commits.append(error_commit) @@ -978,4 +979,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/design/special-key-space.md b/design/special-key-space.md index 15386de508..e6bc0796f6 100644 --- a/design/special-key-space.md +++ b/design/special-key-space.md @@ -7,7 +7,7 @@ Currently, there are several client functions implemented as FDB calls by passin - **cluster_file_path**: `get("\xff\xff/cluster_file_path)` - **connection_string**: `get("\xff\xff/connection_string)` - **worker_interfaces**: `getRange("\xff\xff/worker_interfaces", )` -- **conflicting-keys**: `getRange("\xff\xff/transaction/conflicting_keys/", "\xff\xff/transaction/conflicting_keys/\xff")` +- **conflicting_keys**: `getRange("\xff\xff/transaction/conflicting_keys/", "\xff\xff/transaction/conflicting_keys/\xff")` At present, implementions are hard-coded and the pain points are obvious: - **Maintainability**: As more features added, the hard-coded snippets are hard to maintain @@ -78,4 +78,21 @@ ASSERT( res2[0].value == LiteralStringRef("London") && res2[1].value == LiteralStringRef("Washington, D.C.") ); -``` \ No newline at end of file +``` + +## Module +We introduce this `module` concept after a [discussion](https://forums.foundationdb.org/t/versioning-of-special-key-space/2068) on cross module read on special-key-space. By default, range reads cover more than one module will not be allowed with `special_keys_cross_module_read` errors. In addition, range reads touch no modules will come with `special_keys_no_module_found` errors. The motivation here is to avoid unexpected blocking or errors happen in a wide-scope range read. In particular, you write code `getRange("A", "Z")` when all registered calls between `[A, Z)` happen locally, thus your code does not have any error-handling. However, if in the future, anyone register a new call in `[A, Z)` and sometimes throw errors like `time_out()`, then your original code is broken. The `module` is like a top-level directory where inside the module, calls are homogeneous. So we allow cross range read inside each module by default but cross module reads are forbidden. Right now, there are two modules available to use: + +- TRANSACTION : `\xff\xff/transaction/, \xff\xff/transaction0`, all transaction related information like *read_conflict_range*, *write_conflict_range*, *conflicting_keys*.(All happen locally). Right now we have: + - `\xff\xff/transaction/conflicting_keys/, \xff\xff/transaction/conflicting_keys0` : conflicting keys that caused conflicts + - `\xff\xff/transaction/read_conflict_range/, \xff\xff/transaction/read_conflict_range0` : read conflict ranges of the transaction + - `\xff\xff/transaction/write_conflict_range/, \xff\xff/transaction/write_conflict_range0` : write conflict ranges of the transaction +- METRICS: `\xff\xff/metrics/, \xff\xff/metrics0`, all metrics like data-distribution metrics or healthy metrics are planned to put here. All need to call the rpc, so time_out error s may happen. Right now we have: + - `\xff\xff/metrics/data_distribution_stats, \xff\xff/metrics/data_distribution_stats` : stats info about data-distribution +- WORKERINTERFACE : `\xff\xff/worker_interfaces/, \xff\xff/worker_interfaces0`, which is compatible with previous implementation, thus should not be used to add new functions. + +In addition, all singleKeyRanges are formatted as modules and cannot be used again. In particular, you should call `get` not `getRange` on these keys. Below are existing ones: + +- STATUSJSON : `\xff\xff/status/json` +- CONNECTIONSTRING : `\xff\xff/connection_string` +- CLUSTERFILEPATH : `\xff\xff/cluster_file_path` \ No newline at end of file diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 3b8df02949..31ccb629fc 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -284,8 +284,6 @@ }, "limiting_queue_bytes_storage_server":0, "worst_queue_bytes_storage_server":0, - "limiting_version_lag_storage_server":0, - "worst_version_lag_storage_server":0, "limiting_data_lag_storage_server":{ "versions":0, "seconds":0.0 diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index f7de6d70ff..f294f724c9 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -219,8 +219,9 @@ struct VersionedMutations { */ struct DecodeProgress { DecodeProgress() = default; - DecodeProgress(const LogFile& file, std::vector> values) - : file(file), keyValues(values) {} + template + DecodeProgress(const LogFile& file, U &&values) + : file(file), keyValues(std::forward(values)) {} // If there are no more mutations to pull from the file. // However, we could have unfinished version in the buffer when EOF is true, @@ -228,7 +229,7 @@ struct DecodeProgress { // should call getUnfinishedBuffer() to get these left data. bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); } - std::vector>&& getUnfinishedBuffer() { return std::move(keyValues); } + std::vector>&& getUnfinishedBuffer() && { return std::move(keyValues); } // Returns all mutations of the next version in a batch. Future getNextBatch() { return getNextBatchImpl(this); } @@ -448,7 +449,7 @@ ACTOR Future decode_logs(DecodeParams params) { for (; i < logs.size(); i++) { if (logs[i].fileSize == 0) continue; - state DecodeProgress progress(logs[i], left); + state DecodeProgress progress(logs[i], std::move(left)); wait(progress.openFile(container)); while (!progress.finished()) { VersionedMutations vms = wait(progress.getNextBatch()); @@ -456,7 +457,7 @@ ACTOR Future decode_logs(DecodeParams params) { std::cout << vms.version << " " << m.toString() << "\n"; } } - left = progress.getUnfinishedBuffer(); + left = std::move(progress).getUnfinishedBuffer(); if (!left.empty()) { TraceEvent("UnfinishedFile").detail("File", logs[i].fileName).detail("Q", left.size()); } diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 54ef9fbb06..128470674d 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -63,7 +63,7 @@ using std::endl; #endif #endif -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #include "flow/SimpleOpt.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -593,9 +593,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_RESTORECONTAINER,"-r", SO_REQ_SEP }, - { OPT_PREFIX_ADD, "-add_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 { OPT_PREFIX_ADD, "--add_prefix", SO_REQ_SEP }, - { OPT_PREFIX_REMOVE, "-remove_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 { OPT_PREFIX_REMOVE, "--remove_prefix", SO_REQ_SEP }, { OPT_TAGNAME, "-t", SO_REQ_SEP }, { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, @@ -2709,7 +2707,13 @@ extern uint8_t *g_extra_memory; int main(int argc, char* argv[]) { platformInit(); - int status = FDB_EXIT_SUCCESS; + int status = FDB_EXIT_SUCCESS; + + std::string commandLine; + for(int a=0; a getTransaction(Database db, Reference& lc) { +void compGenerator(const char* text, bool help, std::vector& lc) { std::map::const_iterator iter; int len = strlen(text); @@ -2438,7 +2438,7 @@ void comp_generator(const char* text, bool help, std::vector& lc) { for (auto iter = helpMap.begin(); iter != helpMap.end(); ++iter) { const char* name = (*iter).first.c_str(); if (!strncmp(name, text, len)) { - lc.push_back( new_completion(help ? "help " : "", name) ); + lc.push_back( newCompletion(help ? "help " : "", name) ); } } @@ -2447,31 +2447,31 @@ void comp_generator(const char* text, bool help, std::vector& lc) { const char* name = *he; he++; if (!strncmp(name, text, len)) - lc.push_back( new_completion("help ", name) ); + lc.push_back( newCompletion("help ", name) ); } } } -void cmd_generator(const char* text, std::vector& lc) { - comp_generator(text, false, lc); +void cmdGenerator(const char* text, std::vector& lc) { + compGenerator(text, false, lc); } -void help_generator(const char* text, std::vector& lc) { - comp_generator(text, true, lc); +void helpGenerator(const char* text, std::vector& lc) { + compGenerator(text, true, lc); } -void option_generator(const char* text, const char *line, std::vector& lc) { +void optionGenerator(const char* text, const char *line, std::vector& lc) { int len = strlen(text); for (auto iter = validOptions.begin(); iter != validOptions.end(); ++iter) { const char* name = (*iter).c_str(); if (!strncmp(name, text, len)) { - lc.push_back( new_completion(line, name) ); + lc.push_back( newCompletion(line, name) ); } } } -void array_generator(const char* text, const char *line, const char** options, std::vector& lc) { +void arrayGenerator(const char* text, const char *line, const char** options, std::vector& lc) { const char** iter = options; int len = strlen(text); @@ -2479,32 +2479,57 @@ void array_generator(const char* text, const char *line, const char** options, s const char* name = *iter; iter++; if (!strncmp(name, text, len)) { - lc.push_back( new_completion(line, name) ); + lc.push_back( newCompletion(line, name) ); } } } -void onoff_generator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = {"on", "off", NULL}; - array_generator(text, line, opts, lc); +void onOffGenerator(const char* text, const char *line, std::vector& lc) { + const char* opts[] = {"on", "off", nullptr}; + arrayGenerator(text, line, opts, lc); } -void configure_generator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "memory-radixtree-beta", "proxies=", "logs=", "resolvers=", NULL}; - array_generator(text, line, opts, lc); +void configureGenerator(const char* text, const char *line, std::vector& lc) { + const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "memory-radixtree-beta", "proxies=", "logs=", "resolvers=", nullptr}; + arrayGenerator(text, line, opts, lc); } -void status_generator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = {"minimal", "details", "json", NULL}; - array_generator(text, line, opts, lc); +void statusGenerator(const char* text, const char *line, std::vector& lc) { + const char* opts[] = {"minimal", "details", "json", nullptr}; + arrayGenerator(text, line, opts, lc); } -void kill_generator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = {"all", "list", NULL}; - array_generator(text, line, opts, lc); +void killGenerator(const char* text, const char *line, std::vector& lc) { + const char* opts[] = {"all", "list", nullptr}; + arrayGenerator(text, line, opts, lc); } -void fdbcli_comp_cmd(std::string const& text, std::vector& lc) { +void throttleGenerator(const char* text, const char *line, std::vector& lc, std::vector const& tokens) { + if(tokens.size() == 1) { + const char* opts[] = { "on tag", "off", "enable auto", "disable auto", "list", nullptr }; + arrayGenerator(text, line, opts, lc); + } + else if(tokens.size() >= 2 && tokencmp(tokens[1], "on")) { + if(tokens.size() == 2) { + const char* opts[] = { "tag", nullptr }; + arrayGenerator(text, line, opts, lc); + } + else if(tokens.size() == 6) { + const char* opts[] = { "default", "immediate", "batch", nullptr }; + arrayGenerator(text, line, opts, lc); + } + } + else if(tokens.size() >= 2 && tokencmp(tokens[1], "off") && !tokencmp(tokens[tokens.size()-1], "tag")) { + const char* opts[] = { "all", "auto", "manual", "tag", "default", "immediate", "batch", nullptr }; + arrayGenerator(text, line, opts, lc); + } + else if(tokens.size() == 2 && tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { + const char* opts[] = { "auto", nullptr }; + arrayGenerator(text, line, opts, lc); + } +} + +void fdbcliCompCmd(std::string const& text, std::vector& lc) { bool err, partial; std::string whole_line = text; auto parsed = parseLine(whole_line, err, partial); @@ -2531,37 +2556,102 @@ void fdbcli_comp_cmd(std::string const& text, std::vector& lc) { // printf("final text (%d tokens): `%s' & `%s'\n", count, base_input.c_str(), ntext.c_str()); if (!count) { - cmd_generator(ntext.c_str(), lc); + cmdGenerator(ntext.c_str(), lc); return; } if (tokencmp(tokens[0], "help") && count == 1) { - help_generator(ntext.c_str(), lc); + helpGenerator(ntext.c_str(), lc); return; } if (tokencmp(tokens[0], "option")) { if (count == 1) - onoff_generator(ntext.c_str(), base_input.c_str(), lc); + onOffGenerator(ntext.c_str(), base_input.c_str(), lc); if (count == 2) - option_generator(ntext.c_str(), base_input.c_str(), lc); + optionGenerator(ntext.c_str(), base_input.c_str(), lc); } if (tokencmp(tokens[0], "writemode") && count == 1) { - onoff_generator(ntext.c_str(), base_input.c_str(), lc); + onOffGenerator(ntext.c_str(), base_input.c_str(), lc); } if (tokencmp(tokens[0], "configure")) { - configure_generator(ntext.c_str(), base_input.c_str(), lc); + configureGenerator(ntext.c_str(), base_input.c_str(), lc); } if (tokencmp(tokens[0], "status") && count == 1) { - status_generator(ntext.c_str(), base_input.c_str(), lc); + statusGenerator(ntext.c_str(), base_input.c_str(), lc); } if (tokencmp(tokens[0], "kill") && count == 1) { - kill_generator(ntext.c_str(), base_input.c_str(), lc); + killGenerator(ntext.c_str(), base_input.c_str(), lc); } + + if (tokencmp(tokens[0], "throttle")) { + throttleGenerator(ntext.c_str(), base_input.c_str(), lc, tokens); + } +} + +std::vector throttleHintGenerator(std::vector const& tokens, bool inArgument) { + if(tokens.size() == 1) { + return { "", "[ARGS]" }; + } + else if(tokencmp(tokens[1], "on")) { + std::vector opts = { "tag", "", "[RATE]", "[DURATION]", "[default|immediate|batch]" }; + if(tokens.size() == 2) { + return opts; + } + else if(((tokens.size() == 3 && inArgument) || tokencmp(tokens[2], "tag")) && tokens.size() < 7) { + return std::vector(opts.begin() + tokens.size() - 2, opts.end()); + } + } + else if(tokencmp(tokens[1], "off")) { + if(tokencmp(tokens[tokens.size()-1], "tag")) { + return { "" }; + } + else { + bool hasType = false; + bool hasTag = false; + bool hasPriority = false; + for(int i = 2; i < tokens.size(); ++i) { + if(tokencmp(tokens[i], "all") || tokencmp(tokens[i], "auto") || tokencmp(tokens[i], "manual")) { + hasType = true; + } + else if(tokencmp(tokens[i], "default") || tokencmp(tokens[i], "immediate") || tokencmp(tokens[i], "batch")) { + hasPriority = true; + } + else if(tokencmp(tokens[i], "tag")) { + hasTag = true; + ++i; + } + else { + return {}; + } + } + + std::vector options; + if(!hasType) { + options.push_back("[all|auto|manual]"); + } + if(!hasTag) { + options.push_back("[tag ]"); + } + if(!hasPriority) { + options.push_back("[default|immediate|batch]"); + } + + return options; + } + } + else if((tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) && tokens.size() == 2) { + return { "auto" }; + } + else if(tokens.size() == 2 && inArgument) { + return { "[ARGS]" }; + } + + return std::vector(); } void LogCommand(std::string line, UID randomID, std::string errMsg) { @@ -3919,7 +4009,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { (int)(itr->tpsRate), std::min((int)(itr->expirationTime-now()), (int)(itr->initialDuration)), transactionPriorityToString(itr->priority, false), - itr->autoThrottled ? "auto" : "manual", + itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual", itr->tag.toString().c_str()); } } @@ -3932,19 +4022,21 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { printf("There are no throttled tags\n"); } } - else if(tokencmp(tokens[1], "on") && tokens.size() <=6) { - if(tokens.size() < 4 || !tokencmp(tokens[2], "tag")) { - printf("Usage: throttle on tag [RATE] [DURATION]\n"); + else if(tokencmp(tokens[1], "on")) { + if(tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) { + printf("Usage: throttle on tag [RATE] [DURATION] [PRIORITY]\n"); printf("\n"); printf("Enables throttling for transactions with the specified tag.\n"); printf("An optional transactions per second rate can be specified (default 0).\n"); printf("An optional duration can be specified, which must include a time suffix (s, m, h, d) (default 1h).\n"); + printf("An optional priority can be specified. Choices are `default', `immediate', and `batch' (default `default').\n"); is_error = true; continue; } double tpsRate = 0.0; uint64_t duration = 3600; + TransactionPriority priority = TransactionPriority::DEFAULT; if(tokens.size() >= 5) { char *end; @@ -3968,70 +4060,145 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } duration = parsedDuration.get(); - } - if(duration == 0) { - printf("ERROR: throttle duration cannot be 0\n"); - is_error = true; - continue; + if(duration == 0) { + printf("ERROR: throttle duration cannot be 0\n"); + is_error = true; + continue; + } + } + if(tokens.size() == 7) { + if(tokens[6] == LiteralStringRef("default")) { + priority = TransactionPriority::DEFAULT; + } + else if(tokens[6] == LiteralStringRef("immediate")) { + priority = TransactionPriority::IMMEDIATE; + } + else if(tokens[6] == LiteralStringRef("batch")) { + priority = TransactionPriority::BATCH; + } + else { + printf("ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', or `batch'.\n", tokens[6].toString().c_str()); + is_error = true; + continue; + } } TagSet tags; tags.addTag(tokens[3]); - wait(ThrottleApi::throttleTags(db, tags, tpsRate, duration, false, TransactionPriority::DEFAULT)); + wait(ThrottleApi::throttleTags(db, tags, tpsRate, duration, TagThrottleType::MANUAL, priority)); printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str()); } else if(tokencmp(tokens[1], "off")) { - if(tokencmp(tokens[2], "tag") && tokens.size() == 4) { - TagSet tags; - tags.addTag(tokens[3]); - bool success = wait(ThrottleApi::unthrottleTags(db, tags, false, TransactionPriority::DEFAULT)); // TODO: Allow targeting priority and auto/manual - if(success) { - printf("Unthrottled tag `%s'\n", tokens[3].toString().c_str()); + int nextIndex = 2; + TagSet tags; + bool throttleTypeSpecified = false; + Optional throttleType = TagThrottleType::MANUAL; + Optional priority; + + if(tokens.size() == 2) { + is_error = true; + } + + while(nextIndex < tokens.size() && !is_error) { + if(tokencmp(tokens[nextIndex], "all")) { + if(throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = Optional(); + ++nextIndex; } - else { - printf("Tag `%s' was not throttled\n", tokens[3].toString().c_str()); + else if(tokencmp(tokens[nextIndex], "auto")) { + if(throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::AUTO; + ++nextIndex; + } + else if(tokencmp(tokens[nextIndex], "manual")) { + if(throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::MANUAL; + ++nextIndex; + } + else if(tokencmp(tokens[nextIndex], "default")) { + if(priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::DEFAULT; + ++nextIndex; + } + else if(tokencmp(tokens[nextIndex], "immediate")) { + if(priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::IMMEDIATE; + ++nextIndex; + } + else if(tokencmp(tokens[nextIndex], "batch")) { + if(priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::BATCH; + ++nextIndex; + } + else if(tokencmp(tokens[nextIndex], "tag")) { + if(tags.size() > 0 || nextIndex == tokens.size()-1) { + is_error = true; + continue; + } + tags.addTag(tokens[nextIndex+1]); + nextIndex += 2; } } - else if(tokencmp(tokens[2], "all") && tokens.size() == 3) { - bool unthrottled = wait(ThrottleApi::unthrottleAll(db)); - if(unthrottled) { - printf("Unthrottled all tags\n"); + + if(!is_error) { + state const char *throttleTypeString = !throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually "); + state std::string priorityString = priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : ""; + + if(tags.size() > 0) { + bool success = wait(ThrottleApi::unthrottleTags(db, tags, throttleType, priority)); + if(success) { + printf("Unthrottled tag `%s'%s\n", tokens[3].toString().c_str(), priorityString.c_str()); + } + else { + printf("Tag `%s' was not %sthrottled%s\n", tokens[3].toString().c_str(), throttleTypeString, priorityString.c_str()); + } } else { - printf("There were no tags being throttled\n"); - } - } - else if(tokencmp(tokens[2], "auto") && tokens.size() == 3) { - bool unthrottled = wait(ThrottleApi::unthrottleAuto(db)); - if(unthrottled) { - printf("Unthrottled all auto-throttled tags\n"); - } - else { - printf("There were no tags being throttled\n"); - } - } - else if(tokencmp(tokens[2], "manual") && tokens.size() == 3) { - bool unthrottled = wait(ThrottleApi::unthrottleManual(db)); - if(unthrottled) { - printf("Unthrottled all manually throttled tags\n"); - } - else { - printf("There were no tags being throttled\n"); + bool unthrottled = wait(ThrottleApi::unthrottleAll(db, throttleType, priority)); + if(unthrottled) { + printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str()); + } + else { + printf("There were no tags being %sthrottled%s\n", throttleTypeString, priorityString.c_str()); + } } } else { - printf("Usage: throttle off [TAG]\n"); + printf("Usage: throttle off [all|auto|manual] [tag ] [PRIORITY]\n"); printf("\n"); - printf("Disables throttling for the specified tag(s).\n"); - printf("Use `all' to turn off all tag throttles, `auto' to turn off throttles created by\n"); - printf("the cluster, and `manual' to turn off throttles created manually. Use `tag '\n"); - printf("to turn off throttles for a specific tag\n"); - is_error = true; + printf("Disables throttling for throttles matching the specified filters. At least one filter must be used.\n\n"); + printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type of throttle\n"); + printf("affected. `all' targets all throttles, `auto' targets those created by the cluster, and\n"); + printf("`manual' targets those created manually (default `manual').\n\n"); + printf("The `tag' filter can be use to turn off only a specific tag.\n\n"); + printf("The priority filter can be used to turn off only throttles at specific priorities. Choices are\n"); + printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n"); } } - else if((tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) && tokens.size() == 3 && tokencmp(tokens[2], "auto")) { + else if(tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { if(tokens.size() != 3 || !tokencmp(tokens[2], "auto")) { printf("Usage: throttle auto\n"); printf("\n"); @@ -4077,7 +4244,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { ACTOR Future runCli(CLIOptions opt) { state LineNoise linenoise( [](std::string const& line, std::vector& completions) { - fdbcli_comp_cmd(line, completions); + fdbcliCompCmd(line, completions); }, [enabled=opt.cliHints](std::string const& line)->LineNoise::Hint { if (!enabled) { @@ -4098,18 +4265,32 @@ ACTOR Future runCli(CLIOptions opt) { // being entered. if (error && line.back() != '\\') return LineNoise::Hint(std::string(" {malformed escape sequence}"), 90, false); - auto iter = helpMap.find(command.toString()); - if (iter != helpMap.end()) { - std::string helpLine = iter->second.usage; - std::vector> parsedHelp = parseLine(helpLine, error, partial); - std::string hintLine = (*(line.end() - 1) == ' ' ? "" : " "); - for (int i = finishedParameters; i < parsedHelp.back().size(); i++) { - hintLine = hintLine + parsedHelp.back()[i].toString() + " "; + bool inArgument = *(line.end() - 1) != ' '; + std::string hintLine = inArgument ? " " : ""; + if(tokencmp(command, "throttle")) { + std::vector hintItems = throttleHintGenerator(parsed.back(), inArgument); + if(hintItems.empty()) { + return LineNoise::Hint(); + } + for(auto item : hintItems) { + hintLine = hintLine + item + " "; } - return LineNoise::Hint(hintLine, 90, false); - } else { - return LineNoise::Hint(); } + else { + auto iter = helpMap.find(command.toString()); + if(iter != helpMap.end()) { + std::string helpLine = iter->second.usage; + std::vector> parsedHelp = parseLine(helpLine, error, partial); + for (int i = finishedParameters; i < parsedHelp.back().size(); i++) { + hintLine = hintLine + parsedHelp.back()[i].toString() + " "; + } + } + else { + return LineNoise::Hint(); + } + } + + return LineNoise::Hint(hintLine, 90, false); }, 1000, false); diff --git a/fdbclient/ClientLogEvents.h b/fdbclient/ClientLogEvents.h index 614c8cf7c7..67c4ba64d0 100644 --- a/fdbclient/ClientLogEvents.h +++ b/fdbclient/ClientLogEvents.h @@ -44,19 +44,28 @@ namespace FdbClientLogEvents { }; struct Event { - Event(EventType t, double ts) : type(t), startTs(ts) { } + Event(EventType t, double ts, const Optional> &dc) : type(t), startTs(ts){ + if (dc.present()) + dcId = dc.get(); + } Event() { } - template Ar& serialize(Ar &ar) { return serializer(ar, type, startTs); } + template Ar& serialize(Ar &ar) { + if (ar.protocolVersion().version() >= (uint64_t) 0x0FDB00B063010001LL) { + return serializer(ar, type, startTs, dcId); + } else { + return serializer(ar, type, startTs); + } + } EventType type{ EVENTTYPEEND }; double startTs{ 0 }; + Key dcId{}; void logEvent(std::string id, int maxFieldLength) const {} }; struct EventGetVersion : public Event { - EventGetVersion(double ts, double lat) : Event(GET_VERSION_LATENCY, ts), latency(lat) { } EventGetVersion() { } template Ar& serialize(Ar &ar) { @@ -77,22 +86,6 @@ namespace FdbClientLogEvents { // Version V2 of EventGetVersion starting at 6.2 struct EventGetVersion_V2 : public Event { - EventGetVersion_V2(double ts, double lat, TransactionPriority priority) : Event(GET_VERSION_LATENCY, ts), latency(lat) { - switch(priority) { - // Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority - case TransactionPriority::IMMEDIATE: - priorityType = PRIORITY_IMMEDIATE; - break; - case TransactionPriority::DEFAULT: - priorityType = PRIORITY_DEFAULT; - break; - case TransactionPriority::BATCH: - priorityType = PRIORITY_BATCH; - break; - default: - ASSERT(false); - } - } EventGetVersion_V2() { } template Ar& serialize(Ar &ar) { @@ -115,7 +108,7 @@ namespace FdbClientLogEvents { // Version V3 of EventGetVersion starting at 6.3 struct EventGetVersion_V3 : public Event { - EventGetVersion_V3(double ts, double lat, TransactionPriority priority, Version version) : Event(GET_VERSION_LATENCY, ts), latency(lat), readVersion(version) { + EventGetVersion_V3(double ts, const Optional> &dcId, double lat, TransactionPriority priority, Version version) : Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) { switch(priority) { // Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority case TransactionPriority::IMMEDIATE: @@ -154,7 +147,7 @@ namespace FdbClientLogEvents { }; struct EventGet : public Event { - EventGet(double ts, double lat, int size, const KeyRef &in_key) : Event(GET_LATENCY, ts), latency(lat), valueSize(size), key(in_key) { } + EventGet(double ts, const Optional> &dcId, double lat, int size, const KeyRef &in_key) : Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) { } EventGet() { } template Ar& serialize(Ar &ar) { @@ -180,7 +173,7 @@ namespace FdbClientLogEvents { }; struct EventGetRange : public Event { - EventGetRange(double ts, double lat, int size, const KeyRef &start_key, const KeyRef & end_key) : Event(GET_RANGE_LATENCY, ts), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) { } + EventGetRange(double ts, const Optional> &dcId, double lat, int size, const KeyRef &start_key, const KeyRef & end_key) : Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) { } EventGetRange() { } template Ar& serialize(Ar &ar) { @@ -208,7 +201,6 @@ namespace FdbClientLogEvents { }; struct EventCommit : public Event { - EventCommit(double ts, double lat, int mut, int bytes, const CommitTransactionRequest &commit_req) : Event(COMMIT_LATENCY, ts), latency(lat), numMutations(mut), commitBytes(bytes), req(commit_req) { } EventCommit() { } template Ar& serialize(Ar &ar) { @@ -260,8 +252,8 @@ namespace FdbClientLogEvents { // Version V2 of EventGetVersion starting at 6.3 struct EventCommit_V2 : public Event { - EventCommit_V2(double ts, double lat, int mut, int bytes, Version version, const CommitTransactionRequest &commit_req) - : Event(COMMIT_LATENCY, ts), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version), req(commit_req) { } + EventCommit_V2(double ts, const Optional> &dcId, double lat, int mut, int bytes, Version version, const CommitTransactionRequest &commit_req) + : Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version), req(commit_req) { } EventCommit_V2() { } template Ar& serialize(Ar &ar) { @@ -314,7 +306,7 @@ namespace FdbClientLogEvents { }; struct EventGetError : public Event { - EventGetError(double ts, int err_code, const KeyRef &in_key) : Event(ERROR_GET, ts), errCode(err_code), key(in_key) { } + EventGetError(double ts, const Optional> &dcId, int err_code, const KeyRef &in_key) : Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) { } EventGetError() { } template Ar& serialize(Ar &ar) { @@ -338,7 +330,7 @@ namespace FdbClientLogEvents { }; struct EventGetRangeError : public Event { - EventGetRangeError(double ts, int err_code, const KeyRef &start_key, const KeyRef & end_key) : Event(ERROR_GET_RANGE, ts), errCode(err_code), startKey(start_key), endKey(end_key) { } + EventGetRangeError(double ts, const Optional> &dcId, int err_code, const KeyRef &start_key, const KeyRef & end_key) : Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) { } EventGetRangeError() { } template Ar& serialize(Ar &ar) { @@ -364,7 +356,7 @@ namespace FdbClientLogEvents { }; struct EventCommitError : public Event { - EventCommitError(double ts, int err_code, const CommitTransactionRequest &commit_req) : Event(ERROR_COMMIT, ts), errCode(err_code), req(commit_req) { } + EventCommitError(double ts, const Optional> &dcId, int err_code, const CommitTransactionRequest &commit_req) : Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) { } EventCommitError() { } template Ar& serialize(Ar &ar) { diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 5a3ace0532..7d4e450c30 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -1019,6 +1019,21 @@ struct HealthMetrics { } }; +struct DDMetricsRef { + int64_t shardBytes; + KeyRef beginKey; + + DDMetricsRef() : shardBytes(0) {} + DDMetricsRef(int64_t bytes, KeyRef begin) : shardBytes(bytes), beginKey(begin) {} + DDMetricsRef(Arena& a, const DDMetricsRef& copyFrom) + : shardBytes(copyFrom.shardBytes), beginKey(a, copyFrom.beginKey) {} + + template + void serialize(Ar& ar) { + serializer(ar, shardBytes, beginKey); + } +}; + struct WorkerBackupStatus { LogEpoch epoch; Version version; diff --git a/fdbclient/IncludeVersions.h b/fdbclient/IncludeVersions.h deleted file mode 100644 index 66bdccf43d..0000000000 --- a/fdbclient/IncludeVersions.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * IncludeVersions.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This is a simple header to isolate the stupidity that results out of two -// build systems and versions.h include directives - -#if defined(CMAKE_BUILD) -# include "fdbclient/versions.h" -#elif !defined(WIN32) -# include "versions.h" -#endif diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index a7d62ad684..c334d3c500 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -92,6 +92,7 @@ void ClientKnobs::initialize(bool randomize) { init( STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, 15.0 ); init( AGGREGATE_HEALTH_METRICS_MAX_STALENESS, 0.5 ); init( DETAILED_HEALTH_METRICS_MAX_STALENESS, 5.0 ); + init( TAG_ENCODE_KEY_SERVERS, false ); if( randomize && BUGGIFY ) TAG_ENCODE_KEY_SERVERS = true; //KeyRangeMap init( KRM_GET_RANGE_LIMIT, 1e5 ); if( randomize && BUGGIFY ) KRM_GET_RANGE_LIMIT = 10; diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index a28c05e19a..31919811f3 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -85,6 +85,7 @@ public: double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY; double AGGREGATE_HEALTH_METRICS_MAX_STALENESS; double DETAILED_HEALTH_METRICS_MAX_STALENESS; + bool TAG_ENCODE_KEY_SERVERS; //KeyRangeMap int KRM_GET_RANGE_LIMIT; diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h index abb3069490..7216015535 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/MasterProxyInterface.h @@ -42,7 +42,6 @@ struct MasterProxyInterface { Optional processId; bool provisional; - Endpoint base; RequestStream< struct CommitTransactionRequest > commit; RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent // (at some point between when this request is sent and when its response is received, the latest version reported committed) @@ -56,6 +55,7 @@ struct MasterProxyInterface { RequestStream< struct GetHealthMetricsRequest > getHealthMetrics; RequestStream< struct ProxySnapRequest > proxySnapReq; RequestStream< struct ExclusionSafetyCheckRequest > exclusionSafetyCheckReq; + RequestStream< struct GetDDMetricsRequest > getDDMetrics; UID id() const { return commit.getEndpoint().token; } std::string toString() const { return id().shortString(); } @@ -65,18 +65,18 @@ struct MasterProxyInterface { template void serialize(Archive& ar) { - serializer(ar, processId, provisional, base); + serializer(ar, processId, provisional, commit); if( Archive::isDeserializing ) { - commit = RequestStream< struct CommitTransactionRequest >( base.getAdjustedEndpoint(0) ); - getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( base.getAdjustedEndpoint(1) ); - getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( base.getAdjustedEndpoint(2) ); - getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( base.getAdjustedEndpoint(3) ); - waitFailure = RequestStream>( base.getAdjustedEndpoint(4) ); - getRawCommittedVersion = RequestStream< struct GetRawCommittedVersionRequest >( base.getAdjustedEndpoint(5) ); - txnState = RequestStream< struct TxnStateRequest >( base.getAdjustedEndpoint(6) ); - getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( base.getAdjustedEndpoint(7) ); - proxySnapReq = RequestStream< struct ProxySnapRequest >( base.getAdjustedEndpoint(8) ); - exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( base.getAdjustedEndpoint(9) ); + getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) ); + getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) ); + getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) ); + waitFailure = RequestStream>( commit.getEndpoint().getAdjustedEndpoint(4) ); + getRawCommittedVersion = RequestStream< struct GetRawCommittedVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(5) ); + txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(6) ); + getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(7) ); + proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(8) ); + exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(9) ); + getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(10) ); } } @@ -92,7 +92,8 @@ struct MasterProxyInterface { streams.push_back(getHealthMetrics.getReceiver()); streams.push_back(proxySnapReq.getReceiver()); streams.push_back(exclusionSafetyCheckReq.getReceiver()); - base = FlowTransport::transport().addEndpoints(streams); + streams.push_back(getDDMetrics.getReceiver()); + FlowTransport::transport().addEndpoints(streams); } }; @@ -391,6 +392,34 @@ struct GetHealthMetricsRequest } }; +struct GetDDMetricsReply +{ + constexpr static FileIdentifier file_identifier = 7277713; + Standalone> storageMetricsList; + + GetDDMetricsReply() {} + + template + void serialize(Ar& ar) { + serializer(ar, storageMetricsList); + } +}; + +struct GetDDMetricsRequest { + constexpr static FileIdentifier file_identifier = 14536812; + KeyRange keys; + int shardLimit; + ReplyPromise reply; + + GetDDMetricsRequest() {} + explicit GetDDMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {} + + template + void serialize(Ar& ar) { + serializer(ar, keys, shardLimit, reply); + } +}; + struct ProxySnapRequest { constexpr static FileIdentifier file_identifier = 22204900; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 72ab5d2878..25899fcfda 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -49,7 +49,7 @@ #include "flow/TLSConfig.actor.h" #include "flow/UnitTest.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #ifdef WIN32 #define WIN32_LEAN_AND_MEAN @@ -607,6 +607,8 @@ DatabaseContext::DatabaseContext(Reference(conflictingKeysRange)); registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, std::make_unique(readConflictRangeKeysRange)); registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, std::make_unique(writeConflictRangeKeysRange)); + registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::METRICS, + std::make_unique(ddStatsRange)); registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::WORKERINTERFACE, std::make_unique(KeyRangeRef( LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0")))); registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::STATUSJSON, std::make_unique( @@ -738,7 +740,7 @@ Reference DatabaseContext::setCachedLocation( const KeyRangeRef& k locationCache.insert( KeyRangeRef(begin, end), Reference() ); } locationCache.insert( keys, loc ); - return std::move(loc); + return loc; } void DatabaseContext::invalidateCache( const KeyRef& key, bool isBackward ) { @@ -1518,7 +1520,7 @@ ACTOR Future> getValue( Future version, Key key, Databa cx->readLatencies.addSample(latency); if (trLogInfo) { int valueSize = reply.value.present() ? reply.value.get().size() : 0; - trLogInfo->addLog(FdbClientLogEvents::EventGet(startTimeD, latency, valueSize, key)); + trLogInfo->addLog(FdbClientLogEvents::EventGet(startTimeD, cx->clientLocality.dcId(), latency, valueSize, key)); } cx->getValueCompleted->latency = timer_int() - startTime; cx->getValueCompleted->log(); @@ -1550,7 +1552,7 @@ ACTOR Future> getValue( Future version, Key key, Databa wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID)); } else { if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventGetError(startTimeD, static_cast(e.code()), key)); + trLogInfo->addLog(FdbClientLogEvents::EventGetError(startTimeD, cx->clientLocality.dcId(), static_cast(e.code()), key)); throw e; } } @@ -1955,7 +1957,7 @@ void getRangeFinished(Database cx, Reference trLogInfo, doub cx->transactionKeysRead += result.size(); if( trLogInfo ) { - trLogInfo->addLog(FdbClientLogEvents::EventGetRange(startTime, now()-startTime, bytes, begin.getKey(), end.getKey())); + trLogInfo->addLog(FdbClientLogEvents::EventGetRange(startTime, cx->clientLocality.dcId(), now()-startTime, bytes, begin.getKey(), end.getKey())); } if( !snapshot ) { @@ -2195,7 +2197,7 @@ ACTOR Future> getRange( Database cx, ReferenceWRONG_SHARD_SERVER_DELAY, info.taskID)); } else { if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventGetRangeError(startTime, static_cast(e.code()), begin.getKey(), end.getKey())); + trLogInfo->addLog(FdbClientLogEvents::EventGetRangeError(startTime, cx->clientLocality.dcId(), static_cast(e.code()), begin.getKey(), end.getKey())); throw e; } @@ -2449,7 +2451,7 @@ ACTOR Future< Key > getKeyAndConflictRange( conflictRange.send( std::make_pair( rep, k.orEqual ? keyAfter( k.getKey() ) : Key(k.getKey(), k.arena()) ) ); else conflictRange.send( std::make_pair( k.orEqual ? keyAfter( k.getKey() ) : Key(k.getKey(), k.arena()), keyAfter( rep ) ) ); - return std::move(rep); + return rep; } catch( Error&e ) { conflictRange.send(std::make_pair(Key(), Key())); throw; @@ -2975,7 +2977,7 @@ ACTOR static Future tryCommit( Database cx, Reference cx->commitLatencies.addSample(latency); cx->latencies.addSample(now() - tr->startTime); if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventCommit_V2(startTime, latency, req.transaction.mutations.size(), req.transaction.mutations.expectedSize(), ci.version, req)); + trLogInfo->addLog(FdbClientLogEvents::EventCommit_V2(startTime, cx->clientLocality.dcId(), latency, req.transaction.mutations.size(), req.transaction.mutations.expectedSize(), ci.version, req)); return Void(); } else { // clear the RYW transaction which contains previous conflicting keys @@ -3038,7 +3040,7 @@ ACTOR static Future tryCommit( Database cx, Reference TraceEvent(SevError, "TryCommitError").error(e); } if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventCommitError(startTime, static_cast(e.code()), req)); + trLogInfo->addLog(FdbClientLogEvents::EventCommitError(startTime, cx->clientLocality.dcId(), static_cast(e.code()), req)); throw; } } @@ -3449,7 +3451,7 @@ ACTOR Future extractReadVersion(DatabaseContext* cx, TransactionPriorit double latency = now() - startTime; cx->GRVLatencies.addSample(latency); if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(startTime, latency, priority, rep.version)); + trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(startTime, cx->clientLocality.dcId(), latency, priority, rep.version)); if (rep.version == 1 && rep.locked) { throw proxy_memory_limit_exceeded(); } @@ -3858,6 +3860,25 @@ Future< StorageMetrics > Transaction::getStorageMetrics( KeyRange const& keys, i } } +ACTOR Future>> waitDataDistributionMetricsList(Database cx, KeyRange keys, + int shardLimit) { + state Future clientTimeout = delay(5.0); + loop { + choose { + when(wait(cx->onMasterProxiesChanged())) {} + when(ErrorOr rep = + wait(errorOr(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::getDDMetrics, + GetDDMetricsRequest(keys, shardLimit))))) { + if (rep.isError()) { + throw rep.getError(); + } + return rep.get().storageMetricsList; + } + when(wait(clientTimeout)) { throw timed_out(); } + } + } +} + Future>> Transaction::getReadHotRanges(KeyRange const& keys) { return ::getReadHotRanges(cx, keys); } diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index b32e980c85..ac252345fc 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -330,6 +330,8 @@ private: }; ACTOR Future waitForCommittedVersion(Database cx, Version version); +ACTOR Future>> waitDataDistributionMetricsList(Database cx, KeyRange keys, + int shardLimit); std::string unprintable( const std::string& ); diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 74b410efb2..6111ed1114 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -312,8 +312,6 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( }, "limiting_queue_bytes_storage_server":0, "worst_queue_bytes_storage_server":0, - "limiting_version_lag_storage_server":0, - "worst_version_lag_storage_server":0, "limiting_data_lag_storage_server":{ "versions":0, "seconds":0.0 diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 86bc2a6c49..00e05118fe 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -29,7 +29,9 @@ std::unordered_map SpecialKeySpace::moduleToB KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0")) }, { SpecialKeySpace::MODULE::STATUSJSON, singleKeyRange(LiteralStringRef("\xff\xff/status/json")) }, { SpecialKeySpace::MODULE::CONNECTIONSTRING, singleKeyRange(LiteralStringRef("\xff\xff/connection_string")) }, - { SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange(LiteralStringRef("\xff\xff/cluster_file_path")) } + { SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange(LiteralStringRef("\xff\xff/cluster_file_path")) }, + { SpecialKeySpace::MODULE::METRICS, + KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) } }; // This function will move the given KeySelector as far as possible to the standard form: @@ -164,7 +166,6 @@ SpecialKeySpace::getRangeAggregationActor(SpecialKeySpace* sks, Reference lastModuleRead; wait(normalizeKeySelectorActor(sks, ryw, &begin, &lastModuleRead, &actualBeginOffset, &result)); - // TODO : check if end the boundary of a module wait(normalizeKeySelectorActor(sks, ryw, &end, &lastModuleRead, &actualEndOffset, &result)); // Handle all corner cases like what RYW does // return if range inverted @@ -314,6 +315,37 @@ Future> ConflictingKeysImpl::getRange(Reference> ddStatsGetRangeActor(Reference ryw, + KeyRangeRef kr) { + try { + auto keys = kr.removePrefix(ddStatsRange.begin); + Standalone> resultWithoutPrefix = + wait(waitDataDistributionMetricsList(ryw->getDatabase(), keys, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT)); + Standalone result; + for (const auto& ddMetricsRef : resultWithoutPrefix) { + // each begin key is the previous end key, thus we only encode the begin key in the result + KeyRef beginKey = ddMetricsRef.beginKey.withPrefix(ddStatsRange.begin, result.arena()); + // Use json string encoded in utf-8 to encode the values, easy for adding more fields in the future + json_spirit::mObject statsObj; + statsObj["ShardBytes"] = ddMetricsRef.shardBytes; + std::string statsString = + json_spirit::write_string(json_spirit::mValue(statsObj), json_spirit::Output_options::raw_utf8); + ValueRef bytes(result.arena(), statsString); + result.push_back(result.arena(), KeyValueRef(beginKey, bytes)); + } + return result; + } catch (Error& e) { + throw; + } +} + +DDStatsRangeImpl::DDStatsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {} + +Future> DDStatsRangeImpl::getRange(Reference ryw, + KeyRangeRef kr) const { + return ddStatsGetRangeActor(ryw, kr); +} + class SpecialKeyRangeTestImpl : public SpecialKeyRangeBaseImpl { public: explicit SpecialKeyRangeTestImpl(KeyRangeRef kr, const std::string& prefix, int size) diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 45ed78cc00..a7b03a4ff5 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -51,13 +51,14 @@ protected: class SpecialKeySpace { public: enum class MODULE { - UNKNOWN, // default value for all unregistered range - TESTONLY, // only used by correctness tests - TRANSACTION, - WORKERINTERFACE, - STATUSJSON, CLUSTERFILEPATH, - CONNECTIONSTRING + CONNECTIONSTRING, + METRICS, // data-distribution metrics + TESTONLY, // only used by correctness tests + TRANSACTION, // transaction related info, conflicting keys, read/write conflict range + STATUSJSON, + UNKNOWN, // default value for all unregistered range + WORKERINTERFACE, }; Future> get(Reference ryw, const Key& key); @@ -152,5 +153,12 @@ public: KeyRangeRef kr) const override; }; +class DDStatsRangeImpl : public SpecialKeyRangeBaseImpl { +public: + explicit DDStatsRangeImpl(KeyRangeRef kr); + Future> getRange(Reference ryw, + KeyRangeRef kr) const override; +}; + #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 575d234159..b8970c86e0 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -54,7 +54,6 @@ struct StorageServerInterface { LocalityData locality; UID uniqueID; - Endpoint base; RequestStream getValue; RequestStream getKey; @@ -87,20 +86,19 @@ struct StorageServerInterface { // versioned carefully! if (ar.protocolVersion().hasSmallEndpoints()) { - serializer(ar, uniqueID, locality, base); + serializer(ar, uniqueID, locality, getValue); if( Ar::isDeserializing ) { - getValue = RequestStream( base.getAdjustedEndpoint(0) ); - getKey = RequestStream( base.getAdjustedEndpoint(1) ); - getKeyValues = RequestStream( base.getAdjustedEndpoint(2) ); - getShardState = RequestStream( base.getAdjustedEndpoint(3) ); - waitMetrics = RequestStream( base.getAdjustedEndpoint(4) ); - splitMetrics = RequestStream( base.getAdjustedEndpoint(5) ); - getStorageMetrics = RequestStream( base.getAdjustedEndpoint(6) ); - waitFailure = RequestStream>( base.getAdjustedEndpoint(7) ); - getQueuingMetrics = RequestStream( base.getAdjustedEndpoint(8) ); - getKeyValueStoreType = RequestStream>( base.getAdjustedEndpoint(9) ); - watchValue = RequestStream( base.getAdjustedEndpoint(10) ); - getReadHotRanges = RequestStream( base.getAdjustedEndpoint(11) ); + getKey = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(1) ); + getKeyValues = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(2) ); + getShardState = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(3) ); + waitMetrics = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(4) ); + splitMetrics = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(5) ); + getStorageMetrics = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(6) ); + waitFailure = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(7) ); + getQueuingMetrics = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(8) ); + getKeyValueStoreType = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(9) ); + watchValue = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(10) ); + getReadHotRanges = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(11) ); } } else { ASSERT(Ar::isDeserializing); @@ -110,7 +108,6 @@ struct StorageServerInterface { serializer(ar, uniqueID, locality, getValue, getKey, getKeyValues, getShardState, waitMetrics, splitMetrics, getStorageMetrics, waitFailure, getQueuingMetrics, getKeyValueStoreType); if (ar.protocolVersion().hasWatches()) serializer(ar, watchValue); - base = getValue.getEndpoint(); } } bool operator == (StorageServerInterface const& s) const { return uniqueID == s.uniqueID; } @@ -129,7 +126,7 @@ struct StorageServerInterface { streams.push_back(getKeyValueStoreType.getReceiver()); streams.push_back(watchValue.getReceiver()); streams.push_back(getReadHotRanges.getReceiver()); - base = FlowTransport::transport().addEndpoints(streams); + FlowTransport::transport().addEndpoints(streams); } }; @@ -320,6 +317,8 @@ struct GetShardStateRequest { struct StorageMetrics { constexpr static FileIdentifier file_identifier = 13622226; int64_t bytes = 0; // total storage + // FIXME: currently, neither of bytesPerKSecond or iosPerKSecond are actually used in DataDistribution calculations. + // This may change in the future, but this comment is left here to avoid any confusion for the time being. int64_t bytesPerKSecond = 0; // network bandwidth (average over 10s) int64_t iosPerKSecond = 0; int64_t bytesReadPerKSecond = 0; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index a8cc2851e1..7c27227483 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -46,6 +46,11 @@ const KeyRef keyServersKey( const KeyRef& k, Arena& arena ) { return k.withPrefix( keyServersPrefix, arena ); } const Value keyServersValue( Standalone result, const std::vector& src, const std::vector& dest ) { + if(!CLIENT_KNOBS->TAG_ENCODE_KEY_SERVERS) { + BinaryWriter wr(IncludeVersion()); wr << src << dest; + return wr.toValue(); + } + std::vector srcTag; std::vector destTag; @@ -203,6 +208,9 @@ const KeyRangeRef writeConflictRangeKeysRange = KeyRangeRef(LiteralStringRef("\xff\xff/transaction/write_conflict_range/"), LiteralStringRef("\xff\xff/transaction/write_conflict_range/\xff\xff")); +const KeyRangeRef ddStatsRange = KeyRangeRef(LiteralStringRef("\xff\xff/metrics/data_distribution_stats/"), + LiteralStringRef("\xff\xff/metrics/data_distribution_stats/\xff\xff")); + // "\xff/storageCache/[[begin]]" := "[[vector]]" const KeyRangeRef storageCacheKeys( LiteralStringRef("\xff/storageCache/"), LiteralStringRef("\xff/storageCache0") ); const KeyRef storageCachePrefix = storageCacheKeys.begin; diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 54e7271456..fd588a6e94 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -81,6 +81,7 @@ extern const KeyRangeRef conflictingKeysRange; extern const ValueRef conflictingKeysTrue, conflictingKeysFalse; extern const KeyRangeRef writeConflictRangeKeysRange; extern const KeyRangeRef readConflictRangeKeysRange; +extern const KeyRangeRef ddStatsRange; extern const KeyRef cacheKeysPrefix; diff --git a/fdbclient/TagThrottle.actor.cpp b/fdbclient/TagThrottle.actor.cpp index 074d39e158..40de79d325 100644 --- a/fdbclient/TagThrottle.actor.cpp +++ b/fdbclient/TagThrottle.actor.cpp @@ -73,7 +73,7 @@ Key TagThrottleKey::toKey() const { memcpy(str, tagThrottleKeysPrefix.begin(), tagThrottleKeysPrefix.size()); str += tagThrottleKeysPrefix.size(); - *(str++) = autoThrottled ? 1 : 0; + *(str++) = (uint8_t)throttleType; *(str++) = (uint8_t)priority; for(auto tag : tags) { @@ -89,7 +89,7 @@ Key TagThrottleKey::toKey() const { TagThrottleKey TagThrottleKey::fromKey(const KeyRef& key) { const uint8_t *str = key.substr(tagThrottleKeysPrefix.size()).begin(); - bool autoThrottled = *(str++) != 0; + TagThrottleType throttleType = TagThrottleType(*(str++)); TransactionPriority priority = TransactionPriority(*(str++)); TagSet tags; @@ -99,7 +99,7 @@ TagThrottleKey TagThrottleKey::fromKey(const KeyRef& key) { str += size; } - return TagThrottleKey(tags, autoThrottled, priority); + return TagThrottleKey(tags, throttleType, priority); } TagThrottleValue TagThrottleValue::fromValue(const ValueRef& value) { @@ -164,9 +164,9 @@ namespace ThrottleApi { } } - ACTOR Future throttleTags(Database db, TagSet tags, double tpsRate, double initialDuration, bool autoThrottled, TransactionPriority priority, Optional expirationTime) { + ACTOR Future throttleTags(Database db, TagSet tags, double tpsRate, double initialDuration, TagThrottleType throttleType, TransactionPriority priority, Optional expirationTime) { state Transaction tr(db); - state Key key = TagThrottleKey(tags, autoThrottled, priority).toKey(); + state Key key = TagThrottleKey(tags, throttleType, priority).toKey(); ASSERT(initialDuration > 0); @@ -177,7 +177,7 @@ namespace ThrottleApi { loop { try { - if(!autoThrottled) { + if(throttleType == TagThrottleType::MANUAL) { Optional oldThrottle = wait(tr.get(key)); if(!oldThrottle.present()) { wait(updateThrottleCount(&tr, 1)); @@ -186,7 +186,7 @@ namespace ThrottleApi { tr.set(key, value); - if(!autoThrottled) { + if(throttleType == TagThrottleType::MANUAL) { signalThrottleChange(tr); } @@ -199,28 +199,54 @@ namespace ThrottleApi { } } - ACTOR Future unthrottleTags(Database db, TagSet tags, bool autoThrottled, TransactionPriority priority) { + ACTOR Future unthrottleTags(Database db, TagSet tags, Optional throttleType, Optional priority) { state Transaction tr(db); - state Key key = TagThrottleKey(tags, autoThrottled, priority).toKey(); - state bool removed = false; + state std::vector keys; + for(auto p : allTransactionPriorities) { + if(!priority.present() || priority.get() == p) { + if(!throttleType.present() || throttleType.get() == TagThrottleType::AUTO) { + keys.push_back(TagThrottleKey(tags, TagThrottleType::AUTO, p).toKey()); + } + if(!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) { + keys.push_back(TagThrottleKey(tags, TagThrottleType::MANUAL, p).toKey()); + } + } + } + + state bool removed = false; loop { try { - state Optional value = wait(tr.get(key)); - if(value.present()) { - if(!autoThrottled) { - wait(updateThrottleCount(&tr, -1)); + state std::vector>> values; + for(auto key : keys) { + values.push_back(tr.get(key)); + } + + wait(waitForAll(values)); + + int delta = 0; + for(int i = 0; i < values.size(); ++i) { + if(values[i].get().present()) { + if(TagThrottleKey::fromKey(keys[i]).throttleType == TagThrottleType::MANUAL) { + delta -= 1; + } + + tr.clear(keys[i]); + + // Report that we are removing this tag if we ever see it present. + // This protects us from getting confused if the transaction is maybe committed. + // It's ok if someone else actually ends up removing this tag at the same time + // and we aren't the ones to actually do it. + removed = true; } + } - tr.clear(key); + if(delta != 0) { + wait(updateThrottleCount(&tr, delta)); + } + if(removed) { signalThrottleChange(tr); - - // Report that we are removing this tag if we ever see it present. - // This protects us from getting confused if the transaction is maybe committed. - // It's ok if someone else actually ends up removing this tag at the same time - // and we aren't the ones to actually do it. - removed = true; wait(tr.commit()); } @@ -232,7 +258,7 @@ namespace ThrottleApi { } } - ACTOR Future unthrottleTags(Database db, KeyRef beginKey, KeyRef endKey, bool onlyExpiredThrottles) { + ACTOR Future unthrottleMatchingThrottles(Database db, KeyRef beginKey, KeyRef endKey, Optional priority, bool onlyExpiredThrottles) { state Transaction tr(db); state KeySelector begin = firstGreaterOrEqual(beginKey); @@ -253,8 +279,12 @@ namespace ThrottleApi { } } - bool autoThrottled = TagThrottleKey::fromKey(tag.key).autoThrottled; - if(!autoThrottled) { + TagThrottleKey key = TagThrottleKey::fromKey(tag.key); + if(priority.present() && key.priority != priority.get()) { + continue; + } + + if(key.throttleType == TagThrottleType::MANUAL) { ++manualUnthrottledTags; } @@ -285,20 +315,22 @@ namespace ThrottleApi { } } - Future unthrottleManual(Database db) { - return unthrottleTags(db, tagThrottleKeysPrefix, tagThrottleAutoKeysPrefix, false); - } + Future unthrottleAll(Database db, Optional tagThrottleType, Optional priority) { + KeyRef begin = tagThrottleKeys.begin; + KeyRef end = tagThrottleKeys.end; - Future unthrottleAuto(Database db) { - return unthrottleTags(db, tagThrottleAutoKeysPrefix, tagThrottleKeys.end, false); - } + if(tagThrottleType.present() && tagThrottleType == TagThrottleType::AUTO) { + begin = tagThrottleAutoKeysPrefix; + } + else if(tagThrottleType.present() && tagThrottleType == TagThrottleType::MANUAL) { + end = tagThrottleAutoKeysPrefix; + } - Future unthrottleAll(Database db) { - return unthrottleTags(db, tagThrottleKeys.begin, tagThrottleKeys.end, false); + return unthrottleMatchingThrottles(db, begin, end, priority, false); } Future expire(Database db) { - return unthrottleTags(db, tagThrottleKeys.begin, tagThrottleKeys.end, true); + return unthrottleMatchingThrottles(db, tagThrottleKeys.begin, tagThrottleKeys.end, Optional(), true); } ACTOR Future enableAuto(Database db, bool enabled) { diff --git a/fdbclient/TagThrottle.h b/fdbclient/TagThrottle.h index 944e307152..a79c962fb1 100644 --- a/fdbclient/TagThrottle.h +++ b/fdbclient/TagThrottle.h @@ -107,14 +107,19 @@ struct dynamic_size_traits : std::true_type { } }; +enum class TagThrottleType : uint8_t { + MANUAL, + AUTO +}; + struct TagThrottleKey { TagSet tags; - bool autoThrottled; + TagThrottleType throttleType; TransactionPriority priority; - TagThrottleKey() : autoThrottled(false), priority(TransactionPriority::DEFAULT) {} - TagThrottleKey(TagSet tags, bool autoThrottled, TransactionPriority priority) - : tags(tags), autoThrottled(autoThrottled), priority(priority) {} + TagThrottleKey() : throttleType(TagThrottleType::MANUAL), priority(TransactionPriority::DEFAULT) {} + TagThrottleKey(TagSet tags, TagThrottleType throttleType, TransactionPriority priority) + : tags(tags), throttleType(throttleType), priority(priority) {} Key toKey() const; static TagThrottleKey fromKey(const KeyRef& key); @@ -139,17 +144,17 @@ struct TagThrottleValue { struct TagThrottleInfo { TransactionTag tag; - bool autoThrottled; + TagThrottleType throttleType; TransactionPriority priority; double tpsRate; double expirationTime; double initialDuration; - TagThrottleInfo(TransactionTag tag, bool autoThrottled, TransactionPriority priority, double tpsRate, double expirationTime, double initialDuration) - : tag(tag), autoThrottled(autoThrottled), priority(priority), tpsRate(tpsRate), expirationTime(expirationTime), initialDuration(initialDuration) {} + TagThrottleInfo(TransactionTag tag, TagThrottleType throttleType, TransactionPriority priority, double tpsRate, double expirationTime, double initialDuration) + : tag(tag), throttleType(throttleType), priority(priority), tpsRate(tpsRate), expirationTime(expirationTime), initialDuration(initialDuration) {} TagThrottleInfo(TagThrottleKey key, TagThrottleValue value) - : autoThrottled(key.autoThrottled), priority(key.priority), tpsRate(value.tpsRate), expirationTime(value.expirationTime), initialDuration(value.initialDuration) + : throttleType(key.throttleType), priority(key.priority), tpsRate(value.tpsRate), expirationTime(value.expirationTime), initialDuration(value.initialDuration) { ASSERT(key.tags.size() == 1); // Multiple tags per throttle is not currently supported tag = *key.tags.begin(); @@ -160,13 +165,11 @@ namespace ThrottleApi { Future> getThrottledTags(Database const& db, int const& limit); Future throttleTags(Database const& db, TagSet const& tags, double const& tpsRate, double const& initialDuration, - bool const& autoThrottled, TransactionPriority const& priority, Optional const& expirationTime = Optional()); + TagThrottleType const& throttleType, TransactionPriority const& priority, Optional const& expirationTime = Optional()); - Future unthrottleTags(Database const& db, TagSet const& tags, bool const& autoThrottled, TransactionPriority const& priority); + Future unthrottleTags(Database const& db, TagSet const& tags, Optional const& throttleType, Optional const& priority); - Future unthrottleManual(Database db); - Future unthrottleAuto(Database db); - Future unthrottleAll(Database db); + Future unthrottleAll(Database db, Optional throttleType, Optional priority); Future expire(Database db); Future enableAuto(Database const& db, bool const& enabled); diff --git a/fdbclient/ThreadSafeTransaction.actor.cpp b/fdbclient/ThreadSafeTransaction.actor.cpp index 5b69c1656c..d26ff900fd 100644 --- a/fdbclient/ThreadSafeTransaction.actor.cpp +++ b/fdbclient/ThreadSafeTransaction.actor.cpp @@ -21,7 +21,7 @@ #include "fdbclient/ThreadSafeTransaction.h" #include "fdbclient/ReadYourWrites.h" #include "fdbclient/DatabaseContext.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" // Users of ThreadSafeTransaction might share Reference between different threads as long as they don't call addRef (e.g. C API follows this). // Therefore, it is unsafe to call (explicitly or implicitly) this->addRef in any of these functions. diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp index 4f5d061d82..feeff186e2 100644 --- a/fdbmonitor/fdbmonitor.cpp +++ b/fdbmonitor/fdbmonitor.cpp @@ -77,7 +77,7 @@ #include "flow/SimpleOpt.h" #include "SimpleIni.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #ifdef __linux__ typedef fd_set* fdb_fd_set; diff --git a/fdbrpc/ActorFuzz.actor.cpp b/fdbrpc/ActorFuzz.actor.cpp index 88a51e9343..f622504a1d 100644 --- a/fdbrpc/ActorFuzz.actor.cpp +++ b/fdbrpc/ActorFuzz.actor.cpp @@ -802,36 +802,36 @@ ACTOR Future actorFuzz29( FutureStream inputStream, PromiseStream std::pair actorFuzzTests() { int testsOK = 0; - testsOK += testFuzzActor( &actorFuzz0, "actorFuzz0", (vector(),390229,596271,574865) ); - testsOK += testFuzzActor( &actorFuzz1, "actorFuzz1", (vector(),477566,815578,477566,815578,477566,815578,477566,815578,477566,815578,917160) ); - testsOK += testFuzzActor( &actorFuzz2, "actorFuzz2", (vector(),476677,930237) ); - testsOK += testFuzzActor( &actorFuzz3, "actorFuzz3", (vector(),1000) ); - testsOK += testFuzzActor( &actorFuzz4, "actorFuzz4", (vector(),180600,177605,177605,177605,954508,810052) ); - testsOK += testFuzzActor( &actorFuzz5, "actorFuzz5", (vector(),1000) ); - testsOK += testFuzzActor( &actorFuzz6, "actorFuzz6", (vector(),320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,945289) ); - testsOK += testFuzzActor( &actorFuzz7, "actorFuzz7", (vector(),406152,478841,609181,634881,253861,592023,240597,253861,593023,240597,253861,594023,240597,415949,169335,478331,634881,253861,596023,240597,253861,597023,240597,253861,598023,240597,415949,173335,478331,634881,253861,600023,240597,253861,601023,240597,253861,602023,240597,415949,177335,478331,634881,253861,604023,240597,253861,605023,240597,253861,606023,240597,415949,181335,478331,634881,253861,608023,240597,253861,609023,240597,253861,610023,240597,415949,185335,478331,331905,946924,663973,797073,971923,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,534407,814172,949658) ); - testsOK += testFuzzActor( &actorFuzz8, "actorFuzz8", (vector(),285937,696473) ); - testsOK += testFuzzActor( &actorFuzz9, "actorFuzz9", (vector(),141463,397424) ); - testsOK += testFuzzActor( &actorFuzz10, "actorFuzz10", (vector(),543113,1000) ); - testsOK += testFuzzActor( &actorFuzz11, "actorFuzz11", (vector(),1000) ); - testsOK += testFuzzActor( &actorFuzz12, "actorFuzz12", (vector(),970588,981887) ); - testsOK += testFuzzActor( &actorFuzz13, "actorFuzz13", (vector(),861219) ); - testsOK += testFuzzActor( &actorFuzz14, "actorFuzz14", (vector(),527098,527098,527098,628047) ); - testsOK += testFuzzActor( &actorFuzz15, "actorFuzz15", (vector(),582389,240216,732317,582389,240216,732317,582389,240216,732317,582389,240216,732317,582389,240216,732317,884781) ); - testsOK += testFuzzActor( &actorFuzz16, "actorFuzz16", (vector(),943071,492690,908751,198776,537939) ); - testsOK += testFuzzActor( &actorFuzz17, "actorFuzz17", (vector(),249436,416782,249436,416782,249436,416782,299183) ); - testsOK += testFuzzActor( &actorFuzz18, "actorFuzz18", (vector(),337649,395297,807261,517901) ); - testsOK += testFuzzActor( &actorFuzz19, "actorFuzz19", (vector(),492598,139186,742053,492598,140186,742053,492598,141186,742053,592919) ); - testsOK += testFuzzActor( &actorFuzz20, "actorFuzz20", (vector(),760082,1000) ); - testsOK += testFuzzActor( &actorFuzz21, "actorFuzz21", (vector(),806394) ); - testsOK += testFuzzActor( &actorFuzz22, "actorFuzz22", (vector(),722878,369302,416748) ); - testsOK += testFuzzActor( &actorFuzz23, "actorFuzz23", (vector(),562792,231437) ); - testsOK += testFuzzActor( &actorFuzz24, "actorFuzz24", (vector(),847672,835175) ); - testsOK += testFuzzActor( &actorFuzz25, "actorFuzz25", (vector(),843261,327560,592398) ); - testsOK += testFuzzActor( &actorFuzz26, "actorFuzz26", (vector(),520263,306397,944232,366272,700651,146918,191890) ); - testsOK += testFuzzActor( &actorFuzz27, "actorFuzz27", (vector(),313322,196907) ); - testsOK += testFuzzActor( &actorFuzz28, "actorFuzz28", (vector(),715827,529509,449273,715827,529509,449273,715827,529509,449273,715827,529509,449273,715827,529509,449273,743922) ); - testsOK += testFuzzActor( &actorFuzz29, "actorFuzz29", (vector(),821092,901028,617942,821092,902028,617942,821092,903028,617942,821092,904028,617942,821092,905028,617942,560881) ); + testsOK += testFuzzActor( &actorFuzz0, "actorFuzz0", {390229,596271,574865}); + testsOK += testFuzzActor( &actorFuzz1, "actorFuzz1", {477566,815578,477566,815578,477566,815578,477566,815578,477566,815578,917160}); + testsOK += testFuzzActor( &actorFuzz2, "actorFuzz2", {476677,930237}); + testsOK += testFuzzActor( &actorFuzz3, "actorFuzz3", {1000}); + testsOK += testFuzzActor( &actorFuzz4, "actorFuzz4", {180600,177605,177605,177605,954508,810052}); + testsOK += testFuzzActor( &actorFuzz5, "actorFuzz5", {1000}); + testsOK += testFuzzActor( &actorFuzz6, "actorFuzz6", {320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,320321,266526,762336,463730,945289}); + testsOK += testFuzzActor( &actorFuzz7, "actorFuzz7", {406152,478841,609181,634881,253861,592023,240597,253861,593023,240597,253861,594023,240597,415949,169335,478331,634881,253861,596023,240597,253861,597023,240597,253861,598023,240597,415949,173335,478331,634881,253861,600023,240597,253861,601023,240597,253861,602023,240597,415949,177335,478331,634881,253861,604023,240597,253861,605023,240597,253861,606023,240597,415949,181335,478331,634881,253861,608023,240597,253861,609023,240597,253861,610023,240597,415949,185335,478331,331905,946924,663973,797073,971923,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,295772,923567,559259,559259,559259,325678,679187,534407,814172,949658}); + testsOK += testFuzzActor( &actorFuzz8, "actorFuzz8", {285937,696473}); + testsOK += testFuzzActor( &actorFuzz9, "actorFuzz9", {141463,397424}); + testsOK += testFuzzActor( &actorFuzz10, "actorFuzz10", {543113,1000}); + testsOK += testFuzzActor( &actorFuzz11, "actorFuzz11", {1000}); + testsOK += testFuzzActor( &actorFuzz12, "actorFuzz12", {970588,981887}); + testsOK += testFuzzActor( &actorFuzz13, "actorFuzz13", {861219}); + testsOK += testFuzzActor( &actorFuzz14, "actorFuzz14", {527098,527098,527098,628047}); + testsOK += testFuzzActor( &actorFuzz15, "actorFuzz15", {582389,240216,732317,582389,240216,732317,582389,240216,732317,582389,240216,732317,582389,240216,732317,884781}); + testsOK += testFuzzActor( &actorFuzz16, "actorFuzz16", {943071,492690,908751,198776,537939}); + testsOK += testFuzzActor( &actorFuzz17, "actorFuzz17", {249436,416782,249436,416782,249436,416782,299183}); + testsOK += testFuzzActor( &actorFuzz18, "actorFuzz18", {337649,395297,807261,517901}); + testsOK += testFuzzActor( &actorFuzz19, "actorFuzz19", {492598,139186,742053,492598,140186,742053,492598,141186,742053,592919}); + testsOK += testFuzzActor( &actorFuzz20, "actorFuzz20", {760082,1000}); + testsOK += testFuzzActor( &actorFuzz21, "actorFuzz21", {806394}); + testsOK += testFuzzActor( &actorFuzz22, "actorFuzz22", {722878,369302,416748}); + testsOK += testFuzzActor( &actorFuzz23, "actorFuzz23", {562792,231437}); + testsOK += testFuzzActor( &actorFuzz24, "actorFuzz24", {847672,835175}); + testsOK += testFuzzActor( &actorFuzz25, "actorFuzz25", {843261,327560,592398}); + testsOK += testFuzzActor( &actorFuzz26, "actorFuzz26", {520263,306397,944232,366272,700651,146918,191890}); + testsOK += testFuzzActor( &actorFuzz27, "actorFuzz27", {313322,196907}); + testsOK += testFuzzActor( &actorFuzz28, "actorFuzz28", {715827,529509,449273,715827,529509,449273,715827,529509,449273,715827,529509,449273,715827,529509,449273,743922}); + testsOK += testFuzzActor( &actorFuzz29, "actorFuzz29", {821092,901028,617942,821092,902028,617942,821092,903028,617942,821092,904028,617942,821092,905028,617942,560881}); return std::make_pair(testsOK, 30); } #endif // WIN32 diff --git a/fdbrpc/ActorFuzz.h b/fdbrpc/ActorFuzz.h index 74289b06e3..e718f344e5 100644 --- a/fdbrpc/ActorFuzz.h +++ b/fdbrpc/ActorFuzz.h @@ -24,14 +24,6 @@ using std::vector; -inline vector& operator , (vector& v, int a) { - v.push_back(a); - return v; -} - -inline vector& operator , (vector const& v, int a) { - return (const_cast&>(v), a); -} inline void throw_operation_failed() { throw operation_failed(); } // This is in dsltest.actor.cpp: diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp index ec5b6f6e73..86d8141273 100644 --- a/fdbrpc/AsyncFileCached.actor.cpp +++ b/fdbrpc/AsyncFileCached.actor.cpp @@ -80,16 +80,17 @@ Future> AsyncFileCached::open_impl( std::string filename, return open_impl(filename, flags, mode, pageCache); } -Future AsyncFileCached::read_write_impl( AsyncFileCached* self, void* data, int length, int64_t offset, bool writing ) { - if (writing) { +template +Future AsyncFileCached::read_write_impl(AsyncFileCached* self, + typename std::conditional_t data, + int length, int64_t offset) { + if constexpr (writing) { if (offset + length > self->length) self->length = offset + length; } std::vector> actors; - uint8_t* cdata = static_cast(data); - int offsetInPage = offset % self->pageCache->pageSize; int64_t pageOffset = offset - offsetInPage; @@ -108,13 +109,16 @@ Future AsyncFileCached::read_write_impl( AsyncFileCached* self, void* data int bytesInPage = std::min(self->pageCache->pageSize - offsetInPage, remaining); - auto w = writing - ? p->second->write( cdata, bytesInPage, offsetInPage ) - : p->second->read( cdata, bytesInPage, offsetInPage ); + Future w; + if constexpr (writing) { + w = p->second->write(data, bytesInPage, offsetInPage); + } else { + w = p->second->read(data, bytesInPage, offsetInPage); + } if (!w.isReady() || w.isError()) actors.push_back( w ); - cdata += bytesInPage; + data += bytesInPage; pageOffset += self->pageCache->pageSize; offsetInPage = 0; diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h index d9b192b662..66599e6fe9 100644 --- a/fdbrpc/AsyncFileCached.actor.h +++ b/fdbrpc/AsyncFileCached.actor.h @@ -28,6 +28,7 @@ #define FLOW_ASYNCFILECACHED_ACTOR_H #include +#include #include "flow/flow.h" #include "fdbrpc/IAsyncFile.h" @@ -166,7 +167,7 @@ public: length = int(this->length - offset); ASSERT(length >= 0); } - auto f = read_write_impl(this, data, length, offset, false); + auto f = read_write_impl(this, static_cast(data), length, offset); if( f.isReady() && !f.isError() ) return length; ++countFileCacheReadsBlocked; ++countCacheReadsBlocked; @@ -180,7 +181,7 @@ public: wait(self->currentTruncate); ++self->countFileCacheWrites; ++self->countCacheWrites; - Future f = read_write_impl(self, const_cast(data), length, offset, true); + Future f = read_write_impl(self, static_cast(data), length, offset); if (!f.isReady()) { ++self->countFileCacheWritesBlocked; ++self->countCacheWritesBlocked; @@ -346,7 +347,10 @@ private: return Void(); } - static Future read_write_impl( AsyncFileCached* self, void* data, int length, int64_t offset, bool writing ); + template + static Future read_write_impl(AsyncFileCached* self, + typename std::conditional_t data, + int length, int64_t offset); void remove_page( AFCPage* page ); }; diff --git a/fdbrpc/FailureMonitor.actor.cpp b/fdbrpc/FailureMonitor.actor.cpp index 799c4fda77..ceb709a6c7 100644 --- a/fdbrpc/FailureMonitor.actor.cpp +++ b/fdbrpc/FailureMonitor.actor.cpp @@ -121,7 +121,8 @@ void SimpleFailureMonitor::endpointNotFound(Endpoint const& endpoint) { .suppressFor(1.0) .detail("Address", endpoint.getPrimaryAddress()) .detail("Token", endpoint.token); - endpointKnownFailed.set(endpoint, true); + failedEndpoints.insert(endpoint); + endpointKnownFailed.trigger(endpoint); } void SimpleFailureMonitor::notifyDisconnect(NetworkAddress const& address) { @@ -132,7 +133,7 @@ void SimpleFailureMonitor::notifyDisconnect(NetworkAddress const& address) { Future SimpleFailureMonitor::onDisconnectOrFailure(Endpoint const& endpoint) { // If the endpoint or address is already failed, return right away auto i = addressStatus.find(endpoint.getPrimaryAddress()); - if (i == addressStatus.end() || i->second.isFailed() || endpointKnownFailed.get(endpoint)) { + if (i == addressStatus.end() || i->second.isFailed() || failedEndpoints.count(endpoint)) { TraceEvent("AlreadyDisconnected").detail("Addr", endpoint.getPrimaryAddress()).detail("Tok", endpoint.token); return Void(); } @@ -149,14 +150,14 @@ Future SimpleFailureMonitor::onStateChanged(Endpoint const& endpoint) { // failure status for that endpoint can never change (and we could be spuriously triggered by setStatus) // Also returns spuriously when notifyDisconnect is called (which doesn't actually change the state), but callers // check the state so it's OK - if (endpointKnownFailed.get(endpoint)) + if (failedEndpoints.count(endpoint)) return Never(); else return endpointKnownFailed.onChange(endpoint); } FailureStatus SimpleFailureMonitor::getState(Endpoint const& endpoint) { - if (endpointKnownFailed.get(endpoint)) + if (failedEndpoints.count(endpoint)) return FailureStatus(true); else { auto a = addressStatus.find(endpoint.getPrimaryAddress()); @@ -178,7 +179,7 @@ FailureStatus SimpleFailureMonitor::getState(NetworkAddress const& address) { } bool SimpleFailureMonitor::onlyEndpointFailed(Endpoint const& endpoint) { - if (!endpointKnownFailed.get(endpoint)) return false; + if (!failedEndpoints.count(endpoint)) return false; auto a = addressStatus.find(endpoint.getPrimaryAddress()); if (a == addressStatus.end()) return true; @@ -187,10 +188,11 @@ bool SimpleFailureMonitor::onlyEndpointFailed(Endpoint const& endpoint) { } bool SimpleFailureMonitor::permanentlyFailed(Endpoint const& endpoint) { - return endpointKnownFailed.get(endpoint); + return failedEndpoints.count(endpoint); } void SimpleFailureMonitor::reset() { addressStatus = std::unordered_map(); + failedEndpoints = std::unordered_set(); endpointKnownFailed.resetNoWaiting(); } diff --git a/fdbrpc/FailureMonitor.h b/fdbrpc/FailureMonitor.h index d6d11e6e3e..434f0f9a91 100644 --- a/fdbrpc/FailureMonitor.h +++ b/fdbrpc/FailureMonitor.h @@ -25,6 +25,7 @@ #include "flow/flow.h" #include "fdbrpc/FlowTransport.h" // Endpoint #include +#include using std::vector; @@ -153,6 +154,7 @@ public: private: std::unordered_map addressStatus; YieldedAsyncMap endpointKnownFailed; + std::unordered_set failedEndpoints; friend class OnStateChangedActorActor; }; diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp index 65ef314e9f..7c9e3ed912 100644 --- a/fdbrpc/FlowTransport.actor.cpp +++ b/fdbrpc/FlowTransport.actor.cpp @@ -122,10 +122,11 @@ const Endpoint& EndpointMap::insert( NetworkAddressList localAddresses, std::vec } UID base = deterministicRandom()->randomUniqueID(); - for(int i=0; isetEndpoint( Endpoint( localAddresses, UID( base.first() | TOKEN_STREAM_FLAG, (base.second()&0xffffffff00000000LL) | index) ) ); - data[index].token() = Endpoint::Token( base.first() | TOKEN_STREAM_FLAG, (base.second()&0xffffffff00000000LL) | static_cast(streams[i].second) ); + uint64_t first = (base.first()+(i<<32)) | TOKEN_STREAM_FLAG; + streams[i].first->setEndpoint( Endpoint( localAddresses, UID( first, (base.second()&0xffffffff00000000LL) | index) ) ); + data[index].token() = Endpoint::Token( first, (base.second()&0xffffffff00000000LL) | static_cast(streams[i].second) ); data[index].receiver = (NetworkMessageReceiver*) streams[i].first; } @@ -1277,8 +1278,8 @@ void FlowTransport::addEndpoint( Endpoint& endpoint, NetworkMessageReceiver* rec self->endpoints.insert( receiver, endpoint.token, taskID ); } -const Endpoint& FlowTransport::addEndpoints( std::vector> const& streams ) { - return self->endpoints.insert( self->localAddresses, streams ); +void FlowTransport::addEndpoints( std::vector> const& streams ) { + self->endpoints.insert( self->localAddresses, streams ); } void FlowTransport::removeEndpoint( const Endpoint& endpoint, NetworkMessageReceiver* receiver ) { diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h index 1573577ee6..0f7326b35e 100644 --- a/fdbrpc/FlowTransport.h +++ b/fdbrpc/FlowTransport.h @@ -68,23 +68,17 @@ public: Endpoint getAdjustedEndpoint( uint32_t index ) { uint32_t newIndex = token.second(); newIndex += index; - return Endpoint( addresses, UID(token.first(), (token.second()&0xffffffff00000000LL) | newIndex) ); + return Endpoint( addresses, UID(token.first()+(uint64_t(index)<<32), (token.second()&0xffffffff00000000LL) | newIndex) ); } bool operator == (Endpoint const& r) const { - return getPrimaryAddress() == r.getPrimaryAddress() && token == r.token; + return token == r.token && getPrimaryAddress() == r.getPrimaryAddress(); } bool operator != (Endpoint const& r) const { return !(*this == r); } - bool operator < (Endpoint const& r) const { - const NetworkAddress& left = getPrimaryAddress(); - const NetworkAddress& right = r.getPrimaryAddress(); - if (left != right) - return left < right; - else - return token < r.token; + return addresses.address < r.addresses.address || (addresses.address == r.addresses.address && token < r.token); } template @@ -109,6 +103,18 @@ public: }; #pragma pack(pop) +namespace std +{ + template <> + struct hash + { + size_t operator()(const Endpoint& ep) const + { + return ep.token.hash() + ep.addresses.address.hash(); + } + }; +} + class ArenaObjectReader; class NetworkMessageReceiver { public: @@ -186,7 +192,7 @@ public: void addEndpoint( Endpoint& endpoint, NetworkMessageReceiver*, TaskPriority taskID ); // Sets endpoint to be a new local endpoint which delivers messages to the given receiver - const Endpoint& addEndpoints( std::vector> const& streams ); + void addEndpoints( std::vector> const& streams ); void removeEndpoint( const Endpoint&, NetworkMessageReceiver* ); // The given local endpoint no longer delivers messages to the given receiver or uses resources diff --git a/fdbrpc/actorFuzz.py b/fdbrpc/actorFuzz.py old mode 100644 new mode 100755 index 05eb22e4de..dc83b7dbaa --- a/fdbrpc/actorFuzz.py +++ b/fdbrpc/actorFuzz.py @@ -449,7 +449,7 @@ for actor in actors: print("std::pair actorFuzzTests() {\n\tint testsOK = 0;", file=outputFile) for actor in actors: - print('\ttestsOK += testFuzzActor( &%s, "%s", (vector(),%s) );' % (actor.name, actor.name, ','.join(str(e) for e in actor.ecx.output)), + print('\ttestsOK += testFuzzActor( &%s, "%s", {%s} );' % (actor.name, actor.name, ','.join(str(e) for e in actor.ecx.output)), file=outputFile) print("\treturn std::make_pair(testsOK, %d);\n}" % len(actors), file=outputFile) print('#endif // WIN32\n', file=outputFile) diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index ddfc66c42c..6114956dc9 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -85,17 +85,6 @@ void ISimulator::displayWorkers() const return; } -namespace std { -template<> -class hash { -public: - size_t operator()(const Endpoint &s) const - { - return crc32c_append(0, (const uint8_t*)&s, sizeof(s)); - } -}; -} - const UID TOKEN_ENDPOINT_NOT_FOUND(-1, -1); ISimulator* g_pSimulator = 0; diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp index 985fcb7f93..898ce31b70 100644 --- a/fdbserver/BackupProgress.actor.cpp +++ b/fdbserver/BackupProgress.actor.cpp @@ -83,21 +83,24 @@ std::map, std::map> BackupProgr auto progressIt = progress.lower_bound(epoch); if (progressIt != progress.end() && progressIt->first == epoch) { - if (progressIt != progress.begin()) { + std::set toCheck = tags; + for (auto current = progressIt; current != progress.begin() && !toCheck.empty();) { + auto prev = std::prev(current); // Previous epoch is gone, consolidate the progress. - auto prev = std::prev(progressIt); for (auto [tag, version] : prev->second) { - if (tags.count(tag) > 0) { + if (toCheck.count(tag) > 0) { progressIt->second[tag] = std::max(version, progressIt->second[tag]); + toCheck.erase(tag); } } + current = prev; } updateTagVersions(&tagVersions, &tags, progressIt->second, info.epochEnd, adjustedBeginVersion, epoch); } else { auto rit = std::find_if( progress.rbegin(), progress.rend(), [epoch = epoch](const std::pair>& p) { return p.first < epoch; }); - if (!(rit == progress.rend())) { + while (!(rit == progress.rend())) { // A partial recovery can result in empty epoch that copies previous // epoch's version range. In this case, we should check previous // epoch's savedVersion. @@ -112,7 +115,9 @@ std::map, std::map> BackupProgr // ASSERT(info.logRouterTags == epochTags[rit->first]); updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, adjustedBeginVersion, epoch); + break; } + rit++; } } diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index b1d666e0f4..e1e0aee474 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -34,14 +34,17 @@ #include "flow/actorcompiler.h" // This must be the last #include. +#define SevDebugMemory SevVerbose + struct VersionedMessage { LogMessageVersion version; StringRef message; VectorRef tags; Arena arena; // Keep a reference to the memory containing the message + size_t bytes; // arena's size when inserted, which can grow afterwards VersionedMessage(LogMessageVersion v, StringRef m, const VectorRef& t, const Arena& a) - : version(v), message(m), tags(t), arena(a) {} + : version(v), message(m), tags(t), arena(a), bytes(a.getSize()) {} const Version getVersion() const { return version.version; } const uint32_t getSubVersion() const { return version.sub; } @@ -64,6 +67,10 @@ struct VersionedMessage { } }; +static bool sameArena(const Arena& a, const Arena& b) { + return a.impl.getPtr() == b.impl.getPtr(); +} + struct BackupData { const UID myId; const Tag tag; // LogRouter tag for this worker, i.e., (-2, i) @@ -84,6 +91,7 @@ struct BackupData { bool stopped = false; bool exitEarly = false; // If the worker is on an old epoch and all backups starts a version >= the endVersion AsyncVar paused; // Track if "backupPausedKey" is set. + Reference lock; struct PerBackupInfo { PerBackupInfo() = default; @@ -231,12 +239,14 @@ struct BackupData { : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion), endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch), minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), popVersion(req.startVersion - 1), - cc("BackupWorker", myId.toString()), pulledVersion(0), paused(false) { + cc("BackupWorker", myId.toString()), pulledVersion(0), paused(false), + lock(new FlowLock(SERVER_KNOBS->BACKUP_LOCK_BYTES)) { cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true); specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; }); specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; }); specialCounter(cc, "MsgQ", [this]() { return this->messages.size(); }); + specialCounter(cc, "BufferedBytes", [this]() { return this->lock->activePermits(); }); logger = traceCounters("BackupWorkerMetrics", myId, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "BackupWorkerMetrics"); } @@ -310,6 +320,34 @@ struct BackupData { doneTrigger.trigger(); } + // Erases messages and updates lock with memory released. + void eraseMessages(int num) { + ASSERT(num <= messages.size()); + if (num == 0) return; + + if (messages.size() == num) { + messages.clear(); + TraceEvent(SevDebugMemory, "BackupWorkerMemory", myId).detail("ReleaseAll", lock->activePermits()); + lock->release(lock->activePermits()); + return; + } + + // keep track of each arena and accumulate their sizes + int64_t bytes = 0; + for (int i = 0; i < num; i++) { + const Arena& a = messages[i].arena; + const Arena& b = messages[i + 1].arena; + if (!sameArena(a, b)) { + bytes += messages[i].bytes; + TraceEvent(SevDebugMemory, "BackupWorkerMemory", myId) + .detail("Release", messages[i].bytes) + .detail("Arena", (void*)a.impl.getPtr()); + } + } + lock->release(bytes); + messages.erase(messages.begin(), messages.begin() + num); + } + void eraseMessagesAfterEndVersion() { ASSERT(endVersion.present()); const Version ver = endVersion.get(); @@ -637,6 +675,7 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int state std::vector> logFiles; state std::vector blockEnds; state std::vector activeUids; // active Backups' UIDs + state std::vector beginVersions; // logFiles' begin versions state KeyRangeMap> keyRangeMap; // range to index in logFileFutures, logFiles, & blockEnds state std::vector> mutations; state int idx; @@ -655,15 +694,20 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int const int index = logFileFutures.size(); activeUids.push_back(it->first); self->insertRanges(keyRangeMap, it->second.ranges.get(), index); + if (it->second.lastSavedVersion == invalidVersion) { if (it->second.startVersion > self->startVersion && !self->messages.empty()) { // True-up first mutation log's begin version it->second.lastSavedVersion = self->messages[0].getVersion(); } else { - it->second.lastSavedVersion = - std::max(self->popVersion, std::max(self->savedVersion, self->startVersion)); + it->second.lastSavedVersion = std::max({ self->popVersion, self->savedVersion, self->startVersion }); } + TraceEvent("BackupWorkerTrueUp", self->myId).detail("LastSavedVersion", it->second.lastSavedVersion); } + // The true-up version can be larger than first message version, so keep + // the begin versions for later muation filtering. + beginVersions.push_back(it->second.lastSavedVersion); + logFileFutures.push_back(it->second.container.get().get()->writeTaggedLogFile( it->second.lastSavedVersion, popVersion + 1, blockSize, self->tag.id, self->totalTags)); it++; @@ -675,7 +719,7 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int std::transform(logFileFutures.begin(), logFileFutures.end(), std::back_inserter(logFiles), [](const Future>& f) { return f.get(); }); - ASSERT(activeUids.size() == logFiles.size()); + ASSERT(activeUids.size() == logFiles.size() && beginVersions.size() == logFiles.size()); for (int i = 0; i < logFiles.size(); i++) { TraceEvent("OpenMutationFile", self->myId) .detail("BackupID", activeUids[i]) @@ -698,7 +742,10 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int std::vector> adds; if (m.type != MutationRef::Type::ClearRange) { for (int index : keyRangeMap[m.param1]) { - adds.push_back(addMutation(logFiles[index], message, message.message, &blockEnds[index], blockSize)); + if (message.getVersion() >= beginVersions[index]) { + adds.push_back( + addMutation(logFiles[index], message, message.message, &blockEnds[index], blockSize)); + } } } else { KeyRangeRef mutationRange(m.param1, m.param2); @@ -713,8 +760,10 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int wr << subm; mutations.push_back(wr.toValue()); for (int index : range.value()) { - adds.push_back( - addMutation(logFiles[index], message, mutations.back(), &blockEnds[index], blockSize)); + if (message.getVersion() >= beginVersions[index]) { + adds.push_back( + addMutation(logFiles[index], message, mutations.back(), &blockEnds[index], blockSize)); + } } } } @@ -791,12 +840,12 @@ ACTOR Future uploadData(BackupData* self) { .detail("MsgQ", self->messages.size()); // save an empty file for old epochs so that log file versions are continuous wait(saveMutationsToFile(self, popVersion, numMsg)); - self->messages.erase(self->messages.begin(), self->messages.begin() + numMsg); + self->eraseMessages(numMsg); } // If transition into NOOP mode, should clear messages if (!self->pulling) { - self->messages.clear(); + self->eraseMessages(self->messages.size()); } if (popVersion > self->savedVersion && popVersion > self->popVersion) { @@ -810,7 +859,7 @@ ACTOR Future uploadData(BackupData* self) { } if (self->allMessageSaved()) { - self->messages.clear(); + self->eraseMessages(self->messages.size()); return Void(); } @@ -825,6 +874,7 @@ ACTOR Future pullAsyncData(BackupData* self) { state Future logSystemChange = Void(); state Reference r; state Version tagAt = std::max(self->pulledVersion.get(), std::max(self->startVersion, self->savedVersion)); + state Arena prev; TraceEvent("BackupWorkerPull", self->myId); loop { @@ -850,6 +900,15 @@ ACTOR Future pullAsyncData(BackupData* self) { // Note we aggressively peek (uncommitted) messages, but only committed // messages/mutations will be flushed to disk/blob in uploadData(). while (r->hasMessage()) { + if (!sameArena(prev, r->arena())) { + TraceEvent(SevDebugMemory, "BackupWorkerMemory", self->myId) + .detail("Take", r->arena().getSize()) + .detail("Arena", (void*)r->arena().impl.getPtr()) + .detail("Current", self->lock->activePermits()); + + wait(self->lock->take(TaskPriority::DefaultYield, r->arena().getSize())); + prev = r->arena(); + } self->messages.emplace_back(r->version(), r->getMessage(), r->getTags(), r->arena()); r->nextMessage(); } diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 0a12adedf4..52a08a6ef4 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -134,6 +134,7 @@ set(FDBSERVER_SRCS workloads/ConsistencyCheck.actor.cpp workloads/CpuProfiler.actor.cpp workloads/Cycle.actor.cpp + workloads/DataDistributionMetrics.actor.cpp workloads/DDBalance.actor.cpp workloads/DDMetrics.actor.cpp workloads/DDMetricsExclude.actor.cpp diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 7f4549ed76..65bbe60f3b 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -4429,7 +4429,7 @@ ACTOR Future monitorBatchLimitedTime(Reference> db, } } -ACTOR Future dataDistribution(Reference self) +ACTOR Future dataDistribution(Reference self, PromiseStream getShardMetricsList) { state double lastLimited = 0; self->addActor.send( monitorBatchLimitedTime(self->dbInfo, &lastLimited) ); @@ -4605,7 +4605,7 @@ ACTOR Future dataDistribution(Reference self) } actors.push_back( pollMoveKeysLock(cx, lock) ); - actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, output, shardsAffectedByTeamFailure, getShardMetrics, getAverageShardBytes.getFuture(), readyToStart, anyZeroHealthyTeams, self->ddId ), "DDTracker", self->ddId, &normalDDQueueErrors() ) ); + actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, output, shardsAffectedByTeamFailure, getShardMetrics, getShardMetricsList, getAverageShardBytes.getFuture(), readyToStart, anyZeroHealthyTeams, self->ddId ), "DDTracker", self->ddId, &normalDDQueueErrors() ) ); actors.push_back( reportErrorsExcept( dataDistributionQueue( cx, output, input.getFuture(), getShardMetrics, processingUnhealthy, tcis, shardsAffectedByTeamFailure, lock, getAverageShardBytes, self->ddId, storageTeamSize, configuration.storageTeamSize, &lastLimited ), "DDQueue", self->ddId, &normalDDQueueErrors() ) ); vector teamCollectionsPtrs; @@ -4856,6 +4856,7 @@ ACTOR Future ddExclusionSafetyCheck(DistributorExclusionSafetyCheckRequest ACTOR Future dataDistributor(DataDistributorInterface di, Reference> db ) { state Reference self( new DataDistributorData(db, di.id()) ); state Future collection = actorCollection( self->addActor.getFuture() ); + state PromiseStream getShardMetricsList; state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true); state ActorCollection actors(false); self->addActor.send(actors.getResult()); @@ -4864,7 +4865,7 @@ ACTOR Future dataDistributor(DataDistributorInterface di, ReferenceaddActor.send( waitFailureServer(di.waitFailure.getFuture()) ); - state Future distributor = reportErrorsExcept( dataDistribution(self), "DataDistribution", di.id(), &normalDataDistributorErrors() ); + state Future distributor = reportErrorsExcept( dataDistribution(self, getShardMetricsList), "DataDistribution", di.id(), &normalDataDistributorErrors() ); loop choose { when ( wait(distributor || collection) ) { @@ -4876,6 +4877,17 @@ ACTOR Future dataDistributor(DataDistributorInterface di, Reference>> result = wait(errorOr(brokenPromiseToNever( + getShardMetricsList.getReply(GetMetricsListRequest(req.keys, req.shardLimit))))); + if ( result.isError() ) { + req.reply.sendError(result.getError()); + } else { + GetDataDistributorMetricsReply rep; + rep.storageMetricsList = result.get(); + req.reply.send(rep); + } + } when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) { actors.add(ddSnapCreate(snapReq, db)); } diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h index f07a15dbfd..116d6a9234 100644 --- a/fdbserver/DataDistribution.actor.h +++ b/fdbserver/DataDistribution.actor.h @@ -107,6 +107,15 @@ struct GetMetricsRequest { GetMetricsRequest( KeyRange const& keys ) : keys(keys) {} }; +struct GetMetricsListRequest { + KeyRange keys; + int shardLimit; + Promise>> reply; + + GetMetricsListRequest() {} + GetMetricsListRequest( KeyRange const& keys, const int shardLimit ) : keys(keys), shardLimit(shardLimit) {} +}; + struct TeamCollectionInterface { PromiseStream< GetTeamRequest > getTeam; }; @@ -203,6 +212,7 @@ Future dataDistributionTracker( PromiseStream const& output, Reference const& shardsAffectedByTeamFailure, PromiseStream const& getShardMetrics, + PromiseStream const& getShardMetricsList, FutureStream> const& getAverageShardBytes, Promise const& readyToStart, Reference> const& zeroHealthyTeams, diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp index dae7942057..01d2a34ab9 100644 --- a/fdbserver/DataDistributionTracker.actor.cpp +++ b/fdbserver/DataDistributionTracker.actor.cpp @@ -813,12 +813,60 @@ ACTOR Future fetchShardMetrics( DataDistributionTracker* self, GetMetricsR return Void(); } + +ACTOR Future fetchShardMetricsList_impl( DataDistributionTracker* self, GetMetricsListRequest req ) { + try { + loop { + // used to control shard limit + int shardNum = 0; + // list of metrics, regenerate on loop when full range unsuccessful + Standalone> result; + Future onChange; + for (auto t : self->shards.containedRanges(req.keys)) { + auto &stats = t.value().stats; + if( !stats->get().present() ) { + onChange = stats->onChange(); + break; + } + result.push_back_deep(result.arena(), + DDMetricsRef(stats->get().get().metrics.bytes, KeyRef(t.begin().toString()))); + ++shardNum; + if (shardNum >= req.shardLimit) { + break; + } + } + + if( !onChange.isValid() ) { + req.reply.send( result ); + return Void(); + } + + wait( onChange ); + } + } catch( Error &e ) { + if( e.code() != error_code_actor_cancelled && !req.reply.isSet() ) + req.reply.sendError(e); + throw; + } +} + +ACTOR Future fetchShardMetricsList( DataDistributionTracker* self, GetMetricsListRequest req ) { + choose { + when( wait( fetchShardMetricsList_impl( self, req ) ) ) {} + when( wait( delay( SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT ) ) ) { + req.reply.sendError(timed_out()); + } + } + return Void(); +} + ACTOR Future dataDistributionTracker( Reference initData, Database cx, PromiseStream output, Reference shardsAffectedByTeamFailure, PromiseStream getShardMetrics, + PromiseStream getShardMetricsList, FutureStream> getAverageShardBytes, Promise readyToStart, Reference> anyZeroHealthyTeams, @@ -847,6 +895,9 @@ ACTOR Future dataDistributionTracker( when( GetMetricsRequest req = waitNext( getShardMetrics.getFuture() ) ) { self.sizeChanges.add( fetchShardMetrics( &self, req ) ); } + when( GetMetricsListRequest req = waitNext( getShardMetricsList.getFuture() ) ) { + self.sizeChanges.add( fetchShardMetricsList( &self, req ) ); + } when( wait( self.sizeChanges.getResult() ) ) {} } } catch (Error& e) { diff --git a/fdbserver/DataDistributorInterface.h b/fdbserver/DataDistributorInterface.h index a1e0ffb35e..063772f02b 100644 --- a/fdbserver/DataDistributorInterface.h +++ b/fdbserver/DataDistributorInterface.h @@ -32,6 +32,7 @@ struct DataDistributorInterface { struct LocalityData locality; RequestStream distributorSnapReq; RequestStream distributorExclCheckReq; + RequestStream dataDistributorMetrics; DataDistributorInterface() {} explicit DataDistributorInterface(const struct LocalityData& l) : locality(l) {} @@ -48,7 +49,7 @@ struct DataDistributorInterface { template void serialize(Archive& ar) { - serializer(ar, waitFailure, haltDataDistributor, locality, distributorSnapReq, distributorExclCheckReq); + serializer(ar, waitFailure, haltDataDistributor, locality, distributorSnapReq, distributorExclCheckReq, dataDistributorMetrics); } }; @@ -66,6 +67,33 @@ struct HaltDataDistributorRequest { } }; +struct GetDataDistributorMetricsReply { + constexpr static FileIdentifier file_identifier = 1284337; + Standalone> storageMetricsList; + + GetDataDistributorMetricsReply() {} + + template + void serialize(Ar& ar) { + serializer(ar,storageMetricsList); + } +}; + +struct GetDataDistributorMetricsRequest { + constexpr static FileIdentifier file_identifier = 1059267; + KeyRange keys; + int shardLimit; + ReplyPromise reply; + + GetDataDistributorMetricsRequest() {} + explicit GetDataDistributorMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {} + + template + void serialize(Ar& ar) { + serializer(ar, keys, shardLimit, reply); + } +}; + struct DistributorSnapRequest { constexpr static FileIdentifier file_identifier = 22204900; diff --git a/fdbserver/FDBExecHelper.actor.cpp b/fdbserver/FDBExecHelper.actor.cpp index 0aa2332fc2..69d73c24d4 100644 --- a/fdbserver/FDBExecHelper.actor.cpp +++ b/fdbserver/FDBExecHelper.actor.cpp @@ -7,7 +7,7 @@ #include "fdbserver/FDBExecHelper.actor.h" #include "flow/Trace.h" #include "flow/flow.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #include "fdbserver/Knobs.h" #include "flow/actorcompiler.h" // This must be the last #include. diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index f1b1b26034..12fdadf4b3 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -387,7 +387,8 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi init( BACKUP_TIMEOUT, 0.4 ); init( BACKUP_NOOP_POP_DELAY, 5.0 ); init( BACKUP_FILE_BLOCK_BYTES, 1024 * 1024 ); - init( BACKUP_UPLOAD_DELAY, 10.0 ); if( randomize && BUGGIFY ) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 20; // TODO: Increase delay range + init( BACKUP_LOCK_BYTES, 3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 1024; + init( BACKUP_UPLOAD_DELAY, 10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60; //Cluster Controller init( CLUSTER_CONTROLLER_LOGGING_DELAY, 5.0 ); @@ -629,6 +630,13 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi init( REDWOOD_DEFAULT_PAGE_SIZE, 4096 ); init( REDWOOD_KVSTORE_CONCURRENT_READS, 64 ); init( REDWOOD_PAGE_REBUILD_FILL_FACTOR, 0.66 ); + init( REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES, 10 ); + init( REDWOOD_LAZY_CLEAR_MIN_PAGES, 0 ); + init( REDWOOD_LAZY_CLEAR_MAX_PAGES, 1e6 ); + init( REDWOOD_REMAP_CLEANUP_BATCH_SIZE, 5000 ); + init( REDWOOD_REMAP_CLEANUP_VERSION_LAG_MIN, 4 ); + init( REDWOOD_REMAP_CLEANUP_VERSION_LAG_MAX, 15 ); + init( REDWOOD_LOGGING_INTERVAL, 5.0 ); // clang-format on diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h index 0569660c40..e91a03a42b 100644 --- a/fdbserver/Knobs.h +++ b/fdbserver/Knobs.h @@ -179,7 +179,7 @@ public: int64_t DD_SS_FAILURE_VERSIONLAG; // Allowed SS version lag from the current read version before marking it as failed. int64_t DD_SS_ALLOWED_VERSIONLAG; // SS will be marked as healthy if it's version lag goes below this value. double DD_SS_STUCK_TIME_LIMIT; // If a storage server is not getting new versions for this amount of time, then it becomes undesired. - + // TeamRemover to remove redundant teams bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor double TR_REMOVE_MACHINE_TEAM_DELAY; // wait for the specified time before try to remove next machine team @@ -313,6 +313,7 @@ public: double BACKUP_TIMEOUT; // master's reaction time for backup failure double BACKUP_NOOP_POP_DELAY; int BACKUP_FILE_BLOCK_BYTES; + int64_t BACKUP_LOCK_BYTES; double BACKUP_UPLOAD_DELAY; //Cluster Controller @@ -561,6 +562,13 @@ public: int REDWOOD_DEFAULT_PAGE_SIZE; // Page size for new Redwood files int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress. double REDWOOD_PAGE_REBUILD_FILL_FACTOR; // When rebuilding pages, start a new page after this capacity + int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at once + int REDWOOD_LAZY_CLEAR_MIN_PAGES; // Minimum number of pages to free before ending a lazy clear cycle, unless the queue is empty + int REDWOOD_LAZY_CLEAR_MAX_PAGES; // Maximum number of pages to free before ending a lazy clear cycle, unless the queue is empty + int REDWOOD_REMAP_CLEANUP_BATCH_SIZE; // Number of queue entries for remap cleanup to process and potentially coalesce at once. + int REDWOOD_REMAP_CLEANUP_VERSION_LAG_MIN; // Number of versions between head of remap queue and oldest retained version before remap cleanup starts + int REDWOOD_REMAP_CLEANUP_VERSION_LAG_MAX; // Number of versions between head of remap queue and oldest retained version before remap cleanup may stop + double REDWOOD_LOGGING_INTERVAL; ServerKnobs(); void initialize(bool randomize = false, ClientKnobs* clientKnobs = NULL, bool isSimulated = false); diff --git a/fdbserver/MasterInterface.h b/fdbserver/MasterInterface.h index ccf08e4f70..df76a8fcf0 100644 --- a/fdbserver/MasterInterface.h +++ b/fdbserver/MasterInterface.h @@ -33,7 +33,6 @@ typedef uint64_t DBRecoveryCount; struct MasterInterface { constexpr static FileIdentifier file_identifier = 5979145; LocalityData locality; - Endpoint base; RequestStream< ReplyPromise > waitFailure; RequestStream< struct TLogRejoinRequest > tlogRejoin; // sent by tlog (whether or not rebooted) to communicate with a new master RequestStream< struct ChangeCoordinatorsRequest > changeCoordinators; @@ -49,13 +48,12 @@ struct MasterInterface { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, locality, base); + serializer(ar, locality, waitFailure); if( Archive::isDeserializing ) { - waitFailure = RequestStream< ReplyPromise >( base.getAdjustedEndpoint(0) ); - tlogRejoin = RequestStream< struct TLogRejoinRequest >( base.getAdjustedEndpoint(1) ); - changeCoordinators = RequestStream< struct ChangeCoordinatorsRequest >( base.getAdjustedEndpoint(2) ); - getCommitVersion = RequestStream< struct GetCommitVersionRequest >( base.getAdjustedEndpoint(3) ); - notifyBackupWorkerDone = RequestStream( base.getAdjustedEndpoint(4) ); + tlogRejoin = RequestStream< struct TLogRejoinRequest >( waitFailure.getEndpoint().getAdjustedEndpoint(1) ); + changeCoordinators = RequestStream< struct ChangeCoordinatorsRequest >( waitFailure.getEndpoint().getAdjustedEndpoint(2) ); + getCommitVersion = RequestStream< struct GetCommitVersionRequest >( waitFailure.getEndpoint().getAdjustedEndpoint(3) ); + notifyBackupWorkerDone = RequestStream( waitFailure.getEndpoint().getAdjustedEndpoint(4) ); } } @@ -66,7 +64,7 @@ struct MasterInterface { streams.push_back(changeCoordinators.getReceiver()); streams.push_back(getCommitVersion.getReceiver(TaskPriority::GetConsistentReadVersion)); streams.push_back(notifyBackupWorkerDone.getReceiver()); - base = FlowTransport::transport().addEndpoints(streams); + FlowTransport::transport().addEndpoints(streams); } }; diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index fc70d975fe..af91c46105 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -1756,6 +1756,25 @@ ACTOR Future healthMetricsRequestServer(MasterProxyInterface proxy, GetHea } } +ACTOR Future ddMetricsRequestServer(MasterProxyInterface proxy, Reference> db) +{ + loop { + choose { + when(state GetDDMetricsRequest req = waitNext(proxy.getDDMetrics.getFuture())) + { + ErrorOr reply = wait(errorOr(db->get().distributor.get().dataDistributorMetrics.getReply(GetDataDistributorMetricsRequest(req.keys, req.shardLimit)))); + if ( reply.isError() ) { + req.reply.sendError(reply.getError()); + } else { + GetDDMetricsReply newReply; + newReply.storageMetricsList = reply.get().storageMetricsList; + req.reply.send(newReply); + } + } + } + } +} + ACTOR Future monitorRemoteCommitted(ProxyCommitData* self) { loop { wait(delay(0)); //allow this actor to be cancelled if we are removed after db changes. @@ -1996,6 +2015,7 @@ ACTOR Future masterProxyServerCore( addActor.send(readRequestServer(proxy, addActor, &commitData)); addActor.send(rejoinServer(proxy, &commitData)); addActor.send(healthMetricsRequestServer(proxy, &healthMetricsReply, &detailedHealthMetricsReply)); + addActor.send(ddMetricsRequestServer(proxy, db)); // wait for txnStateStore recovery wait(success(commitData.txnStateStore->readValue(StringRef()))); diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index 6f3ce7c2d7..b65fe21f52 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -782,7 +782,7 @@ ACTOR Future monitorThrottlingChanges(RatekeeperData *self) { TransactionTag tag = *tagKey.tags.begin(); Optional oldLimits = self->throttledTags.getManualTagThrottleLimits(tag, tagKey.priority); - if(tagKey.autoThrottled) { + if(tagKey.throttleType == TagThrottleType::AUTO) { updatedTagThrottles.autoThrottleTag(self->id, tag, 0, tagValue.tpsRate, tagValue.expirationTime); } else { @@ -819,7 +819,7 @@ void tryAutoThrottleTag(RatekeeperData *self, StorageQueueInfo const& ss, RkTagT TagSet tags; tags.addTag(ss.busiestTag.get()); - self->addActor.send(ThrottleApi::throttleTags(self->db, tags, clientRate.get(), SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION, true, TransactionPriority::DEFAULT, now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION)); + self->addActor.send(ThrottleApi::throttleTags(self->db, tags, clientRate.get(), SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION, TagThrottleType::AUTO, TransactionPriority::DEFAULT, now() + SERVER_KNOBS->AUTO_TAG_THROTTLE_DURATION)); } } } diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index 570d46d4bd..ccc1dfa9f3 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -217,6 +217,9 @@ ACTOR static Future _parsePartitionedLogFileOnLoader( VersionedMutationsMap::iterator it; bool inserted; std::tie(it, inserted) = kvOps.emplace(msgVersion, MutationsVec()); + // A clear mutation can be split into multiple mutations with the same (version, sub). + // See saveMutationsToFile(). Current tests only use one key range per backup, thus + // only one clear mutation is generated (i.e., always inserted). ASSERT(inserted); ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion)); diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 0a5ad07a37..ed381389c6 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -31,7 +31,7 @@ #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/BackupAgent.actor.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #include "flow/actorcompiler.h" // This must be the last #include. #undef max diff --git a/fdbserver/SkipList.cpp b/fdbserver/SkipList.cpp index 0cb3636aed..8cf6a5977e 100644 --- a/fdbserver/SkipList.cpp +++ b/fdbserver/SkipList.cpp @@ -728,7 +728,7 @@ StringRef setK(Arena& arena, int i) { #include "fdbserver/ConflictSet.h" struct ConflictSet { - ConflictSet() : oldestVersion(0) {} + ConflictSet() : oldestVersion(0), removalKey(makeString(0)) {} ~ConflictSet() {} SkipList versionHistory; diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index e1db7b9512..2e7f17b21c 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -377,9 +377,9 @@ JsonBuilderObject getLagObject(int64_t versions) { struct MachineMemoryInfo { double memoryUsage; - double numProcesses; + double aggregateLimit; - MachineMemoryInfo() : memoryUsage(0), numProcesses(0) {} + MachineMemoryInfo() : memoryUsage(0), aggregateLimit(0) {} bool valid() { return memoryUsage >= 0; } void invalidate() { memoryUsage = -1; } @@ -613,11 +613,12 @@ ACTOR static Future processStatusFetcher( try { ASSERT(pMetrics.count(workerItr->interf.address())); const TraceEventFields& processMetrics = pMetrics[workerItr->interf.address()]; + const TraceEventFields& programStart = programStarts[workerItr->interf.address()]; if(memInfo->second.valid()) { - if(processMetrics.size() > 0) { + if(processMetrics.size() > 0 && programStart.size() > 0) { memInfo->second.memoryUsage += processMetrics.getDouble("Memory"); - ++memInfo->second.numProcesses; + memInfo->second.aggregateLimit += programStart.getDouble("MemoryLimit"); } else memInfo->second.invalidate(); @@ -789,19 +790,21 @@ ACTOR static Future processStatusFetcher( memoryObj.setKeyRawNumber("unused_allocated_memory", processMetrics.getValue("UnusedAllocatedMemory")); } + int64_t memoryLimit = 0; if (programStarts.count(address)) { - auto const& psxml = programStarts.at(address); + auto const& programStartEvent = programStarts.at(address); - if(psxml.size() > 0) { - memoryObj.setKeyRawNumber("limit_bytes",psxml.getValue("MemoryLimit")); + if(programStartEvent.size() > 0) { + memoryLimit = programStartEvent.getInt64("MemoryLimit"); + memoryObj.setKey("limit_bytes", memoryLimit); std::string version; - if (psxml.tryGetValue("Version", version)) { + if (programStartEvent.tryGetValue("Version", version)) { statusObj["version"] = version; } std::string commandLine; - if (psxml.tryGetValue("CommandLine", commandLine)) { + if (programStartEvent.tryGetValue("CommandLine", commandLine)) { statusObj["command_line"] = commandLine; } } @@ -813,10 +816,10 @@ ACTOR static Future processStatusFetcher( availableMemory = mMetrics[address].getDouble("AvailableMemory"); auto machineMemInfo = machineMemoryUsage[workerItr->interf.locality.machineId()]; - if (machineMemInfo.valid()) { - ASSERT(machineMemInfo.numProcesses > 0); - int64_t memory = (availableMemory + machineMemInfo.memoryUsage) / machineMemInfo.numProcesses; - memoryObj["available_bytes"] = std::max(memory, 0); + if (machineMemInfo.valid() && memoryLimit > 0) { + ASSERT(machineMemInfo.aggregateLimit > 0); + int64_t memory = (availableMemory + machineMemInfo.memoryUsage) * memoryLimit / machineMemInfo.aggregateLimit; + memoryObj["available_bytes"] = std::min(std::max(memory, 0), memoryLimit); } } @@ -1725,10 +1728,6 @@ ACTOR static Future workloadStatusFetcher(Reference peekMessages; RequestStream< struct TLogPopRequest > popMessages; @@ -75,7 +74,7 @@ struct TLogInterface { streams.push_back(disablePopRequest.getReceiver()); streams.push_back(enablePopRequest.getReceiver()); streams.push_back(snapRequest.getReceiver()); - base = FlowTransport::transport().addEndpoints(streams); + FlowTransport::transport().addEndpoints(streams); } template @@ -83,19 +82,18 @@ struct TLogInterface { if constexpr (!is_fb_function) { ASSERT(ar.isDeserializing || uniqueID != UID()); } - serializer(ar, uniqueID, sharedTLogID, filteredLocality, base); + serializer(ar, uniqueID, sharedTLogID, filteredLocality, peekMessages); if( Ar::isDeserializing ) { - peekMessages = RequestStream< struct TLogPeekRequest >( base.getAdjustedEndpoint(0) ); - popMessages = RequestStream< struct TLogPopRequest >( base.getAdjustedEndpoint(1) ); - commit = RequestStream< struct TLogCommitRequest >( base.getAdjustedEndpoint(2) ); - lock = RequestStream< ReplyPromise< struct TLogLockResult > >( base.getAdjustedEndpoint(3) ); - getQueuingMetrics = RequestStream< struct TLogQueuingMetricsRequest >( base.getAdjustedEndpoint(4) ); - confirmRunning = RequestStream< struct TLogConfirmRunningRequest >( base.getAdjustedEndpoint(5) ); - waitFailure = RequestStream< ReplyPromise >( base.getAdjustedEndpoint(6) ); - recoveryFinished = RequestStream< struct TLogRecoveryFinishedRequest >( base.getAdjustedEndpoint(7) ); - disablePopRequest = RequestStream< struct TLogDisablePopRequest >( base.getAdjustedEndpoint(8) ); - enablePopRequest = RequestStream< struct TLogEnablePopRequest >( base.getAdjustedEndpoint(9) ); - snapRequest = RequestStream< struct TLogSnapRequest >( base.getAdjustedEndpoint(10) ); + popMessages = RequestStream< struct TLogPopRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(1) ); + commit = RequestStream< struct TLogCommitRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(2) ); + lock = RequestStream< ReplyPromise< struct TLogLockResult > >( peekMessages.getEndpoint().getAdjustedEndpoint(3) ); + getQueuingMetrics = RequestStream< struct TLogQueuingMetricsRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(4) ); + confirmRunning = RequestStream< struct TLogConfirmRunningRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(5) ); + waitFailure = RequestStream< ReplyPromise >( peekMessages.getEndpoint().getAdjustedEndpoint(6) ); + recoveryFinished = RequestStream< struct TLogRecoveryFinishedRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(7) ); + disablePopRequest = RequestStream< struct TLogDisablePopRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(8) ); + enablePopRequest = RequestStream< struct TLogEnablePopRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(9) ); + snapRequest = RequestStream< struct TLogSnapRequest >( peekMessages.getEndpoint().getAdjustedEndpoint(10) ); } } }; diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index ded6da82f2..2959c06b2a 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -542,6 +542,16 @@ public: Future>> peekAll() { return peekAll_impl(this); } + ACTOR static Future> peek_impl(FIFOQueue* self) { + state Cursor c; + c.initReadOnly(self->headReader); + + Optional x = wait(c.readNext()); + return x; + } + + Future> peek() { return peek_impl(this); } + // Pop the next item on front of queue if it is <= upperBound or if upperBound is not present Future> pop(Optional upperBound = {}) { return headReader.readNext(upperBound); } @@ -730,6 +740,201 @@ private: uint8_t* buffer; }; +struct RedwoodMetrics { + static constexpr int btreeLevels = 5; + + RedwoodMetrics() { clear(); } + + void clear() { + memset(this, 0, sizeof(RedwoodMetrics)); + for (auto& level : levels) { + level = {}; + } + startTime = g_network ? now() : 0; + } + + struct Level { + unsigned int pageRead; + unsigned int pageReadExt; + unsigned int pageBuild; + unsigned int pageBuildExt; + unsigned int pageCommitStart; + unsigned int pageModify; + unsigned int pageModifyExt; + unsigned int lazyClearRequeue; + unsigned int lazyClearRequeueExt; + unsigned int lazyClearFree; + unsigned int lazyClearFreeExt; + double buildStoredPct; + double buildFillPct; + unsigned int buildItemCount; + double modifyStoredPct; + double modifyFillPct; + unsigned int modifyItemCount; + }; + + Level levels[btreeLevels]; + + unsigned int opSet; + unsigned int opSetKeyBytes; + unsigned int opSetValueBytes; + unsigned int opClear; + unsigned int opClearKey; + unsigned int opCommit; + unsigned int opGet; + unsigned int opGetRange; + unsigned int pagerDiskWrite; + unsigned int pagerDiskRead; + unsigned int pagerRemapFree; + unsigned int pagerRemapCopy; + unsigned int pagerRemapSkip; + unsigned int pagerCacheHit; + unsigned int pagerCacheMiss; + unsigned int pagerProbeHit; + unsigned int pagerProbeMiss; + unsigned int pagerEvictUnhit; + unsigned int pagerEvictFail; + unsigned int btreeLeafPreload; + unsigned int btreeLeafPreloadExt; + + double startTime; + + Level& level(unsigned int level) { + static Level outOfBound; + if (level == 0 || level > btreeLevels) { + return outOfBound; + } + return levels[level - 1]; + } + + // This will populate a trace event and/or a string with Redwood metrics. The string is a + // reasonably well formatted page of information + void getFields(TraceEvent* e, std::string* s = nullptr) { + std::pair metrics[] = { { "BTreePreload", btreeLeafPreload }, + { "BTreePreloadExt", btreeLeafPreloadExt }, + { "", 0 }, + { "OpSet", opSet }, + { "OpSetKeyBytes", opSetKeyBytes }, + { "OpSetValueBytes", opSetValueBytes }, + { "OpClear", opClear }, + { "OpClearKey", opClearKey }, + { "", 0 }, + { "OpGet", opGet }, + { "OpGetRange", opGetRange }, + { "OpCommit", opCommit }, + { "", 0 }, + { "PagerDiskWrite", pagerDiskWrite }, + { "PagerDiskRead", pagerDiskRead }, + { "PagerCacheHit", pagerCacheHit }, + { "PagerCacheMiss", pagerCacheMiss }, + { "", 0 }, + { "PagerProbeHit", pagerProbeHit }, + { "PagerProbeMiss", pagerProbeMiss }, + { "PagerEvictUnhit", pagerEvictUnhit }, + { "PagerEvictFail", pagerEvictFail }, + { "", 0 }, + { "PagerRemapFree", pagerRemapFree }, + { "PagerRemapCopy", pagerRemapCopy }, + { "PagerRemapSkip", pagerRemapSkip } }; + double elapsed = now() - startTime; + for (auto& m : metrics) { + if (*m.first == '\0') { + if (s != nullptr) { + *s += "\n"; + } + } else { + if (s != nullptr) { + *s += format("%-15s %-8u %8u/s ", m.first, m.second, int(m.second / elapsed)); + } + if (e != nullptr) { + e->detail(m.first, m.second); + } + } + } + + for (int i = 0; i < btreeLevels; ++i) { + auto& level = levels[i]; + std::pair metrics[] = { + { "PageBuild", level.pageBuild }, + { "PageBuildExt", level.pageBuildExt }, + { "PageModify", level.pageModify }, + { "PageModifyExt", level.pageModifyExt }, + { "", 0 }, + { "PageRead", level.pageRead }, + { "PageReadExt", level.pageReadExt }, + { "PageCommitStart", level.pageCommitStart }, + { "", 0 }, + { "LazyClearInt", level.lazyClearRequeue }, + { "LazyClearIntExt", level.lazyClearRequeueExt }, + { "LazyClear", level.lazyClearFree }, + { "LazyClearExt", level.lazyClearFreeExt }, + { "", 0 }, + { "-BldAvgCount", level.pageBuild ? level.buildItemCount / level.pageBuild : 0 }, + { "-BldAvgFillPct", level.pageBuild ? level.buildFillPct / level.pageBuild * 100 : 0 }, + { "-BldAvgStoredPct", level.pageBuild ? level.buildStoredPct / level.pageBuild * 100 : 0 }, + { "", 0 }, + { "-ModAvgCount", level.pageModify ? level.modifyItemCount / level.pageModify : 0 }, + { "-ModAvgFillPct", level.pageModify ? level.modifyFillPct / level.pageModify * 100 : 0 }, + { "-ModAvgStoredPct", level.pageModify ? level.modifyStoredPct / level.pageModify * 100 : 0 } + }; + + if (s != nullptr) { + *s += format("\nLevel %d\n\t", i + 1); + } + for (auto& m : metrics) { + const char* name = m.first; + bool rate = elapsed != 0; + if (*name == '-') { + ++name; + rate = false; + } + + if (*name == '\0') { + if (s != nullptr) { + *s += "\n\t"; + } + } else { + if (s != nullptr) { + *s += format("%-15s %8u %8u/s ", name, m.second, rate ? int(m.second / elapsed) : 0); + } + if (e != nullptr) { + e->detail(format("L%d%s", i + 1, name), m.second); + } + } + } + } + } + + std::string toString(bool clearAfter) { + std::string s; + getFields(nullptr, &s); + + if (clearAfter) { + clear(); + } + + return s; + } +}; + +// Using a global for Redwood metrics because a single process shouldn't normally have multiple storage engines +RedwoodMetrics g_redwoodMetrics = {}; +Future g_redwoodMetricsActor; + +ACTOR Future redwoodMetricsLogger() { + g_redwoodMetrics.clear(); + + loop { + wait(delay(SERVER_KNOBS->REDWOOD_LOGGING_INTERVAL)); + + TraceEvent e("RedwoodMetrics"); + double elapsed = now() - g_redwoodMetrics.startTime; + e.detail("Elapsed", elapsed); + g_redwoodMetrics.getFields(&e); + g_redwoodMetrics.clear(); + } +} + // Holds an index of recently used objects. // ObjectType must have the methods // bool evictable() const; // return true if the entry can be evicted @@ -748,8 +953,7 @@ class ObjectCache : NonCopyable { typedef boost::intrusive::list EvictionOrderT; public: - ObjectCache(int sizeLimit = 1) - : sizeLimit(sizeLimit), cacheHits(0), cacheMisses(0), noHitEvictions(0), failedEvictions(0) {} + ObjectCache(int sizeLimit = 1) : sizeLimit(sizeLimit) {} void setSizeLimit(int n) { ASSERT(n > 0); @@ -762,52 +966,64 @@ public: auto i = cache.find(index); if (i != cache.end()) { ++i->second.hits; + ++g_redwoodMetrics.pagerProbeHit; return &i->second.item; } + ++g_redwoodMetrics.pagerProbeMiss; return nullptr; } // Get the object for i or create a new one. // After a get(), the object for i is the last in evictionOrder. - ObjectType& get(const IndexType& index, bool noHit = false) { + // If noHit is set, do not consider this access to be cache hit if the object is present + // If noMiss is set, do not consider this access to be a cache miss if the object is not present + ObjectType& get(const IndexType& index, bool noHit = false, bool noMiss = false) { Entry& entry = cache[index]; // If entry is linked into evictionOrder then move it to the back of the order if (entry.is_linked()) { if (!noHit) { ++entry.hits; - ++cacheHits; + ++g_redwoodMetrics.pagerCacheHit; + + // Move the entry to the back of the eviction order + evictionOrder.erase(evictionOrder.iterator_to(entry)); + evictionOrder.push_back(entry); } - // Move the entry to the back of the eviction order - evictionOrder.erase(evictionOrder.iterator_to(entry)); - evictionOrder.push_back(entry); } else { - ++cacheMisses; + if (!noMiss) { + ++g_redwoodMetrics.pagerCacheMiss; + } // Finish initializing entry entry.index = index; - entry.hits = noHit ? 0 : 1; + entry.hits = 0; // Insert the newly created Entry at the back of the eviction order evictionOrder.push_back(entry); // While the cache is too big, evict the oldest entry until the oldest entry can't be evicted. while (cache.size() > sizeLimit) { Entry& toEvict = evictionOrder.front(); + + // It's critical that we do not evict the item we just added because it would cause the reference + // returned to be invalid. An eviction could happen with a no-hit access to a cache resident page + // that is currently evictable and exists in the oversized portion of the cache eviction order due + // to previously failed evictions. + if (&entry == &toEvict) { + debug_printf("Cannot evict target index %s\n", toString(index).c_str()); + break; + } + debug_printf("Trying to evict %s to make room for %s\n", toString(toEvict.index).c_str(), toString(index).c_str()); - // It's critical that we do not evict the item we just added (or the reference we return would be - // invalid) but since sizeLimit must be > 0, entry was just added to the end of the evictionOrder, and - // this loop will end if we move anything to the end of the eviction order, we can be guaraunted that - // entry != toEvict, so we do not need to check. If the item is not evictable then move it to the back - // of the eviction order and stop. if (!toEvict.item.evictable()) { evictionOrder.erase(evictionOrder.iterator_to(toEvict)); evictionOrder.push_back(toEvict); - ++failedEvictions; + ++g_redwoodMetrics.pagerEvictFail; break; } else { if (toEvict.hits == 0) { - ++noHitEvictions; + ++g_redwoodMetrics.pagerEvictUnhit; } debug_printf("Evicting %s to make room for %s\n", toString(toEvict.index).c_str(), toString(index).c_str()); @@ -858,10 +1074,6 @@ public: private: int64_t sizeLimit; - int64_t cacheHits; - int64_t cacheMisses; - int64_t noHitEvictions; - int64_t failedEvictions; CacheT cache; EvictionOrderT evictionOrder; @@ -910,6 +1122,9 @@ public: }; struct RemappedPage { + RemappedPage() : version(invalidVersion) {} + RemappedPage(Version v, LogicalPageID o, LogicalPageID n) : version(v), originalPageID(o), newPageID(n) {} + Version version; LogicalPageID originalPageID; LogicalPageID newPageID; @@ -933,6 +1148,11 @@ public: DWALPager(int desiredPageSize, std::string filename, int64_t pageCacheSizeBytes, bool memoryOnly = false) : desiredPageSize(desiredPageSize), filename(filename), pHeader(nullptr), pageCacheBytes(pageCacheSizeBytes), memoryOnly(memoryOnly) { + + if (!g_redwoodMetricsActor.isValid()) { + g_redwoodMetricsActor = redwoodMetricsLogger(); + } + if (pageCacheBytes == 0) { pageCacheBytes = g_network->isSimulated() ? (BUGGIFY ? FLOW_KNOBS->BUGGIFY_SIM_PAGE_CACHE_4K : FLOW_KNOBS->SIM_PAGE_CACHE_4K) @@ -961,7 +1181,6 @@ public: ACTOR static Future recover(DWALPager* self) { ASSERT(!self->recoverFuture.isValid()); - self->remapUndoFuture = Void(); state bool exists = false; if (!self->memoryOnly) { @@ -1067,6 +1286,7 @@ public: // header) self->updateCommittedHeader(); self->addLatestSnapshot(); + self->remapCleanupFuture = remapCleanup(self); } else { // Note: If the file contains less than 2 pages but more than 0 bytes then the pager was never successfully // committed. A new pager will be created in its place. @@ -1111,6 +1331,7 @@ public: // Since there is no previously committed header use the initial header for the initial commit. self->updateCommittedHeader(); + self->remapCleanupFuture = Void(); wait(self->commit()); } @@ -1170,6 +1391,7 @@ public: debug_printf("DWALPager(%s) op=%s %s ptr=%p\n", filename.c_str(), (header ? "writePhysicalHeader" : "writePhysical"), toString(pageID).c_str(), page->begin()); + ++g_redwoodMetrics.pagerDiskWrite; VALGRIND_MAKE_MEM_DEFINED(page->begin(), page->size()); ((Page*)page.getPtr())->updateChecksum(pageID); @@ -1196,7 +1418,8 @@ public: void updatePage(LogicalPageID pageID, Reference data) override { // Get the cache entry for this page, without counting it as a cache hit as we're replacing its contents now - PageCacheEntry& cacheEntry = pageCache.get(pageID, true); + // or as a cache miss because there is no benefit to the page already being in cache + PageCacheEntry& cacheEntry = pageCache.get(pageID, true, true); debug_printf("DWALPager(%s) op=write %s cached=%d reading=%d writing=%d\n", filename.c_str(), toString(pageID).c_str(), cacheEntry.initialized(), cacheEntry.initialized() && cacheEntry.reading(), @@ -1234,7 +1457,6 @@ public: Future atomicUpdatePage(LogicalPageID pageID, Reference data, Version v) override { debug_printf("DWALPager(%s) op=writeAtomic %s @%" PRId64 "\n", filename.c_str(), toString(pageID).c_str(), v); - // This pager does not support atomic update, so it always allocates and uses a new pageID Future f = map(newPageID(), [=](LogicalPageID newPageID) { updatePage(newPageID, data); // TODO: Possibly limit size of remap queue since it must be recovered on cold start @@ -1249,16 +1471,7 @@ public: return f; } - void freePage(LogicalPageID pageID, Version v) override { - // If pageID has been remapped, then it can't be freed until all existing remaps for that page have been undone, - // so queue it for later deletion - if (remappedPages.find(pageID) != remappedPages.end()) { - debug_printf("DWALPager(%s) op=freeRemapped %s @%" PRId64 " oldestVersion=%" PRId64 "\n", filename.c_str(), - toString(pageID).c_str(), v, pLastCommittedHeader->oldestVersion); - remapQueue.pushBack(RemappedPage{ v, pageID, invalidLogicalPageID }); - return; - } - + void freeUnmappedPage(LogicalPageID pageID, Version v) { // If v is older than the oldest version still readable then mark pageID as free as of the next commit if (v < effectiveOldestVersion()) { debug_printf("DWALPager(%s) op=freeNow %s @%" PRId64 " oldestVersion=%" PRId64 "\n", filename.c_str(), @@ -1270,6 +1483,19 @@ public: toString(pageID).c_str(), v, pLastCommittedHeader->oldestVersion); delayedFreeList.pushBack({ v, pageID }); } + } + + void freePage(LogicalPageID pageID, Version v) override { + // If pageID has been remapped, then it can't be freed until all existing remaps for that page have been undone, + // so queue it for later deletion + if (remappedPages.find(pageID) != remappedPages.end()) { + debug_printf("DWALPager(%s) op=freeRemapped %s @%" PRId64 " oldestVersion=%" PRId64 "\n", filename.c_str(), + toString(pageID).c_str(), v, pLastCommittedHeader->oldestVersion); + remapQueue.pushBack(RemappedPage{ v, pageID, invalidLogicalPageID }); + return; + } + + freeUnmappedPage(pageID, v); }; // Read a physical page from the page file. Note that header pages use a page size of smallestPhysicalBlock @@ -1278,6 +1504,7 @@ public: ACTOR static Future> readPhysicalPage(DWALPager* self, PhysicalPageID pageID, bool header = false) { ASSERT(!self->memoryOnly); + ++g_redwoodMetrics.pagerDiskRead; if (g_network->getCurrentTask() > TaskPriority::DiskRead) { wait(delay(0, TaskPriority::DiskRead)); @@ -1320,7 +1547,8 @@ public: return readPhysicalPage(self, pageID, true); } - // Reads the most recent version of pageID either committed or written using updatePage() + // Reads the most recent version of pageID, either previously committed or written using updatePage() in the current + // commit Future> readPage(LogicalPageID pageID, bool cacheable, bool noHit = false) override { // Use cached page if present, without triggering a cache hit. // Otherwise, read the page and return it but don't add it to the cache @@ -1393,56 +1621,134 @@ public: return std::min(pLastCommittedHeader->oldestVersion, snapshots.front().version); } - ACTOR static Future undoRemaps(DWALPager* self) { + ACTOR static Future remapCopyAndFree(DWALPager* self, RemappedPage m) { + debug_printf("DWALPager(%s) remapCleanup copyAndFree %s\n", self->filename.c_str(), m.toString().c_str()); + + // Read the data from the page that the original was mapped to + Reference data = wait(self->readPage(m.newPageID, false)); + + // Write the data to the original page so it can be read using its original pageID + self->updatePage(m.originalPageID, data); + ++g_redwoodMetrics.pagerRemapCopy; + + // Remove all remaps for the original page ID up through version + auto i = self->remappedPages.find(m.originalPageID); + i->second.erase(i->second.begin(), i->second.upper_bound(m.version)); + // If the version map for this page is now empty, erase it + if (i->second.empty()) { + self->remappedPages.erase(i); + } + + // Now that the remap has been undone nothing will read this page so it can be freed as of the next + // commit. + self->freeUnmappedPage(m.newPageID, 0); + ++g_redwoodMetrics.pagerRemapFree; + + return Void(); + } + + ACTOR static Future getRemapLag(DWALPager* self) { + Optional head = wait(self->remapQueue.peek()); + if (head.present()) { + return self->effectiveOldestVersion() - head.get().version; + } + return 0; + } + + ACTOR static Future remapCleanup(DWALPager* self) { + self->remapCleanupStop = false; + + // Cutoff is the version we can pop to state RemappedPage cutoff; cutoff.version = self->effectiveOldestVersion(); - // TODO: Use parallel reads - // TODO: One run of this actor might write to the same original page more than once, in which case just unmap - // the latest + // Each page is only updated at most once per version, so in order to coalesce multiple updates + // to the same page and skip some page writes we have to accumulate multiple versions worth of + // poppable entries. + Version lag = wait(getRemapLag(self)); + debug_printf("DWALPager(%s) remapCleanup versionLag=%" PRId64 "\n", self->filename.c_str(), lag); + if (lag < SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_VERSION_LAG_MIN) { + debug_printf("DWALPager(%s) not starting, lag too low\n", self->filename.c_str()); + return Void(); + } + loop { - if (self->remapUndoStop) { - break; - } - state Optional p = wait(self->remapQueue.pop(cutoff)); - if (!p.present()) { - break; - } - debug_printf("DWALPager(%s) undoRemaps popped %s\n", self->filename.c_str(), p.get().toString().c_str()); + // Pop up to the pop size limit from the queue, but only keep the latest remap queue entry per + // original page ID. This will coalesce multiple remaps of the same LogicalPageID within the + // interval of pages being unmapped to a single page copy. + state int toPop = SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_BATCH_SIZE; + state std::unordered_map toCopy; + toCopy.reserve(toPop); - if (p.get().newPageID == invalidLogicalPageID) { - debug_printf("DWALPager(%s) undoRemaps freeing %s\n", self->filename.c_str(), - p.get().toString().c_str()); - self->freePage(p.get().originalPageID, p.get().version); - } else { - // Read the data from the page that the original was mapped to - Reference data = wait(self->readPage(p.get().newPageID, false)); - - // Write the data to the original page so it can be read using its original pageID - self->updatePage(p.get().originalPageID, data); - - // Remove the remap from this page, deleting the entry for the pageID if its map becomes empty - auto i = self->remappedPages.find(p.get().originalPageID); - if (i->second.size() == 1) { - self->remappedPages.erase(i); - } else { - i->second.erase(p.get().version); + // Take up to batch size pages from front of queue + while (toPop > 0) { + state Optional p = wait(self->remapQueue.pop(cutoff)); + debug_printf("DWALPager(%s) remapCleanup popped %s\n", self->filename.c_str(), ::toString(p).c_str()); + if (!p.present()) { + break; } - // Now that the remap has been undone nothing will read this page so it can be freed as of the next - // commit. - self->freePage(p.get().newPageID, 0); + // Get the existing remap entry for the original page, which could be newly initialized + auto& m = toCopy[p.get().originalPageID]; + // If version is invalid then this is a newly constructed RemappedPage, so copy p.get() over it + if (m.version != invalidVersion) { + ASSERT(m.version < p.get().version); + ASSERT(m.newPageID != invalidLogicalPageID); + // We're replacing a previously popped item so we can avoid copying it over the original. + debug_printf("DWALPager(%s) remapCleanup elided %s\n", self->filename.c_str(), + m.toString().c_str()); + // The remapped pages entries will be cleaned up below. + self->freeUnmappedPage(m.newPageID, 0); + ++g_redwoodMetrics.pagerRemapFree; + ++g_redwoodMetrics.pagerRemapSkip; + } + m = p.get(); + + --toPop; + } + + std::vector> copies; + + for (auto& e : toCopy) { + const RemappedPage& m = e.second; + // If newPageID is invalid, originalPageID page was freed at version, not remapped + if (m.newPageID == invalidLogicalPageID) { + debug_printf("DWALPager(%s) remapCleanup freeNoCopy %s\n", self->filename.c_str(), + m.toString().c_str()); + self->remappedPages.erase(m.originalPageID); + self->freeUnmappedPage(m.originalPageID, 0); + ++g_redwoodMetrics.pagerRemapFree; + } else { + copies.push_back(remapCopyAndFree(self, m)); + } + } + + wait(waitForAll(copies)); + + // Stop if there was nothing more that could be popped + if (toPop > 0) { + break; + } + + // If the stop flag is set then stop but only if the remap lag is below the maximum allowed + if (self->remapCleanupStop) { + Version lag = wait(getRemapLag(self)); + if (lag <= SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_VERSION_LAG_MAX) { + break; + } else { + debug_printf("DWALPager(%s) remapCleanup refusing to stop, versionLag=%" PRId64 "\n", + self->filename.c_str(), lag); + } } } - debug_printf("DWALPager(%s) undoRemaps stopped, remapQueue size is %d\n", self->filename.c_str(), - self->remapQueue.numEntries); + debug_printf("DWALPager(%s) remapCleanup stopped (stop=%d)\n", self->filename.c_str(), self->remapCleanupStop); return Void(); } // Flush all queues so they have no operations pending. ACTOR static Future flushQueues(DWALPager* self) { - ASSERT(self->remapUndoFuture.isReady()); + ASSERT(self->remapCleanupFuture.isReady()); // Flush remap queue separately, it's not involved in free page management wait(self->remapQueue.flush()); @@ -1472,8 +1778,8 @@ public: self->writeHeaderPage(1, self->lastCommittedHeaderPage); // Trigger the remap eraser to stop and then wait for it. - self->remapUndoStop = true; - wait(self->remapUndoFuture); + self->remapCleanupStop = true; + wait(self->remapCleanupFuture); wait(flushQueues(self)); @@ -1518,8 +1824,7 @@ public: self->expireSnapshots(self->pHeader->oldestVersion); // Start unmapping pages for expired versions - self->remapUndoStop = false; - self->remapUndoFuture = undoRemaps(self); + self->remapCleanupFuture = remapCleanup(self); return Void(); } @@ -1543,7 +1848,7 @@ public: debug_printf("DWALPager(%s) shutdown cancel commit\n", self->filename.c_str()); self->commitFuture.cancel(); debug_printf("DWALPager(%s) shutdown cancel remap\n", self->filename.c_str()); - self->remapUndoFuture.cancel(); + self->remapCleanupFuture.cancel(); if (self->errorPromise.canBeSet()) { debug_printf("DWALPager(%s) shutdown sending error\n", self->filename.c_str()); @@ -1601,7 +1906,7 @@ public: ACTOR static Future getUserPageCount_cleanup(DWALPager* self) { // Wait for the remap eraser to finish all of its work (not triggering stop) - wait(self->remapUndoFuture); + wait(self->remapCleanupFuture); // Flush queues so there are no pending freelist operations wait(flushQueues(self)); @@ -1712,8 +2017,8 @@ private: Future commitFuture; SignalableActorCollection operations; Future recoverFuture; - Future remapUndoFuture; - bool remapUndoStop; + Future remapCleanupFuture; + bool remapCleanupStop; Reference pageFile; @@ -1984,6 +2289,7 @@ struct RedwoodRecordRef { Version version; int expectedSize() const { return key.expectedSize() + value.expectedSize(); } + int kvBytes() const { return expectedSize(); } class Reader { public: @@ -2433,8 +2739,8 @@ struct BTreePage { #pragma pack(pop) int size() const { - const BinaryTree* t = &tree(); - return (uint8_t*)t - (uint8_t*)this + t->size(); + auto& t = tree(); + return (uint8_t*)&t - (uint8_t*)this + t.size(); } bool isLeaf() const { return height == 1; } @@ -2557,11 +2863,11 @@ public: // A record which is greater than the last possible record in the tree static RedwoodRecordRef dbEnd; - struct LazyDeleteQueueEntry { + struct LazyClearQueueEntry { Version version; Standalone pageID; - bool operator<(const LazyDeleteQueueEntry& rhs) const { return version < rhs.version; } + bool operator<(const LazyClearQueueEntry& rhs) const { return version < rhs.version; } int readFromBytes(const uint8_t* src) { version = *(Version*)src; @@ -2584,15 +2890,15 @@ public: std::string toString() const { return format("{%s @%" PRId64 "}", ::toString(pageID).c_str(), version); } }; - typedef FIFOQueue LazyDeleteQueueT; + typedef FIFOQueue LazyClearQueueT; #pragma pack(push, 1) struct MetaKey { - static constexpr int FORMAT_VERSION = 7; + static constexpr int FORMAT_VERSION = 8; // This serves as the format version for the entire tree, individual pages will not be versioned uint16_t formatVersion; uint8_t height; - LazyDeleteQueueT::QueueState lazyDeleteQueue; + LazyClearQueueT::QueueState lazyDeleteQueue; InPlaceArray root; KeyRef asKeyRef() const { return KeyRef((uint8_t*)this, sizeof(MetaKey) + root.extraSize()); } @@ -2609,68 +2915,6 @@ public: }; #pragma pack(pop) - struct Counts { - Counts() { - memset(this, 0, sizeof(Counts)); - startTime = g_network ? now() : 0; - } - - void clear() { *this = Counts(); } - - int64_t pageReads; - int64_t extPageReads; - int64_t pagePreloads; - int64_t extPagePreloads; - int64_t setBytes; - int64_t pageWrites; - int64_t extPageWrites; - int64_t sets; - int64_t clears; - int64_t clearSingleKey; - int64_t commits; - int64_t gets; - int64_t getRanges; - int64_t commitSubtreeStart; - int64_t pageUpdates; - double startTime; - - std::string toString(bool clearAfter = false) { - const char* labels[] = { "set", - "clear", - "clearSingleKey", - "get", - "getRange", - "commit", - "pageReads", - "extPageRead", - "pagePreloads", - "extPagePreloads", - "pageWrites", - "pageUpdates", - "extPageWrites", - "commitSubtreeStart" }; - const int64_t values[] = { - sets, clears, clearSingleKey, gets, getRanges, commits, pageReads, - extPageReads, pagePreloads, extPagePreloads, pageWrites, pageUpdates, extPageWrites, commitSubtreeStart - }; - - double elapsed = now() - startTime; - std::string s; - for (int i = 0; i < sizeof(values) / sizeof(int64_t); ++i) { - s += format("%s=%" PRId64 " (%d/s) ", labels[i], values[i], int(values[i] / elapsed)); - } - - if (clearAfter) { - clear(); - } - - return s; - } - }; - - // Using a static for metrics because a single process shouldn't normally have multiple storage engines - static Counts counts; - // All async opts on the btree are based on pager reads, writes, and commits, so // we can mostly forward these next few functions to the pager Future getError() { return m_pager->getError(); } @@ -2700,7 +2944,9 @@ public: // setWriteVersion() A write shall not become durable until the following call to commit() begins, and shall be // durable once the following call to commit() returns void set(KeyValueRef keyValue) { - ++counts.sets; + ++g_redwoodMetrics.opSet; + ++g_redwoodMetrics.opSetKeyBytes += keyValue.key.size(); + ++g_redwoodMetrics.opSetValueBytes += keyValue.value.size(); m_pBuffer->insert(keyValue.key).mutation().setBoundaryValue(m_pBuffer->copyToArena(keyValue.value)); } @@ -2708,13 +2954,13 @@ public: // Optimization for single key clears to create just one mutation boundary instead of two if (clearedRange.begin.size() == clearedRange.end.size() - 1 && clearedRange.end[clearedRange.end.size() - 1] == 0 && clearedRange.end.startsWith(clearedRange.begin)) { - ++counts.clears; - ++counts.clearSingleKey; + ++g_redwoodMetrics.opClear; + ++g_redwoodMetrics.opClearKey; m_pBuffer->insert(clearedRange.begin).mutation().clearBoundary(); return; } - ++counts.clears; + ++g_redwoodMetrics.opClear; MutationBuffer::iterator iBegin = m_pBuffer->insert(clearedRange.begin); MutationBuffer::iterator iEnd = m_pBuffer->insert(clearedRange.end); @@ -2743,42 +2989,46 @@ public: VersionedBTree(IPager2* pager, std::string name) : m_pager(pager), m_writeVersion(invalidVersion), m_lastCommittedVersion(invalidVersion), m_pBuffer(nullptr), m_name(name) { + m_lazyClearActor = 0; m_init = init_impl(this); m_latestCommit = m_init; } - ACTOR static Future incrementalSubtreeClear(VersionedBTree* self, bool* pStop = nullptr, int batchSize = 10, - unsigned int minPages = 0, - int maxPages = std::numeric_limits::max()) { + ACTOR static Future incrementalLazyClear(VersionedBTree* self) { + ASSERT(self->m_lazyClearActor.isReady()); + self->m_lazyClearStop = false; + // TODO: Is it contractually okay to always to read at the latest version? state Reference snapshot = self->m_pager->getReadSnapshot(self->m_pager->getLatestVersion()); state int freedPages = 0; loop { - state std::vector>>> entries; + state int toPop = SERVER_KNOBS->REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; + state std::vector>>> entries; + entries.reserve(toPop); // Take up to batchSize pages from front of queue - while (entries.size() < batchSize) { - Optional q = wait(self->m_lazyDeleteQueue.pop()); - debug_printf("LazyDelete: popped %s\n", toString(q).c_str()); + while (toPop > 0) { + Optional q = wait(self->m_lazyClearQueue.pop()); + debug_printf("LazyClear: popped %s\n", toString(q).c_str()); if (!q.present()) { break; } // Start reading the page, without caching entries.push_back( std::make_pair(q.get(), self->readPage(snapshot, q.get().pageID, nullptr, nullptr, true))); - } - if (entries.empty()) { - break; + --toPop; } state int i; for (i = 0; i < entries.size(); ++i) { Reference p = wait(entries[i].second); - const LazyDeleteQueueEntry& entry = entries[i].first; + const LazyClearQueueEntry& entry = entries[i].first; const BTreePage& btPage = *(BTreePage*)p->begin(); - debug_printf("LazyDelete: processing %s\n", toString(entry).c_str()); + auto& metrics = g_redwoodMetrics.level(btPage.height); + + debug_printf("LazyClear: processing %s\n", toString(entry).c_str()); // Level 1 (leaf) nodes should never be in the lazy delete queue ASSERT(btPage.height > 1); @@ -2792,15 +3042,19 @@ public: while (1) { if (c.get().value.present()) { BTreePageIDRef btChildPageID = c.get().getChildPage(); - // If this page is height 2, then the children are leaves so free + // If this page is height 2, then the children are leaves so free them directly if (btPage.height == 2) { - debug_printf("LazyDelete: freeing child %s\n", toString(btChildPageID).c_str()); + debug_printf("LazyClear: freeing child %s\n", toString(btChildPageID).c_str()); self->freeBtreePage(btChildPageID, v); freedPages += btChildPageID.size(); + metrics.lazyClearFree += 1; + metrics.lazyClearFreeExt += (btChildPageID.size() - 1); } else { // Otherwise, queue them for lazy delete. - debug_printf("LazyDelete: queuing child %s\n", toString(btChildPageID).c_str()); - self->m_lazyDeleteQueue.pushFront(LazyDeleteQueueEntry{ v, btChildPageID }); + debug_printf("LazyClear: queuing child %s\n", toString(btChildPageID).c_str()); + self->m_lazyClearQueue.pushFront(LazyClearQueueEntry{ v, btChildPageID }); + metrics.lazyClearRequeue += 1; + metrics.lazyClearRequeueExt += (btChildPageID.size() - 1); } } if (!c.moveNext()) { @@ -2809,25 +3063,32 @@ public: } // Free the page, now that its children have either been freed or queued - debug_printf("LazyDelete: freeing queue entry %s\n", toString(entry.pageID).c_str()); + debug_printf("LazyClear: freeing queue entry %s\n", toString(entry.pageID).c_str()); self->freeBtreePage(entry.pageID, v); freedPages += entry.pageID.size(); + metrics.lazyClearFree += 1; + metrics.lazyClearFreeExt += entry.pageID.size() - 1; } - // If stop is set and we've freed the minimum number of pages required, or the maximum is exceeded, return. - if ((freedPages >= minPages && pStop != nullptr && *pStop) || freedPages >= maxPages) { + // Stop if + // - the poppable items in the queue have already been exhausted + // - stop flag is set and we've freed the minimum number of pages required + // - maximum number of pages to free met or exceeded + if (toPop > 0 || (freedPages >= SERVER_KNOBS->REDWOOD_LAZY_CLEAR_MIN_PAGES && self->m_lazyClearStop) || + (freedPages >= SERVER_KNOBS->REDWOOD_LAZY_CLEAR_MAX_PAGES)) { break; } } - debug_printf("LazyDelete: freed %d pages, %s has %" PRId64 " entries\n", freedPages, - self->m_lazyDeleteQueue.name.c_str(), self->m_lazyDeleteQueue.numEntries); + debug_printf("LazyClear: freed %d pages, %s has %" PRId64 " entries\n", freedPages, + self->m_lazyClearQueue.name.c_str(), self->m_lazyClearQueue.numEntries); return freedPages; } ACTOR static Future init_impl(VersionedBTree* self) { wait(self->m_pager->init()); + self->m_blockSize = self->m_pager->getUsablePageSize(); state Version latest = self->m_pager->getLatestVersion(); self->m_newOldestVersion = self->m_pager->getOldestVersion(); @@ -2849,19 +3110,20 @@ public: self->m_pager->setCommitVersion(latest); LogicalPageID newQueuePage = wait(self->m_pager->newPageID()); - self->m_lazyDeleteQueue.create(self->m_pager, newQueuePage, "LazyDeleteQueue"); - self->m_header.lazyDeleteQueue = self->m_lazyDeleteQueue.getState(); + self->m_lazyClearQueue.create(self->m_pager, newQueuePage, "LazyClearQueue"); + self->m_header.lazyDeleteQueue = self->m_lazyClearQueue.getState(); self->m_pager->setMetaKey(self->m_header.asKeyRef()); wait(self->m_pager->commit()); debug_printf("Committed initial commit.\n"); } else { self->m_header.fromKeyRef(meta); - self->m_lazyDeleteQueue.recover(self->m_pager, self->m_header.lazyDeleteQueue, "LazyDeleteQueueRecovered"); + self->m_lazyClearQueue.recover(self->m_pager, self->m_header.lazyDeleteQueue, "LazyClearQueueRecovered"); } debug_printf("Recovered btree at version %" PRId64 ": %s\n", latest, self->m_header.toString().c_str()); self->m_lastCommittedVersion = latest; + self->m_lazyClearActor = incrementalLazyClear(self); return Void(); } @@ -2910,15 +3172,24 @@ public: ACTOR static Future destroyAndCheckSanity_impl(VersionedBTree* self) { ASSERT(g_network->isSimulated()); + // This isn't pretty but remap cleanup is controlled by knobs and for this test we need the entire remap queue + // to be processed. + const_cast(SERVER_KNOBS)->REDWOOD_REMAP_CLEANUP_VERSION_LAG_MIN = 0; + const_cast(SERVER_KNOBS)->REDWOOD_REMAP_CLEANUP_VERSION_LAG_MAX = 0; + debug_printf("Clearing tree.\n"); self->setWriteVersion(self->getLatestVersion() + 1); self->clear(KeyRangeRef(dbBegin.key, dbEnd.key)); + wait(self->commit()); + // Loop commits until the the lazy delete queue is completely processed. loop { - state int freedPages = wait(self->incrementalSubtreeClear(self)); wait(self->commit()); - // Keep looping until the last commit doesn't do anything at all - if (self->m_lazyDeleteQueue.numEntries == 0 && freedPages == 0) { + + // If the lazy delete queue is completely processed then the last time the lazy delete actor + // was started it, after the last commit, it would exist immediately and do no work, so its + // future would be ready and its value would be 0. + if (self->m_lazyClearActor.isReady() && self->m_lazyClearActor.get() == 0) { break; } self->setWriteVersion(self->getLatestVersion() + 1); @@ -2932,7 +3203,7 @@ public: // The lazy delete queue should now be empty and contain only the new page to start writing to // on the next commit. - LazyDeleteQueueT::QueueState s = self->m_lazyDeleteQueue.getState(); + LazyClearQueueT::QueueState s = self->m_lazyClearQueue.getState(); ASSERT(s.numEntries == 0); ASSERT(s.numPages == 1); @@ -3167,6 +3438,7 @@ private: Future m_latestCommit; Future m_init; std::string m_name; + int m_blockSize; // MetaKey changes size so allocate space for it to expand into union { @@ -3174,7 +3446,9 @@ private: MetaKey m_header; }; - LazyDeleteQueueT m_lazyDeleteQueue; + LazyClearQueueT m_lazyClearQueue; + Future m_lazyClearActor; + bool m_lazyClearStop; // Writes entries to 1 or more pages and return a vector of boundary keys with their IPage(s) ACTOR static Future>> writePages( @@ -3184,7 +3458,7 @@ private: state Standalone> records; // This is how much space for the binary tree exists in the page, after the header - state int blockSize = self->m_pager->getUsablePageSize(); + state int blockSize = self->m_blockSize; state int pageSize = blockSize - sizeof(BTreePage); state int pageFillTarget = pageSize * SERVER_KNOBS->REDWOOD_PAGE_REBUILD_FILL_FACTOR; state int blockCount = 1; @@ -3223,14 +3497,12 @@ private: // overhead for the delta size must be assumed. int deltaSize = entry.deltaSize(base, skip, true); - int keySize = entry.key.size(); - int valueSize = entry.value.present() ? entry.value.get().size() : 0; - int nodeSize = BTreePage::BinaryTree::Node::headerSize(largeTree) + deltaSize; debug_printf("Adding %3d of %3lu (i=%3d) klen %4d vlen %5d nodeSize %5d deltaSize %5d page usage: " "%d/%d (%.2f%%) record=%s\n", - i + 1, entries.size(), i, keySize, valueSize, nodeSize, deltaSize, compressedBytes, - pageSize, (float)compressedBytes / pageSize * 100, entry.toString(height == 1).c_str()); + i + 1, entries.size(), i, entry.key.size(), entry.value.orDefault(StringRef()).size(), + nodeSize, deltaSize, compressedBytes, pageSize, (float)compressedBytes / pageSize * 100, + entry.toString(height == 1).c_str()); // While the node doesn't fit, expand the page. // This is a loop because if the page size moves into "large" range for DeltaTree @@ -3261,7 +3533,7 @@ private: pageFillTarget = pageSize * SERVER_KNOBS->REDWOOD_PAGE_REBUILD_FILL_FACTOR; } - kvBytes += keySize + valueSize; + kvBytes += entry.kvBytes(); compressedBytes += nodeSize; ++i; } @@ -3289,14 +3561,14 @@ private: state std::vector> pages; BTreePage* btPage; + int capacity = blockSize * blockCount; if (blockCount == 1) { Reference page = self->m_pager->newPageBuffer(); btPage = (BTreePage*)page->mutate(); pages.push_back(std::move(page)); } else { ASSERT(blockCount > 1); - int size = blockSize * blockCount; - btPage = (BTreePage*)new uint8_t[size]; + btPage = (BTreePage*)new uint8_t[capacity]; } btPage->height = height; @@ -3318,6 +3590,13 @@ private: ASSERT(false); } + auto& metrics = g_redwoodMetrics.level(btPage->height); + metrics.pageBuild += 1; + metrics.pageBuildExt += blockCount - 1; + metrics.buildFillPct += (double)written / capacity; + metrics.buildStoredPct += (double)btPage->kvBytes / capacity; + metrics.buildItemCount += btPage->tree().numItems; + // Create chunked pages // TODO: Avoid copying page bytes, but this is not trivial due to how pager checksums are currently handled. if (blockCount != 1) { @@ -3362,12 +3641,6 @@ private: wait(yield()); - // Update activity counts - ++counts.pageWrites; - if (pages.size() > 1) { - counts.extPageWrites += pages.size() - 1; - } - debug_printf("Flushing %s lastPage=%d original=%s start=%d i=%d count=%d page usage: %d/%d (%.2f%%) " "bytes\nlower: %s\nupper: %s\n", toString(childPageID).c_str(), isLastPage, toString(previousID).c_str(), start, i, i - start, @@ -3466,8 +3739,8 @@ private: ACTOR static Future> readPage(Reference snapshot, BTreePageIDRef id, const RedwoodRecordRef* lowerBound, const RedwoodRecordRef* upperBound, - bool forLazyDelete = false) { - if (!forLazyDelete) { + bool forLazyClear = false) { + if (!forLazyClear) { debug_printf("readPage() op=read %s @%" PRId64 " lower=%s upper=%s\n", toString(id).c_str(), snapshot->getVersion(), lowerBound->toString(false).c_str(), upperBound->toString(false).c_str()); @@ -3480,16 +3753,14 @@ private: state Reference page; - ++counts.pageReads; if (id.size() == 1) { - Reference p = wait(snapshot->getPhysicalPage(id.front(), !forLazyDelete, false)); + Reference p = wait(snapshot->getPhysicalPage(id.front(), !forLazyClear, false)); page = p; } else { ASSERT(!id.empty()); - counts.extPageReads += (id.size() - 1); std::vector>> reads; for (auto& pageID : id) { - reads.push_back(snapshot->getPhysicalPage(pageID, !forLazyDelete, false)); + reads.push_back(snapshot->getPhysicalPage(pageID, !forLazyClear, false)); } std::vector> pages = wait(getAll(reads)); // TODO: Cache reconstituted super pages somehow, perhaps with help from the Pager. @@ -3498,8 +3769,11 @@ private: debug_printf("readPage() op=readComplete %s @%" PRId64 " \n", toString(id).c_str(), snapshot->getVersion()); const BTreePage* pTreePage = (const BTreePage*)page->begin(); + auto& metrics = g_redwoodMetrics.level(pTreePage->height); + metrics.pageRead += 1; + metrics.pageReadExt += (id.size() - 1); - if (!forLazyDelete && page->userData == nullptr) { + if (!forLazyClear && page->userData == nullptr) { debug_printf("readPage() Creating Reader for %s @%" PRId64 " lower=%s upper=%s\n", toString(id).c_str(), snapshot->getVersion(), lowerBound->toString(false).c_str(), upperBound->toString(false).c_str()); @@ -3507,7 +3781,7 @@ private: page->userDataDestructor = [](void* ptr) { delete (BTreePage::BinaryTree::Mirror*)ptr; }; } - if (!forLazyDelete) { + if (!forLazyClear) { debug_printf("readPage() %s\n", pTreePage->toString(false, id, snapshot->getVersion(), lowerBound, upperBound).c_str()); } @@ -3516,8 +3790,8 @@ private: } static void preLoadPage(IPagerSnapshot* snapshot, BTreePageIDRef id) { - ++counts.pagePreloads; - counts.extPagePreloads += (id.size() - 1); + g_redwoodMetrics.btreeLeafPreload += 1; + g_redwoodMetrics.btreeLeafPreloadExt += (id.size() - 1); for (auto pageID : id) { snapshot->getPhysicalPage(pageID, true, true); @@ -3563,12 +3837,6 @@ private: } } - // Update activity counts - ++counts.pageWrites; - if (newID.size() > 1) { - counts.extPageWrites += newID.size() - 1; - } - return newID; } @@ -3646,7 +3914,14 @@ private: } // Page was updated in-place through edits and written to maybeNewID - void updatedInPlace(BTreePageIDRef maybeNewID) { + void updatedInPlace(BTreePageIDRef maybeNewID, BTreePage* btPage, int capacity) { + auto& metrics = g_redwoodMetrics.level(btPage->height); + metrics.pageModify += 1; + metrics.pageModify += (maybeNewID.size() - 1); + metrics.modifyFillPct += (double)btPage->size() / capacity; + metrics.modifyStoredPct += (double)btPage->kvBytes / capacity; + metrics.modifyItemCount += btPage->tree().numItems; + // The boundaries can't have changed, but the child page link may have. if (maybeNewID != decodeLowerBound->getChildPage()) { // Add page's decode lower bound to newLinks set without its child page, intially @@ -3704,10 +3979,11 @@ private: struct InternalPageModifier { InternalPageModifier() {} - InternalPageModifier(BTreePage::BinaryTree::Mirror* m, bool updating) - : m(m), updating(updating), changesMade(false) {} + InternalPageModifier(BTreePage* p, BTreePage::BinaryTree::Mirror* m, bool updating) + : btPage(p), m(m), updating(updating), changesMade(false) {} bool updating; + BTreePage* btPage; BTreePage::BinaryTree::Mirror* m; Standalone> rebuild; bool changesMade; @@ -3747,6 +4023,7 @@ private: updating = false; break; } + btPage->kvBytes += rec.kvBytes(); ++i; } } @@ -3789,6 +4066,7 @@ private: auto c = u.cBegin; while (c != u.cEnd) { debug_printf("internal page (updating) erasing: %s\n", c.get().toString(false).c_str()); + btPage->kvBytes -= c.get().kvBytes(); c.erase(); } // [cBegin, cEnd) is now erased, and cBegin is invalid, so cEnd represents the end @@ -3847,12 +4125,12 @@ private: debug_printf("%s -------------------------------------\n", context.c_str()); } - ++self->counts.commitSubtreeStart; state Version writeVersion = self->getLastCommittedVersion() + 1; state Reference page = wait(readPage(snapshot, rootID, update->decodeLowerBound, update->decodeUpperBound)); state BTreePage* btPage = (BTreePage*)page->begin(); ASSERT(isLeaf == btPage->isLeaf()); + g_redwoodMetrics.level(btPage->height).pageCommitStart += 1; // TODO: Decide if it is okay to update if the subtree boundaries are expanded. It can result in // records in a DeltaTree being outside its decode boundary range, which isn't actually invalid @@ -3943,6 +4221,7 @@ private: if (updating) { debug_printf("%s Erasing %s [existing, boundary start]\n", context.c_str(), cursor.get().toString().c_str()); + btPage->kvBytes -= cursor.get().kvBytes(); cursor.erase(); } else { debug_printf("%s Skipped %s [existing, boundary start]\n", context.c_str(), @@ -3964,6 +4243,7 @@ private: // If updating, add to the page, else add to the output set if (updating) { if (cursor.mirror->insert(rec, update->skipLen, maxHeightAllowed)) { + btPage->kvBytes += rec.kvBytes(); debug_printf("%s Inserted %s [mutation, boundary start]\n", context.c_str(), rec.toString().c_str()); } else { @@ -4012,6 +4292,7 @@ private: if (updating) { debug_printf("%s Erasing %s [existing, boundary start]\n", context.c_str(), cursor.get().toString().c_str()); + btPage->kvBytes -= cursor.get().kvBytes(); cursor.erase(); changesMade = true; } else { @@ -4046,6 +4327,7 @@ private: debug_printf( "%s Erasing %s and beyond [existing, matches changed upper mutation boundary]\n", context.c_str(), cursor.get().toString().c_str()); + btPage->kvBytes -= cursor.get().kvBytes(); cursor.erase(); } else { merged.push_back(merged.arena(), cursor.get()); @@ -4086,8 +4368,7 @@ private: BTreePageIDRef newID = wait(self->updateBtreePage(self, rootID, &update->newLinks.arena(), page.castTo(), writeVersion)); - update->updatedInPlace(newID); - ++counts.pageUpdates; + update->updatedInPlace(newID, btPage, newID.size() * self->m_blockSize); debug_printf("%s Page updated in-place, returning %s\n", context.c_str(), toString(*update).c_str()); } @@ -4123,8 +4404,10 @@ private: cursor.moveFirst(); bool first = true; + while (cursor.valid()) { InternalPageSliceUpdate& u = *new (arena) InternalPageSliceUpdate(); + slices.push_back(&u); // At this point we should never be at a null child page entry because the first entry of a page // can't be null and this loop will skip over null entries that come after non-null entries. @@ -4136,8 +4419,16 @@ private: if (first) { u.subtreeLowerBound = update->subtreeLowerBound; first = false; + // mbegin is already the first mutation that could affect this subtree described by update } else { u.subtreeLowerBound = u.decodeLowerBound; + mBegin = mEnd; + // mBegin is either at or greater than subtreeLowerBound->key, which was the subtreeUpperBound->key + // for the previous subtree slice. But we need it to be at or *before* subtreeLowerBound->key + // so if mBegin.key() is not exactly the subtree lower bound key then decrement it. + if (mBegin.key() != u.subtreeLowerBound->key) { + --mBegin; + } } BTreePageIDRef pageID = cursor.get().getChildPage(); @@ -4166,28 +4457,21 @@ private: } u.subtreeUpperBound = cursor.valid() ? &cursor.get() : update->subtreeUpperBound; u.cEnd = cursor; - u.skipLen = 0; // TODO: set this - slices.push_back(&u); - // Find the mutation buffer range that includes all changes to the range described by u - MutationBuffer::const_iterator mBegin = mutationBuffer->upper_bound(u.subtreeLowerBound->key); - MutationBuffer::const_iterator mEnd = mutationBuffer->lower_bound(u.subtreeUpperBound->key); + mEnd = mutationBuffer->lower_bound(u.subtreeUpperBound->key); - // If mutation boundaries are the same, the range is fully described by (mBegin - 1).mutation() - bool fullyCovered = (mBegin == mEnd); - --mBegin; - - // If mBegin describes the entire subtree range, see if there are either no changes or if the entire - // range is cleared. - if (fullyCovered) { + // If the mutation range described by mBegin extends to mEnd, then see if the part of that range + // that overlaps with u's subtree range is being fully cleared or fully unchanged. + auto next = mBegin; + ++next; + if (next == mEnd) { + // Check for uniform clearedness or unchangedness for the range mutation where it overlaps u's + // subtree + const KeyRef& mutationBoundaryKey = mBegin.key(); const RangeMutation& range = mBegin.mutation(); - - // Check for uniform clearedness or unchangedness for the range mutation - KeyRef mutationBoundaryKey = mBegin.key(); bool uniform; - if (range.clearAfterBoundary) { // If the mutation range after the boundary key is cleared, then the mutation boundary key must // be cleared or must be different than the subtree lower bound key so that it doesn't matter @@ -4199,11 +4483,13 @@ private: uniform = !range.boundaryChanged || mutationBoundaryKey != u.subtreeLowerBound->key; } - // If the subtree range described by u is either uniformly changed or unchanged + // If u's subtree is either all cleared or all unchanged if (uniform) { - // See if we can expand the subtree range to include more subtrees which are also covered by the - // same mutation range - if (cursor.valid() && mEnd.key() != cursor.get().key) { + // We do not need to recurse to this subtree. Next, let's see if we can embiggen u's range to + // include sibling subtrees also covered by (mBegin, mEnd) so we can not recurse to those, too. + // If the cursor is valid, u.subtreeUpperBound is the cursor's position, which is >= mEnd.key(). + // If equal, no range expansion is possible. + if (cursor.valid() && mEnd.key() != u.subtreeUpperBound->key) { cursor.seekLessThanOrEqual(mEnd.key(), update->skipLen, &cursor, 1); // If this seek moved us ahead, to something other than cEnd, then update subtree range @@ -4250,8 +4536,8 @@ private: } else { debug_printf("%s: queuing subtree deletion cleared subtree range: %s\n", context.c_str(), ::toString(rec.getChildPage()).c_str()); - self->m_lazyDeleteQueue.pushFront( - LazyDeleteQueueEntry{ writeVersion, rec.getChildPage() }); + self->m_lazyClearQueue.pushFront( + LazyClearQueueEntry{ writeVersion, rec.getChildPage() }); } } c.moveNext(); @@ -4260,15 +4546,18 @@ private: // Subtree range unchanged } - debug_printf("%s: MutationBuffer covers this range in a single mutation: %s\n", context.c_str(), - u.toString().c_str()); + debug_printf("%s: MutationBuffer covers this range in a single mutation, not recursing: %s\n", + context.c_str(), u.toString().c_str()); + + // u has already been initialized with the correct result, no recursion needed, so restart the + // loop. continue; } } // If this page has height of 2 then its children are leaf nodes - recursions.push_back(self->commitSubtree(self, snapshot, mutationBuffer, pageID, btPage->height == 2, - mBegin, mEnd, slices.back())); + recursions.push_back( + self->commitSubtree(self, snapshot, mutationBuffer, pageID, btPage->height == 2, mBegin, mEnd, &u)); } debug_printf( @@ -4279,7 +4568,7 @@ private: wait(waitForAll(recursions)); debug_printf("%s Recursions done, processing slice updates.\n", context.c_str()); - state InternalPageModifier m(cursor.mirror, tryToUpdate); + state InternalPageModifier m(btPage, cursor.mirror, tryToUpdate); // Apply the possible changes for each subtree range recursed to, except the last one. // For each range, the expected next record, if any, is checked against the first boundary @@ -4310,8 +4599,7 @@ private: BTreePageIDRef newID = wait(self->updateBtreePage(self, rootID, &update->newLinks.arena(), page.castTo(), writeVersion)); - update->updatedInPlace(newID); - ++counts.pageUpdates; + update->updatedInPlace(newID, btPage, newID.size() * self->m_blockSize); debug_printf("%s Internal page updated in-place, returning %s\n", context.c_str(), toString(*update).c_str()); } else { @@ -4357,9 +4645,6 @@ private: debug_printf("%s: Beginning commit of version %" PRId64 ", new oldest version set to %" PRId64 "\n", self->m_name.c_str(), writeVersion, self->m_newOldestVersion); - state bool lazyDeleteStop = false; - state Future lazyDelete = incrementalSubtreeClear(self, &lazyDeleteStop); - // Get the latest version from the pager, which is what we will read at state Version latestVersion = self->m_pager->getLatestVersion(); debug_printf("%s: pager latestVersion %" PRId64 "\n", self->m_name.c_str(), latestVersion); @@ -4405,14 +4690,14 @@ private: self->m_header.root.set(rootPageID, sizeof(headerSpace) - sizeof(m_header)); - lazyDeleteStop = true; - wait(success(lazyDelete)); - debug_printf("Lazy delete freed %u pages\n", lazyDelete.get()); + self->m_lazyClearStop = true; + wait(success(self->m_lazyClearActor)); + debug_printf("Lazy delete freed %u pages\n", self->m_lazyClearActor.get()); self->m_pager->setCommitVersion(writeVersion); - wait(self->m_lazyDeleteQueue.flush()); - self->m_header.lazyDeleteQueue = self->m_lazyDeleteQueue.getState(); + wait(self->m_lazyClearQueue.flush()); + self->m_header.lazyDeleteQueue = self->m_lazyClearQueue.getState(); debug_printf("Setting metakey\n"); self->m_pager->setMetaKey(self->m_header.asKeyRef()); @@ -4427,9 +4712,10 @@ private: self->m_mutationBuffers.erase(self->m_mutationBuffers.begin()); self->m_lastCommittedVersion = writeVersion; - ++counts.commits; - committed.send(Void()); + ++g_redwoodMetrics.opCommit; + self->m_lazyClearActor = incrementalLazyClear(self); + committed.send(Void()); return Void(); } @@ -4899,9 +5185,8 @@ public: #include "fdbserver/art_impl.h" -RedwoodRecordRef VersionedBTree::dbBegin(StringRef(), 0); +RedwoodRecordRef VersionedBTree::dbBegin(LiteralStringRef("")); RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff")); -VersionedBTree::Counts VersionedBTree::counts; class KeyValueStoreRedwoodUnversioned : public IKeyValueStore { public: @@ -4982,7 +5267,7 @@ public: wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - self->m_tree->counts.getRanges++; + ++g_redwoodMetrics.opGetRange; state Standalone result; state int accumulatedBytes = 0; ASSERT(byteLimit > 0); @@ -5034,7 +5319,7 @@ public: wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - self->m_tree->counts.gets++; + ++g_redwoodMetrics.opGet; state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); wait(cur->findEqual(key)); @@ -5053,7 +5338,7 @@ public: wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - self->m_tree->counts.gets++; + ++g_redwoodMetrics.opGet; state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); wait(cur->findEqual(key)); @@ -6184,6 +6469,9 @@ TEST_CASE("!/redwood/performance/mutationBuffer") { } TEST_CASE("!/redwood/correctness/btree") { + g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting + g_redwoodMetrics.clear(); + state std::string pagerFile = "unittest_pageFile.redwood"; IPager2* pager; @@ -6229,6 +6517,7 @@ TEST_CASE("!/redwood/correctness/btree") { printf("Initializing...\n"); state double startTime = now(); + pager = new DWALPager(pageSize, pagerFile, cacheSizeBytes, pagerMemoryOnly); state VersionedBTree* btree = new VersionedBTree(pager, pagerFile); wait(btree->init()); @@ -6379,7 +6668,7 @@ TEST_CASE("!/redwood/correctness/btree") { } commit = map(btree->commit(), [=](Void) { - printf("Committed: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Committed:\n%s\n", g_redwoodMetrics.toString(true).c_str()); // Notify the background verifier that version is committed and therefore readable committedVersions.send(v); return Void(); @@ -6533,7 +6822,9 @@ TEST_CASE("!/redwood/correctness/pager/cow") { TEST_CASE("!/redwood/performance/set") { state SignalableActorCollection actors; - VersionedBTree::counts.clear(); + + g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting + g_redwoodMetrics.clear(); // If a test file is passed in by environment then don't write new data to it. state bool reload = getenv("TESTFILE") == nullptr; @@ -6544,7 +6835,7 @@ TEST_CASE("!/redwood/performance/set") { deleteFile(pagerFile); } - state int pageSize = 4096; + state int pageSize = SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE; state int64_t pageCacheBytes = FLOW_KNOBS->PAGE_CACHE_4K; DWALPager* pager = new DWALPager(pageSize, pagerFile, pageCacheBytes); state VersionedBTree* btree = new VersionedBTree(pager, pagerFile); @@ -6594,7 +6885,8 @@ TEST_CASE("!/redwood/performance/set") { Version lastVer = btree->getLatestVersion(); state Version version = lastVer + 1; btree->setWriteVersion(version); - int changesThisVersion = deterministicRandom()->randomInt(0, maxRecordsPerCommit - recordsThisCommit + 1); + state int changesThisVersion = + deterministicRandom()->randomInt(0, maxRecordsPerCommit - recordsThisCommit + 1); while (changesThisVersion > 0 && kvBytesThisCommit < maxKVBytesPerCommit) { KeyValue kv; @@ -6617,6 +6909,8 @@ TEST_CASE("!/redwood/performance/set") { kvBytesThisCommit += kv.key.size() + kv.value.size(); ++recordsThisCommit; } + + wait(yield()); } if (kvBytesThisCommit >= maxKVBytesPerCommit || recordsThisCommit >= maxRecordsPerCommit) { @@ -6634,7 +6928,7 @@ TEST_CASE("!/redwood/performance/set") { double* pIntervalStart = &intervalStart; commit = map(btree->commit(), [=](Void result) { - printf("Committed: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Committed:\n%s\n", g_redwoodMetrics.toString(true).c_str()); double elapsed = timer() - *pIntervalStart; printf("Committed %d keyValueBytes in %d records in %f seconds, %.2f MB/s\n", kvb, recs, elapsed, kvb / elapsed / 1e6); @@ -6659,46 +6953,46 @@ TEST_CASE("!/redwood/performance/set") { actors.add(randomSeeks(btree, seeks / 3, firstKeyChar, lastKeyChar)); actors.add(randomSeeks(btree, seeks / 3, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); state int ops = 10000; printf("Serial scans with adaptive readAhead...\n"); actors.add(randomScans(btree, ops, 50, -1, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Serial scans with readAhead 3 pages...\n"); actors.add(randomScans(btree, ops, 50, 12000, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Serial scans with readAhead 2 pages...\n"); actors.add(randomScans(btree, ops, 50, 8000, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Serial scans with readAhead 1 page...\n"); actors.add(randomScans(btree, ops, 50, 4000, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Serial scans...\n"); actors.add(randomScans(btree, ops, 50, 0, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Serial seeks...\n"); actors.add(randomSeeks(btree, ops, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); printf("Parallel seeks...\n"); actors.add(randomSeeks(btree, ops, firstKeyChar, lastKeyChar)); actors.add(randomSeeks(btree, ops, firstKeyChar, lastKeyChar)); actors.add(randomSeeks(btree, ops, firstKeyChar, lastKeyChar)); wait(actors.signalAndReset()); - printf("Stats: %s\n", VersionedBTree::counts.toString(true).c_str()); + printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); Future closedFuture = btree->onClosed(); btree->close(); @@ -6991,7 +7285,6 @@ Future closeKVS(IKeyValueStore* kvs) { ACTOR Future doPrefixInsertComparison(int suffixSize, int valueSize, int recordCountTarget, bool usePrefixesInOrder, KVSource source) { - VersionedBTree::counts.clear(); deleteFile("test.redwood"); wait(delay(5)); diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index a23bfcb5a6..b43ddf11b3 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -54,7 +54,7 @@ #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbserver/CoroFlow.h" #include "flow/TLSConfig.actor.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" #include "fdbmonitor/SimpleIni.h" diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 2cdd37a1e1..3c335d753e 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -4101,3 +4101,4 @@ void versionedMapTest() { printf("Memory used: %f MB\n", (after - before)/ 1e6); } + diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp index 283a0d7210..16794b749f 100644 --- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp @@ -399,9 +399,11 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { if (!self->locked && BUGGIFY) { TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag)); try { + // Note the "partitionedLog" must be false, because we change + // the configuration to disable backup workers before restore. extraBackup = backupAgent.submitBackup( cx, LiteralStringRef("file://simfdb/backups/"), deterministicRandom()->randomInt(0, 100), - self->backupTag.toString(), self->backupRanges, true, self->usePartitionedLogs); + self->backupTag.toString(), self->backupRanges, true, false); } catch (Error& e) { TraceEvent("BARW_SubmitBackup2Exception", randomID) .error(e) diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp index 7269457492..4ec2110abc 100644 --- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp +++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp @@ -138,11 +138,9 @@ bool checkTxInfoEntryFormat(BinaryReader &reader) { while (!reader.empty()) { // Get EventType and timestamp - FdbClientLogEvents::EventType event; + FdbClientLogEvents::Event event; reader >> event; - double timeStamp; - reader >> timeStamp; - switch (event) + switch (event.type) { case FdbClientLogEvents::GET_VERSION_LATENCY: parser->parseGetVersion(reader); @@ -166,7 +164,7 @@ bool checkTxInfoEntryFormat(BinaryReader &reader) { parser->parseErrorCommit(reader); break; default: - TraceEvent(SevError, "ClientTransactionProfilingUnknownEvent").detail("EventType", event); + TraceEvent(SevError, "ClientTransactionProfilingUnknownEvent").detail("EventType", event.type); return false; } } diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp index e17ead6c94..4349e09619 100644 --- a/fdbserver/workloads/ConfigureDatabase.actor.cpp +++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp @@ -34,6 +34,7 @@ static const char* logTypes[] = { "log_version:=2", "log_version:=3", "log_version:=4" }; static const char* redundancies[] = { "single", "double", "triple" }; +static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_enabled:=1" }; std::string generateRegions() { std::string result; @@ -271,7 +272,7 @@ struct ConfigureDatabaseWorkload : TestWorkload { if(g_simulator.speedUpSimulation) { return Void(); } - state int randomChoice = deterministicRandom()->randomInt(0, 7); + state int randomChoice = deterministicRandom()->randomInt(0, 8); if( randomChoice == 0 ) { wait( success( runRYWTransaction(cx, [=](Reference tr) -> Future> @@ -322,6 +323,10 @@ struct ConfigureDatabaseWorkload : TestWorkload { else if ( randomChoice == 6 ) { // Some configurations will be invalid, and that's fine. wait(success( IssueConfigurationChange( cx, logTypes[deterministicRandom()->randomInt( 0, sizeof(logTypes)/sizeof(logTypes[0]))], false ) )); + } else if (randomChoice == 7) { + wait(success(IssueConfigurationChange( + cx, backupTypes[deterministicRandom()->randomInt(0, sizeof(backupTypes) / sizeof(backupTypes[0]))], + false))); } else { ASSERT(false); } diff --git a/fdbserver/workloads/DataDistributionMetrics.actor.cpp b/fdbserver/workloads/DataDistributionMetrics.actor.cpp new file mode 100644 index 0000000000..96a0d37510 --- /dev/null +++ b/fdbserver/workloads/DataDistributionMetrics.actor.cpp @@ -0,0 +1,108 @@ +/* + * DataDistributionMetrics.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "fdbclient/ReadYourWrites.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "flow/actorcompiler.h" // This must be the last include + +struct DataDistributionMetricsWorkload : KVWorkload { + + int numTransactions; + int writesPerTransaction; + int transactionsCommitted; + int numShards; + int64_t avgBytes; + + DataDistributionMetricsWorkload(WorkloadContext const& wcx) + : KVWorkload(wcx), transactionsCommitted(0), numShards(0), avgBytes(0) { + numTransactions = getOption(options, LiteralStringRef("numTransactions"), 100); + writesPerTransaction = getOption(options, LiteralStringRef("writesPerTransaction"), 1000); + } + + static Value getRandomValue() { + return Standalone(format("Value/%08d", deterministicRandom()->randomInt(0, 10e6))); + } + + ACTOR static Future _start(Database cx, DataDistributionMetricsWorkload* self) { + state int tNum; + for (tNum = 0; tNum < self->numTransactions; ++tNum) { + loop { + state ReadYourWritesTransaction tr(cx); + try { + state int i; + for (i = 0; i < self->writesPerTransaction; ++i) { + tr.set(StringRef(format("Key/%08d", tNum * self->writesPerTransaction + i)), getRandomValue()); + } + wait(tr.commit()); + ++self->transactionsCommitted; + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + return Void(); + } + + ACTOR static Future _check(Database cx, DataDistributionMetricsWorkload* self) { + if (self->transactionsCommitted == 0) { + TraceEvent(SevError, "NoTransactionsCommitted"); + return false; + } + state Reference tr = + Reference(new ReadYourWritesTransaction(cx)); + try { + state Standalone result = wait(tr->getRange(ddStatsRange, 100)); + ASSERT(!result.more); + self->numShards = result.size(); + if (self->numShards < 1) return false; + state int64_t totalBytes = 0; + for (int i = 0; i < result.size(); ++i) { + ASSERT(result[i].key.startsWith(ddStatsRange.begin)); + totalBytes += readJSONStrictly(result[i].value.toString()).get_obj()["ShardBytes"].get_int64(); + } + self->avgBytes = totalBytes / self->numShards; + // fetch data-distribution stats for a smalller range + state int idx = deterministicRandom()->randomInt(0, result.size()); + Standalone res = wait(tr->getRange( + KeyRangeRef(result[idx].key, idx + 1 < result.size() ? result[idx + 1].key : ddStatsRange.end), 100)); + ASSERT_WE_THINK(res.size() == 1 && + res[0] == result[idx]); // It works good now. However, not sure in any case of data-distribution, the number changes + } catch (Error& e) { + TraceEvent(SevError, "FailedToRetrieveDDMetrics").detail("Error", e.what()); + return false; + } + return true; + } + + virtual std::string description() { return "DataDistributionMetrics"; } + virtual Future setup(Database const& cx) { return Void(); } + virtual Future start(Database const& cx) { return _start(cx, this); } + virtual Future check(Database const& cx) { return _check(cx, this); } + + virtual void getMetrics(vector& m) { + m.push_back(PerfMetric("NumShards", numShards, true)); + m.push_back(PerfMetric("AvgBytes", avgBytes, true)); + } +}; + +WorkloadFactory DataDistributionMetricsWorkloadFactory("DataDistributionMetrics"); diff --git a/fdbserver/workloads/TagThrottleApi.actor.cpp b/fdbserver/workloads/TagThrottleApi.actor.cpp index 7ec90868c9..812f369882 100644 --- a/fdbserver/workloads/TagThrottleApi.actor.cpp +++ b/fdbserver/workloads/TagThrottleApi.actor.cpp @@ -50,6 +50,22 @@ struct TagThrottleApiWorkload : TestWorkload { virtual void getMetrics(vector& m) {} + static Optional randomTagThrottleType() { + Optional throttleType; + switch(deterministicRandom()->randomInt(0, 3)) { + case 0: + throttleType = TagThrottleType::AUTO; + break; + case 1: + throttleType = TagThrottleType::MANUAL; + break; + default: + break; + } + + return throttleType; + } + ACTOR Future throttleTag(Database cx, std::map, TagThrottleInfo> *manuallyThrottledTags) { state TransactionTag tag = TransactionTagRef(deterministicRandom()->randomChoice(DatabaseContext::debugTransactionTagChoices)); state TransactionPriority priority = deterministicRandom()->randomChoice(allTransactionPriorities); @@ -60,7 +76,7 @@ struct TagThrottleApiWorkload : TestWorkload { tagSet.addTag(tag); try { - wait(ThrottleApi::throttleTags(cx, tagSet, rate, duration, false, priority)); + wait(ThrottleApi::throttleTags(cx, tagSet, rate, duration, TagThrottleType::MANUAL, priority)); } catch(Error &e) { state Error err = e; @@ -72,7 +88,7 @@ struct TagThrottleApiWorkload : TestWorkload { throw err; } - manuallyThrottledTags->insert_or_assign(std::make_pair(tag, priority), TagThrottleInfo(tag, false, priority, rate, now() + duration, duration)); + manuallyThrottledTags->insert_or_assign(std::make_pair(tag, priority), TagThrottleInfo(tag, TagThrottleType::MANUAL, priority, rate, now() + duration, duration)); return Void(); } @@ -82,26 +98,30 @@ struct TagThrottleApiWorkload : TestWorkload { TagSet tagSet; tagSet.addTag(tag); - state bool autoThrottled = deterministicRandom()->coinflip(); - TransactionPriority priority = deterministicRandom()->randomChoice(allTransactionPriorities); + state Optional throttleType = TagThrottleApiWorkload::randomTagThrottleType(); + Optional priority = deterministicRandom()->coinflip() ? Optional() : deterministicRandom()->randomChoice(allTransactionPriorities); state bool erased = false; - state double expiration = 0; - if(!autoThrottled) { - auto itr = manuallyThrottledTags->find(std::make_pair(tag, priority)); - if(itr != manuallyThrottledTags->end()) { - expiration = itr->second.expirationTime; - erased = true; - manuallyThrottledTags->erase(itr); + state double maxExpiration = 0; + if(!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) { + for(auto p : allTransactionPriorities) { + if(!priority.present() || priority.get() == p) { + auto itr = manuallyThrottledTags->find(std::make_pair(tag, p)); + if(itr != manuallyThrottledTags->end()) { + maxExpiration = std::max(maxExpiration, itr->second.expirationTime); + erased = true; + manuallyThrottledTags->erase(itr); + } + } } } - bool removed = wait(ThrottleApi::unthrottleTags(cx, tagSet, autoThrottled, priority)); + bool removed = wait(ThrottleApi::unthrottleTags(cx, tagSet, throttleType, priority)); if(removed) { - ASSERT(erased || autoThrottled); + ASSERT(erased || !throttleType.present() || throttleType.get() == TagThrottleType::AUTO); } else { - ASSERT(expiration < now()); + ASSERT(maxExpiration < now()); } return Void(); @@ -113,7 +133,7 @@ struct TagThrottleApiWorkload : TestWorkload { int manualThrottledTags = 0; int activeAutoThrottledTags = 0; for(auto &tag : tags) { - if(!tag.autoThrottled) { + if(tag.throttleType == TagThrottleType::MANUAL) { ASSERT(manuallyThrottledTags->find(std::make_pair(tag.tag, tag.priority)) != manuallyThrottledTags->end()); ++manualThrottledTags; } @@ -139,34 +159,32 @@ struct TagThrottleApiWorkload : TestWorkload { } ACTOR Future unthrottleTagGroup(Database cx, std::map, TagThrottleInfo> *manuallyThrottledTags) { - state int choice = deterministicRandom()->randomInt(0, 3); + state Optional throttleType = TagThrottleApiWorkload::randomTagThrottleType(); + state Optional priority = deterministicRandom()->coinflip() ? Optional() : deterministicRandom()->randomChoice(allTransactionPriorities); - if(choice == 0) { - bool unthrottled = wait(ThrottleApi::unthrottleAll(cx)); + bool unthrottled = wait(ThrottleApi::unthrottleAll(cx, throttleType, priority)); + if(!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) { bool unthrottleExpected = false; - for(auto itr = manuallyThrottledTags->begin(); itr != manuallyThrottledTags->end(); ++itr) { - if(itr->second.expirationTime > now()) { - unthrottleExpected = true; + bool empty = manuallyThrottledTags->empty(); + for(auto itr = manuallyThrottledTags->begin(); itr != manuallyThrottledTags->end();) { + if(!priority.present() || priority.get() == itr->first.second) { + if(itr->second.expirationTime > now()) { + unthrottleExpected = true; + } + + itr = manuallyThrottledTags->erase(itr); + } + else { + ++itr; } } - ASSERT(!unthrottleExpected || unthrottled); - manuallyThrottledTags->clear(); - } - else if(choice == 1) { - bool unthrottled = wait(ThrottleApi::unthrottleManual(cx)); - bool unthrottleExpected = false; - for(auto itr = manuallyThrottledTags->begin(); itr != manuallyThrottledTags->end(); ++itr) { - if(itr->second.expirationTime > now()) { - unthrottleExpected = true; - } + if(throttleType.present()) { + ASSERT((unthrottled && !empty) || (!unthrottled && !unthrottleExpected)); + } + else { + ASSERT(unthrottled || !unthrottleExpected); } - - ASSERT((unthrottled && !manuallyThrottledTags->empty()) || (!unthrottled && !unthrottleExpected)); - manuallyThrottledTags->clear(); - } - else { - bool unthrottled = wait(ThrottleApi::unthrottleAuto(cx)); } return Void(); @@ -176,7 +194,7 @@ struct TagThrottleApiWorkload : TestWorkload { if(deterministicRandom()->coinflip()) { wait(ThrottleApi::enableAuto(cx, true)); if(deterministicRandom()->coinflip()) { - bool unthrottled = wait(ThrottleApi::unthrottleAuto(cx)); + bool unthrottled = wait(ThrottleApi::unthrottleAll(cx, TagThrottleType::AUTO, Optional())); } } else { diff --git a/fdbservice/FDBService.cpp b/fdbservice/FDBService.cpp index 59ef5c8045..fe761a0109 100644 --- a/fdbservice/FDBService.cpp +++ b/fdbservice/FDBService.cpp @@ -30,7 +30,7 @@ #include "flow/SimpleOpt.h" #include "fdbmonitor/SimpleIni.h" -#include "fdbclient/IncludeVersions.h" +#include "fdbclient/versions.h" // For PathFileExists #include "Shlwapi.h" diff --git a/flow/IThreadPool.cpp b/flow/IThreadPool.cpp index 362eee4598..6dc79f8c05 100644 --- a/flow/IThreadPool.cpp +++ b/flow/IThreadPool.cpp @@ -73,7 +73,7 @@ class ThreadPool : public IThreadPool, public ReferenceCounted { void operator()() { Thread::dispatch(action); action = NULL; } ~ActionWrapper() { if (action) { action->cancel(); } } private: - void operator=(ActionWrapper const&); + ActionWrapper &operator=(ActionWrapper const&); }; public: ThreadPool() : dontstop(ios), mode(Run) {} diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.actor.cpp index 73a336e38a..0b33550104 100644 --- a/flow/TLSConfig.actor.cpp +++ b/flow/TLSConfig.actor.cpp @@ -287,7 +287,7 @@ ACTOR static Future readEntireFile( std::string filename, std::string* des throw file_too_large(); } destination->resize(filesize); - wait(success(file->read(const_cast(destination->c_str()), filesize, 0))); + wait(success(file->read(&destination[0], filesize, 0))); return Void(); } diff --git a/flow/network.h b/flow/network.h index 92980c97f4..0ce7b7d5fb 100644 --- a/flow/network.h +++ b/flow/network.h @@ -235,6 +235,17 @@ struct NetworkAddress { bool isTLS() const { return (flags & FLAG_TLS) != 0; } bool isV6() const { return ip.isV6(); } + size_t hash() const { + size_t result = 0; + if (ip.isV6()) { + uint16_t* ptr = (uint16_t*)ip.toV6().data(); + result = ((size_t)ptr[5] << 32) | ((size_t)ptr[6] << 16) | ptr[7]; + } else { + result = ip.toV4(); + } + return (result << 16) + port; + } + static NetworkAddress parse(std::string const&); // May throw connection_string_invalid static Optional parseOptional(std::string const&); static std::vector parseList( std::string const& ); @@ -270,14 +281,7 @@ namespace std { size_t operator()(const NetworkAddress& na) const { - size_t result = 0; - if (na.ip.isV6()) { - uint16_t* ptr = (uint16_t*)na.ip.toV6().data(); - result = ((size_t)ptr[5] << 32) | ((size_t)ptr[6] << 16) | ptr[7]; - } else { - result = na.ip.toV4(); - } - return (result << 16) + na.port; + return na.hash(); } }; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d60859be79..c4e8697fb7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,6 +46,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES BlobStore.txt IGNORE) add_fdb_test(TEST_FILES ConsistencyCheck.txt IGNORE) add_fdb_test(TEST_FILES DDMetricsExclude.txt IGNORE) + add_fdb_test(TEST_FILES DataDistributionMetrics.txt IGNORE) add_fdb_test(TEST_FILES DiskDurability.txt IGNORE) add_fdb_test(TEST_FILES FileSystem.txt IGNORE) add_fdb_test(TEST_FILES Happy.txt IGNORE) diff --git a/tests/DataDistributionMetrics.txt b/tests/DataDistributionMetrics.txt new file mode 100644 index 0000000000..77c83b0eb6 --- /dev/null +++ b/tests/DataDistributionMetrics.txt @@ -0,0 +1,21 @@ +testTitle=DataDistributionMetrics + testName=Cycle + transactionsPerSecond=2500.0 + testDuration=10.0 + expectedRate=0.025 + + testName=DataDistributionMetrics + numTransactions=100 + writesPerTransaction=1000 + + testName=Attrition + machinesToKill=1 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + testName=Attrition + machinesToKill=1 + machinesToLeave=3 + reboot=true + testDuration=10.0 \ No newline at end of file From 8a98ea332099d5459e06e07f8aa9c67d164c5c07 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 23 May 2020 22:57:41 -0700 Subject: [PATCH 44/89] Changed non_flow_ref to flow_ref for clarity. Also updated comment to explain that std::is_trivially_copyable cannot be used to determine whether or not a type is a flow ref. --- fdbclient/FDBTypes.h | 2 +- flow/Arena.h | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 26934c77ca..2183581135 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -80,7 +80,7 @@ struct Tag { }; template <> -struct non_flow_ref : std::integral_constant {}; +struct flow_ref : std::integral_constant {}; #pragma pack(pop) diff --git a/flow/Arena.h b/flow/Arena.h index a8412bf6c5..01abf8dc32 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -700,16 +700,18 @@ inline bool operator >= ( const StringRef& lhs, const StringRef& rhs ) { return // This trait is used by VectorRef to determine if deep copy constructor should recursively // call deep copies of each element. -// TODO: There should be an easier way to identify the difference between -// flow_ref and non-flow_ref types. +// +// TODO: There should be an easier way to identify the difference between flow_ref and non-flow_ref types. +// std::is_trivially_copyable does not work because some flow_ref types are trivially copyable +// and some non-flow_ref types are not trivially copyable. template -struct non_flow_ref : std::is_fundamental {}; +struct flow_ref : std::integral_constant> {}; template <> -struct non_flow_ref : std::integral_constant {}; +struct flow_ref : std::integral_constant {}; template -struct non_flow_ref> : std::integral_constant {}; +struct flow_ref> : std::integral_constant {}; template struct string_serialized_traits : std::false_type { @@ -800,9 +802,9 @@ public: return *this; } - // Arena constructor for non-Ref types, identified by non_flow_ref + // Arena constructor for non-Ref types, identified by !flow_ref template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(toCopy), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { if (m_size > 0) { @@ -812,7 +814,7 @@ public: // Arena constructor for Ref types, which must have an Arena constructor template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { for (int i = 0; i < m_size; i++) { auto ptr = new (&data[i]) T(p, toCopy[i]); @@ -942,15 +944,15 @@ public: if (size > m_capacity) reallocate(p, size); } - // expectedSize() for non-Ref types, identified by non_flow_ref + // expectedSize() for non-Ref types, identified by !flow_ref template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { return sizeof(T) * m_size; } // expectedSize() for Ref types, which must in turn have expectedSize() implemented. template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { size_t t = sizeof(T) * m_size; for (int i = 0; i < m_size; i++) t += data[i].expectedSize(); return t; From b7160bab2dd1d53927deb6979929c00ccc810109 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 24 May 2020 13:24:52 -0700 Subject: [PATCH 45/89] Added -Woverloaded-virtual warning for clang, and fixed accidental overloads in flow.h --- cmake/ConfigureCompiler.cmake | 3 ++- flow/flow.h | 21 +++++++-------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index ddb2f38792..5e95f0328f 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -241,7 +241,8 @@ else() -Wno-delete-non-virtual-dtor -Wno-undefined-var-template -Wno-tautological-pointer-compare - -Wno-format) + -Wno-format + -Woverloaded-virtual) if (USE_CCACHE) add_compile_options( -Wno-register diff --git a/flow/flow.h b/flow/flow.h index 8890ad13b8..b58916ea44 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -560,10 +560,8 @@ public: cb->insertChain(this); } - virtual void unwait() { - delFutureRef(); - } - virtual void fire() { ASSERT(false); } + virtual void unwait() override { delFutureRef(); } + virtual void fire(T const&) override { ASSERT(false); } }; template @@ -644,10 +642,9 @@ struct NotifiedQueue : private SingleCallback, FastAllocated ASSERT(SingleCallback::next == this); cb->insert(this); } - virtual void unwait() { - delFutureRef(); - } - virtual void fire() { ASSERT(false); } + virtual void unwait() override { delFutureRef(); } + virtual void fire(T const&) override { ASSERT(false); } + virtual void fire(T&&) override { ASSERT(false); } }; @@ -1006,12 +1003,8 @@ struct Actor { template struct ActorCallback : Callback { - virtual void fire(ValueType const& value) { - static_cast(this)->a_callback_fire(this, value); - } - virtual void error(Error e) { - static_cast(this)->a_callback_error(this, e); - } + virtual void fire(ValueType const& value) override { static_cast(this)->a_callback_fire(this, value); } + virtual void error(Error e) override { static_cast(this)->a_callback_error(this, e); } }; template From 36ad1a95f4f65e20ac774940f291b6ce83bcbd0b Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Mon, 25 May 2020 12:11:52 -0700 Subject: [PATCH 46/89] Resolve conflicts when merge release-6.3 into master --- design/special-key-space.md | 8 -------- fdbcli/fdbcli.actor.cpp | 4 ---- fdbclient/SpecialKeySpace.actor.cpp | 10 ---------- fdbclient/SpecialKeySpace.actor.h | 7 ------- fdbclient/SystemData.cpp | 4 ---- fdbserver/BackupWorker.actor.cpp | 7 ------- fdbservice/FDBService.cpp | 5 ----- 7 files changed, 45 deletions(-) diff --git a/design/special-key-space.md b/design/special-key-space.md index d9fb328c65..c54cfb065f 100644 --- a/design/special-key-space.md +++ b/design/special-key-space.md @@ -88,19 +88,11 @@ We introduce this `module` concept after a [discussion](https://forums.foundatio - `\xff\xff/transaction/read_conflict_range/, \xff\xff/transaction/read_conflict_range0` : read conflict ranges of the transaction - `\xff\xff/transaction/write_conflict_range/, \xff\xff/transaction/write_conflict_range0` : write conflict ranges of the transaction - METRICS: `\xff\xff/metrics/, \xff\xff/metrics0`, all metrics like data-distribution metrics or healthy metrics are planned to put here. All need to call the rpc, so time_out error s may happen. Right now we have: -<<<<<<< HEAD - `\xff\xff/metrics/data_distribution_stats/, \xff\xff/metrics/data_distribution_stats0` : stats info about data-distribution -======= - - `\xff\xff/metrics/data_distribution_stats, \xff\xff/metrics/data_distribution_stats` : stats info about data-distribution ->>>>>>> master - WORKERINTERFACE : `\xff\xff/worker_interfaces/, \xff\xff/worker_interfaces0`, which is compatible with previous implementation, thus should not be used to add new functions. In addition, all singleKeyRanges are formatted as modules and cannot be used again. In particular, you should call `get` not `getRange` on these keys. Below are existing ones: - STATUSJSON : `\xff\xff/status/json` - CONNECTIONSTRING : `\xff\xff/connection_string` -<<<<<<< HEAD - CLUSTERFILEPATH : `\xff\xff/cluster_file_path` -======= -- CLUSTERFILEPATH : `\xff\xff/cluster_file_path` ->>>>>>> master diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 37ee20ec8c..1af54f9d48 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -2523,11 +2523,7 @@ void throttleGenerator(const char* text, const char *line, std::vector>>>>>> master const char* opts[] = { "auto", nullptr }; arrayGenerator(text, line, opts, lc); } diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 6950f8ca74..b23b5b8d87 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -316,12 +316,7 @@ Future> ConflictingKeysImpl::getRange(ReadYourWritesT return result; } -<<<<<<< HEAD ACTOR Future> ddStatsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) { -======= -ACTOR Future> ddStatsGetRangeActor(Reference ryw, - KeyRangeRef kr) { ->>>>>>> master try { auto keys = kr.removePrefix(ddStatsRange.begin); Standalone> resultWithoutPrefix = @@ -346,12 +341,7 @@ ACTOR Future> ddStatsGetRangeActor(Reference> DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { -======= -Future> DDStatsRangeImpl::getRange(Reference ryw, - KeyRangeRef kr) const { ->>>>>>> master return ddStatsGetRangeActor(ryw, kr); } diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 2c8242935f..a33ff666a4 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -154,12 +154,5 @@ public: Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; }; -class DDStatsRangeImpl : public SpecialKeyRangeBaseImpl { -public: - explicit DDStatsRangeImpl(KeyRangeRef kr); - Future> getRange(Reference ryw, - KeyRangeRef kr) const override; -}; - #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 1dc20884a5..e72b99dce7 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -47,11 +47,7 @@ const KeyRef keyServersKey( const KeyRef& k, Arena& arena ) { } const Value keyServersValue( Standalone result, const std::vector& src, const std::vector& dest ) { if(!CLIENT_KNOBS->TAG_ENCODE_KEY_SERVERS) { -<<<<<<< HEAD BinaryWriter wr(IncludeVersion(ProtocolVersion::withKeyServerValue())); wr << src << dest; -======= - BinaryWriter wr(IncludeVersion()); wr << src << dest; ->>>>>>> master return wr.toValue(); } diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index b07beb4cfe..2c38671eae 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -247,10 +247,7 @@ struct BackupData { specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; }); specialCounter(cc, "MsgQ", [this]() { return this->messages.size(); }); specialCounter(cc, "BufferedBytes", [this]() { return this->lock->activePermits(); }); -<<<<<<< HEAD specialCounter(cc, "AvailableBytes", [this]() { return this->lock->available(); }); -======= ->>>>>>> master logger = traceCounters("BackupWorkerMetrics", myId, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "BackupWorkerMetrics"); } @@ -848,11 +845,7 @@ ACTOR Future uploadData(BackupData* self) { } // If transition into NOOP mode, should clear messages -<<<<<<< HEAD if (!self->pulling && self->backupEpoch == self->recruitedEpoch) { -======= - if (!self->pulling) { ->>>>>>> master self->eraseMessages(self->messages.size()); } diff --git a/fdbservice/FDBService.cpp b/fdbservice/FDBService.cpp index 22e4ddc5ab..fe761a0109 100644 --- a/fdbservice/FDBService.cpp +++ b/fdbservice/FDBService.cpp @@ -28,13 +28,8 @@ #include #include -<<<<<<< HEAD -#include "..\flow\SimpleOpt.h" -#include "..\fdbmonitor\SimpleIni.h" -======= #include "flow/SimpleOpt.h" #include "fdbmonitor/SimpleIni.h" ->>>>>>> master #include "fdbclient/versions.h" // For PathFileExists From 856fb7b38fb22c2fcb5b9cb8d3f3f0610c2ab979 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Wed, 27 May 2020 09:46:20 -0700 Subject: [PATCH 47/89] Revert "Merge pull request #3197 from tclinken/optimize-deque-copy-ctor" This reverts commit e898cb7f38ba7a5ba501001c5dc748ded8bfa192, reversing changes made to 18c2c3d346c48fb62e532647f09e4c5d1987d427. --- flow/Deque.h | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/flow/Deque.h b/flow/Deque.h index 6148e00d0c..c5c05fb895 100644 --- a/flow/Deque.h +++ b/flow/Deque.h @@ -41,25 +41,21 @@ public: Deque() : arr(0), begin(0), end(0), mask(-1) {} // TODO: iterator construction, other constructors - Deque(Deque const& r) : arr(nullptr), begin(0), end(r.size()), mask(r.mask) { + Deque(Deque const& r) : arr(0), begin(0), end(r.size()), mask(r.mask) { if (r.capacity() > 0) { arr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)), capacity() * sizeof(T)); ASSERT(arr != nullptr); } ASSERT(capacity() >= end || end == 0); - if (r.end >= r.begin) { - std::copy(r.arr + r.begin, r.arr + r.begin + r.size(), arr); - } else { - auto partOneSize = r.capacity() - r.begin; - std::copy(r.arr + r.begin, r.arr + r.begin + partOneSize, arr); - std::copy(r.arr, r.arr + r.end, arr + partOneSize); - } + for (uint32_t i=0; i= end || end == 0); - if (r.end >= r.begin) { - std::copy(r.arr + r.begin, r.arr + r.begin + r.size(), arr); - } else { - auto partOneSize = r.capacity() - r.begin; - std::copy(r.arr + r.begin, r.arr + r.begin + partOneSize, arr); - std::copy(r.arr, r.arr + r.end, arr + partOneSize); - } + for (uint32_t i=0; i Date: Wed, 27 May 2020 19:51:17 -0700 Subject: [PATCH 48/89] Update api version to 700 --- bindings/bindingtester/__init__.py | 2 +- bindings/bindingtester/bindingtester.py | 2 +- bindings/bindingtester/known_testers.py | 2 +- bindings/bindingtester/tests/scripted.py | 2 +- bindings/c/fdb_c.cpp | 2 +- bindings/c/foundationdb/fdb_c.h | 6 +++--- bindings/c/test/mako/mako.h | 2 +- bindings/c/test/performance_test.c | 2 +- bindings/c/test/ryw_benchmark.c | 2 +- bindings/c/test/test.h | 2 +- bindings/c/test/txn_size_test.c | 2 +- bindings/c/test/workloads/SimpleWorkload.cpp | 4 ++-- bindings/flow/fdb_flow.actor.cpp | 4 ++-- bindings/flow/fdb_flow.h | 2 +- bindings/flow/tester/Tester.actor.cpp | 2 +- bindings/go/README.md | 2 +- bindings/go/src/fdb/cluster.go | 2 +- bindings/go/src/fdb/database.go | 2 +- bindings/go/src/fdb/doc.go | 2 +- bindings/go/src/fdb/errors.go | 2 +- bindings/go/src/fdb/fdb.go | 8 ++++---- bindings/go/src/fdb/futures.go | 2 +- bindings/go/src/fdb/range.go | 2 +- bindings/go/src/fdb/transaction.go | 2 +- bindings/java/JavaWorkload.cpp | 4 ++-- bindings/java/fdbJNI.cpp | 2 +- bindings/java/src/main/com/apple/foundationdb/FDB.java | 6 +++--- bindings/java/src/main/overview.html.in | 4 ++-- .../test/com/apple/foundationdb/test/AbstractTester.java | 2 +- .../com/apple/foundationdb/test/BlockingBenchmark.java | 2 +- .../com/apple/foundationdb/test/ConcurrentGetSetGet.java | 2 +- .../test/com/apple/foundationdb/test/DirectoryTest.java | 2 +- .../src/test/com/apple/foundationdb/test/Example.java | 2 +- .../test/com/apple/foundationdb/test/IterableTest.java | 2 +- .../test/com/apple/foundationdb/test/LocalityTests.java | 2 +- .../com/apple/foundationdb/test/ParallelRandomScan.java | 2 +- .../src/test/com/apple/foundationdb/test/RangeTest.java | 2 +- .../test/com/apple/foundationdb/test/SerialInsertion.java | 2 +- .../test/com/apple/foundationdb/test/SerialIteration.java | 2 +- .../src/test/com/apple/foundationdb/test/SerialTest.java | 2 +- .../apple/foundationdb/test/SnapshotTransactionTest.java | 2 +- .../src/test/com/apple/foundationdb/test/TupleTest.java | 2 +- .../apple/foundationdb/test/VersionstampSmokeTest.java | 2 +- .../src/test/com/apple/foundationdb/test/WatchTest.java | 2 +- bindings/python/fdb/__init__.py | 2 +- bindings/python/fdb/impl.py | 2 +- bindings/python/tests/size_limit_tests.py | 2 +- bindings/ruby/lib/fdb.rb | 2 +- build/cmake/package_tester/fdb_c_app/app.c | 4 ++-- build/cmake/package_tester/modules/tests.sh | 2 +- documentation/sphinx/source/api-c.rst | 2 +- documentation/sphinx/source/api-common.rst.inc | 2 +- documentation/sphinx/source/api-python.rst | 2 +- documentation/sphinx/source/api-ruby.rst | 2 +- documentation/sphinx/source/api-version-upgrade-guide.rst | 4 ++-- documentation/sphinx/source/class-scheduling-go.rst | 6 +++--- documentation/sphinx/source/class-scheduling-java.rst | 6 +++--- documentation/sphinx/source/class-scheduling-ruby.rst | 6 +++--- documentation/sphinx/source/class-scheduling.rst | 8 ++++---- .../sphinx/source/hierarchical-documents-java.rst | 2 +- documentation/sphinx/source/multimaps-java.rst | 2 +- documentation/sphinx/source/priority-queues-java.rst | 2 +- documentation/sphinx/source/queues-java.rst | 2 +- documentation/sphinx/source/release-notes.rst | 2 +- documentation/sphinx/source/simple-indexes-java.rst | 2 +- documentation/sphinx/source/tables-java.rst | 2 +- documentation/sphinx/source/vector-java.rst | 2 +- fdbclient/MultiVersionTransaction.actor.cpp | 2 +- recipes/go-recipes/blob.go | 2 +- recipes/go-recipes/doc.go | 2 +- recipes/go-recipes/graph.go | 2 +- recipes/go-recipes/indirect.go | 2 +- recipes/go-recipes/multi.go | 2 +- recipes/go-recipes/priority.go | 2 +- recipes/go-recipes/queue.go | 2 +- recipes/go-recipes/table.go | 2 +- 76 files changed, 98 insertions(+), 98 deletions(-) diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py index 0adababb92..f8ad0030e2 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -26,7 +26,7 @@ sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', import util -FDB_API_VERSION = 630 +FDB_API_VERSION = 700 LOGGING = { 'version': 1, diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index 6feed3b283..58db70f5db 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -157,7 +157,7 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers api_version = min_version elif random.random() < 0.9: api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, - 440, 450, 460, 500, 510, 520, 600, 610, 620, 630] if v >= min_version and v <= max_version]) + 440, 450, 460, 500, 510, 520, 600, 610, 620, 630, 700] if v >= min_version and v <= max_version]) else: api_version = random.randint(min_version, max_version) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index ee82663411..e1522039db 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -20,7 +20,7 @@ import os -MAX_API_VERSION = 630 +MAX_API_VERSION = 700 COMMON_TYPES = ['null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple'] ALL_TYPES = COMMON_TYPES + ['versionstamp'] diff --git a/bindings/bindingtester/tests/scripted.py b/bindings/bindingtester/tests/scripted.py index 60b1959864..c113ebc07f 100644 --- a/bindings/bindingtester/tests/scripted.py +++ b/bindings/bindingtester/tests/scripted.py @@ -34,7 +34,7 @@ fdb.api_version(FDB_API_VERSION) class ScriptedTest(Test): - TEST_API_VERSION = 630 + TEST_API_VERSION = 700 def __init__(self, subspace): super(ScriptedTest, self).__init__(subspace, ScriptedTest.TEST_API_VERSION, ScriptedTest.TEST_API_VERSION) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index dae60c7ea8..ba56744dc7 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #define FDB_INCLUDE_LEGACY_TYPES #include "fdbclient/MultiVersionTransaction.h" diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index a930434819..b5dfa63d13 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -28,10 +28,10 @@ #endif #if !defined(FDB_API_VERSION) -#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 630) +#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 700) #elif FDB_API_VERSION < 13 #error API version no longer supported (upgrade to 13) -#elif FDB_API_VERSION > 630 +#elif FDB_API_VERSION > 700 #error Requested API version requires a newer version of this header #endif @@ -91,7 +91,7 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter); #pragma pack(push, 4) -#if FDB_API_VERSION >= 630 +#if FDB_API_VERSION >= 700 typedef struct keyvalue { const uint8_t* key; int key_length; diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index 4f703e7271..792dd1d6dc 100644 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -3,7 +3,7 @@ #pragma once #ifndef FDB_API_VERSION -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #endif #include diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 7a265e7d0f..319895554d 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -603,7 +603,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index cbb7fcf304..4604dc41a8 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -244,7 +244,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, keySize); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 5fb4268b78..7169689f76 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -29,7 +29,7 @@ #include #ifndef FDB_API_VERSION -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #endif #include diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index 4f2744d199..7aa5a18576 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -97,7 +97,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, KEY_SIZE); diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 35b18f71a3..7b19654874 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include "foundationdb/fdb_c.h" #undef DLLEXPORT #include "workloads.h" @@ -258,7 +258,7 @@ struct SimpleWorkload : FDBWorkload { insertsPerTx = context->getOption("insertsPerTx", 100ul); opsPerTx = context->getOption("opsPerTx", 100ul); runFor = context->getOption("runFor", 10.0); - auto err = fdb_select_api_version(630); + auto err = fdb_select_api_version(700); if (err) { context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 3ed3d93700..5fd56e653d 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -36,7 +36,7 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); @@ -79,7 +79,7 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index e261052fae..66049cae0c 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -23,7 +23,7 @@ #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include #undef DLLEXPORT diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index a190299747..578f159f8c 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -1817,7 +1817,7 @@ ACTOR void _test_versionstamp() { try { g_network = newNet2(TLSConfig()); - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/go/README.md b/bindings/go/README.md index 7a03ea1d6f..8619e1692a 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -9,7 +9,7 @@ This package requires: - [Mono](http://www.mono-project.com/) (macOS or Linux) or [Visual Studio](https://www.visualstudio.com/) (Windows) (build-time only) - FoundationDB C API 2.0.x-6.1.x (part of the [FoundationDB client packages](https://apple.github.io/foundationdb/downloads.html#c)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-630. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-700. To install this package, you can run the "fdb-go-install.sh" script (for versions 5.0.x and greater): diff --git a/bindings/go/src/fdb/cluster.go b/bindings/go/src/fdb/cluster.go index df895e9a51..5ab17b5273 100644 --- a/bindings/go/src/fdb/cluster.go +++ b/bindings/go/src/fdb/cluster.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go index 23cb4f19be..60f3f03d06 100644 --- a/bindings/go/src/fdb/database.go +++ b/bindings/go/src/fdb/database.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go index 5cfb157ad6..e1759701ff 100644 --- a/bindings/go/src/fdb/doc.go +++ b/bindings/go/src/fdb/doc.go @@ -46,7 +46,7 @@ A basic interaction with the FoundationDB API is demonstrated below: func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) // Open the default database from the system cluster db := fdb.MustOpenDefault() diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go index 94e699c89e..9c9f75b566 100644 --- a/bindings/go/src/fdb/errors.go +++ b/bindings/go/src/fdb/errors.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go index d0bfd5f699..bc05a05dba 100644 --- a/bindings/go/src/fdb/fdb.go +++ b/bindings/go/src/fdb/fdb.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include // #include import "C" @@ -108,7 +108,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // library, an error will be returned. APIVersion must be called prior to any // other functions in the fdb package. // -// Currently, this package supports API versions 200 through 630. +// Currently, this package supports API versions 200 through 700. // // Warning: When using the multi-version client API, setting an API version that // is not supported by a particular client library will prevent that client from @@ -116,7 +116,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // the API version of your application after upgrading your client until the // cluster has also been upgraded. func APIVersion(version int) error { - headerVersion := 630 + headerVersion := 700 networkMutex.Lock() defer networkMutex.Unlock() @@ -128,7 +128,7 @@ func APIVersion(version int) error { return errAPIVersionAlreadySet } - if version < 200 || version > 630 { + if version < 200 || version > 700 { return errAPIVersionNotSupported } diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 17ae1d70a4..43718fe738 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -23,7 +23,7 @@ package fdb // #cgo LDFLAGS: -lfdb_c -lm -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include // #include // diff --git a/bindings/go/src/fdb/range.go b/bindings/go/src/fdb/range.go index 67a45c63b2..584f23cb2b 100644 --- a/bindings/go/src/fdb/range.go +++ b/bindings/go/src/fdb/range.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index 4102a0556b..6bd198b0da 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 808485486b..e47208b6e6 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -19,7 +19,7 @@ */ #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include #include @@ -370,7 +370,7 @@ struct JVM { jmethodID selectMethod = env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(I)Lcom/apple/foundationdb/FDB;"); checkException(); - auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(630)); + auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(700)); checkException(); env->CallObjectMethod(fdbInstance, getMethod(fdbClass, "disableShutdownHook", "()V")); checkException(); diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 938ac498f3..a127a47864 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -21,7 +21,7 @@ #include #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java index ba96814cef..b945a5dc69 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDB.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This call is required before using any other part of the API. The call allows * an error to be thrown at this point to prevent client code from accessing a later library * with incorrect assumptions from the current version. The API version documented here is version - * {@code 630}.

+ * {@code 700}.

* FoundationDB encapsulates multiple versions of its interface by requiring * the client to explicitly specify the version of the API it uses. The purpose * of this design is to allow you to upgrade the server, client libraries, or @@ -181,8 +181,8 @@ public class FDB { } if(version < 510) throw new IllegalArgumentException("API version not supported (minimum 510)"); - if(version > 630) - throw new IllegalArgumentException("API version not supported (maximum 630)"); + if(version > 700) + throw new IllegalArgumentException("API version not supported (maximum 700)"); Select_API_version(version); singleton = new FDB(version); diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in index fd7c6ac80d..adaedd1a03 100644 --- a/bindings/java/src/main/overview.html.in +++ b/bindings/java/src/main/overview.html.in @@ -13,7 +13,7 @@ and then added to your classpath.

Getting started

To start using FoundationDB from Java, create an instance of the {@link com.apple.foundationdb.FDB FoundationDB API interface} with the version of the -API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 630}). +API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 700}). With this API object you can then open {@link com.apple.foundationdb.Cluster Cluster}s and {@link com.apple.foundationdb.Database Database}s and start using {@link com.apple.foundationdb.Transaction Transaction}s. @@ -29,7 +29,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java index 3a153e3582..e27e80b082 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java @@ -27,7 +27,7 @@ import com.apple.foundationdb.Database; import com.apple.foundationdb.FDB; public abstract class AbstractTester { - public static final int API_VERSION = 630; + public static final int API_VERSION = 700; protected static final int NUM_RUNS = 25; protected static final Charset ASCII = Charset.forName("ASCII"); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java index f21aabeb6a..68f7d74a95 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java @@ -33,7 +33,7 @@ public class BlockingBenchmark { private static final int PARALLEL = 100; public static void main(String[] args) throws InterruptedException { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); // The cluster file DOES NOT need to be valid, although it must exist. // This is because the database is never really contacted in this test. diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java index 53f13695c1..bddfd6f57d 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java @@ -48,7 +48,7 @@ public class ConcurrentGetSetGet { } public static void main(String[] args) { - try(Database database = FDB.selectAPIVersion(630).open()) { + try(Database database = FDB.selectAPIVersion(700).open()) { new ConcurrentGetSetGet().apply(database); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java index c43dd71809..9f838d8eeb 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java @@ -33,7 +33,7 @@ import com.apple.foundationdb.directory.DirectorySubspace; public class DirectoryTest { public static void main(String[] args) throws Exception { try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Example.java b/bindings/java/src/test/com/apple/foundationdb/test/Example.java index 74090eccc0..44e9087b3e 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Example.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Example.java @@ -26,7 +26,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java index aca9e918d2..ce1f623f4c 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java @@ -31,7 +31,7 @@ public class IterableTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java index 70f688e46a..29abab1471 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.tuple.ByteArrayUtil; public class LocalityTests { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database database = fdb.open(args[0])) { try(Transaction tr = database.createTransaction()) { String[] keyAddresses = LocalityUtil.getAddressesForKey(tr, "a".getBytes()).join(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java index 014f1f038d..624566964a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java @@ -43,7 +43,7 @@ public class ParallelRandomScan { private static final int PARALLELISM_STEP = 5; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open(args[0])) { for(int i = PARALLELISM_MIN; i <= PARALLELISM_MAX; i += PARALLELISM_STEP) { runTest(database, i, ROWS, DURATION_MS); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java index 3a99c68d56..81365ff44f 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncIterable; public class RangeTest { - private static final int API_VERSION = 630; + private static final int API_VERSION = 700; public static void main(String[] args) { System.out.println("About to use version " + API_VERSION); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java index f873e954e1..8ad91314c2 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java @@ -34,7 +34,7 @@ public class SerialInsertion { private static final int NODES = 1000000; public static void main(String[] args) { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open()) { long start = System.currentTimeMillis(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java index cbcc2d713a..db63999daa 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java @@ -39,7 +39,7 @@ public class SerialIteration { private static final int THREAD_COUNT = 1; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open(args[0])) { for(int i = 1; i <= THREAD_COUNT; i++) { runThreadedTest(database, i); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java index 2aad1eb1bb..df084d564f 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java @@ -30,7 +30,7 @@ public class SerialTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java index d324463408..78de1ae3db 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java @@ -39,7 +39,7 @@ public class SnapshotTransactionTest { private static final Subspace SUBSPACE = new Subspace(Tuple.from("test", "conflict_ranges")); public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { snapshotReadShouldNotConflict(db); snapshotShouldNotAddConflictRange(db); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java index c7aa190ce7..066cf43383 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java @@ -50,7 +50,7 @@ public class TupleTest { public static void main(String[] args) throws NoSuchFieldException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); addMethods(); comparisons(); emptyTuple(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java index 12bef587d2..e50bc9c031 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java @@ -32,7 +32,7 @@ import com.apple.foundationdb.tuple.Versionstamp; public class VersionstampSmokeTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { db.run(tr -> { tr.clear(Tuple.from("prefix").range()); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java index b204e842e5..14c0aa1d43 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; public class WatchTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database database = fdb.open(args[0])) { database.options().setLocationCacheSize(42); try(Transaction tr = database.createTransaction()) { diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 0d54c96b5f..c969b6c70c 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -52,7 +52,7 @@ def get_api_version(): def api_version(ver): - header_version = 630 + header_version = 700 if '_version' in globals(): if globals()['_version'] != ver: diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 3dd5e87077..91bdc2f3a0 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -253,7 +253,7 @@ def transactional(*tr_args, **tr_kwargs): @functools.wraps(func) def wrapper(*args, **kwargs): # We can't throw this from the decorator, as when a user runs - # >>> import fdb ; fdb.api_version(630) + # >>> import fdb ; fdb.api_version(700) # the code above uses @transactional before the API version is set if fdb.get_api_version() >= 630 and inspect.isgeneratorfunction(func): raise ValueError("Generators can not be wrapped with fdb.transactional") diff --git a/bindings/python/tests/size_limit_tests.py b/bindings/python/tests/size_limit_tests.py index 446f787bc1..756d9422e0 100644 --- a/bindings/python/tests/size_limit_tests.py +++ b/bindings/python/tests/size_limit_tests.py @@ -22,7 +22,7 @@ import fdb import sys if __name__ == '__main__': - fdb.api_version(630) + fdb.api_version(700) @fdb.transactional def setValue(tr, key, value): diff --git a/bindings/ruby/lib/fdb.rb b/bindings/ruby/lib/fdb.rb index b1b72d38d7..df8448ea0b 100644 --- a/bindings/ruby/lib/fdb.rb +++ b/bindings/ruby/lib/fdb.rb @@ -36,7 +36,7 @@ module FDB end end def self.api_version(version) - header_version = 630 + header_version = 700 if self.is_api_version_selected?() if @@chosen_version != version raise "FDB API already loaded at version #{@@chosen_version}." diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c index a15c1193e7..f26b2513c1 100644 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ b/build/cmake/package_tester/fdb_c_app/app.c @@ -1,7 +1,7 @@ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include int main(int argc, char* argv[]) { - fdb_select_api_version(630); + fdb_select_api_version(700); return 0; } diff --git a/build/cmake/package_tester/modules/tests.sh b/build/cmake/package_tester/modules/tests.sh index 88709a7953..35ff098a6f 100644 --- a/build/cmake/package_tester/modules/tests.sh +++ b/build/cmake/package_tester/modules/tests.sh @@ -65,7 +65,7 @@ then python setup.py install successOr "Installing python bindings failed" popd - python -c 'import fdb; fdb.api_version(630)' + python -c 'import fdb; fdb.api_version(700)' successOr "Loading python bindings failed" # Test cmake and pkg-config integration: https://github.com/apple/foundationdb/issues/1483 diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index 5c7cdd2c5d..40482d3b0b 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -133,7 +133,7 @@ API versioning Prior to including ``fdb_c.h``, you must define the ``FDB_API_VERSION`` macro. This, together with the :func:`fdb_select_api_version()` function, allows programs written against an older version of the API to compile and run with newer versions of the C library. The current version of the FoundationDB C API is |api-version|. :: - #define FDB_API_VERSION 630 + #define FDB_API_VERSION 700 #include .. function:: fdb_error_t fdb_select_api_version(int version) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 6bce920a45..6ab190a052 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -147,7 +147,7 @@ .. |atomic-versionstamps-tuple-warning-value| replace:: At this time, versionstamped values are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. -.. |api-version| replace:: 630 +.. |api-version| replace:: 700 .. |streaming-mode-blurb1| replace:: When using |get-range-func| and similar interfaces, API clients can request large ranges of the database to iterate over. Making such a request doesn't necessarily mean that the client will consume all of the data in the range - sometimes the client doesn't know how far it intends to iterate in advance. FoundationDB tries to balance latency and bandwidth by requesting data for iteration in batches. diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index fb75d3516d..69c6dab28a 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -108,7 +108,7 @@ Opening a database After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() .. function:: open( cluster_file=None, event_model=None ) diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index df73cf6dc4..84078b02d4 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -93,7 +93,7 @@ Opening a database After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: require 'fdb' - FDB.api_version 630 + FDB.api_version 700 db = FDB.open .. function:: open( cluster_file=nil ) -> Database diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst index 16fa55100a..12643cc082 100644 --- a/documentation/sphinx/source/api-version-upgrade-guide.rst +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -9,9 +9,9 @@ This document provides an overview of changes that an application developer may For more details about API versions, see :ref:`api-versions`. -.. _api-version-upgrade-guide-630: +.. _api-version-upgrade-guide-700: -API version 630 +API version 700 =============== General diff --git a/documentation/sphinx/source/class-scheduling-go.rst b/documentation/sphinx/source/class-scheduling-go.rst index d8ea0a5b19..77d9c01e90 100644 --- a/documentation/sphinx/source/class-scheduling-go.rst +++ b/documentation/sphinx/source/class-scheduling-go.rst @@ -29,7 +29,7 @@ Before using the API, we need to specify the API version. This allows programs t .. code-block:: go - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. @@ -78,7 +78,7 @@ If this is all working, it looks like we are ready to start building a real appl func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -666,7 +666,7 @@ Here's the code for the scheduling tutorial: } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() db.Options().SetTransactionTimeout(60000) // 60,000 ms = 1 minute db.Options().SetTransactionRetryLimit(100) diff --git a/documentation/sphinx/source/class-scheduling-java.rst b/documentation/sphinx/source/class-scheduling-java.rst index c899c546dc..c5dda17d55 100644 --- a/documentation/sphinx/source/class-scheduling-java.rst +++ b/documentation/sphinx/source/class-scheduling-java.rst @@ -30,7 +30,7 @@ Before using the API, we need to specify the API version. This allows programs t private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); } @@ -66,7 +66,7 @@ If this is all working, it looks like we are ready to start building a real appl private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); } @@ -441,7 +441,7 @@ Here's the code for the scheduling tutorial: private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); db.options().setTransactionTimeout(60000); // 60,000 ms = 1 minute db.options().setTransactionRetryLimit(100); diff --git a/documentation/sphinx/source/class-scheduling-ruby.rst b/documentation/sphinx/source/class-scheduling-ruby.rst index d1f79c3725..c8d8483aad 100644 --- a/documentation/sphinx/source/class-scheduling-ruby.rst +++ b/documentation/sphinx/source/class-scheduling-ruby.rst @@ -23,7 +23,7 @@ Open a Ruby interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - > FDB.api_version 630 + > FDB.api_version 700 => nil Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -46,7 +46,7 @@ If this is all working, it looks like we are ready to start building a real appl .. code-block:: ruby require 'fdb' - FDB.api_version 630 + FDB.api_version 700 @db = FDB.open @db['hello'] = 'world' print 'hello ', @db['hello'] @@ -373,7 +373,7 @@ Here's the code for the scheduling tutorial: require 'fdb' - FDB.api_version 630 + FDB.api_version 700 #################################### ## Initialization ## diff --git a/documentation/sphinx/source/class-scheduling.rst b/documentation/sphinx/source/class-scheduling.rst index b516bc9f7c..23615a08a6 100644 --- a/documentation/sphinx/source/class-scheduling.rst +++ b/documentation/sphinx/source/class-scheduling.rst @@ -30,7 +30,7 @@ Open a Python interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - >>> fdb.api_version(630) + >>> fdb.api_version(700) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -48,7 +48,7 @@ When this command returns without exception, the modification is durably stored If this is all working, it looks like we are ready to start building a real application. For reference, here's the full code for "hello world":: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() db[b'hello'] = b'world' print 'hello', db[b'hello'] @@ -91,7 +91,7 @@ FoundationDB includes a few tools that make it easy to model data using this app opening a :ref:`directory ` in the database:: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() scheduling = fdb.directory.create_or_open(db, ('scheduling',)) @@ -337,7 +337,7 @@ Here's the code for the scheduling tutorial:: import fdb import fdb.tuple - fdb.api_version(630) + fdb.api_version(700) #################################### diff --git a/documentation/sphinx/source/hierarchical-documents-java.rst b/documentation/sphinx/source/hierarchical-documents-java.rst index c2631e5b36..db33abd4ef 100644 --- a/documentation/sphinx/source/hierarchical-documents-java.rst +++ b/documentation/sphinx/source/hierarchical-documents-java.rst @@ -69,7 +69,7 @@ Here’s a basic implementation of the recipe. private static final long EMPTY_ARRAY = -1; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); docSpace = new Subspace(Tuple.from("D")); } diff --git a/documentation/sphinx/source/multimaps-java.rst b/documentation/sphinx/source/multimaps-java.rst index 4ce8e1f3ba..3c9a46ad3c 100644 --- a/documentation/sphinx/source/multimaps-java.rst +++ b/documentation/sphinx/source/multimaps-java.rst @@ -74,7 +74,7 @@ Here’s a simple implementation of multimaps with multisets as described: private static final int N = 100; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); multi = new Subspace(Tuple.from("M")); } diff --git a/documentation/sphinx/source/priority-queues-java.rst b/documentation/sphinx/source/priority-queues-java.rst index 068349d680..0fafb08b4b 100644 --- a/documentation/sphinx/source/priority-queues-java.rst +++ b/documentation/sphinx/source/priority-queues-java.rst @@ -74,7 +74,7 @@ Here's a basic implementation of the model: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); pq = new Subspace(Tuple.from("P")); diff --git a/documentation/sphinx/source/queues-java.rst b/documentation/sphinx/source/queues-java.rst index 1ed636146d..b4b60df48b 100644 --- a/documentation/sphinx/source/queues-java.rst +++ b/documentation/sphinx/source/queues-java.rst @@ -73,7 +73,7 @@ The following is a simple implementation of the basic pattern: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); queue = new Subspace(Tuple.from("Q")); randno = new Random(); diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 2297176944..c8ee3f42b5 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -67,7 +67,7 @@ Status Bindings -------- -* API version updated to 630. See the :ref:`API version upgrade guide ` for upgrade details. +* API version updated to 700. See the :ref:`API version upgrade guide ` for upgrade details. * Python: The ``@fdb.transactional`` decorator will now throw an error if the decorated function returns a generator. `(PR #1724) `_ * Java: Add caching for various JNI objects to improve performance. `(PR #2809) `_ * Java: Optimize byte array comparisons in ``ByteArrayUtil``. `(PR #2823) `_ diff --git a/documentation/sphinx/source/simple-indexes-java.rst b/documentation/sphinx/source/simple-indexes-java.rst index 709bc4bc7c..c5edf02e71 100644 --- a/documentation/sphinx/source/simple-indexes-java.rst +++ b/documentation/sphinx/source/simple-indexes-java.rst @@ -87,7 +87,7 @@ In this example, we’re storing user data based on user ID but sometimes need t private static final Subspace index; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); main = new Subspace(Tuple.from("user")); index = new Subspace(Tuple.from("zipcode_index")); diff --git a/documentation/sphinx/source/tables-java.rst b/documentation/sphinx/source/tables-java.rst index 0f13cebd65..235dbd5b47 100644 --- a/documentation/sphinx/source/tables-java.rst +++ b/documentation/sphinx/source/tables-java.rst @@ -62,7 +62,7 @@ Here’s a simple implementation of the basic table pattern: private static final Subspace colIndex; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); table = new Subspace(Tuple.from("T")); rowIndex = table.subspace(Tuple.from("R")); diff --git a/documentation/sphinx/source/vector-java.rst b/documentation/sphinx/source/vector-java.rst index 254ca26cc2..17da6ebed8 100644 --- a/documentation/sphinx/source/vector-java.rst +++ b/documentation/sphinx/source/vector-java.rst @@ -77,7 +77,7 @@ Here’s the basic pattern: private static final Subspace vector; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); vector = new Subspace(Tuple.from("V")); } diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index bb1ef53260..40c8616d12 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -321,7 +321,7 @@ void DLApi::init() { loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset"); loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel"); loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); - loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); + loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 700); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); diff --git a/recipes/go-recipes/blob.go b/recipes/go-recipes/blob.go index d3b9a2b052..2ac8681803 100644 --- a/recipes/go-recipes/blob.go +++ b/recipes/go-recipes/blob.go @@ -78,7 +78,7 @@ func read_blob(t fdb.ReadTransactor, blob_subspace subspace.Subspace) ([]byte, e } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/doc.go b/recipes/go-recipes/doc.go index 418040f84f..5595f3b799 100644 --- a/recipes/go-recipes/doc.go +++ b/recipes/go-recipes/doc.go @@ -219,7 +219,7 @@ func (doc Doc) GetDoc(trtr fdb.Transactor, doc_id int) interface{} { } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/graph.go b/recipes/go-recipes/graph.go index 371ea9b1a2..966b3e5c5f 100644 --- a/recipes/go-recipes/graph.go +++ b/recipes/go-recipes/graph.go @@ -124,7 +124,7 @@ func (graph *Graph) get_in_neighbors(trtr fdb.Transactor, node int) ([]int, erro } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/indirect.go b/recipes/go-recipes/indirect.go index 4945b67b50..e354a1af2f 100644 --- a/recipes/go-recipes/indirect.go +++ b/recipes/go-recipes/indirect.go @@ -93,7 +93,7 @@ func (wrkspc Workspace) Session(foo func(directory.DirectorySubspace)) (err erro } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/multi.go b/recipes/go-recipes/multi.go index 4fa06a8622..58cbfd2ba1 100644 --- a/recipes/go-recipes/multi.go +++ b/recipes/go-recipes/multi.go @@ -132,7 +132,7 @@ func (multi MultiMap) MultiIsElement(trtr fdb.Transactor, index, value interface func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/priority.go b/recipes/go-recipes/priority.go index 3ad3762f79..b4f455716a 100644 --- a/recipes/go-recipes/priority.go +++ b/recipes/go-recipes/priority.go @@ -117,7 +117,7 @@ func (prty Priority) Peek(trtr fdb.Transactor, max bool) interface{} { } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/queue.go b/recipes/go-recipes/queue.go index 3028e5ff79..6e6c1cee69 100644 --- a/recipes/go-recipes/queue.go +++ b/recipes/go-recipes/queue.go @@ -107,7 +107,7 @@ func (q *Queue) FirstItem(trtr fdb.Transactor) (interface{}, error) { func main() { fmt.Println("Queue Example Program") - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() diff --git a/recipes/go-recipes/table.go b/recipes/go-recipes/table.go index 50272df14b..b037699b95 100644 --- a/recipes/go-recipes/table.go +++ b/recipes/go-recipes/table.go @@ -144,7 +144,7 @@ func (tbl Table) TableGetCol(tr fdb.ReadTransactor, col int) ([]interface{}, err } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() From 7bcabbe4be9d44b020e828ea2ffd20d2ba336fce Mon Sep 17 00:00:00 2001 From: Balachandar Namasivayam Date: Thu, 28 May 2020 12:21:28 -0700 Subject: [PATCH 49/89] Addressed review comments and update go bindings version. --- bindings/go/src/fdb/fdb_test.go | 12 ++++++------ .../sphinx/source/api-version-upgrade-guide.rst | 4 ++++ documentation/sphinx/source/release-notes.rst | 2 +- fdbclient/MultiVersionTransaction.actor.cpp | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go index 7bcd588de8..e455dba473 100644 --- a/bindings/go/src/fdb/fdb_test.go +++ b/bindings/go/src/fdb/fdb_test.go @@ -32,7 +32,7 @@ import ( func ExampleOpenDefault() { var e error - e = fdb.APIVersion(630) + e = fdb.APIVersion(700) if e != nil { fmt.Printf("Unable to set API version: %v\n", e) return @@ -52,7 +52,7 @@ func ExampleOpenDefault() { } func TestVersionstamp(t *testing.T) { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() setVs := func(t fdb.Transactor, key fdb.Key) (fdb.FutureKey, error) { @@ -98,7 +98,7 @@ func TestVersionstamp(t *testing.T) { } func ExampleTransactor() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() setOne := func(t fdb.Transactor, key fdb.Key, value []byte) error { @@ -149,7 +149,7 @@ func ExampleTransactor() { } func ExampleReadTransactor() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() getOne := func(rt fdb.ReadTransactor, key fdb.Key) ([]byte, error) { @@ -202,7 +202,7 @@ func ExampleReadTransactor() { } func ExamplePrefixRange() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() tr, e := db.CreateTransaction() @@ -241,7 +241,7 @@ func ExamplePrefixRange() { } func ExampleRangeIterator() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() tr, e := db.CreateTransaction() diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst index 12643cc082..7c6e298352 100644 --- a/documentation/sphinx/source/api-version-upgrade-guide.rst +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -14,6 +14,10 @@ For more details about API versions, see :ref:`api-versions`. API version 700 =============== + +API version 630 +=============== + General ------- diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index c8ee3f42b5..2297176944 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -67,7 +67,7 @@ Status Bindings -------- -* API version updated to 700. See the :ref:`API version upgrade guide ` for upgrade details. +* API version updated to 630. See the :ref:`API version upgrade guide ` for upgrade details. * Python: The ``@fdb.transactional`` decorator will now throw an error if the decorated function returns a generator. `(PR #1724) `_ * Java: Add caching for various JNI objects to improve performance. `(PR #2809) `_ * Java: Optimize byte array comparisons in ``ByteArrayUtil``. `(PR #2823) `_ diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 40c8616d12..bb1ef53260 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -321,7 +321,7 @@ void DLApi::init() { loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset"); loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel"); loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); - loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 700); + loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); From a8bfd62f834b78987f5d82c580d5b8f44c0792eb Mon Sep 17 00:00:00 2001 From: Balachandar Namasivayam Date: Thu, 28 May 2020 12:32:55 -0700 Subject: [PATCH 50/89] Fix mistake. --- documentation/sphinx/source/api-version-upgrade-guide.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst index 7c6e298352..16fa55100a 100644 --- a/documentation/sphinx/source/api-version-upgrade-guide.rst +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -9,11 +9,7 @@ This document provides an overview of changes that an application developer may For more details about API versions, see :ref:`api-versions`. -.. _api-version-upgrade-guide-700: - -API version 700 -=============== - +.. _api-version-upgrade-guide-630: API version 630 =============== From 3108102f26637e3fdd22bd0acd923d0e06cc8f7c Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Sun, 31 May 2020 14:15:12 -0700 Subject: [PATCH 51/89] Fix a backup progress true-up bug Sometimes, the true-up has to go backup multiple epochs for saved versions, because a tag's progress can be missing in an epoch. In other words, we need to check progress for all tags. --- fdbserver/BackupProgress.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp index 898ce31b70..037c0bb6e3 100644 --- a/fdbserver/BackupProgress.actor.cpp +++ b/fdbserver/BackupProgress.actor.cpp @@ -115,7 +115,7 @@ std::map, std::map> BackupProgr // ASSERT(info.logRouterTags == epochTags[rit->first]); updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, adjustedBeginVersion, epoch); - break; + if (tags.empty()) break; } rit++; } From b39cdc4633d4765448199ab1bb4b47697c122bf1 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Fri, 24 Apr 2020 15:37:26 -0700 Subject: [PATCH 52/89] check-in unmodified rte_memcpy.h from dpdk 19.11 --- flow/rte_memcpy.h | 876 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 876 insertions(+) create mode 100644 flow/rte_memcpy.h diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h new file mode 100644 index 0000000000..ba44c4a328 --- /dev/null +++ b/flow/rte_memcpy.h @@ -0,0 +1,876 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation + */ + +#ifndef _RTE_MEMCPY_X86_64_H_ +#define _RTE_MEMCPY_X86_64_H_ + +/** + * @file + * + * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy(). + */ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Copy bytes from one location to another. The locations must not overlap. + * + * @note This is implemented as a macro, so it's address should not be taken + * and care is needed as parameter expressions may be evaluated multiple times. + * + * @param dst + * Pointer to the destination of the data. + * @param src + * Pointer to the source data. + * @param n + * Number of bytes to copy. + * @return + * Pointer to the destination data. + */ +static __rte_always_inline void * +rte_memcpy(void *dst, const void *src, size_t n); + +#ifdef RTE_MACHINE_CPUFLAG_AVX512F + +#define ALIGNMENT_MASK 0x3F + +/** + * AVX512 implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + __m256i ymm0; + + ymm0 = _mm256_loadu_si256((const __m256i *)src); + _mm256_storeu_si256((__m256i *)dst, ymm0); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + __m512i zmm0; + + zmm0 = _mm512_loadu_si512((const void *)src); + _mm512_storeu_si512((void *)dst, zmm0); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov64(dst + 0 * 64, src + 0 * 64); + rte_mov64(dst + 1 * 64, src + 1 * 64); +} + +/** + * Copy 256 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov256(uint8_t *dst, const uint8_t *src) +{ + rte_mov64(dst + 0 * 64, src + 0 * 64); + rte_mov64(dst + 1 * 64, src + 1 * 64); + rte_mov64(dst + 2 * 64, src + 2 * 64); + rte_mov64(dst + 3 * 64, src + 3 * 64); +} + +/** + * Copy 128-byte blocks from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m512i zmm0, zmm1; + + while (n >= 128) { + zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64)); + n -= 128; + zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64)); + src = src + 128; + _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0); + _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1); + dst = dst + 128; + } +} + +/** + * Copy 512-byte blocks from one location to another, + * locations should not overlap. + */ +static inline void +rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m512i zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7; + + while (n >= 512) { + zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64)); + n -= 512; + zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64)); + zmm2 = _mm512_loadu_si512((const void *)(src + 2 * 64)); + zmm3 = _mm512_loadu_si512((const void *)(src + 3 * 64)); + zmm4 = _mm512_loadu_si512((const void *)(src + 4 * 64)); + zmm5 = _mm512_loadu_si512((const void *)(src + 5 * 64)); + zmm6 = _mm512_loadu_si512((const void *)(src + 6 * 64)); + zmm7 = _mm512_loadu_si512((const void *)(src + 7 * 64)); + src = src + 512; + _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0); + _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1); + _mm512_storeu_si512((void *)(dst + 2 * 64), zmm2); + _mm512_storeu_si512((void *)(dst + 3 * 64), zmm3); + _mm512_storeu_si512((void *)(dst + 4 * 64), zmm4); + _mm512_storeu_si512((void *)(dst + 5 * 64), zmm5); + _mm512_storeu_si512((void *)(dst + 6 * 64), zmm6); + _mm512_storeu_si512((void *)(dst + 7 * 64), zmm7); + dst = dst + 512; + } +} + +static __rte_always_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t bits; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) + *(uint64_t *)dstu = *(const uint64_t *)srcu; + return ret; + } + + /** + * Fast way when copy size doesn't exceed 512 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n <= 512) { + if (n >= 256) { + n -= 256; + rte_mov256((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 256; + dst = (uint8_t *)dst + 256; + } + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK63: + if (n > 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + return ret; + } + if (n > 0) + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + return ret; + } + + /** + * Make store aligned when copy size exceeds 512 bytes + */ + dstofss = ((uintptr_t)dst & 0x3F); + if (dstofss > 0) { + dstofss = 64 - dstofss; + n -= dstofss; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + + /** + * Copy 512-byte blocks. + * Use copy block function for better instruction order control, + * which is important when load is unaligned. + */ + rte_mov512blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 511; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + + /** + * Copy 128-byte blocks. + * Use copy block function for better instruction order control, + * which is important when load is unaligned. + */ + if (n >= 128) { + rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 127; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + } + + /** + * Copy whatever left + */ + goto COPY_BLOCK_128_BACK63; +} + +#elif defined RTE_MACHINE_CPUFLAG_AVX2 + +#define ALIGNMENT_MASK 0x1F + +/** + * AVX2 implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + __m256i ymm0; + + ymm0 = _mm256_loadu_si256((const __m256i *)src); + _mm256_storeu_si256((__m256i *)dst, ymm0); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); + rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); + rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32); + rte_mov32((uint8_t *)dst + 2 * 32, (const uint8_t *)src + 2 * 32); + rte_mov32((uint8_t *)dst + 3 * 32, (const uint8_t *)src + 3 * 32); +} + +/** + * Copy 128-byte blocks from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m256i ymm0, ymm1, ymm2, ymm3; + + while (n >= 128) { + ymm0 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 0 * 32)); + n -= 128; + ymm1 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 1 * 32)); + ymm2 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 2 * 32)); + ymm3 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 3 * 32)); + src = (const uint8_t *)src + 128; + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 0 * 32), ymm0); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 1 * 32), ymm1); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 2 * 32), ymm2); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 3 * 32), ymm3); + dst = (uint8_t *)dst + 128; + } +} + +static __rte_always_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t bits; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) { + *(uint64_t *)dstu = *(const uint64_t *)srcu; + } + return ret; + } + + /** + * Fast way when copy size doesn't exceed 256 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 48) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst + 16, (const uint8_t *)src + 16); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n <= 256) { + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK31: + if (n >= 64) { + n -= 64; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 64; + dst = (uint8_t *)dst + 64; + } + if (n > 32) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n > 0) { + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + } + return ret; + } + + /** + * Make store aligned when copy size exceeds 256 bytes + */ + dstofss = (uintptr_t)dst & 0x1F; + if (dstofss > 0) { + dstofss = 32 - dstofss; + n -= dstofss; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + + /** + * Copy 128-byte blocks + */ + rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 127; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + + /** + * Copy whatever left + */ + goto COPY_BLOCK_128_BACK31; +} + +#else /* RTE_MACHINE_CPUFLAG */ + +#define ALIGNMENT_MASK 0x0F + +/** + * SSE & AVX implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)(const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static __rte_always_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); + rte_mov16((uint8_t *)dst + 4 * 16, (const uint8_t *)src + 4 * 16); + rte_mov16((uint8_t *)dst + 5 * 16, (const uint8_t *)src + 5 * 16); + rte_mov16((uint8_t *)dst + 6 * 16, (const uint8_t *)src + 6 * 16); + rte_mov16((uint8_t *)dst + 7 * 16, (const uint8_t *)src + 7 * 16); +} + +/** + * Copy 256 bytes from one location to another, + * locations should not overlap. + */ +static inline void +rte_mov256(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); + rte_mov16((uint8_t *)dst + 4 * 16, (const uint8_t *)src + 4 * 16); + rte_mov16((uint8_t *)dst + 5 * 16, (const uint8_t *)src + 5 * 16); + rte_mov16((uint8_t *)dst + 6 * 16, (const uint8_t *)src + 6 * 16); + rte_mov16((uint8_t *)dst + 7 * 16, (const uint8_t *)src + 7 * 16); + rte_mov16((uint8_t *)dst + 8 * 16, (const uint8_t *)src + 8 * 16); + rte_mov16((uint8_t *)dst + 9 * 16, (const uint8_t *)src + 9 * 16); + rte_mov16((uint8_t *)dst + 10 * 16, (const uint8_t *)src + 10 * 16); + rte_mov16((uint8_t *)dst + 11 * 16, (const uint8_t *)src + 11 * 16); + rte_mov16((uint8_t *)dst + 12 * 16, (const uint8_t *)src + 12 * 16); + rte_mov16((uint8_t *)dst + 13 * 16, (const uint8_t *)src + 13 * 16); + rte_mov16((uint8_t *)dst + 14 * 16, (const uint8_t *)src + 14 * 16); + rte_mov16((uint8_t *)dst + 15 * 16, (const uint8_t *)src + 15 * 16); +} + +/** + * Macro for copying unaligned block from one location to another with constant load offset, + * 47 bytes leftover maximum, + * locations should not overlap. + * Requirements: + * - Store is aligned + * - Load offset is , which must be immediate value within [1, 15] + * - For , make sure bit backwards & <16 - offset> bit forwards are available for loading + * - , , must be variables + * - __m128i ~ must be pre-defined + */ +#define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \ +__extension__ ({ \ + size_t tmp; \ + while (len >= 128 + 16 - offset) { \ + xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ + len -= 128; \ + xmm1 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 1 * 16)); \ + xmm2 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 2 * 16)); \ + xmm3 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 3 * 16)); \ + xmm4 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 4 * 16)); \ + xmm5 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 5 * 16)); \ + xmm6 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 6 * 16)); \ + xmm7 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 7 * 16)); \ + xmm8 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 8 * 16)); \ + src = (const uint8_t *)src + 128; \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 2 * 16), _mm_alignr_epi8(xmm3, xmm2, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 3 * 16), _mm_alignr_epi8(xmm4, xmm3, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 4 * 16), _mm_alignr_epi8(xmm5, xmm4, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 5 * 16), _mm_alignr_epi8(xmm6, xmm5, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 6 * 16), _mm_alignr_epi8(xmm7, xmm6, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 7 * 16), _mm_alignr_epi8(xmm8, xmm7, offset)); \ + dst = (uint8_t *)dst + 128; \ + } \ + tmp = len; \ + len = ((len - 16 + offset) & 127) + 16 - offset; \ + tmp -= len; \ + src = (const uint8_t *)src + tmp; \ + dst = (uint8_t *)dst + tmp; \ + if (len >= 32 + 16 - offset) { \ + while (len >= 32 + 16 - offset) { \ + xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ + len -= 32; \ + xmm1 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 1 * 16)); \ + xmm2 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 2 * 16)); \ + src = (const uint8_t *)src + 32; \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \ + dst = (uint8_t *)dst + 32; \ + } \ + tmp = len; \ + len = ((len - 16 + offset) & 31) + 16 - offset; \ + tmp -= len; \ + src = (const uint8_t *)src + tmp; \ + dst = (uint8_t *)dst + tmp; \ + } \ +}) + +/** + * Macro for copying unaligned block from one location to another, + * 47 bytes leftover maximum, + * locations should not overlap. + * Use switch here because the aligning instruction requires immediate value for shift count. + * Requirements: + * - Store is aligned + * - Load offset is , which must be within [1, 15] + * - For , make sure bit backwards & <16 - offset> bit forwards are available for loading + * - , , must be variables + * - __m128i ~ used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined + */ +#define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \ +__extension__ ({ \ + switch (offset) { \ + case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \ + case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \ + case 0x03: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x03); break; \ + case 0x04: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x04); break; \ + case 0x05: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x05); break; \ + case 0x06: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x06); break; \ + case 0x07: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x07); break; \ + case 0x08: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x08); break; \ + case 0x09: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x09); break; \ + case 0x0A: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0A); break; \ + case 0x0B: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0B); break; \ + case 0x0C: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0C); break; \ + case 0x0D: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0D); break; \ + case 0x0E: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0E); break; \ + case 0x0F: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0F); break; \ + default:; \ + } \ +}) + +static __rte_always_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t srcofs; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) { + *(uint64_t *)dstu = *(const uint64_t *)srcu; + } + return ret; + } + + /** + * Fast way when copy size doesn't exceed 512 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 48) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst + 32, (const uint8_t *)src + 32); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 128) { + goto COPY_BLOCK_128_BACK15; + } + if (n <= 512) { + if (n >= 256) { + n -= 256; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128); + src = (const uint8_t *)src + 256; + dst = (uint8_t *)dst + 256; + } +COPY_BLOCK_255_BACK15: + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK15: + if (n >= 64) { + n -= 64; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 64; + dst = (uint8_t *)dst + 64; + } +COPY_BLOCK_64_BACK15: + if (n >= 32) { + n -= 32; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 32; + dst = (uint8_t *)dst + 32; + } + if (n > 16) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n > 0) { + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + } + return ret; + } + + /** + * Make store aligned when copy size exceeds 512 bytes, + * and make sure the first 15 bytes are copied, because + * unaligned copy functions require up to 15 bytes + * backwards access. + */ + dstofss = (uintptr_t)dst & 0x0F; + if (dstofss > 0) { + dstofss = 16 - dstofss + 16; + n -= dstofss; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + srcofs = ((uintptr_t)src & 0x0F); + + /** + * For aligned copy + */ + if (srcofs == 0) { + /** + * Copy 256-byte blocks + */ + for (; n >= 256; n -= 256) { + rte_mov256((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 256; + src = (const uint8_t *)src + 256; + } + + /** + * Copy whatever left + */ + goto COPY_BLOCK_255_BACK15; + } + + /** + * For copy with unaligned load + */ + MOVEUNALIGNED_LEFT47(dst, src, n, srcofs); + + /** + * Copy whatever left + */ + goto COPY_BLOCK_64_BACK15; +} + +#endif /* RTE_MACHINE_CPUFLAG */ + +static __rte_always_inline void * +rte_memcpy_aligned(void *dst, const void *src, size_t n) +{ + void *ret = dst; + + /* Copy size <= 16 bytes */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dst = *(const uint8_t *)src; + src = (const uint8_t *)src + 1; + dst = (uint8_t *)dst + 1; + } + if (n & 0x02) { + *(uint16_t *)dst = *(const uint16_t *)src; + src = (const uint16_t *)src + 1; + dst = (uint16_t *)dst + 1; + } + if (n & 0x04) { + *(uint32_t *)dst = *(const uint32_t *)src; + src = (const uint32_t *)src + 1; + dst = (uint32_t *)dst + 1; + } + if (n & 0x08) + *(uint64_t *)dst = *(const uint64_t *)src; + + return ret; + } + + /* Copy 16 <= size <= 32 bytes */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + + return ret; + } + + /* Copy 32 < size <= 64 bytes */ + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + + return ret; + } + + /* Copy 64 bytes blocks */ + for (; n >= 64; n -= 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; + } + + /* Copy whatever left */ + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + + return ret; +} + +static __rte_always_inline void * +rte_memcpy(void *dst, const void *src, size_t n) +{ + if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK)) + return rte_memcpy_aligned(dst, src, n); + else + return rte_memcpy_generic(dst, src, n); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMCPY_X86_64_H_ */ From 89e6c2f57a12e54a7c7fcb0114dcabe33dadcff1 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Fri, 24 Apr 2020 15:39:52 -0700 Subject: [PATCH 53/89] copy relevant DPDK license into rte_memcpy.h header --- flow/rte_memcpy.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index ba44c4a328..764e81c5d3 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -1,5 +1,16 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2014 Intel Corporation +/* +SPDX-License-Identifier: BSD-3-Clause +Copyright(c) 2010-2014 Intel Corporation + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _RTE_MEMCPY_X86_64_H_ From a910fa9ac75db6e321b9fb4717b45d656f4c5e9a Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Fri, 24 Apr 2020 15:44:27 -0700 Subject: [PATCH 54/89] memcpy tests from dpdk --- flow/test_memcpy.c | 133 +++++++++++++++ flow/test_memcpy_perf.c | 352 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 485 insertions(+) create mode 100644 flow/test_memcpy.c create mode 100644 flow/test_memcpy_perf.c diff --git a/flow/test_memcpy.c b/flow/test_memcpy.c new file mode 100644 index 0000000000..2c69ad9647 --- /dev/null +++ b/flow/test_memcpy.c @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "test.h" + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, + 256, 257, 320, 384, 511, 512, 513, 1023, 1024, 1025, 1518, 1522, 1600, + 2048, 3072, 4096, 5120, 6144, 7168, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + +/* Data is aligned on this many bytes (power of 2) */ +#define ALIGNMENT_UNIT 32 + + +/* + * Create two buffers, and initialise one with random values. These are copied + * to the second buffer and then compared to see if the copy was successful. + * The bytes outside the copied area are also checked to make sure they were not + * changed. + */ +static int +test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) +{ + unsigned int i; + uint8_t dest[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + uint8_t src[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + void * ret; + + /* Setup buffers */ + for (i = 0; i < SMALL_BUFFER_SIZE + ALIGNMENT_UNIT; i++) { + dest[i] = 0; + src[i] = (uint8_t) rte_rand(); + } + + /* Do the copy */ + ret = rte_memcpy(dest + off_dst, src + off_src, size); + if (ret != (dest + off_dst)) { + printf("rte_memcpy() returned %p, not %p\n", + ret, dest + off_dst); + } + + /* Check nothing before offset is affected */ + for (i = 0; i < off_dst; i++) { + if (dest[i] != 0) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[modified before start of dst].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + + /* Check everything was copied */ + for (i = 0; i < size; i++) { + if (dest[i + off_dst] != src[i + off_src]) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[didn't copy byte %u].\n", + (unsigned)size, off_src, off_dst, i); + return -1; + } + } + + /* Check nothing after copy was affected */ + for (i = size; i < SMALL_BUFFER_SIZE; i++) { + if (dest[i + off_dst] != 0) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[copied too many].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + return 0; +} + +/* + * Check functionality for various buffer sizes and data offsets/alignments. + */ +static int +func_test(void) +{ + unsigned int off_src, off_dst, i; + unsigned int num_buf_sizes = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + int ret; + + for (off_src = 0; off_src < ALIGNMENT_UNIT; off_src++) { + for (off_dst = 0; off_dst < ALIGNMENT_UNIT; off_dst++) { + for (i = 0; i < num_buf_sizes; i++) { + ret = test_single_memcpy(off_src, off_dst, + buf_sizes[i]); + if (ret != 0) + return -1; + } + } + } + return 0; +} + +static int +test_memcpy(void) +{ + int ret; + + ret = func_test(); + if (ret != 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(memcpy_autotest, test_memcpy); diff --git a/flow/test_memcpy_perf.c b/flow/test_memcpy_perf.c new file mode 100644 index 0000000000..6f436f3ef3 --- /dev/null +++ b/flow/test_memcpy_perf.c @@ -0,0 +1,352 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "test.h" + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, + 129, 191, 192, 193, 255, 256, 257, 319, 320, 321, 383, 384, 385, 447, 448, + 449, 511, 512, 513, 767, 768, 769, 1023, 1024, 1025, 1518, 1522, 1536, 1600, + 2048, 2560, 3072, 3584, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + + +/* + * Arrays of this size are used for measuring uncached memory accesses by + * picking a random location within the buffer. Make this smaller if there are + * memory allocation errors. + */ +#define LARGE_BUFFER_SIZE (100 * 1024 * 1024) + +/* How many times to run timing loop for performance tests */ +#define TEST_ITERATIONS 1000000 +#define TEST_BATCH_SIZE 100 + +/* Data is aligned on this many bytes (power of 2) */ +#ifdef RTE_MACHINE_CPUFLAG_AVX512F +#define ALIGNMENT_UNIT 64 +#elif defined RTE_MACHINE_CPUFLAG_AVX2 +#define ALIGNMENT_UNIT 32 +#else /* RTE_MACHINE_CPUFLAG */ +#define ALIGNMENT_UNIT 16 +#endif /* RTE_MACHINE_CPUFLAG */ + +/* + * Pointers used in performance tests. The two large buffers are for uncached + * access where random addresses within the buffer are used for each + * memcpy. The two small buffers are for cached access. + */ +static uint8_t *large_buf_read, *large_buf_write; +static uint8_t *small_buf_read, *small_buf_write; + +/* Initialise data buffers. */ +static int +init_buffers(void) +{ + unsigned i; + + large_buf_read = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_read == NULL) + goto error_large_buf_read; + + large_buf_write = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_write == NULL) + goto error_large_buf_write; + + small_buf_read = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_read == NULL) + goto error_small_buf_read; + + small_buf_write = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_write == NULL) + goto error_small_buf_write; + + for (i = 0; i < LARGE_BUFFER_SIZE; i++) + large_buf_read[i] = rte_rand(); + for (i = 0; i < SMALL_BUFFER_SIZE; i++) + small_buf_read[i] = rte_rand(); + + return 0; + +error_small_buf_write: + rte_free(small_buf_read); +error_small_buf_read: + rte_free(large_buf_write); +error_large_buf_write: + rte_free(large_buf_read); +error_large_buf_read: + printf("ERROR: not enough memory\n"); + return -1; +} + +/* Cleanup data buffers */ +static void +free_buffers(void) +{ + rte_free(large_buf_read); + rte_free(large_buf_write); + rte_free(small_buf_read); + rte_free(small_buf_write); +} + +/* + * Get a random offset into large array, with enough space needed to perform + * max copy size. Offset is aligned, uoffset is used for unalignment setting. + */ +static inline size_t +get_rand_offset(size_t uoffset) +{ + return ((rte_rand() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & + ~(ALIGNMENT_UNIT - 1)) + uoffset; +} + +/* Fill in source and destination addresses. */ +static inline void +fill_addr_arrays(size_t *dst_addr, int is_dst_cached, size_t dst_uoffset, + size_t *src_addr, int is_src_cached, size_t src_uoffset) +{ + unsigned int i; + + for (i = 0; i < TEST_BATCH_SIZE; i++) { + dst_addr[i] = (is_dst_cached) ? dst_uoffset : get_rand_offset(dst_uoffset); + src_addr[i] = (is_src_cached) ? src_uoffset : get_rand_offset(src_uoffset); + } +} + +/* + * WORKAROUND: For some reason the first test doing an uncached write + * takes a very long time (~25 times longer than is expected). So we do + * it once without timing. + */ +static void +do_uncached_write(uint8_t *dst, int is_dst_cached, + const uint8_t *src, int is_src_cached, size_t size) +{ + unsigned i, j; + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; + + for (i = 0; i < (TEST_ITERATIONS / TEST_BATCH_SIZE); i++) { + fill_addr_arrays(dst_addrs, is_dst_cached, 0, + src_addrs, is_src_cached, 0); + for (j = 0; j < TEST_BATCH_SIZE; j++) { + rte_memcpy(dst+dst_addrs[j], src+src_addrs[j], size); + } + } +} + +/* + * Run a single memcpy performance test. This is a macro to ensure that if + * the "size" parameter is a constant it won't be converted to a variable. + */ +#define SINGLE_PERF_TEST(dst, is_dst_cached, dst_uoffset, \ + src, is_src_cached, src_uoffset, size) \ +do { \ + unsigned int iter, t; \ + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; \ + uint64_t start_time, total_time = 0; \ + uint64_t total_time2 = 0; \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + rte_memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time += rte_rdtsc() - start_time; \ + } \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time2 += rte_rdtsc() - start_time; \ + } \ + printf("%3.0f -", (double)total_time / TEST_ITERATIONS); \ + printf("%3.0f", (double)total_time2 / TEST_ITERATIONS); \ + printf("(%6.2f%%) ", ((double)total_time - total_time2)*100/total_time2); \ +} while (0) + +/* Run aligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, large_buf_read, 0, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, large_buf_read, 0, 0, n); \ +} while (0) + +/* Run unaligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE_UNALIGNED(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, large_buf_read, 0, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, large_buf_read, 0, 5, n); \ +} while (0) + +/* Run memcpy tests for constant length */ +#define ALL_PERF_TEST_FOR_CONSTANT \ +do { \ + TEST_CONSTANT(6U); TEST_CONSTANT(64U); TEST_CONSTANT(128U); \ + TEST_CONSTANT(192U); TEST_CONSTANT(256U); TEST_CONSTANT(512U); \ + TEST_CONSTANT(768U); TEST_CONSTANT(1024U); TEST_CONSTANT(1536U); \ +} while (0) + +/* Run all memcpy tests for aligned constant cases */ +static inline void +perf_test_constant_aligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for unaligned constant cases */ +static inline void +perf_test_constant_unaligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE_UNALIGNED + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for aligned variable cases */ +static inline void +perf_test_variable_aligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests for unaligned variable cases */ +static inline void +perf_test_variable_unaligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE_UNALIGNED((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests */ +static int +perf_test(void) +{ + int ret; + struct timeval tv_begin, tv_end; + double time_aligned, time_unaligned; + double time_aligned_const, time_unaligned_const; + + ret = init_buffers(); + if (ret != 0) + return ret; + +#if TEST_VALUE_RANGE != 0 + /* Set up buf_sizes array, if required */ + unsigned i; + for (i = 0; i < TEST_VALUE_RANGE; i++) + buf_sizes[i] = i; +#endif + + /* See function comment */ + do_uncached_write(large_buf_write, 0, small_buf_read, 1, SMALL_BUFFER_SIZE); + + printf("\n** rte_memcpy() - memcpy perf. tests (C = compile-time constant) **\n" + "======= ================= ================= ================= =================\n" + " Size Cache to cache Cache to mem Mem to cache Mem to mem\n" + "(bytes) (ticks) (ticks) (ticks) (ticks)\n" + "------- ----------------- ----------------- ----------------- -----------------"); + + printf("\n================================= %2dB aligned =================================", + ALIGNMENT_UNIT); + /* Do aligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do aligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n================================== Unaligned =================================="); + /* Do unaligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do unaligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n======= ================= ================= ================= =================\n\n"); + + printf("Test Execution Time (seconds):\n"); + printf("Aligned variable copy size = %8.3f\n", time_aligned); + printf("Aligned constant copy size = %8.3f\n", time_aligned_const); + printf("Unaligned variable copy size = %8.3f\n", time_unaligned); + printf("Unaligned constant copy size = %8.3f\n", time_unaligned_const); + free_buffers(); + + return 0; +} + +static int +test_memcpy_perf(void) +{ + int ret; + + ret = perf_test(); + if (ret != 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(memcpy_perf_autotest, test_memcpy_perf); From 678b57c0d9c2279e6cbc9ceb2be5d6fe423b468f Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Mon, 27 Apr 2020 11:00:46 -0700 Subject: [PATCH 55/89] port rte_memcpy to flow; add -mavx compiler flag --- cmake/ConfigureCompiler.cmake | 2 +- fdbserver/workloads/UnitTests.actor.cpp | 4 ++ flow/CMakeLists.txt | 5 +- flow/rte_memcpy.h | 64 ++++++++++++------- flow/{test_memcpy.c => test_memcpy.cpp} | 31 +++------ ...est_memcpy_perf.c => test_memcpy_perf.cpp} | 53 +++++++-------- 6 files changed, 81 insertions(+), 78 deletions(-) rename flow/{test_memcpy.c => test_memcpy.cpp} (89%) rename flow/{test_memcpy_perf.c => test_memcpy_perf.cpp} (94%) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 5e95f0328f..5be685b34f 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -254,7 +254,7 @@ else() endif() if (GCC) add_compile_options(-Wno-pragmas) - + add_compile_options(-mavx) # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 91692fd6eb..479ac8c7cc 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -26,6 +26,8 @@ void forceLinkIndexedSetTests(); void forceLinkDequeTests(); void forceLinkFlowTests(); void forceLinkVersionedMapTests(); +void forceLinkMemcpyTests(); +void forceLinkMemcpyPerfTests(); struct UnitTestWorkload : TestWorkload { bool enabled; @@ -45,6 +47,8 @@ struct UnitTestWorkload : TestWorkload { forceLinkDequeTests(); forceLinkFlowTests(); forceLinkVersionedMapTests(); + forceLinkMemcpyTests(); + forceLinkMemcpyPerfTests(); } virtual std::string description() { return "UnitTests"; } diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 6c3b8eca24..080d483edd 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -67,7 +67,7 @@ set(FLOW_SRCS XmlTraceLogFormatter.cpp XmlTraceLogFormatter.h actorcompiler.h - crc32c.h + crc32c.h crc32c.cpp error_definitions.h ${CMAKE_CURRENT_BINARY_DIR}/SourceVersion.h @@ -79,10 +79,13 @@ set(FLOW_SRCS genericactors.actor.h network.cpp network.h + rte_memcpy.h serialize.cpp serialize.h stacktrace.amalgamation.cpp stacktrace.h + test_memcpy.cpp + test_memcpy_perf.cpp version.cpp) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/SourceVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/SourceVersion.h) diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index 764e81c5d3..ba0ab3cf6c 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -25,9 +25,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #include #include #include -#include -#include -#include + +#include #ifdef __cplusplus extern "C" { @@ -48,7 +47,7 @@ extern "C" { * @return * Pointer to the destination data. */ -static __rte_always_inline void * +static force_inline void * rte_memcpy(void *dst, const void *src, size_t n); #ifdef RTE_MACHINE_CPUFLAG_AVX512F @@ -63,7 +62,7 @@ rte_memcpy(void *dst, const void *src, size_t n); * Copy 16 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov16(uint8_t *dst, const uint8_t *src) { __m128i xmm0; @@ -76,7 +75,7 @@ rte_mov16(uint8_t *dst, const uint8_t *src) * Copy 32 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov32(uint8_t *dst, const uint8_t *src) { __m256i ymm0; @@ -89,7 +88,7 @@ rte_mov32(uint8_t *dst, const uint8_t *src) * Copy 64 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov64(uint8_t *dst, const uint8_t *src) { __m512i zmm0; @@ -102,7 +101,7 @@ rte_mov64(uint8_t *dst, const uint8_t *src) * Copy 128 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov128(uint8_t *dst, const uint8_t *src) { rte_mov64(dst + 0 * 64, src + 0 * 64); @@ -113,7 +112,7 @@ rte_mov128(uint8_t *dst, const uint8_t *src) * Copy 256 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov256(uint8_t *dst, const uint8_t *src) { rte_mov64(dst + 0 * 64, src + 0 * 64); @@ -126,7 +125,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * Copy 128-byte blocks from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) { __m512i zmm0, zmm1; @@ -174,7 +173,7 @@ rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n) } } -static __rte_always_inline void * +static force_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; @@ -304,7 +303,7 @@ COPY_BLOCK_128_BACK63: * Copy 16 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov16(uint8_t *dst, const uint8_t *src) { __m128i xmm0; @@ -317,7 +316,7 @@ rte_mov16(uint8_t *dst, const uint8_t *src) * Copy 32 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov32(uint8_t *dst, const uint8_t *src) { __m256i ymm0; @@ -330,7 +329,7 @@ rte_mov32(uint8_t *dst, const uint8_t *src) * Copy 64 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov64(uint8_t *dst, const uint8_t *src) { rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); @@ -341,7 +340,7 @@ rte_mov64(uint8_t *dst, const uint8_t *src) * Copy 128 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov128(uint8_t *dst, const uint8_t *src) { rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); @@ -354,7 +353,7 @@ rte_mov128(uint8_t *dst, const uint8_t *src) * Copy 128-byte blocks from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) { __m256i ymm0, ymm1, ymm2, ymm3; @@ -374,7 +373,7 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) } } -static __rte_always_inline void * +static force_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; @@ -497,7 +496,7 @@ COPY_BLOCK_128_BACK31: * Copy 16 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov16(uint8_t *dst, const uint8_t *src) { __m128i xmm0; @@ -510,7 +509,7 @@ rte_mov16(uint8_t *dst, const uint8_t *src) * Copy 32 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov32(uint8_t *dst, const uint8_t *src) { rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); @@ -521,7 +520,7 @@ rte_mov32(uint8_t *dst, const uint8_t *src) * Copy 64 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov64(uint8_t *dst, const uint8_t *src) { rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); @@ -534,7 +533,7 @@ rte_mov64(uint8_t *dst, const uint8_t *src) * Copy 128 bytes from one location to another, * locations should not overlap. */ -static __rte_always_inline void +static force_inline void rte_mov128(uint8_t *dst, const uint8_t *src) { rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); @@ -666,7 +665,7 @@ __extension__ ({ \ } \ }) -static __rte_always_inline void * +static force_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; @@ -811,7 +810,7 @@ COPY_BLOCK_64_BACK15: #endif /* RTE_MACHINE_CPUFLAG */ -static __rte_always_inline void * +static force_inline void * rte_memcpy_aligned(void *dst, const void *src, size_t n) { void *ret = dst; @@ -871,7 +870,7 @@ rte_memcpy_aligned(void *dst, const void *src, size_t n) return ret; } -static __rte_always_inline void * +static force_inline void * rte_memcpy(void *dst, const void *src, size_t n) { if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK)) @@ -880,6 +879,23 @@ rte_memcpy(void *dst, const void *src, size_t n) return rte_memcpy_generic(dst, src, n); } +static inline uint64_t +rte_rdtsc(void) +{ + union { + uint64_t tsc_64; + struct { + uint32_t lo_32; + uint32_t hi_32; + }; + } tsc; + + asm volatile("rdtsc" : + "=a" (tsc.lo_32), + "=d" (tsc.hi_32)); + return tsc.tsc_64; +} + #ifdef __cplusplus } #endif diff --git a/flow/test_memcpy.c b/flow/test_memcpy.cpp similarity index 89% rename from flow/test_memcpy.c rename to flow/test_memcpy.cpp index 2c69ad9647..19ab66e35b 100644 --- a/flow/test_memcpy.c +++ b/flow/test_memcpy.cpp @@ -7,11 +7,10 @@ #include #include -#include -#include -#include +#include "flow/rte_memcpy.h" +#include "flow/IRandom.h" -#include "test.h" +#include "flow/UnitTest.h" /* * Set this to the maximum buffer size you want to test. If it is 0, then the @@ -54,7 +53,7 @@ test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) /* Setup buffers */ for (i = 0; i < SMALL_BUFFER_SIZE + ALIGNMENT_UNIT; i++) { dest[i] = 0; - src[i] = (uint8_t) rte_rand(); + src[i] = (uint8_t) deterministicRandom()->randomUInt32(); } /* Do the copy */ @@ -99,9 +98,7 @@ test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) /* * Check functionality for various buffer sizes and data offsets/alignments. */ -static int -func_test(void) -{ +TEST_CASE("/rte/memcpy") { unsigned int off_src, off_dst, i; unsigned int num_buf_sizes = sizeof(buf_sizes) / sizeof(buf_sizes[0]); int ret; @@ -111,23 +108,11 @@ func_test(void) for (i = 0; i < num_buf_sizes; i++) { ret = test_single_memcpy(off_src, off_dst, buf_sizes[i]); - if (ret != 0) - return -1; + ASSERT(ret == 0); } } } - return 0; + return Void(); } -static int -test_memcpy(void) -{ - int ret; - - ret = func_test(); - if (ret != 0) - return -1; - return 0; -} - -REGISTER_TEST_COMMAND(memcpy_autotest, test_memcpy); +void forceLinkMemcpyTests() { } \ No newline at end of file diff --git a/flow/test_memcpy_perf.c b/flow/test_memcpy_perf.cpp similarity index 94% rename from flow/test_memcpy_perf.c rename to flow/test_memcpy_perf.cpp index 6f436f3ef3..b2750b3585 100644 --- a/flow/test_memcpy_perf.c +++ b/flow/test_memcpy_perf.cpp @@ -8,14 +8,9 @@ #include #include -#include -#include -#include -#include - -#include - -#include "test.h" +#include "flow/rte_memcpy.h" +#include "flow/IRandom.h" +#include "flow/UnitTest.h" /* * Set this to the maximum buffer size you want to test. If it is 0, then the @@ -67,6 +62,20 @@ static size_t buf_sizes[TEST_VALUE_RANGE]; static uint8_t *large_buf_read, *large_buf_write; static uint8_t *small_buf_read, *small_buf_write; +static size_t round_up(size_t sz, size_t alignment) { + return (((sz - 1) / alignment) + 1) * alignment; +} + +static uint8_t * rte_malloc(char const * ignored, size_t sz, size_t align) { + return (uint8_t*) aligned_alloc(align, round_up(sz, align)); +} + +static void rte_free(void * ptr) { + if (!!ptr) { + free(ptr); + } +} + /* Initialise data buffers. */ static int init_buffers(void) @@ -90,9 +99,9 @@ init_buffers(void) goto error_small_buf_write; for (i = 0; i < LARGE_BUFFER_SIZE; i++) - large_buf_read[i] = rte_rand(); + large_buf_read[i] = deterministicRandom()->randomUInt32(); for (i = 0; i < SMALL_BUFFER_SIZE; i++) - small_buf_read[i] = rte_rand(); + small_buf_read[i] = deterministicRandom()->randomUInt32(); return 0; @@ -124,7 +133,7 @@ free_buffers(void) static inline size_t get_rand_offset(size_t uoffset) { - return ((rte_rand() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & + return ((deterministicRandom()->randomUInt32() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & ~(ALIGNMENT_UNIT - 1)) + uoffset; } @@ -269,17 +278,14 @@ perf_test_variable_unaligned(void) } /* Run all memcpy tests */ -static int -perf_test(void) -{ +TEST_CASE("performance/memcpy/rte") { int ret; struct timeval tv_begin, tv_end; double time_aligned, time_unaligned; double time_aligned_const, time_unaligned_const; ret = init_buffers(); - if (ret != 0) - return ret; + ASSERT(ret == 0); #if TEST_VALUE_RANGE != 0 /* Set up buf_sizes array, if required */ @@ -335,18 +341,7 @@ perf_test(void) printf("Unaligned constant copy size = %8.3f\n", time_unaligned_const); free_buffers(); - return 0; + return Void(); } -static int -test_memcpy_perf(void) -{ - int ret; - - ret = perf_test(); - if (ret != 0) - return -1; - return 0; -} - -REGISTER_TEST_COMMAND(memcpy_perf_autotest, test_memcpy_perf); +void forceLinkMemcpyPerfTests() {} \ No newline at end of file From 937baedd446f59c7d50bfd67ac96458c178afbb7 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Mon, 27 Apr 2020 11:51:30 -0700 Subject: [PATCH 56/89] add memcpy implementation from libfolly --- flow/folly_memcpy.S | 178 ++++++++++++++++++++++++++++++++++++++++++++ flow/folly_memcpy.h | 29 ++++++++ 2 files changed, 207 insertions(+) create mode 100644 flow/folly_memcpy.S create mode 100644 flow/folly_memcpy.h diff --git a/flow/folly_memcpy.S b/flow/folly_memcpy.S new file mode 100644 index 0000000000..e6e95371eb --- /dev/null +++ b/flow/folly_memcpy.S @@ -0,0 +1,178 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * memcpy: An optimized memcpy implementation for x86_64. It uses AVX when + * __AVX__ is defined, and uses SSE2 otherwise. + * + * @author Bin Liu + */ + +#if defined(__x86_64__) && defined(__linux__) && !defined(__CYGWIN__) + + .file "memcpy.S" + .text + +/* + * _memcpy_short is a local helper used when length < 8. It cannot be called + * from outside, because it expects a non-standard calling convention: + * + * %rax: destination buffer address. + * %rsi: source buffer address. + * %edx: length, in the range of [0, 7] + */ + .type _memcpy_short, @function +_memcpy_short: +.LSHORT: + .cfi_startproc + // if (length == 0) return; + test %edx, %edx + jz .LEND + + movzbl (%rsi), %ecx + // if (length - 4 < 0) goto LS4; + sub $4, %edx + jb .LS4 + + mov (%rsi), %ecx + mov (%rsi, %rdx), %edi + mov %ecx, (%rax) + mov %edi, (%rax, %rdx) +.LEND: + rep + ret + nop + +.LS4: + // At this point, length can be 1 or 2 or 3, and $cl contains + // the first byte. + mov %cl, (%rax) + // if (length - 4 + 2 < 0) return; + add $2, %edx + jnc .LEND + + // length is 2 or 3 here. In either case, just copy the last + // two bytes. + movzwl (%rsi, %rdx), %ecx + mov %cx, (%rax, %rdx) + ret + + .cfi_endproc + .size _memcpy_short, .-_memcpy_short + + +/* + * void* memcpy(void* dst, void* src, uint32_t length); + * + */ + .align 16 + .globl memcpy + .type memcpy, @function +memcpy: + .cfi_startproc + + mov %rdx, %rcx + mov %rdi, %rax + cmp $8, %rdx + jb .LSHORT + + mov -8(%rsi, %rdx), %r8 + mov (%rsi), %r9 + mov %r8, -8(%rdi, %rdx) + and $24, %rcx + jz .L32 + + mov %r9, (%rdi) + mov %rcx, %r8 + sub $16, %rcx + jb .LT32 +#ifndef __AVX__ + movdqu (%rsi, %rcx), %xmm1 + movdqu %xmm1, (%rdi, %rcx) +#else + vmovdqu (%rsi, %rcx), %xmm1 + vmovdqu %xmm1, (%rdi, %rcx) +#endif + // Test if there are 32-byte groups +.LT32: + add %r8, %rsi + and $-32, %rdx + jnz .L32_adjDI + ret + + .align 16 +.L32_adjDI: + add %r8, %rdi +.L32: +#ifndef __AVX__ + movdqu (%rsi), %xmm0 + movdqu 16(%rsi), %xmm1 +#else + vmovdqu (%rsi), %ymm0 +#endif + shr $6, %rdx + jnc .L64_32read +#ifndef __AVX__ + movdqu %xmm0, (%rdi) + movdqu %xmm1, 16(%rdi) +#else + vmovdqu %ymm0, (%rdi) +#endif + lea 32(%rsi), %rsi + jnz .L64_adjDI +#ifdef __AVX__ + vzeroupper +#endif + ret + +.L64_adjDI: + add $32, %rdi + +.L64: +#ifndef __AVX__ + movdqu (%rsi), %xmm0 + movdqu 16(%rsi), %xmm1 +#else + vmovdqu (%rsi), %ymm0 +#endif + +.L64_32read: +#ifndef __AVX__ + movdqu 32(%rsi), %xmm2 + movdqu 48(%rsi), %xmm3 + add $64, %rsi + movdqu %xmm0, (%rdi) + movdqu %xmm1, 16(%rdi) + movdqu %xmm2, 32(%rdi) + movdqu %xmm3, 48(%rdi) +#else + vmovdqu 32(%rsi), %ymm1 + add $64, %rsi + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, 32(%rdi) +#endif + add $64, %rdi + dec %rdx + jnz .L64 +#ifdef __AVX__ + vzeroupper +#endif + ret + + .cfi_endproc + .size memcpy, .-memcpy + +#endif diff --git a/flow/folly_memcpy.h b/flow/folly_memcpy.h new file mode 100644 index 0000000000..e3a7339be5 --- /dev/null +++ b/flow/folly_memcpy.h @@ -0,0 +1,29 @@ +/* + * flow.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLOW_FOLLY_MEMCPY_H +#define FLOW_FOLLY_MEMCPY_H +#pragma once + +extern "C" { + void* folly_memcpy(void* dst, const void* src, uint32_t length); +} + +#endif \ No newline at end of file From b84fcbc828c37de99128263494ce63a92d6fb510 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Tue, 28 Apr 2020 12:41:56 -0700 Subject: [PATCH 57/89] folly_memcpy is ready for benchmarking --- cmake/ConfigureCompiler.cmake | 1 + flow/CMakeLists.txt | 1 + flow/folly_memcpy.S | 8 ++++---- flow/test_memcpy_perf.cpp | 16 ++++++++++------ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 5be685b34f..1c37e8930f 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -255,6 +255,7 @@ else() if (GCC) add_compile_options(-Wno-pragmas) add_compile_options(-mavx) +# add_compile_options(-fno-builtin-memcpy) # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 080d483edd..61ce9ed17b 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -75,6 +75,7 @@ set(FLOW_SRCS flat_buffers.h flow.cpp flow.h + folly_memcpy.S genericactors.actor.cpp genericactors.actor.h network.cpp diff --git a/flow/folly_memcpy.S b/flow/folly_memcpy.S index e6e95371eb..66361a774a 100644 --- a/flow/folly_memcpy.S +++ b/flow/folly_memcpy.S @@ -79,9 +79,9 @@ _memcpy_short: * */ .align 16 - .globl memcpy - .type memcpy, @function -memcpy: + .globl folly_memcpy + .type folly_memcpy, @function +folly_memcpy: .cfi_startproc mov %rdx, %rcx @@ -173,6 +173,6 @@ memcpy: ret .cfi_endproc - .size memcpy, .-memcpy + .size folly_memcpy, .-folly_memcpy #endif diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp index b2750b3585..c54f70aacf 100644 --- a/flow/test_memcpy_perf.cpp +++ b/flow/test_memcpy_perf.cpp @@ -12,6 +12,10 @@ #include "flow/IRandom.h" #include "flow/UnitTest.h" +extern "C" { + void* folly_memcpy(void* dst, void* src, uint32_t length); +} + /* * Set this to the maximum buffer size you want to test. If it is 0, then the * values in the buf_sizes[] array below will be used. @@ -46,13 +50,13 @@ static size_t buf_sizes[TEST_VALUE_RANGE]; #define TEST_BATCH_SIZE 100 /* Data is aligned on this many bytes (power of 2) */ -#ifdef RTE_MACHINE_CPUFLAG_AVX512F +// #ifdef RTE_MACHINE_CPUFLAG_AVX512F #define ALIGNMENT_UNIT 64 -#elif defined RTE_MACHINE_CPUFLAG_AVX2 -#define ALIGNMENT_UNIT 32 -#else /* RTE_MACHINE_CPUFLAG */ -#define ALIGNMENT_UNIT 16 -#endif /* RTE_MACHINE_CPUFLAG */ +// #elif defined RTE_MACHINE_CPUFLAG_AVX2 +// #define ALIGNMENT_UNIT 32 +// #else /* RTE_MACHINE_CPUFLAG */ +// #define ALIGNMENT_UNIT 16 +// #endif /* RTE_MACHINE_CPUFLAG */ /* * Pointers used in performance tests. The two large buffers are for uncached From 8f06d48f79d2ac9f5a2da332599e67c309a39b51 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Tue, 2 Jun 2020 14:59:24 -0700 Subject: [PATCH 58/89] Add documentation for server-side latency band tracking --- .../sphinx/source/administration.rst | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst index e34413b9f3..38120414d7 100644 --- a/documentation/sphinx/source/administration.rst +++ b/documentation/sphinx/source/administration.rst @@ -489,7 +489,7 @@ If a process has had more than 10 TCP segments retransmitted in the last 5 secon 10.0.4.1:4500 ( 3% cpu; 2% machine; 0.004 Gbps; 0% disk; REXMIT! 2.5 GB / 4.1 GB RAM ) Machine-readable status --------------------------------- +----------------------- The status command can provide a complete summary of statistics about the cluster and the database with the ``json`` argument. Full documentation for ``status json`` output can be found :doc:`here `. From the output of ``status json``, operators can find useful health metrics to determine whether or not their cluster is hitting performance limits. @@ -501,6 +501,70 @@ Durable version lag ``cluster.qos.worst_durability_lag_storage_server`` cont Transaction log queue ``cluster.qos.worst_queue_bytes_log_server`` contains the maximum size in bytes of the mutations stored on a transaction log that have not yet been popped by storage servers. A large transaction log queue size can potentially cause the ratekeeper to increase throttling. ====================== ============================================================================================================== +Server-side latency band tracking +--------------------------------- + +As part of the status document, ``status json`` provides some sampled latency metrics obtained by running probe transactions internally. While this can often be useful, it does not necessarily reflect the distribution of latencies for requests originated by clients. + +FoundationDB additionally provides optional functionality to measure the latencies of all incoming get read version (GRV), read, and commit requests and report some basic details about those requests. The latencies are measured from the time the server receives the request to the point when it replies, and will therefore not include time spent in transit between the client and server or delays in the client process itself. + +The latency band tracking works by configuring various latency thresholds and counting the number of requests that occur in each band (i.e. between two consecutive thresholds). For example, if you wanted to define a service-level objective (SLO) for your cluster where 99.9% of read requests were answered within N seconds, you could set a read latency threshold at N. You could then count the number of requests below and above the threshold and determine whether the required percentage of requests are answered sufficiently quickly. + +Configuration of server-side latency bands is performed by setting the ``\xff\x02/latencyBandConfig`` key to a string encoding the following JSON document:: + + { + "get_read_version" : { + "bands" : [ 0.01, 0.1] + }, + "read" : { + "bands" : [ 0.01, 0.1], + "max_key_selector_offset" : 1000, + "max_read_bytes" : 1000000 + }, + "commit" : { + "bands" : [ 0.01, 0.1], + "max_commit_bytes" : 1000000 + } + } + +Every field in this configuration is optional, and any missing fields will be left unset (i.e. no bands will be tracked or limits will not apply). The configuration takes the following arguments: + +* ``bands`` - a list of thresholds (in seconds) to be measured for the given request type (``get_read_version``, ``read``, or ``commit``) +* ``max_key_selector_offset`` - an integer specifying the maximum key selector offset a read request can have and still be counted +* ``max_read_bytes`` - an integer specifying the maximum size in bytes of a read response that will be counted +* ``max_commit_bytes`` - an integer specifying the maximum size in bytes of a commit request that will be counted + +Setting this configuration key to a value that changes the configuration will result in the cluster controller server process logging a ``LatencyBandConfigChanged`` event. This event will indicate whether a configuration is present or not using its ``Present`` field. Specifying an invalid configuration will result in the latency band feature being unconfigured, and the server process running the cluster controller will log a ``InvalidLatencyBandConfiguration`` trace event. + +When configured, the ``status json`` output will include additional fields to report the number of requests in each latency band located at ``cluster.processes..roles[N].*_latency_bands``:: + + "grv_latency_bands" : { + 0.01: 10, + 0.1: 0, + inf: 1, + filtered: 0 + }, + "read_latency_bands" : { + 0.01: 12, + 0.1: 1, + inf: 0, + filtered: 0 + }, + "commit_latency_bands" : { + 0.01: 5, + 0.1: 5, + inf: 2, + filtered: 1 + } + +The ``grv_latency_bands`` and ``commit_latency_bands`` objects will only be logged for ``proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. + +For example, ``0.1: 1`` in ``read_latency_bands`` indicates that there has been 1 read request with a latency in the range ``[0.01, 0.1)``. For the smallest specified threshold, the lower bound is 0 (e.g. ``[0, 0.01)`` in the example above). Requests that took longer than any defined latency band will be reported in the ``inf`` (infinity) band. Requests that were filtered by the configuration (e.g. using ``max_read_bytes``) are reported in the ``filtered`` category. + +Because each threshold reports latencies strictly in the range between the next lower threshold and itself, it may be necessary to sum up the counts for multiple bands to determine the total number of requests below a certain threshold. + +.. note:: No history of request counts is recorded for processes that ran in the past. This includes the history prior to restart for a process that has been restarted, for which the counts get reset to 0. For this reason, it is recommended that you collect this information periodically if you need to be able to track requests from such processes. + .. _administration_fdbmonitor: ``fdbmonitor`` and ``fdbserver`` From 7b638ad2371e03b11a697ad70ef8d7502d8f8b52 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Tue, 2 Jun 2020 15:04:40 -0700 Subject: [PATCH 59/89] Add a note indicating that batch priority GRV requests aren't counted in latency band tracking. --- documentation/sphinx/source/administration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst index 38120414d7..009459d6f5 100644 --- a/documentation/sphinx/source/administration.rst +++ b/documentation/sphinx/source/administration.rst @@ -536,6 +536,8 @@ Every field in this configuration is optional, and any missing fields will be le Setting this configuration key to a value that changes the configuration will result in the cluster controller server process logging a ``LatencyBandConfigChanged`` event. This event will indicate whether a configuration is present or not using its ``Present`` field. Specifying an invalid configuration will result in the latency band feature being unconfigured, and the server process running the cluster controller will log a ``InvalidLatencyBandConfiguration`` trace event. +.. note:: GRV requests are counted only at default and immediate priority. Batch priority GRV requests are ignored for the purposes of latency band tracking. + When configured, the ``status json`` output will include additional fields to report the number of requests in each latency band located at ``cluster.processes..roles[N].*_latency_bands``:: "grv_latency_bands" : { From e77f9701f32b32238d75db1e917cabcb1539ce9f Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Fri, 1 May 2020 13:23:20 -0700 Subject: [PATCH 60/89] Settle on using rte_memcpy when we do not know the copy size at runtime, and builtin memcpy otherwise --- cmake/ConfigureCompiler.cmake | 9 ++++++++- flow/flow.cpp | 11 +++++++++++ flow/rte_memcpy.h | 3 +++ flow/test_memcpy.cpp | 1 + flow/test_memcpy_perf.cpp | 10 +++++++--- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 1c37e8930f..d34a4a52aa 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -255,7 +255,14 @@ else() if (GCC) add_compile_options(-Wno-pragmas) add_compile_options(-mavx) -# add_compile_options(-fno-builtin-memcpy) + # Intentionally using builtin memcpy. G++ does a good job on small memcpy's when the size is known at runtime. + # If the size is not known, then it falls back on the memcpy that's available at runtime (rte_memcpy, as of this + # writing; see flow.cpp). + # + # The downside of the builtin memcpy is that it's slower at large copies, so if we spend a lot of time on large + # copies of sizes that are known at compile time, this might not be a win. See the output of performance/memcpy + # for more information. + #add_compile_options(-fno-builtin-memcpy) # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) diff --git a/flow/flow.cpp b/flow/flow.cpp index fb1cf81f60..c0d8a40f61 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -21,9 +21,20 @@ #include "flow/flow.h" #include "flow/DeterministicRandom.h" #include "flow/UnitTest.h" +#include "flow/rte_memcpy.h" +#include "flow/folly_memcpy.h" #include #include +void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { + return rte_memcpy(__dest, __src, __n); +} + +// This compilation unit will be linked in to the main binary, so this should override glibc memcpy +__attribute__((visibility ("default"))) void *memcpy (void *__restrict __dest, const void *__restrict __src, size_t __n) { + return rte_memcpy(__dest, __src, __n); +} + INetwork *g_network = 0; FILE* randLog = 0; diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index ba0ab3cf6c..b6eedd65b3 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -50,6 +50,9 @@ extern "C" { static force_inline void * rte_memcpy(void *dst, const void *src, size_t n); +//#define RTE_MACHINE_CPUFLAG_AVX512F +#define RTE_MACHINE_CPUFLAG_AVX2 + #ifdef RTE_MACHINE_CPUFLAG_AVX512F #define ALIGNMENT_MASK 0x3F diff --git a/flow/test_memcpy.cpp b/flow/test_memcpy.cpp index 19ab66e35b..bb9fc74343 100644 --- a/flow/test_memcpy.cpp +++ b/flow/test_memcpy.cpp @@ -7,6 +7,7 @@ #include #include +#include "flow/folly_memcpy.h" #include "flow/rte_memcpy.h" #include "flow/IRandom.h" diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp index c54f70aacf..7138d04599 100644 --- a/flow/test_memcpy_perf.cpp +++ b/flow/test_memcpy_perf.cpp @@ -11,11 +11,15 @@ #include "flow/rte_memcpy.h" #include "flow/IRandom.h" #include "flow/UnitTest.h" +#include "flow/flow.h" extern "C" { - void* folly_memcpy(void* dst, void* src, uint32_t length); + void* folly_memcpy(void* dst, const void* src, uint32_t length); } + +void * rte_memcpy_noinline(void* dst, const void* src, size_t length); // for performance comparisons + /* * Set this to the maximum buffer size you want to test. If it is 0, then the * values in the buf_sizes[] array below will be used. @@ -170,7 +174,7 @@ do_uncached_write(uint8_t *dst, int is_dst_cached, fill_addr_arrays(dst_addrs, is_dst_cached, 0, src_addrs, is_src_cached, 0); for (j = 0; j < TEST_BATCH_SIZE; j++) { - rte_memcpy(dst+dst_addrs[j], src+src_addrs[j], size); + memcpy(dst+dst_addrs[j], src+src_addrs[j], size); } } } @@ -191,7 +195,7 @@ do { \ src_addrs, is_src_cached, src_uoffset); \ start_time = rte_rdtsc(); \ for (t = 0; t < TEST_BATCH_SIZE; t++) \ - rte_memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + rte_memcpy_noinline(dst+dst_addrs[t], src+src_addrs[t], size); \ total_time += rte_rdtsc() - start_time; \ } \ for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ From 11f658cfe28c8a8b9a117d89e3a5fa96af3305c0 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Fri, 1 May 2020 13:37:29 -0700 Subject: [PATCH 61/89] Add cmake option to disable AVX --- cmake/ConfigureCompiler.cmake | 13 ++++++++++++- flow/rte_memcpy.h | 4 +++- flow/test_memcpy.cpp | 10 +++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index d34a4a52aa..08712439e1 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -243,6 +243,12 @@ else() -Wno-tautological-pointer-compare -Wno-format -Woverloaded-virtual) + set(USE_AVX ON CACHE BOOL "Enable AVX instructions") + if (USE_AVX) + add_compile_options(-mavx) + else() + add_compile_options(-msse4) + endif() if (USE_CCACHE) add_compile_options( -Wno-register @@ -254,7 +260,12 @@ else() endif() if (GCC) add_compile_options(-Wno-pragmas) - add_compile_options(-mavx) + set(USE_AVX ON CACHE BOOL "Enable AVX instructions") + if (USE_AVX) + add_compile_options(-mavx) + else() + add_compile_options(-msse4) + endif() # Intentionally using builtin memcpy. G++ does a good job on small memcpy's when the size is known at runtime. # If the size is not known, then it falls back on the memcpy that's available at runtime (rte_memcpy, as of this # writing; see flow.cpp). diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index b6eedd65b3..bf8989616c 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -50,8 +50,10 @@ extern "C" { static force_inline void * rte_memcpy(void *dst, const void *src, size_t n); -//#define RTE_MACHINE_CPUFLAG_AVX512F +#ifdef __AVX__ +//#define RTE_MACHINE_CPUFLAG_AVX512F -- our g++ is too old for this #define RTE_MACHINE_CPUFLAG_AVX2 +#endif #ifdef RTE_MACHINE_CPUFLAG_AVX512F diff --git a/flow/test_memcpy.cpp b/flow/test_memcpy.cpp index bb9fc74343..3d8c408205 100644 --- a/flow/test_memcpy.cpp +++ b/flow/test_memcpy.cpp @@ -58,16 +58,16 @@ test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) } /* Do the copy */ - ret = rte_memcpy(dest + off_dst, src + off_src, size); + ret = memcpy(dest + off_dst, src + off_src, size); if (ret != (dest + off_dst)) { - printf("rte_memcpy() returned %p, not %p\n", + printf("memcpy() returned %p, not %p\n", ret, dest + off_dst); } /* Check nothing before offset is affected */ for (i = 0; i < off_dst; i++) { if (dest[i] != 0) { - printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + printf("memcpy() failed for %u bytes (offsets=%u,%u): " "[modified before start of dst].\n", (unsigned)size, off_src, off_dst); return -1; @@ -77,7 +77,7 @@ test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) /* Check everything was copied */ for (i = 0; i < size; i++) { if (dest[i + off_dst] != src[i + off_src]) { - printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + printf("memcpy() failed for %u bytes (offsets=%u,%u): " "[didn't copy byte %u].\n", (unsigned)size, off_src, off_dst, i); return -1; @@ -87,7 +87,7 @@ test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) /* Check nothing after copy was affected */ for (i = size; i < SMALL_BUFFER_SIZE; i++) { if (dest[i + off_dst] != 0) { - printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + printf("memcpy() failed for %u bytes (offsets=%u,%u): " "[copied too many].\n", (unsigned)size, off_src, off_dst); return -1; From c5e2df99a2bdb767a3e804c992427abc2e5ab721 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Mon, 4 May 2020 14:44:42 -0700 Subject: [PATCH 62/89] comments --- flow/flow.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flow/flow.cpp b/flow/flow.cpp index c0d8a40f61..7916accb1f 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -26,12 +26,14 @@ #include #include +// For benchmarking; need a version of rte_memcpy that doesn't live in the same compilation unit as the test. void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { return rte_memcpy(__dest, __src, __n); } // This compilation unit will be linked in to the main binary, so this should override glibc memcpy __attribute__((visibility ("default"))) void *memcpy (void *__restrict __dest, const void *__restrict __src, size_t __n) { + // folly_memcpy is faster for small copies, but rte seems to win out in most other circumstances return rte_memcpy(__dest, __src, __n); } From a99ceb9c42a12b53fa323bf73e16f3b7ebec9d06 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Wed, 13 May 2020 11:59:59 -0700 Subject: [PATCH 63/89] Add cmake option to enable avx512 --- cmake/ConfigureCompiler.cmake | 39 +++++++++++++++++------------------ flow/flow.cpp | 2 ++ flow/rte_memcpy.h | 9 ++++++-- flow/test_memcpy_perf.cpp | 6 ++++-- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 08712439e1..0448425e44 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -209,6 +209,25 @@ else() # -mavx # -msse4.2) + # Tentatively re-enabling vector instructions + set(USE_AVX512F OFF CACHE BOOL "Enable AVX 512F instructions") + if (USE_AVX512F) + add_compile_options(-mavx512f) + endif() + set(USE_AVX ON CACHE BOOL "Enable AVX instructions") + if (USE_AVX) + add_compile_options(-mavx) + endif() + + # Intentionally using builtin memcpy. G++ does a good job on small memcpy's when the size is known at runtime. + # If the size is not known, then it falls back on the memcpy that's available at runtime (rte_memcpy, as of this + # writing; see flow.cpp). + # + # The downside of the builtin memcpy is that it's slower at large copies, so if we spend a lot of time on large + # copies of sizes that are known at compile time, this might not be a win. See the output of performance/memcpy + # for more information. + #add_compile_options(-fno-builtin-memcpy) + if (USE_VALGRIND) add_compile_options(-DVALGRIND -DUSE_VALGRIND) endif() @@ -243,12 +262,6 @@ else() -Wno-tautological-pointer-compare -Wno-format -Woverloaded-virtual) - set(USE_AVX ON CACHE BOOL "Enable AVX instructions") - if (USE_AVX) - add_compile_options(-mavx) - else() - add_compile_options(-msse4) - endif() if (USE_CCACHE) add_compile_options( -Wno-register @@ -260,20 +273,6 @@ else() endif() if (GCC) add_compile_options(-Wno-pragmas) - set(USE_AVX ON CACHE BOOL "Enable AVX instructions") - if (USE_AVX) - add_compile_options(-mavx) - else() - add_compile_options(-msse4) - endif() - # Intentionally using builtin memcpy. G++ does a good job on small memcpy's when the size is known at runtime. - # If the size is not known, then it falls back on the memcpy that's available at runtime (rte_memcpy, as of this - # writing; see flow.cpp). - # - # The downside of the builtin memcpy is that it's slower at large copies, so if we spend a lot of time on large - # copies of sizes that are known at compile time, this might not be a win. See the output of performance/memcpy - # for more information. - #add_compile_options(-fno-builtin-memcpy) # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) diff --git a/flow/flow.cpp b/flow/flow.cpp index 7916accb1f..f9863b21fa 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -26,6 +26,7 @@ #include #include +#if defined (__linux__) || defined (__FreeBSD__) // For benchmarking; need a version of rte_memcpy that doesn't live in the same compilation unit as the test. void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { return rte_memcpy(__dest, __src, __n); @@ -36,6 +37,7 @@ __attribute__((visibility ("default"))) void *memcpy (void *__restrict __dest, c // folly_memcpy is faster for small copies, but rte seems to win out in most other circumstances return rte_memcpy(__dest, __src, __n); } +#endif // defined (__linux__) || defined (__FreeBSD__) INetwork *g_network = 0; diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index bf8989616c..f9c28ce112 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -28,6 +28,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #include +#if defined (__linux__) || defined (__FreeBSD__) + #ifdef __cplusplus extern "C" { #endif @@ -50,8 +52,9 @@ extern "C" { static force_inline void * rte_memcpy(void *dst, const void *src, size_t n); -#ifdef __AVX__ -//#define RTE_MACHINE_CPUFLAG_AVX512F -- our g++ is too old for this +#ifdef __AVX512F__ +#define RTE_MACHINE_CPUFLAG_AVX512F +#elif defined(__AVX__) #define RTE_MACHINE_CPUFLAG_AVX2 #endif @@ -905,4 +908,6 @@ rte_rdtsc(void) } #endif +#endif /* defined (__linux__) || defined (__FreeBSD__) */ + #endif /* _RTE_MEMCPY_X86_64_H_ */ diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp index 7138d04599..e8d9323b09 100644 --- a/flow/test_memcpy_perf.cpp +++ b/flow/test_memcpy_perf.cpp @@ -6,13 +6,13 @@ #include #include #include -#include #include "flow/rte_memcpy.h" #include "flow/IRandom.h" #include "flow/UnitTest.h" #include "flow/flow.h" +#if defined (__linux__) || defined (__FreeBSD__) extern "C" { void* folly_memcpy(void* dst, const void* src, uint32_t length); } @@ -352,4 +352,6 @@ TEST_CASE("performance/memcpy/rte") { return Void(); } -void forceLinkMemcpyPerfTests() {} \ No newline at end of file +#endif // defined (__linux__) || defined (__FreeBSD__) + +void forceLinkMemcpyPerfTests() {} From 3415bbcd09a0389b8dc09e521af2b5a556b5fa17 Mon Sep 17 00:00:00 2001 From: Russell Sears Date: Tue, 2 Jun 2020 16:09:43 -0700 Subject: [PATCH 64/89] Fix ARM build; add ACKNOWLEDGEMENTS entries --- ACKNOWLEDGEMENTS | 48 +++++++++++++++++++++++++++++++++++++++ flow/flow.cpp | 8 +++++-- flow/folly_memcpy.h | 4 ++++ flow/rte_memcpy.h | 2 +- flow/test_memcpy_perf.cpp | 2 +- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index 18ae54aad6..71d04fc55b 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -536,3 +536,51 @@ sse2neon Authors (sse2neon) LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +rte_memcpy.h (from DPDK): + SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2010-2014 Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +folly_memcpy: + + Copyright (c) Facebook, Inc. and its affiliates. + Author: Bin Liu + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/flow/flow.cpp b/flow/flow.cpp index f9863b21fa..5c5450f065 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -26,7 +26,7 @@ #include #include -#if defined (__linux__) || defined (__FreeBSD__) +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) // For benchmarking; need a version of rte_memcpy that doesn't live in the same compilation unit as the test. void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { return rte_memcpy(__dest, __src, __n); @@ -37,7 +37,11 @@ __attribute__((visibility ("default"))) void *memcpy (void *__restrict __dest, c // folly_memcpy is faster for small copies, but rte seems to win out in most other circumstances return rte_memcpy(__dest, __src, __n); } -#endif // defined (__linux__) || defined (__FreeBSD__) +#else +void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { + return memcpy(__dest, __src, __n); +} +#endif // (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) INetwork *g_network = 0; diff --git a/flow/folly_memcpy.h b/flow/folly_memcpy.h index e3a7339be5..9b74507a0d 100644 --- a/flow/folly_memcpy.h +++ b/flow/folly_memcpy.h @@ -22,8 +22,12 @@ #define FLOW_FOLLY_MEMCPY_H #pragma once +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) + extern "C" { void* folly_memcpy(void* dst, const void* src, uint32_t length); } +#endif // linux or bsd and avx + #endif \ No newline at end of file diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h index f9c28ce112..e5986e6500 100644 --- a/flow/rte_memcpy.h +++ b/flow/rte_memcpy.h @@ -28,7 +28,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #include -#if defined (__linux__) || defined (__FreeBSD__) +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) #ifdef __cplusplus extern "C" { diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp index e8d9323b09..b51463534e 100644 --- a/flow/test_memcpy_perf.cpp +++ b/flow/test_memcpy_perf.cpp @@ -12,7 +12,7 @@ #include "flow/UnitTest.h" #include "flow/flow.h" -#if defined (__linux__) || defined (__FreeBSD__) +#if (defined (__linux__) || defined (__FreeBSD__)) && defined (__AVX__) extern "C" { void* folly_memcpy(void* dst, const void* src, uint32_t length); } From d5025a1779ab53e415559bf1cdcf8ea696d3a22b Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Wed, 3 Jun 2020 15:28:59 -0700 Subject: [PATCH 65/89] getAndComputeStagingKeys: Improved handling of not exist keys --- fdbserver/RestoreApplier.actor.cpp | 50 ++++++++++++++++++------------ fdbserver/RestoreApplier.actor.h | 3 +- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 42d1c6511b..52a13d4837 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -224,39 +224,49 @@ ACTOR static Future getAndComputeStagingKeys( .detail("BatchIndex", batchIndex) .detail("GetKeys", incompleteStagingKeys.size()) .detail("DelayTime", delayTime); + state std::set keyNotFounds; + + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + for (auto& key : incompleteStagingKeys) { + fValues.push_back(tr->get(key.first)); + } + + state int i = 0; + state bool hasError = false; loop { - try { - tr->reset(); - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - for (auto& key : incompleteStagingKeys) { - fValues.push_back(tr->get(key.first)); + hasError = false; + for (i = 0; i < incompleteStagingKeys.size(); ++i) { + try { + if (keyNotFounds.count(i)) { + continue; + } + wait(success(fValues[i])); + } catch (Error& e) { + if (e.code() == error_code_key_not_found) { + keyNotFounds.push_back(i); + } else { + hasError = true; + } + wait(tr->onError(e)); } - wait(waitForAll(fValues)); + } + if (!hasError) { break; - } catch (Error& e) { - if (retries++ > 10) { // TODO: Can we stop retry at the first error? - TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysGetKeysStuck", applierID) - .detail("BatchIndex", batchIndex) - .detail("GetKeys", incompleteStagingKeys.size()) - .error(e); - break; - } - wait(tr->onError(e)); - fValues.clear(); } } ASSERT(fValues.size() == incompleteStagingKeys.size()); int i = 0; for (auto& key : incompleteStagingKeys) { - if (!fValues[i].get().present()) { // Debug info to understand which key does not exist in DB + if (keyNotFounds.count(i) || !fValues[i].get().present()) { // Key not exist in DB TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID) .detail("BatchIndex", batchIndex) .detail("Key", key.first) - .detail("Reason", "Not found in DB") + .detail("IsReady", fValues[i].isReady()) .detail("PendingMutations", key.second->second.pendingMutations.size()) - .detail("StagingKeyType", (int)key.second->second.type); + .detail("StagingKeyType", getTypeString(key.second->second.type)); for (auto& vm : key.second->second.pendingMutations) { TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB") .detail("PendingMutationVersion", vm.first.toString()) diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h index 06cff36b75..92c8c1f35f 100644 --- a/fdbserver/RestoreApplier.actor.h +++ b/fdbserver/RestoreApplier.actor.h @@ -126,7 +126,8 @@ struct StagingKey { .detail("Value", val) .detail("MType", type < MutationRef::MAX_ATOMIC_OP ? getTypeString(type) : "[Unset]") .detail("LargestPendingVersion", - (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString())); + (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString())) + .detail("PendingMutations", pendingMutations.size()); std::map>::iterator lb = pendingMutations.lower_bound(version); if (lb == pendingMutations.end()) { return; From 9edc872041300c5ffa75c4a021edb1e5765ad3b3 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 3 Jun 2020 16:05:21 -0700 Subject: [PATCH 66/89] Don't attempt to become a cluster controller on any process with a class that has NeverAssign fitness. --- fdbserver/worker.actor.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index d2147a907b..d69bd5272e 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -1465,7 +1465,12 @@ ACTOR Future fdbd( Promise recoveredDiskFiles; v.push_back(reportErrors(monitorAndWriteCCPriorityInfo(fitnessFilePath, asyncPriorityInfo), "MonitorAndWriteCCPriorityInfo")); - v.push_back( reportErrors( processClass == ProcessClass::TesterClass ? monitorLeader( connFile, cc ) : clusterController( connFile, cc , asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities ), "ClusterController") ); + if(processClass.machineClassFitness(ProcessClass::ClusterController) == ProcessClass::NeverAssign) { + v.push_back(reportErrors(monitorLeader(connFile, cc), "ClusterController")); + } + else { + v.push_back(reportErrors(clusterController(connFile, cc , asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities), "ClusterController")); + } v.push_back( reportErrors(extractClusterInterface( cc, ci ), "ExtractClusterInterface") ); v.push_back( reportErrors(failureMonitorClient( ci, true ), "FailureMonitorClient") ); v.push_back( reportErrorsExcept(workerServer(connFile, cc, localities, asyncPriorityInfo, processClass, dataFolder, memoryLimit, metricsConnFile, metricsPrefix, recoveredDiskFiles, memoryProfileThreshold, coordFolder, whitelistBinPaths), "WorkerServer", UID(), &normalWorkerErrors()) ); From 6d749af3c7ecd79b0988e8fae3d51566d24da098 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 3 Jun 2020 16:08:30 -0700 Subject: [PATCH 67/89] Add a release note --- documentation/sphinx/source/release-notes.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 38b0c415bb..75a8f43654 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -2,6 +2,14 @@ Release Notes ############# +6.2.22 +====== + +Fixes +----- + +* Coordinator class processes could be recruited as the cluster controller. `(PR #3282) `_ + 6.2.21 ====== From 87a557dcb4d63a7e23c3bc2d70e94d508f8c5ede Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Wed, 3 Jun 2020 17:17:47 -0700 Subject: [PATCH 68/89] FastRestore:Applier:Treat future_version as key not exist --- fdbserver/RestoreApplier.actor.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 52a13d4837..58769838b3 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -244,17 +244,25 @@ ACTOR static Future getAndComputeStagingKeys( } wait(success(fValues[i])); } catch (Error& e) { - if (e.code() == error_code_key_not_found) { - keyNotFounds.push_back(i); + if (e.code() == error_code_key_not_found || e.code() == error_code_transaction_too_old || + e.code() == error_code_future_version) { + keyNotFounds.insert(i); } else { hasError = true; } + if (retries > 20) { + TraceEvent(SevError, "GetAndComputeStagingKeys", applierID) + .detail("BatchIndex", batchIndex) + .detail("KeyIndex", i) + .error(e); + } wait(tr->onError(e)); } } if (!hasError) { break; } + retries++; } ASSERT(fValues.size() == incompleteStagingKeys.size()); From 633587a95ae33d559329712b23a5b8c6d6881719 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Wed, 3 Jun 2020 21:17:27 -0700 Subject: [PATCH 69/89] RestoreApplier:getAndComputeStagingKeys:retry for keys that exist in DB Test shows that we cannot just skip the key that exist in DB but has future_version error. --- fdbserver/RestoreApplier.actor.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 58769838b3..ac28724b64 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -216,7 +216,7 @@ ACTOR static Future getAndComputeStagingKeys( std::map::iterator> incompleteStagingKeys, double delayTime, Database cx, UID applierID, int batchIndex) { state Reference tr(new ReadYourWritesTransaction(cx)); - state std::vector>> fValues; + state std::vector>> fValues(incompleteStagingKeys.size(), Never()); state int retries = 0; wait(delay(delayTime + deterministicRandom()->random01() * delayTime)); @@ -226,17 +226,20 @@ ACTOR static Future getAndComputeStagingKeys( .detail("DelayTime", delayTime); state std::set keyNotFounds; - tr->reset(); - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - for (auto& key : incompleteStagingKeys) { - fValues.push_back(tr->get(key.first)); - } - state int i = 0; state bool hasError = false; loop { hasError = false; + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + i = 0; + for (auto& key : incompleteStagingKeys) { + if (!fValues[i].isReady() || !keyNotFounds.count(i)) { + fValues[i] = tr->get(key.first); + } + ++i; + } for (i = 0; i < incompleteStagingKeys.size(); ++i) { try { if (keyNotFounds.count(i)) { @@ -244,13 +247,13 @@ ACTOR static Future getAndComputeStagingKeys( } wait(success(fValues[i])); } catch (Error& e) { - if (e.code() == error_code_key_not_found || e.code() == error_code_transaction_too_old || - e.code() == error_code_future_version) { + if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == + // error_code_future_version keyNotFounds.insert(i); } else { hasError = true; } - if (retries > 20) { + if (retries > incompleteStagingKeys.size()) { TraceEvent(SevError, "GetAndComputeStagingKeys", applierID) .detail("BatchIndex", batchIndex) .detail("KeyIndex", i) From bf072d68ec1bee52be84f127bc71733a11e421b6 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Thu, 4 Jun 2020 01:32:12 -0700 Subject: [PATCH 70/89] Workarounds for strange behaviors in Boost ssl sockets on MacOS and Linux. When writing to the ssl socket, write_some() would sometimes return BrokenPipe instead of WouldBlock unless onWriteable on the raw socket was checked first. On MacOS, even with the onWriteable check using a send size greater than 2016 (determined experimentally) would still result in the error. Also consolidated two identical copies of SendBufferIterator. --- fdbclient/HTTP.actor.cpp | 8 ++-- flow/Net2.actor.cpp | 91 +++++++++++++++------------------------- 2 files changed, 36 insertions(+), 63 deletions(-) diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp index 933dd15fff..3779b26ef5 100644 --- a/fdbclient/HTTP.actor.cpp +++ b/fdbclient/HTTP.actor.cpp @@ -352,6 +352,9 @@ namespace HTTP { send_start = timer(); loop { + wait(conn->onWritable()); + wait( delay( 0, TaskPriority::WriteSocket ) ); + // If we already got a response, before finishing sending the request, then close the connection, // set the Connection header to "close" as a hint to the caller that this connection can't be used // again, and break out of the send loop. @@ -372,11 +375,6 @@ namespace HTTP { pContent->sent(len); if(pContent->empty()) break; - - if(len == 0) { - wait(conn->onWritable()); - wait( delay( 0, TaskPriority::WriteSocket ) ); - } } wait(responseReading); diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index e5078d46dd..c77078dfcb 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -291,6 +291,35 @@ public: } }; +struct SendBufferIterator { + typedef boost::asio::const_buffer value_type; + typedef std::forward_iterator_tag iterator_category; + typedef size_t difference_type; + typedef boost::asio::const_buffer* pointer; + typedef boost::asio::const_buffer& reference; + + SendBuffer const* p; + int limit; + + SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { + ASSERT(limit > 0); + } + + bool operator == (SendBufferIterator const& r) const { return p == r.p; } + bool operator != (SendBufferIterator const& r) const { return p != r.p; } + void operator++() { + limit -= p->bytes_written - p->bytes_sent; + if(limit > 0) + p = p->next; + else + p = NULL; + } + + boost::asio::const_buffer operator*() const { + return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); + } +}; + class Connection : public IConnection, ReferenceCounted { public: virtual void addref() { ReferenceCounted::addref(); } @@ -415,35 +444,6 @@ private: tcp::socket socket; NetworkAddress peer_address; - struct SendBufferIterator { - typedef boost::asio::const_buffer value_type; - typedef std::forward_iterator_tag iterator_category; - typedef size_t difference_type; - typedef boost::asio::const_buffer* pointer; - typedef boost::asio::const_buffer& reference; - - SendBuffer const* p; - int limit; - - SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { - ASSERT(limit > 0); - } - - bool operator == (SendBufferIterator const& r) const { return p == r.p; } - bool operator != (SendBufferIterator const& r) const { return p != r.p; } - void operator++() { - limit -= p->bytes_written - p->bytes_sent; - if(limit > 0) - p = p->next; - else - p = NULL; - } - - boost::asio::const_buffer operator*() const { - return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); - } - }; - void init() { // Socket settings that have to be set after connect or accept succeeds socket.non_blocking(true); @@ -707,6 +707,10 @@ public: // Writes as many bytes as possible from the given SendBuffer chain into the write buffer and returns the number of bytes written (might be 0) virtual int write( SendBuffer const* data, int limit ) { +#ifdef __APPLE__ + // For some reason, writing ssl_sock with more than 2016 bytes when socket is writeable sometimes results in a broken pipe error. + limit = std::min(limit, 2016); +#endif boost::system::error_code err; ++g_net2->countWrites; @@ -749,35 +753,6 @@ private: NetworkAddress peer_address; Reference> sslContext; - struct SendBufferIterator { - typedef boost::asio::const_buffer value_type; - typedef std::forward_iterator_tag iterator_category; - typedef size_t difference_type; - typedef boost::asio::const_buffer* pointer; - typedef boost::asio::const_buffer& reference; - - SendBuffer const* p; - int limit; - - SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { - ASSERT(limit > 0); - } - - bool operator == (SendBufferIterator const& r) const { return p == r.p; } - bool operator != (SendBufferIterator const& r) const { return p != r.p; } - void operator++() { - limit -= p->bytes_written - p->bytes_sent; - if(limit > 0) - p = p->next; - else - p = NULL; - } - - boost::asio::const_buffer operator*() const { - return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); - } - }; - void init() { // Socket settings that have to be set after connect or accept succeeds socket.non_blocking(true); From 71ee7176517fa02f1dd598749dfa16827efdb774 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 4 Jun 2020 16:39:01 -0700 Subject: [PATCH 71/89] updated documentation for 6.2.22 --- documentation/sphinx/source/downloads.rst | 24 +++++++++---------- documentation/sphinx/source/release-notes.rst | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index ee7199e918..13da1f3538 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.2.21.pkg `_ +* `FoundationDB-6.2.22.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.2.21-1_amd64.deb `_ -* `foundationdb-server-6.2.21-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.2.22-1_amd64.deb `_ +* `foundationdb-server-6.2.22-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.2.21-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.2.21-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.22-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.2.22-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.2.21-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.2.21-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.22-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.2.22-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.2.21-x64.msi `_ +* `foundationdb-6.2.22-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package: -* `foundationdb-6.2.21.tar.gz `_ +* `foundationdb-6.2.22.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.2.21.gem `_ +* `fdb-6.2.22.gem `_ Java 8+ ------- -* `fdb-java-6.2.21.jar `_ -* `fdb-java-6.2.21-javadoc.jar `_ +* `fdb-java-6.2.22.jar `_ +* `fdb-java-6.2.22-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 75a8f43654..34ef860311 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -9,6 +9,7 @@ Fixes ----- * Coordinator class processes could be recruited as the cluster controller. `(PR #3282) `_ +* HTTPS requests made by backup would fail (introduced in 6.2.21). `(PR #3284) `_ 6.2.21 ====== From 0965f9e73be4d664c7ca9a8c6e7d2073afef5766 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 4 Jun 2020 19:11:50 -0700 Subject: [PATCH 72/89] update version to 6.2.23 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d21a748bc7..ca479db0da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.12) project(foundationdb - VERSION 6.2.22 + VERSION 6.2.23 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) From 4a8de2910ff1093a4a5c5b2e3fb30ce7d472bc0f Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 4 Jun 2020 19:11:50 -0700 Subject: [PATCH 73/89] update installer WIX GUID following release --- packaging/msi/FDBInstaller.wxs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 30a0fdcd42..26a2af52fd 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -32,7 +32,7 @@ Date: Thu, 4 Jun 2020 19:13:48 -0700 Subject: [PATCH 74/89] update versions.target --- versions.target | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.target b/versions.target index de5dad442d..d18d4b9f45 100644 --- a/versions.target +++ b/versions.target @@ -1,7 +1,7 @@ - 6.2.22 + 6.2.23 6.2 From d199bf5b813f781447a12cd73ceffd12b51ccab4 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 4 Jun 2020 19:27:25 -0700 Subject: [PATCH 75/89] update old release notes --- .../source/old-release-notes/release-notes-620.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/documentation/sphinx/source/old-release-notes/release-notes-620.rst b/documentation/sphinx/source/old-release-notes/release-notes-620.rst index 38b0c415bb..34ef860311 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-620.rst @@ -2,6 +2,15 @@ Release Notes ############# +6.2.22 +====== + +Fixes +----- + +* Coordinator class processes could be recruited as the cluster controller. `(PR #3282) `_ +* HTTPS requests made by backup would fail (introduced in 6.2.21). `(PR #3284) `_ + 6.2.21 ====== From 7ec1d644a61cd38081e45322abd1136d2674a1a1 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 4 Jun 2020 19:38:16 -0700 Subject: [PATCH 76/89] updated documentation for 6.3.1 --- documentation/sphinx/source/downloads.rst | 24 +++++++++---------- documentation/sphinx/source/release-notes.rst | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index b4d30eb629..6b81a98a82 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.3.0.pkg `_ +* `FoundationDB-6.3.1.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.3.0-1_amd64.deb `_ -* `foundationdb-server-6.3.0-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1_amd64.deb `_ +* `foundationdb-server-6.3.1-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.3.0-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.3.0-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.3.1-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.3.0-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.3.0-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.3.1-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.3.0-x64.msi `_ +* `foundationdb-6.3.1-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package: -* `foundationdb-6.3.0.tar.gz `_ +* `foundationdb-6.3.1.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.3.0.gem `_ +* `fdb-6.3.1.gem `_ Java 8+ ------- -* `fdb-java-6.3.0.jar `_ -* `fdb-java-6.3.0-javadoc.jar `_ +* `fdb-java-6.3.1.jar `_ +* `fdb-java-6.3.1-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 2297176944..959623e0de 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -2,7 +2,7 @@ Release Notes ############# -6.3.0 +6.3.1 ===== Features From e9af22085b73bb4c28004c84de93f8a8a5ee57e0 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Thu, 4 Jun 2020 21:26:09 -0700 Subject: [PATCH 77/89] Debug: getAndComputeStagingKeys may be stuck Maybe wait(success(fValues[i])); never return --- fdbserver/RestoreApplier.actor.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index ac28724b64..b87e87cf2d 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -218,9 +218,11 @@ ACTOR static Future getAndComputeStagingKeys( state Reference tr(new ReadYourWritesTransaction(cx)); state std::vector>> fValues(incompleteStagingKeys.size(), Never()); state int retries = 0; + // state UID randomID = deterministicRandom()->randomUniqueID(); wait(delay(delayTime + deterministicRandom()->random01() * delayTime)); TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID) + //.detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) .detail("GetKeys", incompleteStagingKeys.size()) .detail("DelayTime", delayTime); @@ -245,7 +247,7 @@ ACTOR static Future getAndComputeStagingKeys( if (keyNotFounds.count(i)) { continue; } - wait(success(fValues[i])); + wait(success(fValues[i])); // NOTE: This may be waiting for ever! } catch (Error& e) { if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == // error_code_future_version @@ -271,7 +273,8 @@ ACTOR static Future getAndComputeStagingKeys( ASSERT(fValues.size() == incompleteStagingKeys.size()); int i = 0; for (auto& key : incompleteStagingKeys) { - if (keyNotFounds.count(i) || !fValues[i].get().present()) { // Key not exist in DB + if (keyNotFounds.count(i) || (!fValues[i].get().present())) { // Key not exist in DB + // if condition: fValues[i].Valid() && fValues[i].isReady() && !fValues[i].isError() && TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID) .detail("BatchIndex", batchIndex) .detail("Key", key.first) @@ -295,8 +298,10 @@ ACTOR static Future getAndComputeStagingKeys( } TraceEvent("FastRestoreApplierGetAndComputeStagingKeysDone", applierID) + //.detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) - .detail("GetKeys", incompleteStagingKeys.size()); + .detail("GetKeys", incompleteStagingKeys.size()) + .detail("DelayTime", delayTime); return Void(); } @@ -384,7 +389,7 @@ ACTOR static Future precomputeMutationsResult(Reference incompleteStagingKeys.clear(); } } - if (numKeysInBatch > 0) { + if (numKeysInBatch >= 1) { fGetAndComputeKeys.push_back( getAndComputeStagingKeys(incompleteStagingKeys, delayTime, cx, applierID, batchIndex)); } @@ -599,4 +604,4 @@ Value applyAtomicOp(Optional existingValue, Value value, MutationRef: ASSERT(false); } return Value(); -} +} \ No newline at end of file From 949fbd914564fe46abd9e07a5937216a72cc4be3 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Thu, 4 Jun 2020 23:23:14 -0700 Subject: [PATCH 78/89] New low level BTree cursor class which is designed for short lifetimes, does less memory allocation, and can be used to perform getRange() operations with less CPU overhead. --- fdbserver/VersionedBTree.actor.cpp | 555 +++++++++++++++++++++++++++-- 1 file changed, 523 insertions(+), 32 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 2959c06b2a..df06321bde 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -2204,6 +2204,7 @@ struct SplitStringRef { // A BTree "page id" is actually a list of LogicalPageID's whose contents should be concatenated together. // NOTE: Uses host byte order typedef VectorRef BTreePageIDRef; +constexpr LogicalPageID maxPageID = (LogicalPageID)-1; std::string toString(BTreePageIDRef id) { return std::string("BTreePageID") + toString(id.begin(), id.end()); @@ -2246,6 +2247,10 @@ struct RedwoodRecordRef { inline RedwoodRecordRef withoutValue() const { return RedwoodRecordRef(key, version); } + inline RedwoodRecordRef withMaxPageID() const { + return RedwoodRecordRef(key, version, StringRef((uint8_t *)&maxPageID, sizeof(maxPageID))); + } + // Truncate (key, version, part) tuple to len bytes. void truncate(int len) { ASSERT(len <= key.size()); @@ -3872,7 +3877,7 @@ private: // If the decode upper boundary is the subtree upper boundary the pointers will be the same // For the lower boundary, if the pointers are not the same there is still a possibility // that the keys are the same. This happens for the first remaining subtree of an internal page - // after the previous first subtree was cleared. + // after the prior subtree(s) were cleared. return (decodeUpperBound == subtreeUpperBound) && (decodeLowerBound == subtreeLowerBound || decodeLowerBound->sameExceptValue(*subtreeLowerBound)); } @@ -4984,6 +4989,246 @@ public: Future moveLast() { return move_end(this, false); } }; + // Cursor designed for short lifespans. + // Holds references to all pages touched. + // All record references returned from it are valid until the cursor is destroyed. + class BTreeCursor { + Arena arena; + Reference pager; + std::unordered_map> pages; + VersionedBTree* btree; + bool valid; + + struct PathEntry { + BTreePage* btPage; + BTreePage::BinaryTree::Cursor cursor; + }; + VectorRef path; + + public: + BTreeCursor() {} + + bool isValid() const { return valid; } + + std::string toString() const { + std::string r; + for (int i = 0; i < path.size(); ++i) { + r += format("[%d/%d: %s] ", i + 1, path.size(), + path[i].cursor.valid() ? path[i].cursor.get().toString(path[i].btPage->isLeaf()).c_str() + : ""); + } + if (!valid) { + r += " (invalid) "; + } + return r; + } + + const RedwoodRecordRef& get() { return path.back().cursor.get(); } + + bool inRoot() const { return path.size() == 1; } + + // Pop and return the page cursor at the end of the path. + // This is meant to enable range scans to consume the contents of a leaf page more efficiently. + // Can only be used when inRoot() is true. + BTreePage::BinaryTree::Cursor popPath() { + BTreePage::BinaryTree::Cursor c = path.back().cursor; + path.pop_back(); + return c; + } + + Future pushPage(BTreePageIDRef id, const RedwoodRecordRef& lowerBound, + const RedwoodRecordRef& upperBound) { + Reference& page = pages[id.front()]; + if (page.isValid()) { + path.push_back(arena, { (BTreePage*)page->begin(), getCursor(page) }); + return Void(); + } + + return map(readPage(pager, id, &lowerBound, &upperBound), [this, &page, id](Reference p) { + page = p; + path.push_back(arena, { (BTreePage*)p->begin(), getCursor(p) }); + return Void(); + }); + } + + Future pushPage(BTreePage::BinaryTree::Cursor c) { + const RedwoodRecordRef& rec = c.get(); + auto next = c; + next.moveNext(); + BTreePageIDRef id = rec.getChildPage(); + return pushPage(id, rec, next.getOrUpperBound()); + } + + Future init(VersionedBTree* btree_in, Reference pager_in, BTreePageIDRef root) { + btree = btree_in; + pager = pager_in; + path.reserve(arena, 6); + valid = false; + return pushPage(root, dbBegin, dbEnd); + } + + // Seeks cursor to query if it exists, the record before or after it, or an undefined and invalid + // position between those records + // If 0 is returned, then + // If the cursor is valid then it points to query + // If the cursor is not valid then the cursor points to some place in the btree such that + // If there is a record in the tree < query then movePrev() will move to it, and + // If there is a record in the tree > query then moveNext() will move to it. + // If non-zero is returned then the cursor is valid and the return value is logically equivalent + // to query.compare(cursor.get()) + ACTOR Future seek_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + state RedwoodRecordRef internalPageQuery = query.withMaxPageID(); + self->path = self->path.slice(0, 1); + debug_printf("seek(%s, %d) start cursor = %s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + + loop { + auto& entry = self->path.back(); + if (entry.btPage->isLeaf()) { + int cmp = entry.cursor.seek(query); + self->valid = entry.cursor.valid() && !entry.cursor.node->isDeleted(); + debug_printf("seek(%s, %d) loop exit cmp=%d cursor=%s\n", query.toString().c_str(), prefetchBytes, + cmp, self->toString().c_str()); + return self->valid ? cmp : 0; + } + + // Internal page, so seek to the branch where query must be + // Currently, after a subtree deletion internal page boundaries are still strictly adhered + // to and will be updated if anything is inserted into the cleared range, so if the seek fails + // or it finds an entry with a null child page then query does not exist in the BTree. + if (entry.cursor.seekLessThan(internalPageQuery) && entry.cursor.get().value.present()) { + debug_printf("seek(%s, %d) loop seek success cursor=%s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + Future f = self->pushPage(entry.cursor); + + // Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2 + // sibling + if (prefetchBytes != 0 && entry.btPage->height == 2) { + auto c = entry.cursor; + bool fwd = prefetchBytes > 0; + prefetchBytes = abs(prefetchBytes); + // While we should still preload more bytes and a move in the target direction is successful + while (prefetchBytes > 0 && (fwd ? c.moveNext() : c.movePrev())) { + // If there is a page link, preload it. + if (c.get().value.present()) { + BTreePageIDRef childPage = c.get().getChildPage(); + preLoadPage(self->pager.getPtr(), childPage); + prefetchBytes -= self->btree->m_blockSize * childPage.size(); + } + } + } + + wait(f); + } else { + self->valid = false; + debug_printf("seek(%s, %d) loop exit cmp=0 cursor=%s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + return 0; + } + } + } + + Future seek(RedwoodRecordRef query, int prefetchBytes) { return seek_impl(this, query, prefetchBytes); } + + ACTOR Future seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + debug_printf("seekGTE(%s, %d) start\n", query.toString().c_str(), prefetchBytes); + int cmp = wait(self->seek(query, prefetchBytes)); + if (cmp > 0 || (cmp == 0 && !self->isValid())) { + wait(self->moveNext()); + } + return Void(); + } + + Future seekGTE(RedwoodRecordRef query, int prefetchBytes) { + return seekGTE_impl(this, query, prefetchBytes); + } + + ACTOR Future seekLT_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + debug_printf("seekLT(%s, %d) start\n", query.toString().c_str(), prefetchBytes); + int cmp = wait(self->seek(query, prefetchBytes)); + if (cmp <= 0) { + wait(self->movePrev()); + } + return Void(); + } + + Future seekLT(RedwoodRecordRef query, int prefetchBytes) { + return seekLT_impl(this, query, -prefetchBytes); + } + + ACTOR Future move_impl(BTreeCursor* self, bool forward) { + // Try to the move cursor at the end of the path in the correct direction + debug_printf("move%s() start cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + while (1) { + debug_printf("move%s() first loop cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + auto& entry = self->path.back(); + bool success; + if(entry.cursor.valid()) { + success = forward ? entry.cursor.moveNext() : entry.cursor.movePrev(); + } else { + success = forward ? entry.cursor.moveFirst() : false; + } + + // Skip over internal page entries that do not link to child pages. There should never be two in a row. + if (success && !entry.btPage->isLeaf() && !entry.cursor.get().value.present()) { + success = forward ? entry.cursor.moveNext() : entry.cursor.movePrev(); + ASSERT(!success || entry.cursor.get().value.present()); + } + + // Stop if successful + if (success) { + break; + } + + if (self->path.size() == 1) { + self->valid = false; + return Void(); + } + + // Move to parent + self->path = self->path.slice(0, self->path.size() - 1); + } + + // While not on a leaf page, move down to get to one. + while (1) { + debug_printf("move%s() second loop cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + auto& entry = self->path.back(); + if (entry.btPage->isLeaf()) { + break; + } + + // The last entry in an internal page could be a null link, if so move back + if (!forward && !entry.cursor.get().value.present()) { + ASSERT(entry.cursor.movePrev()); + ASSERT(entry.cursor.get().value.present()); + } + + wait(self->pushPage(entry.cursor)); + auto& newEntry = self->path.back(); + ASSERT(forward ? newEntry.cursor.moveFirst() : newEntry.cursor.moveLast()); + } + + self->valid = true; + + debug_printf("move%s() exit cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + return Void(); + } + + Future moveNext() { return move_impl(this, true); } + Future movePrev() { return move_impl(this, false); } + }; + + Future initBTreeCursor(BTreeCursor* cursor, Version snapshotVersion) { + // Only committed versions can be read. + ASSERT(snapshotVersion <= m_lastCommittedVersion); + Reference snapshot = m_pager->getReadSnapshot(snapshotVersion); + + // This is a ref because snapshot will continue to hold the metakey value memory + KeyRef m = snapshot->getMetaKey(); + + return cursor->init(this, snapshot, ((MetaKey*)m.begin())->root.get()); + } + // Cursor is for reading and interating over user visible KV pairs at a specific version // KeyValueRefs returned become invalid once the cursor is moved class Cursor : public IStoreCursor, public ReferenceCounted, public FastAllocated, NonCopyable { @@ -5276,33 +5521,59 @@ public: return result; } - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); - // Prefetch is currently only done in the forward direction - state int prefetchBytes = rowLimit > 1 ? byteLimit : 0; + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + state int prefetchBytes = 0; if (rowLimit > 0) { - wait(cur->findFirstEqualOrGreater(keys.begin, prefetchBytes)); - while (cur->isValid() && cur->getKey() < keys.end) { - KeyValueRef kv(KeyRef(result.arena(), cur->getKey()), ValueRef(result.arena(), cur->getValue())); - accumulatedBytes += kv.expectedSize(); - result.push_back(result.arena(), kv); - if (--rowLimit == 0 || accumulatedBytes >= byteLimit) { + wait(cur.seekGTE(keys.begin, prefetchBytes)); + while (cur.isValid()) { + // Read page contents without using waits + bool isRoot = cur.inRoot(); + BTreePage::BinaryTree::Cursor leafCursor = cur.popPath(); + while(leafCursor.valid()) { + KeyValueRef kv = leafCursor.get().toKeyValueRef(); + if(kv.key >= keys.end) { + break; + } + accumulatedBytes += kv.expectedSize(); + result.push_back_deep(result.arena(), kv); + if (--rowLimit == 0 || accumulatedBytes >= byteLimit) { + break; + } + leafCursor.moveNext(); + } + // Stop if the leaf cursor is still valid which means we hit a key or size limit or + // if we started in the root page + if(leafCursor.valid() || isRoot) { break; } - wait(cur->next()); + wait(cur.moveNext()); } } else { - wait(cur->findLastLessOrEqual(keys.end)); - if (cur->isValid() && cur->getKey() == keys.end) wait(cur->prev()); - - while (cur->isValid() && cur->getKey() >= keys.begin) { - KeyValueRef kv(KeyRef(result.arena(), cur->getKey()), ValueRef(result.arena(), cur->getValue())); - accumulatedBytes += kv.expectedSize(); - result.push_back(result.arena(), kv); - if (++rowLimit == 0 || accumulatedBytes >= byteLimit) { + wait(cur.seekLT(keys.end, prefetchBytes)); + while (cur.isValid()) { + // Read page contents without using waits + bool isRoot = cur.inRoot(); + BTreePage::BinaryTree::Cursor leafCursor = cur.popPath(); + while(leafCursor.valid()) { + KeyValueRef kv = leafCursor.get().toKeyValueRef(); + if(kv.key < keys.begin) { + break; + } + accumulatedBytes += kv.expectedSize(); + result.push_back_deep(result.arena(), kv); + if (++rowLimit == 0 || accumulatedBytes >= byteLimit) { + break; + } + leafCursor.movePrev(); + } + // Stop if the leaf cursor is still valid which means we hit a key or size limit or + // if we started in the root page + if(leafCursor.valid() || isRoot) { break; } - wait(cur->prev()); + wait(cur.movePrev()); } } @@ -5320,11 +5591,12 @@ public: state FlowLock::Releaser releaser(self->m_concurrentReads); ++g_redwoodMetrics.opGet; - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); - wait(cur->findEqual(key)); - if (cur->isValid()) { - return cur->getValue(); + wait(cur.seekGTE(key, 0)); + if (cur.isValid() && cur.get().key == key) { + return cur.get().value.get(); } return Optional(); } @@ -5339,14 +5611,16 @@ public: state FlowLock::Releaser releaser(self->m_concurrentReads); ++g_redwoodMetrics.opGet; - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); - wait(cur->findEqual(key)); - if (cur->isValid()) { - Value v = cur->getValue(); + wait(cur.seekGTE(key, 0)); + if (cur.isValid() && cur.get().key == key) { + Value v = cur.get().value.get(); int len = std::min(v.size(), maxLength); - return Value(cur->getValue().substr(0, len)); + return Value(v.substr(0, len)); } + return Optional(); } @@ -5411,6 +5685,157 @@ KeyValue randomKV(int maxKeySize = 10, int maxValueSize = 5) { return kv; } +// Verify a range using a BTreeCursor. +// Assumes that the BTree holds a single data version and the version is 0. +ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, Key start, Key end, Version v, + std::map, Optional>* written, + int* pErrorCount) { + state int errors = 0; + if (end <= start) end = keyAfter(start); + + state std::map, Optional>::const_iterator i = + written->lower_bound(std::make_pair(start.toString(), 0)); + state std::map, Optional>::const_iterator iEnd = + written->upper_bound(std::make_pair(end.toString(), 0)); + state std::map, Optional>::const_iterator iLast; + + state VersionedBTree::BTreeCursor cur; + wait(btree->initBTreeCursor(&cur, v)); + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Start\n", v, start.printable().c_str(), end.printable().c_str()); + + // Randomly use the cursor for something else first. + if (deterministicRandom()->coinflip()) { + state Key randomKey = randomKV().key; + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Dummy seek to '%s'\n", v, start.printable().c_str(), + end.printable().c_str(), randomKey.toString().c_str()); + wait(success(cur.seek(randomKey, 0))); + } + + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), + end.printable().c_str()); + wait(cur.seekGTE(start, 0)); + + state std::vector results; + + while (cur.isValid() && cur.get().key < end) { + // Find the next written kv pair that would be present at this version + while (1) { + iLast = i; + if (i == iEnd) break; + ++i; + + if (iLast->first.second <= v && iLast->second.present() && + (i == iEnd || i->first.first != iLast->first.first || i->first.second > v)) { + debug_printf("VerifyRange(@%" PRId64 ", %s, %s) Found key in written map: %s\n", v, + start.printable().c_str(), end.printable().c_str(), iLast->first.first.c_str()); + break; + } + } + + if (iLast == iEnd) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' vs nothing in written map.\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str()); + break; + } + + if (cur.get().key != iLast->first.first) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + iLast->first.first.c_str()); + break; + } + if (cur.get().value.get() != iLast->second.get()) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' has tree value '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + cur.get().value.get().toString().c_str(), iLast->second.get().c_str()); + break; + } + + ASSERT(errors == 0); + + results.push_back(KeyValue(KeyValueRef(cur.get().key, cur.get().value.get()))); + wait(cur.moveNext()); + } + + // Make sure there are no further written kv pairs that would be present at this version. + while (1) { + iLast = i; + if (i == iEnd) break; + ++i; + if (iLast->first.second <= v && iLast->second.present() && + (i == iEnd || i->first.first != iLast->first.first || i->first.second > v)) + break; + } + + if (iLast != iEnd) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree range ended but written has @%" PRId64 " '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), iLast->first.second, iLast->first.first.c_str()); + } + + debug_printf("VerifyRangeReverse(@%" PRId64 ", %s, %s): start\n", v, start.printable().c_str(), + end.printable().c_str()); + + // Randomly use a new cursor at the same version for the reverse range read, if the version is still available for + // opening new cursors + if (v >= btree->getOldestVersion() && deterministicRandom()->coinflip()) { + cur = VersionedBTree::BTreeCursor(); + wait(btree->initBTreeCursor(&cur, v)); + } + + // Now read the range from the tree in reverse order and compare to the saved results + wait(cur.seekLT(end, 0)); + + state std::vector::const_reverse_iterator r = results.rbegin(); + + while (cur.isValid() && cur.get().key >= start) { + if (r == results.rend()) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' vs nothing in written map.\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str()); + break; + } + + if (cur.get().key != r->key) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + r->key.toString().c_str()); + break; + } + if (cur.get().value.get() != r->value) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 + ", %s, %s) ERROR: Tree key '%s' has tree value '%s' but expected '%s'\n", + v, start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + cur.get().value.get().toString().c_str(), r->value.toString().c_str()); + break; + } + + ++r; + wait(cur.movePrev()); + } + + if (r != results.rend()) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree range ended but written has '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), r->key.toString().c_str()); + } + + return errors; +} + ACTOR Future verifyRange(VersionedBTree* btree, Key start, Key end, Version v, std::map, Optional>* written, int* pErrorCount) { @@ -5607,6 +6032,58 @@ ACTOR Future seekAll(VersionedBTree* btree, Version v, return errors; } +// Verify the result of point reads for every set or cleared key at the given version +ACTOR Future seekAllBTreeCursor(VersionedBTree* btree, Version v, + std::map, Optional>* written, int* pErrorCount) { + state std::map, Optional>::const_iterator i = written->cbegin(); + state std::map, Optional>::const_iterator iEnd = written->cend(); + state int errors = 0; + state VersionedBTree::BTreeCursor cur; + + wait(btree->initBTreeCursor(&cur, v)); + + while (i != iEnd) { + state std::string key = i->first.first; + state Version ver = i->first.second; + if (ver == v) { + state Optional val = i->second; + debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str()); + state Arena arena; + wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key), 0), 0)); + bool foundKey = cur.isValid() && cur.get().key == key; + bool hasValue = foundKey && cur.get().value.present(); + + if (val.present()) { + bool valueMatch = hasValue && cur.get().value.get() == val.get(); + if (!foundKey || !hasValue || !valueMatch) { + ++errors; + ++*pErrorCount; + if (!foundKey) { + printf("Verify ERROR: key_not_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + val.get().c_str(), ver); + } + else if (!hasValue) { + printf("Verify ERROR: value_not_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + val.get().c_str(), ver); + } + else if (!valueMatch) { + printf("Verify ERROR: value_incorrect: for '%s' found '%s' expected '%s' @%" PRId64 "\n", + key.c_str(), cur.get().value.get().toString().c_str(), val.get().c_str(), + ver); + } + } + } else if (foundKey && hasValue) { + ++errors; + ++*pErrorCount; + printf("Verify ERROR: cleared_key_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + cur.get().value.get().toString().c_str(), ver); + } + } + ++i; + } + return errors; +} + ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, std::map, Optional>* written, int* pErrorCount, bool serial) { @@ -5637,7 +6114,13 @@ ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, state Reference cur = btree->readAtVersion(v); debug_printf("Verifying entire key range at version %" PRId64 "\n", v); - fRangeAll = verifyRange(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fRangeAll = verifyRange(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, + pErrorCount); + } else { + fRangeAll = verifyRangeBTreeCursor(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, + pErrorCount); + } if (serial) { wait(success(fRangeAll)); } @@ -5646,13 +6129,21 @@ ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, Key end = randomKV().key; debug_printf("Verifying range (%s, %s) at version %" PRId64 "\n", toString(begin).c_str(), toString(end).c_str(), v); - fRangeRandom = verifyRange(btree, begin, end, v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fRangeRandom = verifyRange(btree, begin, end, v, written, pErrorCount); + } else { + fRangeRandom = verifyRangeBTreeCursor(btree, begin, end, v, written, pErrorCount); + } if (serial) { wait(success(fRangeRandom)); } debug_printf("Verifying seeks to each changed key at version %" PRId64 "\n", v); - fSeekAll = seekAll(btree, v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fSeekAll = seekAll(btree, v, written, pErrorCount); + } else { + fSeekAll = seekAllBTreeCursor(btree, v, written, pErrorCount); + } if (serial) { wait(success(fSeekAll)); } From 11989c0650a674d920fed243b703d112da61d39b Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Jun 2020 11:10:09 -0700 Subject: [PATCH 79/89] update version to 6.3.2 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e48afd84e..5e11e2ceab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.13) project(foundationdb - VERSION 6.3.1 + VERSION 6.3.2 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) From 3508a0f06a9070a0ff0612a3556e3e8e2d0d3f2e Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Jun 2020 11:10:09 -0700 Subject: [PATCH 80/89] update installer WIX GUID following release --- packaging/msi/FDBInstaller.wxs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 2d2109e696..92aa3fa86e 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -32,7 +32,7 @@ Date: Fri, 5 Jun 2020 12:57:45 -0700 Subject: [PATCH 81/89] Ignore throttling errors in fuzz tester --- fdbserver/workloads/FuzzApiCorrectness.actor.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp index 61010cbaa1..15cc2d3fb2 100644 --- a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp +++ b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp @@ -59,7 +59,9 @@ struct ExceptionContract { e.code() == error_code_transaction_cancelled || e.code() == error_code_key_too_large || e.code() == error_code_value_too_large || - e.code() == error_code_process_behind) + e.code() == error_code_process_behind || + e.code() == error_code_batch_transaction_throttled || + e.code() == error_code_tag_throttled) { return; } From ffe949b04d47257899a65070bd89c86f62db3be0 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Fri, 5 Jun 2020 16:40:19 -0700 Subject: [PATCH 82/89] Applier:getAndComputeStagingKeys:reset txn at first error When tr->onError() is ready, the txn state has been reset. We cannot wait on the get() future from the txn because its state has been deleted. If we do that, it will throw txn_cancelled error, which will be throw all the way up to the RestoreApplier main loop. The batchData->dbApplier, which is assigned by writeMutationsToDB(self->id(), req.batchIndex, batchData, cx), will become ready but isError(). This will make all handleApplyToDBRequest throw error silently. --- fdbserver/RestoreApplier.actor.cpp | 56 +++++++++++++++++------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index b87e87cf2d..5e1566a77c 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -83,6 +83,7 @@ ACTOR Future restoreApplierCore(RestoreApplierInterface applierInterf, int updateProcessStats(self); updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL); } + when(wait(actors.getResult())) {} when(wait(exitRole)) { TraceEvent("RestoreApplierCoreExitRole", self->id()); break; @@ -92,6 +93,7 @@ ACTOR Future restoreApplierCore(RestoreApplierInterface applierInterf, int TraceEvent(SevWarn, "FastRestoreApplierError", self->id()) .detail("RequestType", requestTypeStr) .error(e, true); + actors.clear(false); break; } } @@ -232,38 +234,39 @@ ACTOR static Future getAndComputeStagingKeys( state bool hasError = false; loop { hasError = false; - tr->reset(); - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - i = 0; - for (auto& key : incompleteStagingKeys) { - if (!fValues[i].isReady() || !keyNotFounds.count(i)) { - fValues[i] = tr->get(key.first); + try { + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + i = 0; + for (auto& key : incompleteStagingKeys) { + if (!fValues[i].isReady() || !keyNotFounds.count(i)) { + fValues[i] = tr->get(key.first); + } + ++i; } - ++i; - } - for (i = 0; i < incompleteStagingKeys.size(); ++i) { - try { + for (i = 0; i < incompleteStagingKeys.size(); ++i) { if (keyNotFounds.count(i)) { continue; } wait(success(fValues[i])); // NOTE: This may be waiting for ever! - } catch (Error& e) { - if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == - // error_code_future_version - keyNotFounds.insert(i); - } else { - hasError = true; - } - if (retries > incompleteStagingKeys.size()) { - TraceEvent(SevError, "GetAndComputeStagingKeys", applierID) - .detail("BatchIndex", batchIndex) - .detail("KeyIndex", i) - .error(e); - } - wait(tr->onError(e)); } + } catch (Error& e) { + if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == + // error_code_future_version + keyNotFounds.insert(i); + } else { + hasError = true; + } + if (retries > incompleteStagingKeys.size()) { + TraceEvent(SevError, "GetAndComputeStagingKeys", applierID) + .detail("BatchIndex", batchIndex) + .detail("KeyIndex", i) + .error(e); + } + wait(tr->onError(e)); } + if (!hasError) { break; } @@ -528,6 +531,7 @@ ACTOR static Future handleApplyToDBRequest(RestoreVersionBatchRequest req, .detail("FinishedBatch", self->finishedBatch.get()); // Ensure batch (i-1) is applied before batch i + // TODO: Add a counter to warn when too many requests are waiting on the actor wait(self->finishedBatch.whenAtLeast(req.batchIndex - 1)); state bool isDuplicated = true; @@ -549,6 +553,8 @@ ACTOR static Future handleApplyToDBRequest(RestoreVersionBatchRequest req, } ASSERT(batchData->dbApplier.present()); + ASSERT(batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error. + // We cannot blindly retry because it is not idempodent wait(batchData->dbApplier.get()); From 96c2a164bccce9483bb0388fe5bd40b7e40460c2 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Fri, 5 Jun 2020 16:44:59 -0700 Subject: [PATCH 83/89] RestoreLoader:Wait on actorCollection error so that we will not fail sildently --- fdbserver/RestoreLoader.actor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index c919e77778..ffe5204c58 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -110,13 +110,17 @@ ACTOR Future restoreLoaderCore(RestoreLoaderInterface loaderInterf, int no updateProcessStats(self); updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL); } + when(wait(actors.getResult())) {} when(wait(exitRole)) { TraceEvent("FastRestoreLoaderCoreExitRole", self->id()); break; } } } catch (Error& e) { - TraceEvent(SevWarn, "FastRestoreLoader", self->id()).detail("RequestType", requestTypeStr).error(e, true); + TraceEvent(SevWarn, "FastRestoreLoaderError", self->id()) + .detail("RequestType", requestTypeStr) + .error(e, true); + actors.clear(false); break; } } From f51fca0bf345393c24448cfa63b67280df04805f Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Fri, 5 Jun 2020 17:41:03 -0700 Subject: [PATCH 84/89] FastRestore:Sanity check actors do not throw error silently --- fdbserver/RestoreApplier.actor.cpp | 4 ++-- fdbserver/RestoreMaster.actor.cpp | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 5e1566a77c..909253ae1a 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -553,8 +553,8 @@ ACTOR static Future handleApplyToDBRequest(RestoreVersionBatchRequest req, } ASSERT(batchData->dbApplier.present()); - ASSERT(batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error. - // We cannot blindly retry because it is not idempodent + ASSERT(!batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error. + // We cannot blindly retry because it is not idempodent wait(batchData->dbApplier.get()); diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp index de1fc909f2..6e7ceb961d 100644 --- a/fdbserver/RestoreMaster.actor.cpp +++ b/fdbserver/RestoreMaster.actor.cpp @@ -866,6 +866,7 @@ ACTOR static Future notifyApplierToApplyMutations(ReferenceapplyToDB.present()); + ASSERT(!batchData->applyToDB.get().isError()); wait(batchData->applyToDB.get()); // Sanity check all appliers have applied data to destination DB From a28b5f0a8b862b23188d7a674c43bf68ff0a0c69 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sat, 6 Jun 2020 21:10:13 -0700 Subject: [PATCH 85/89] Possible bug fix, flow locks should be taken after initializing BTree cursor in KVStoreRedwood read functions otherwise it might be possible for the BTree to be closed before the flow lock wait returns, depending on destruction order of some things. --- fdbserver/VersionedBTree.actor.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index df06321bde..bfe4ab4de6 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -5509,10 +5509,13 @@ public: ACTOR static Future> readRange_impl(KeyValueStoreRedwoodUnversioned* self, KeyRange keys, int rowLimit, int byteLimit) { + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGetRange; + state Standalone result; state int accumulatedBytes = 0; ASSERT(byteLimit > 0); @@ -5521,8 +5524,7 @@ public: return result; } - state VersionedBTree::BTreeCursor cur; - wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + // Prefetch is disabled for now pending some decent logic for deciding how much to fetch state int prefetchBytes = 0; if (rowLimit > 0) { @@ -5587,13 +5589,13 @@ public: ACTOR static Future> readValue_impl(KeyValueStoreRedwoodUnversioned* self, Key key, Optional debugID) { - wait(self->m_concurrentReads.take()); - state FlowLock::Releaser releaser(self->m_concurrentReads); - - ++g_redwoodMetrics.opGet; state VersionedBTree::BTreeCursor cur; wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); + state FlowLock::Releaser releaser(self->m_concurrentReads); + ++g_redwoodMetrics.opGet; + wait(cur.seekGTE(key, 0)); if (cur.isValid() && cur.get().key == key) { return cur.get().value.get(); @@ -5607,13 +5609,13 @@ public: ACTOR static Future> readValuePrefix_impl(KeyValueStoreRedwoodUnversioned* self, Key key, int maxLength, Optional debugID) { - wait(self->m_concurrentReads.take()); - state FlowLock::Releaser releaser(self->m_concurrentReads); - - ++g_redwoodMetrics.opGet; state VersionedBTree::BTreeCursor cur; wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); + state FlowLock::Releaser releaser(self->m_concurrentReads); + ++g_redwoodMetrics.opGet; + wait(cur.seekGTE(key, 0)); if (cur.isValid() && cur.get().key == key) { Value v = cur.get().value.get(); From 8fdb81b48de43e68dff5623f7121f4d3396ace9d Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sat, 6 Jun 2020 21:10:52 -0700 Subject: [PATCH 86/89] Tweaked BTree test random parameter limits to avoid test runs which take too long. --- fdbserver/VersionedBTree.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index bfe4ab4de6..5311c0f5b9 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6978,11 +6978,11 @@ TEST_CASE("!/redwood/correctness/btree") { state int maxKeySize = deterministicRandom()->randomInt(1, pageSize * 2); state int maxValueSize = randomSize(pageSize * 25); state int maxCommitSize = shortTest ? 1000 : randomSize(std::min((maxKeySize + maxValueSize) * 20000, 10e6)); - state int mutationBytesTarget = shortTest ? 100000 : randomSize(std::min(maxCommitSize * 100, 100e6)); + state int mutationBytesTarget = shortTest ? 100000 : randomSize(std::min(maxCommitSize * 100, pageSize * 100000)); state double clearProbability = deterministicRandom()->random01() * .1; state double clearSingleKeyProbability = deterministicRandom()->random01(); state double clearPostSetProbability = deterministicRandom()->random01() * .1; - state double coldStartProbability = pagerMemoryOnly ? 0 : deterministicRandom()->random01(); + state double coldStartProbability = pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3); state double advanceOldVersionProbability = deterministicRandom()->random01(); state double maxDuration = 60; state int64_t cacheSizeBytes = From 94be3afcf8e60fae783d82ef68d3f5c5a71a0c8a Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Sat, 6 Jun 2020 21:17:57 -0700 Subject: [PATCH 87/89] RestoreApplier:Costmic change based on review --- fdbserver/RestoreApplier.actor.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 909253ae1a..fa9d8d0640 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -220,15 +220,15 @@ ACTOR static Future getAndComputeStagingKeys( state Reference tr(new ReadYourWritesTransaction(cx)); state std::vector>> fValues(incompleteStagingKeys.size(), Never()); state int retries = 0; - // state UID randomID = deterministicRandom()->randomUniqueID(); + state UID randomID = deterministicRandom()->randomUniqueID(); wait(delay(delayTime + deterministicRandom()->random01() * delayTime)); TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID) - //.detail("RandomUID", randomID) + .detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) .detail("GetKeys", incompleteStagingKeys.size()) .detail("DelayTime", delayTime); - state std::set keyNotFounds; + state std::set keysNotFound; state int i = 0; state bool hasError = false; @@ -240,13 +240,13 @@ ACTOR static Future getAndComputeStagingKeys( tr->setOption(FDBTransactionOptions::LOCK_AWARE); i = 0; for (auto& key : incompleteStagingKeys) { - if (!fValues[i].isReady() || !keyNotFounds.count(i)) { + if (!fValues[i].isReady() || !keysNotFound.count(i)) { fValues[i] = tr->get(key.first); } ++i; } for (i = 0; i < incompleteStagingKeys.size(); ++i) { - if (keyNotFounds.count(i)) { + if (keysNotFound.count(i)) { continue; } wait(success(fValues[i])); // NOTE: This may be waiting for ever! @@ -254,7 +254,7 @@ ACTOR static Future getAndComputeStagingKeys( } catch (Error& e) { if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == // error_code_future_version - keyNotFounds.insert(i); + keysNotFound.insert(i); } else { hasError = true; } @@ -276,7 +276,7 @@ ACTOR static Future getAndComputeStagingKeys( ASSERT(fValues.size() == incompleteStagingKeys.size()); int i = 0; for (auto& key : incompleteStagingKeys) { - if (keyNotFounds.count(i) || (!fValues[i].get().present())) { // Key not exist in DB + if (keysNotFound.count(i) || (!fValues[i].get().present())) { // Key not exist in DB // if condition: fValues[i].Valid() && fValues[i].isReady() && !fValues[i].isError() && TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID) .detail("BatchIndex", batchIndex) @@ -301,7 +301,7 @@ ACTOR static Future getAndComputeStagingKeys( } TraceEvent("FastRestoreApplierGetAndComputeStagingKeysDone", applierID) - //.detail("RandomUID", randomID) + .detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) .detail("GetKeys", incompleteStagingKeys.size()) .detail("DelayTime", delayTime); @@ -392,7 +392,7 @@ ACTOR static Future precomputeMutationsResult(Reference incompleteStagingKeys.clear(); } } - if (numKeysInBatch >= 1) { + if (numKeysInBatch > 0) { fGetAndComputeKeys.push_back( getAndComputeStagingKeys(incompleteStagingKeys, delayTime, cx, applierID, batchIndex)); } From 8c81fedf11ab9856cf7677adaf4fff86184ca70b Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Sun, 7 Jun 2020 20:35:07 -0700 Subject: [PATCH 88/89] RestoreApplier:Better handling of key not exist --- fdbserver/RestoreApplier.actor.cpp | 45 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index fa9d8d0640..5072e45655 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -213,6 +213,21 @@ ACTOR static Future applyClearRangeMutations(Standalone> getValue(Reference tr, Key key, int i, + std::set* keysNotFound) { + try { + Optional v = wait(tr->get(key)); + return v; + } catch (Error& e) { + if (e.code() == error_code_key_not_found) { + keysNotFound->insert(i); + return Optional(); + } else { + throw; + } + } +} + // Get keys in incompleteStagingKeys and precompute the stagingKey which is stored in batchData->stagingKeys ACTOR static Future getAndComputeStagingKeys( std::map::iterator> incompleteStagingKeys, double delayTime, Database cx, @@ -231,46 +246,30 @@ ACTOR static Future getAndComputeStagingKeys( state std::set keysNotFound; state int i = 0; - state bool hasError = false; loop { - hasError = false; try { tr->reset(); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); i = 0; for (auto& key : incompleteStagingKeys) { - if (!fValues[i].isReady() || !keysNotFound.count(i)) { - fValues[i] = tr->get(key.first); + if (!keysNotFound.count(i)) { // only get exist-keys + fValues[i] = getValue(tr, key.first, i, &keysNotFound); } ++i; } - for (i = 0; i < incompleteStagingKeys.size(); ++i) { - if (keysNotFound.count(i)) { - continue; - } - wait(success(fValues[i])); // NOTE: This may be waiting for ever! - } + wait(waitForAll(fValues)); + break; } catch (Error& e) { - if (e.code() == error_code_key_not_found) { // e.code() == error_code_transaction_too_old || e.code() == - // error_code_future_version - keysNotFound.insert(i); - } else { - hasError = true; - } - if (retries > incompleteStagingKeys.size()) { - TraceEvent(SevError, "GetAndComputeStagingKeys", applierID) + bool ok = (e.code() != error_code_key_not_found); + if (!ok || retries++ > incompleteStagingKeys.size()) { + TraceEvent(!ok ? SevError : SevWarnAlways, "GetAndComputeStagingKeys", applierID) .detail("BatchIndex", batchIndex) .detail("KeyIndex", i) .error(e); } wait(tr->onError(e)); } - - if (!hasError) { - break; - } - retries++; } ASSERT(fValues.size() == incompleteStagingKeys.size()); From f00deefd5aa6da98ed10ac46bbc5a4d9ed344ffb Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Mon, 8 Jun 2020 10:10:32 -0700 Subject: [PATCH 89/89] RestoreApplier:Remove unnecessary txn reset --- fdbserver/RestoreApplier.actor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 5072e45655..7df2b61a57 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -248,7 +248,6 @@ ACTOR static Future getAndComputeStagingKeys( state int i = 0; loop { try { - tr->reset(); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); i = 0;