From ea08bc5462a010f7461657ceb263d46f5f9a130b Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Thu, 18 Jun 2020 09:37:49 -0700 Subject: [PATCH 001/458] Added server side code for range split support --- fdbclient/NativeAPI.actor.cpp | 41 ++++++++++++++++++++++++++++++ fdbclient/NativeAPI.actor.h | 2 ++ fdbclient/StorageServerInterface.h | 32 +++++++++++++++++++++++ fdbserver/StorageMetrics.actor.h | 29 +++++++++++++++++++++ fdbserver/storageserver.actor.cpp | 8 ++++++ 5 files changed, 112 insertions(+) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index fc9b1548a8..6768c6ac38 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -4106,6 +4106,47 @@ Future>> Transaction::getReadHotRanges(KeyRang return ::getReadHotRanges(cx, keys); } +ACTOR Future>> getRangeSplitPoints(Database cx, KeyRange keys, int64_t chunkSize) { + loop { + state vector>> locations = + wait(getKeyRangeLocations(cx, keys, 100, false, &StorageServerInterface::getRangeSplitPoints, + TransactionInfo(TaskPriority::DataDistribution))); + try { + state int nLocs = locations.size(); + state vector> fReplies(nLocs); + for (int i = 0; i < nLocs; i++) { + SplitRangeRequest req(locations[i].first, chunkSize); + fReplies[i] = loadBalance(locations[i].second, &StorageServerInterface::getRangeSplitPoints, req, + TaskPriority::DataDistribution); + } + + wait(waitForAll(fReplies)); + Standalone> results; + + for (int i = 0; i < nLocs; i++) { + if (i > 0) { + results.push_back_deep(results.arena(), locations[i].first.begin); // Need this shard boundary + } + results.append_deep(results.arena(), fReplies[i].get().splitPoints.begin(), + fReplies[i].get().splitPoints.size()); + } + + return results; + } catch (Error& e) { + if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { + TraceEvent(SevError, "GetRangeSplitPoints").error(e); + throw; + } + cx->invalidateCache(keys); + wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution)); + } + } +} + +Future>> Transaction::getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize) { + return ::getRangeSplitPoints(cx, keys, chunkSize); +} + ACTOR Future< Standalone> > splitStorageMetrics( Database cx, KeyRange keys, StorageMetrics limit, StorageMetrics estimated ) { loop { diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 5d5d5932da..a026b4b5db 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -263,6 +263,8 @@ public: Future< Standalone> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated ); Future>> getReadHotRanges(KeyRange const& keys); + // Try to split the given range into equally sized chunks based on estimated size. + Future>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize); // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true ); void atomicOp( const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, bool addConflictRange = true ); diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index cfd8c54ec7..a89018ce98 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -72,6 +72,8 @@ struct StorageServerInterface { RequestStream> getKeyValueStoreType; RequestStream watchValue; RequestStream getReadHotRanges; + RequestStream getRangeSplitPoints; + explicit StorageServerInterface(UID uid) : uniqueID( uid ) {} StorageServerInterface() : uniqueID( deterministicRandom()->randomUniqueID() ) {} NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } @@ -98,6 +100,8 @@ struct StorageServerInterface { getKeyValueStoreType = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(9) ); watchValue = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(10) ); getReadHotRanges = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(11) ); + getRangeSplitPoints = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); } } else { ASSERT(Ar::isDeserializing); @@ -449,6 +453,34 @@ struct ReadHotSubRangeRequest { } }; +struct SplitRangeReply { + constexpr static FileIdentifier file_identifier = 21813134; + // If the given range can be divided, contains the split points. + // If the given range cannot be divided(for exmaple its total size is smaller than the chunk size), this would be + // empty + Standalone> splitPoints; + + template + void serialize(Ar& ar) { + serializer(ar, splitPoints); + } +}; +struct SplitRangeRequest { + constexpr static FileIdentifier file_identifier = 30725174; + Arena arena; + KeyRangeRef keys; + int64_t chunkSize; + ReplyPromise reply; + + SplitRangeRequest() {} + SplitRangeRequest(KeyRangeRef const& keys, int64_t chunkSize) : keys(arena, keys), chunkSize(chunkSize) {} + + template + void serialize(Ar& ar) { + serializer(ar, keys, chunkSize, reply, arena); + } +}; + struct GetStorageMetricsReply { constexpr static FileIdentifier file_identifier = 15491478; StorageMetrics load; diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h index f230833df5..9f77d7aed3 100644 --- a/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/StorageMetrics.actor.h @@ -478,6 +478,35 @@ struct StorageServerMetrics { req.reply.send(reply); } + std::vector getSplitPoints(KeyRangeRef range, int64_t chunkSize) { + std::vector toReturn; + KeyRef beginKey = range.begin; + IndexedSet::iterator endKey = + byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize); + while (endKey != byteSample.sample.end()) { + if (*endKey > range.end) { + break; + } + if (*endKey == beginKey) { + ++endKey; + continue; + } + toReturn.push_back(*endKey); + beginKey = *endKey; + endKey = + byteSample.sample.index(byteSample.sample.sumTo(byteSample.sample.lower_bound(beginKey)) + chunkSize); + } + return toReturn; + } + + void getSplitPoints(SplitRangeRequest req) { + SplitRangeReply reply; + std::vector points = getSplitPoints(req.keys, req.chunkSize); + + reply.splitPoints = VectorRef(points.data(), points.size()); + req.reply.send(reply); + } + private: static void collapse( KeyRangeMap& map, KeyRef const& key ) { auto range = map.rangeContaining(key); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index d8eb8f22a6..2aa46d72d1 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -3640,6 +3640,14 @@ ACTOR Future metricsCore( StorageServer* self, StorageServerInterface ssi self->metrics.getReadHotRanges(req); } } + when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) { + if (!self->isReadable(req.keys)) { + TEST(true); // getSplitPoints immediate wrong_shard_server() + self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty()); + } else { + self->metrics.getSplitPoints(req); + } + } when (wait(doPollMetrics) ) { self->metrics.poll(); doPollMetrics = delay(SERVER_KNOBS->STORAGE_SERVER_POLL_METRICS_DELAY); From 2126f46195c4b138651220568cdaf1bc579038e0 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Thu, 18 Jun 2020 09:41:50 -0700 Subject: [PATCH 002/458] Added client side support for range split --- fdbclient/IClientApi.h | 2 ++ fdbclient/MultiVersionTransaction.actor.cpp | 28 +++++++++++++++++++++ fdbclient/MultiVersionTransaction.h | 17 +++++++++++-- fdbclient/ReadYourWrites.actor.cpp | 10 ++++++++ fdbclient/ReadYourWrites.h | 1 + fdbclient/ThreadSafeTransaction.actor.cpp | 10 ++++++++ fdbclient/ThreadSafeTransaction.h | 2 ++ 7 files changed, 68 insertions(+), 2 deletions(-) diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index 154ac9723f..0d7288de52 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -49,6 +49,8 @@ public: virtual void addReadConflictRange(const KeyRangeRef& keys) = 0; virtual ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) = 0; + virtual ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) = 0; virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0; virtual void set(const KeyRef& key, const ValueRef& value) = 0; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 966d0f6504..a3359eb6c8 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -159,6 +159,23 @@ ThreadFuture DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRe }); } +ThreadFuture>> DLTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + if (!api->transactionGetRangeSplitPoints) { + return unsupported_operation(); + } + FdbCApi::FDBFuture* f = api->transactionGetRangeSplitPoints(tr, range.begin.begin(), range.begin.size(), + range.end.begin(), range.end.size(), chunkSize); + + return toThreadFuture>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + const FdbCApi::FDBKey* splitKeys; + int keysArrayLength; + FdbCApi::fdb_error_t error = api->futureGetKeyArray(f, &splitKeys, &keysArrayLength); + ASSERT(!error); + return Standalone>(VectorRef((KeyRef*)splitKeys, keysArrayLength), Arena()); + }); +} + void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ)); } @@ -322,12 +339,15 @@ void DLApi::init() { loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel"); loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); + loadClientFunction(&api->transactionGetRangeSplitPoints, lib, fdbCPath, "fdb_transaction_get_range_split_points", + headerVersion >= 630); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); loadClientFunction(&api->futureGetStringArray, lib, fdbCPath, "fdb_future_get_string_array"); + loadClientFunction(&api->futureGetKeyArray, lib, fdbCPath, "fdb_future_get_key_array"); loadClientFunction(&api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array"); loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback"); loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel"); @@ -568,6 +588,14 @@ ThreadFuture MultiVersionTransaction::getEstimatedRangeSizeBytes(const return abortableFuture(f, tr.onChange); } +ThreadFuture>> MultiVersionTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + auto tr = getTransaction(); + auto f = tr.transaction ? tr.transaction->getRangeSplitPoints(range, chunkSize) + : ThreadFuture>>(Never()); + return abortableFuture(f, tr.onChange); +} + void MultiVersionTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) { auto tr = getTransaction(); if(tr.transaction) { diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index c803032cc7..ebf40726e0 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -35,6 +35,10 @@ struct FdbCApi : public ThreadSafeReferenceCounted { typedef struct transaction FDBTransaction; #pragma pack(push, 4) + typedef struct key { + const uint8_t* key; + int keyLength; + } FDBKey; typedef struct keyvalue { const void *key; int keyLength; @@ -84,7 +88,11 @@ struct FdbCApi : public ThreadSafeReferenceCounted { FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); - + + FDBFuture* (*transactionGetRangeSplitPoints)(FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, + int end_key_name_length, int64_t chunkSize); + FDBFuture* (*transactionCommit)(FDBTransaction *tr); fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion); FDBFuture* (*transactionGetApproximateSize)(FDBTransaction *tr); @@ -103,6 +111,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { fdb_error_t (*futureGetKey)(FDBFuture *f, uint8_t const **outKey, int *outKeyLength); fdb_error_t (*futureGetValue)(FDBFuture *f, fdb_bool_t *outPresent, uint8_t const **outValue, int *outValueLength); fdb_error_t (*futureGetStringArray)(FDBFuture *f, const char ***outStrings, int *outCount); + fdb_error_t (*futureGetKeyArray)(FDBFuture* f, FDBKey const** outKeys, int* outCount); fdb_error_t (*futureGetKeyValueArray)(FDBFuture *f, FDBKeyValue const ** outKV, int *outCount, fdb_bool_t *outMore); fdb_error_t (*futureSetCallback)(FDBFuture *f, FDBCallback callback, void *callback_parameter); void (*futureCancel)(FDBFuture *f); @@ -133,7 +142,9 @@ public: ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; - + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; + void addReadConflictRange(const KeyRangeRef& keys) override; void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; @@ -237,6 +248,8 @@ public: void addReadConflictRange(const KeyRangeRef& keys) override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; void set(const KeyRef& key, const ValueRef& value) override; diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index c69ca87467..0b092c2d58 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1397,6 +1397,16 @@ Future ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyR return map(waitOrError(tr.getStorageMetrics(keys, -1), resetPromise.getFuture()), [](const StorageMetrics& m) { return m.bytes; }); } +Future>> ReadYourWritesTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + if (checkUsedDuringCommit()) { + throw used_during_commit(); + } + if (resetPromise.isSet()) return resetPromise.getFuture().getError(); + + return waitOrError(tr.getRangeSplitPoints(range, chunkSize), resetPromise.getFuture()); +} + void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys ) { if(checkUsedDuringCommit()) { throw used_during_commit(); diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index 16edbb7277..493ff06b27 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -86,6 +86,7 @@ public: [[nodiscard]] Future>> getAddressesForKey(const Key& key); Future getEstimatedRangeSizeBytes( const KeyRangeRef& keys ); + Future>> getRangeSplitPoints(const KeyRangeRef& range, int64_t chunkSize); void addReadConflictRange( KeyRangeRef const& keys ); void makeSelfConflicting() { tr.makeSelfConflicting(); } diff --git a/fdbclient/ThreadSafeTransaction.actor.cpp b/fdbclient/ThreadSafeTransaction.actor.cpp index 8b9d75e2e2..2c375ea28b 100644 --- a/fdbclient/ThreadSafeTransaction.actor.cpp +++ b/fdbclient/ThreadSafeTransaction.actor.cpp @@ -164,6 +164,16 @@ ThreadFuture ThreadSafeTransaction::getEstimatedRangeSizeBytes( const K } ); } +ThreadFuture>> ThreadSafeTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + KeyRange r = range; + + ReadYourWritesTransaction* tr = this->tr; + return onMainThread([tr, r, chunkSize]() -> Future>> { + tr->checkDeferredError(); + return tr->getRangeSplitPoints(r, chunkSize); + }); +} ThreadFuture< Standalone > ThreadSafeTransaction::getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse ) { KeySelector b = begin; diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index 8e364ed3a3..0702daa541 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -72,6 +72,8 @@ public: ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; void addReadConflictRange( const KeyRangeRef& keys ) override; void makeSelfConflicting(); From 440630a0cba642bc8f042eb1e4dcdff75828f228 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Wed, 24 Jun 2020 15:17:57 -0700 Subject: [PATCH 003/458] Added bindings supports --- bindings/c/fdb_c.cpp | 18 +++ bindings/c/foundationdb/fdb_c.h | 13 ++- bindings/flow/fdb_flow.actor.cpp | 11 ++ bindings/flow/fdb_flow.h | 1 + bindings/go/go.mod | 2 +- bindings/go/src/fdb/futures.go | 51 +++++++++ bindings/go/src/fdb/transaction.go | 25 +++++ bindings/java/CMakeLists.txt | 2 + bindings/java/fdbJNI.cpp | 104 ++++++++++++++++++ .../apple/foundationdb/FDBTransaction.java | 26 +++++ .../apple/foundationdb/FutureKeyArray.java | 37 +++++++ .../apple/foundationdb/KeyArrayResult.java | 44 ++++++++ .../apple/foundationdb/ReadTransaction.java | 20 ++++ bindings/python/fdb/impl.py | 32 +++++- bindings/ruby/lib/fdbimpl.rb | 29 +++++ 15 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java create mode 100644 bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index ba56744dc7..ed6f681037 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -281,6 +281,17 @@ fdb_error_t fdb_future_get_string_array( ); } +extern "C" DLLEXPORT +fdb_error_t fdb_future_get_key_array( + FDBFuture* f, FDBKey const** out_key_array, int* out_count) +{ + CATCH_AND_RETURN( + Standalone> na = TSAV(Standalone>, f)->get(); + *out_key_array = (FDBKey*) na.begin(); + *out_count = na.size(); + ); +} + FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) { char *path; if(cluster_file_path) { @@ -634,6 +645,13 @@ FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, u return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); } +extern "C" DLLEXPORT +FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize) { + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunkSize).extractPtr()); +} + #include "fdb_c_function_pointers.g.h" #define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index b5dfa63d13..8eeebd70db 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -91,6 +91,10 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter); #pragma pack(push, 4) + typedef struct key { + const uint8_t* key; + int key_length; + } FDBKey; #if FDB_API_VERSION >= 700 typedef struct keyvalue { const uint8_t* key; @@ -143,9 +147,12 @@ extern "C" { #if FDB_API_VERSION >= 14 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t - fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_kv, + fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_key_array, int* out_count, fdb_bool_t* out_more ); #endif + DLLEXPORT WARN_UNUSED_RESULT fdb_error_t + fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_k, + int* out_count); DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count); @@ -259,6 +266,10 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); + + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* + fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize); #define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0 #define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0 diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 27355138b1..4f03231848 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -134,6 +134,7 @@ namespace FDB { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; + Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override; void addReadConflictRange(KeyRangeRef const& keys) override; void addReadConflictKey(KeyRef const& key) override; @@ -356,6 +357,16 @@ namespace FDB { }); } + Future>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) { + return backToFuture>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference f) { + FDBKey const* ks; + int count; + throw_on_error(fdb_future_get_key_array(f->f, &ks, &count)); + + return FDBStandalone>(f, VectorRef((KeyRef*)ks, count)); + }); + } + void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); } diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 66049cae0c..fe06739529 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -90,6 +90,7 @@ namespace FDB { } virtual Future getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; + virtual Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0; virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; virtual void addReadConflictKey(KeyRef const& key) = 0; diff --git a/bindings/go/go.mod b/bindings/go/go.mod index ec5746bf99..65d3ee7383 100644 --- a/bindings/go/go.mod +++ b/bindings/go/go.mod @@ -3,4 +3,4 @@ module github.com/apple/foundationdb/bindings/go // The FoundationDB go bindings currently have no external golang dependencies outside of // the go standard library. -require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect +require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 43718fe738..31211679a8 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -306,6 +306,57 @@ func (f *futureKeyValueArray) Get() ([]KeyValue, bool, error) { return ret, (more != 0), nil } +// FutureKeyArray represents the asynchronous result of a function +// that returns an array of keys. FutureKeyArray is a lightweight object +// that may be efficiently copied, and is safe for concurrent use by multiple goroutines. +type FutureKeyArray interface { + + // Get returns an array of keys or an error if the asynchronous operation + // associated with this future did not successfully complete. The current + // goroutine will be blocked until the future is ready. + Get() ([]Key, error) + + // MustGet returns an array of keys, or panics if the asynchronous operations + // associated with this future did not successfully complete. The current goroutine + // will be blocked until the future is ready. + MustGet() []Key +} + +type futureKeyArray struct { + *future +} + +func (f *futureKeyArray) Get() ([]Key, error) { + defer runtime.KeepAlive(f.future) + + f.BlockUntilReady() + + var ks *C.FDBKey + var count C.int + + if err:= C.fdb_future_get_key_array(f.ptr, &ks, &count); err != 0 { + return nil, Error{int(err)} + } + + ret := make([]Key, int(count)) + + for i:= 0; iNewByteArray(totalKeySize); + if( !keyArray ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + return JNI_NULL; + } + uint8_t *keys_barr = (uint8_t *)jenv->GetByteArrayElements(keyArray, JNI_NULL); + if (!keys_barr) { + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return JNI_NULL; + } + + jintArray lengthArray = jenv->NewIntArray(count); + if( !lengthArray ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + return JNI_NULL; + } + + jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); + if( !length_barr ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + return JNI_NULL; + } + + int offset = 0; + for(int i = 0; i < count; i++) { + memcpy(keys_barr + offset, ks[i].key, ks[i].key_length); + length_barr[i] = ks[i].key_length; + offset += ks[i].key_length; + } + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0); + + jobject result = jenv->NewObject(key_array_result_class, key_array_result_init, keyArray, lengthArray); + if( jenv->ExceptionOccurred() ) + return JNI_NULL; + + return result; + +} + JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getSummary(JNIEnv *jenv, jobject, jlong future) { if( !future ) { throwParamNotNull(jenv); @@ -669,6 +740,35 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIEnv *jenv, jobject, jlong tPtr, + jbyteArray beginKeyBytes, jbyteArray endKeyBytes, jlong chunkSize) { + if( !tPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction *tr = (FDBTransaction *)tPtr; + + uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL ); + if(!startKey) { + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKey) { + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + FDBFuture *f = fdb_transaction_get_range_split_points( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ), chunkSize ); + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT ); + return (jlong)f; +} + JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { if( !tPtr || !keyBytes || !valueBytes ) { throwParamNotNull(jenv); @@ -1045,6 +1145,10 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { range_result_init = env->GetMethodID(local_range_result_class, "", "([B[IZ)V"); range_result_class = (jclass) (env)->NewGlobalRef(local_range_result_class); + jclass local_key_array_result_class = env->FindClass("com/apple/foundationdb/KeyArrayResult"); + key_array_result_init = env->GetMethodID(local_key_array_result_class, "", "([B[I)V"); + key_array_result_class = (jclass) (env)->NewGlobalRef(local_key_array_result_class); + jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary"); range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "", "([BIZ)V"); range_result_summary_class = (jclass) (env)->NewGlobalRef(local_range_result_summary_class); diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index 09be8a353a..edac8c8775 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -80,6 +80,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC return FDBTransaction.this.getEstimatedRangeSizeBytes(range); } + @Override + public CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) { + return FDBTransaction.this.getRangeSplitPoints(begin, end, chunkSize); + } + + @Override + public CompletableFuture getRangeSplitPoints(Range range, long chunkSize) { + return FDBTransaction.this.getRangeSplitPoints(range, chunkSize); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -282,6 +292,21 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC return this.getEstimatedRangeSizeBytes(range.begin, range.end); } + @Override + public CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) { + pointerReadLock.lock(); + try { + return new FutureKeyArray(Transaction_getRangeSplitPoints(getPtr(), begin, end, chunkSize), executor); + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public CompletableFuture getRangeSplitPoints(Range range, long chunkSize) { + return this.getRangeSplitPoints(range.begin, range.end, chunkSize); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -685,4 +710,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC private native void Transaction_cancel(long cPtr); private native long Transaction_getKeyLocations(long cPtr, byte[] key); private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd); + private native long Transaction_getRangeSplitPoints(long cPtr, byte[] keyBegin, byte[] keyEnd, long chunkSize); } diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java b/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java new file mode 100644 index 0000000000..527d7076b0 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java @@ -0,0 +1,37 @@ +/* + * FutureKeyArray.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.concurrent.Executor; + +class FutureKeyArray extends NativeFuture { + FutureKeyArray(long cPtr, Executor executor) { + super(cPtr); + registerMarshalCallback(executor); + } + + @Override + protected KeyArrayResult getIfDone_internal(long cPtr) throws FDBException { + return FutureKeyArray_get(cPtr); + } + + private native KeyArrayResult FutureKeyArray_get(long cPtr) throws FDBException; +} diff --git a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java new file mode 100644 index 0000000000..244435eb00 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java @@ -0,0 +1,44 @@ +/* + * KeyArrayResult.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.ArrayList; +import java.util.List; + +class KeyArrayResult { + final List keys; + + KeyArrayResult(byte[] keyBytes, int[] keyLengths) { + int count = keyLengths.length; + keys = new ArrayList(count); + + int offset = 0; + for(int i = 0; i < count; i++) { + int keyLength = keyLengths[i]; + + byte[] key = new byte[keyLength]; + System.arraycopy(keyBytes, offset, key, 0, keyLength); + + offset += keyLength; + keys.add(key); + } + } +} diff --git a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java index 3dd11b77ff..8b313c21ee 100644 --- a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java @@ -444,6 +444,26 @@ public interface ReadTransaction extends ReadTransactionContext { */ CompletableFuture getEstimatedRangeSizeBytes(Range range); + /** + * Gets a list of keys that can split the given range into similar sized chunks based on chunkSize + * + * @param begin the beginning of the range (inclusive) + * @param end the end of the range (exclusive) + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize); + + /** + * Gets a list of keys that can split the given range into similar sized chunks based on chunkSize + * + * @param range the range of the keys + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getRangeSplitPoints(Range range, long chunkSize); + + /** * Returns a set of options that can be set on a {@code Transaction} * diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 91bdc2f3a0..d68df5e3e5 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -471,7 +471,16 @@ class TransactionRead(_FDBBase): begin_key, len(begin_key), end_key, len(end_key) )) - + + def get_range_split_points(self, begin_key, end_key, chunkSize): + if begin_key is None or end_key is None: + raise Exception('Invalid begin key or end key') + return FutureKeyArray(self.capi.fdb_transaction_get_range_split_points( + self.tpointer, + begin_key, len(begin_key), + end_key, len(end_key), + chunkSize + )) class Transaction(TransactionRead): """A modifiable snapshot of a Database. @@ -736,6 +745,14 @@ class FutureKeyValueArray(Future): # the KVs on the python side and in most cases we are about to # destroy the future anyway +class FutureKeyArray(Future): + def wait(self): + self.block_until_ready() + ks = ctypes.pointer(KeyStruct()) + count = ctypes.c_int() + self.capi.fdb_future_get_key_array(self.fpointer, ctypes.byref(ks), ctypes.byref(count)) + return ([ctypes.string_at(x.key, x.key_length) for x in ks[0:count.value]], count.value) + class FutureStringArray(Future): def wait(self): @@ -1217,6 +1234,11 @@ class KeyValueStruct(ctypes.Structure): ('value_length', ctypes.c_int)] _pack_ = 4 +class KeyStruct(ctypes.Structure): + _fields_ = [('key', ctypes.POINTER(ctypes.c_byte)), + ('key_length', ctypes.c_int)] + _pack_ = 4 + class KeyValue(object): def __init__(self, key, value): @@ -1406,6 +1428,11 @@ def init_c_api(): _capi.fdb_future_get_keyvalue_array.restype = int _capi.fdb_future_get_keyvalue_array.errcheck = check_error_code + _capi.fdb_future_get_key_array.argtypes = [ctypes.c_void_p, ctypes.POINTER( + ctypes.POINTER(KeyStruct)), ctypes.POINTER(ctypes.c_int)] + _capi.fdb_future_get_key_array.restype = int + _capi.fdb_future_get_key_array.errcheck = check_error_code + _capi.fdb_future_get_string_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)), ctypes.POINTER(ctypes.c_int)] _capi.fdb_future_get_string_array.restype = int _capi.fdb_future_get_string_array.errcheck = check_error_code @@ -1451,6 +1478,9 @@ def init_c_api(): _capi.fdb_transaction_get_estimated_range_size_bytes.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int] _capi.fdb_transaction_get_estimated_range_size_bytes.restype = ctypes.c_void_p + _capi.fdb_transaction_get_range_split_points.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] + _capi.fdb_transaction_get_range_split_points.restype = ctypes.c_void_p + _capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] _capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int _capi.fdb_transaction_add_conflict_range.errcheck = check_error_code diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 043cce23b6..5564db4d0e 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -109,6 +109,7 @@ module FDB attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer attach_function :fdb_transaction_get_estimated_range_size_bytes, [ :pointer, :pointer, :int, :pointer, :int ], :pointer + attach_function :fdb_transaction_get_range_split_points, [ :pointer, :pointer, :int, :pointer, :int, :int64 ], :pointer attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void @@ -129,6 +130,12 @@ module FDB :value_length, :int end + class KeyStruct < FFI::Struct + pack 4 + layout :key, :pointer, + :key_length, :int + end + def self.check_error(code) raise Error.new(code) if code.nonzero? nil @@ -472,6 +479,22 @@ module FDB end end + class FutureKeyArray < Future + def wait + block_until_ready + + ks = FFI::MemoryPointer.new :pointer + count = FFI::MemoryPointer.new :int + FDBC.check_error FDBC.fdb_future_get_key_array(@fpointer, kvs, count) + ks = ks.read_pointer + + [(0..count.read_int-1).map{|i| + x = FDBC::KeyStruct.new(ks + (i * FDBC::KeyStruct.size)) + x[:key].read_bytes(x[:key_length]) + }, count.read_int] + end + end + class FutureStringArray < LazyFuture def getter strings = FFI::MemoryPointer.new :pointer @@ -825,6 +848,12 @@ module FDB Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize)) end + def get_range_split_points(begin_key, end_key, chunkSize) + bkey = FDB.key_to_bytes(begin_key) + ekey = FDB.key_to_bytes(end_key) + FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunkSize)) + end + end TransactionRead.class_variable_set("@@StreamingMode", @@StreamingMode) From dc314ac384c282587aabaa4da0887353928123f4 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Wed, 24 Jun 2020 19:49:56 -0700 Subject: [PATCH 004/458] Fix Go bindings build error --- bindings/go/src/fdb/snapshot.go | 13 +++++++++++++ bindings/go/src/fdb/transaction.go | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bindings/go/src/fdb/snapshot.go b/bindings/go/src/fdb/snapshot.go index ca21818729..2245fca6b9 100644 --- a/bindings/go/src/fdb/snapshot.go +++ b/bindings/go/src/fdb/snapshot.go @@ -87,6 +87,8 @@ func (s Snapshot) GetDatabase() Database { return s.transaction.db } +// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes +// stored in the given range. func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { beginKey, endKey := r.FDBRangeKeys() return s.getEstimatedRangeSizeBytes( @@ -94,3 +96,14 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { endKey.FDBKey(), ) } + +// GetRangeSplitPoints will return a list of keys that can devide the given range into +// chunks based on the chunk size provided. +func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray { + beginKey, endKey := r.FDBRangeKeys() + return s.getRangeSplitPoints( + beginKey.FDBKey(), + endKey.FDBKey(), + chunkSize, + ) +} diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index 314986f65d..73f741a014 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -337,7 +337,7 @@ func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize in C.int(len(beginKey)), byteSliceToPtr(endKey), C.int(len(endKey)), - chunkSize, + C.int64_t(chunkSize), )), } } From eb28492900f64cbe0741233b5a59c4a18fb9fcec Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 26 Jun 2020 11:40:20 -0700 Subject: [PATCH 005/458] Fixed a bug which leads to inaccurate metrics being reported. Added tests for the new API. --- fdbclient/NativeAPI.actor.cpp | 52 +++++++++++++++++++++-- fdbclient/NativeAPI.actor.h | 1 + fdbserver/StorageMetrics.actor.h | 64 +++++++++++++++++++++++++++++ tests/StorageMetricsSampleTests.txt | 2 +- 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 6768c6ac38..a3a5680803 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -1611,6 +1611,7 @@ ACTOR Future< vector< pair> > > getKeyRangeLoca } } +// Returns a vector of pairs. template Future< vector< pair> > > getKeyRangeLocations( Database const& cx, KeyRange const& keys, int limit, bool reverse, F StorageServerInterface::*member, TransactionInfo const& info ) { ASSERT (!keys.empty()); @@ -3896,8 +3897,21 @@ ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRang state int nLocs = locations.size(); state vector> fx(nLocs); state StorageMetrics total; + KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - fx[i] = doGetStorageMetrics(cx, locations[i].first, locations[i].second); + if (i == 0) { + // Use the actual begin key instead of the shard begin + partBegin = keys.begin; + } else { + partBegin = locations[i].first.begin; + } + if (i == nLocs - 1) { + // Use the actual end key instead of the shard end + partEnd = keys.end; + } else { + partEnd = locations[i].first.end; + } + fx[i] = doGetStorageMetrics(cx, KeyRangeRef(partBegin, partEnd), locations[i].second); } wait(waitForAll(fx)); for (int i = 0; i < nLocs; i++) { @@ -3989,9 +4003,22 @@ ACTOR Future>> getReadHotRanges(Database cx, K // .detail("KeysEnd", keys.end.printable().c_str()); // } state vector> fReplies(nLocs); + KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - ReadHotSubRangeRequest req(locations[i].first); - fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getReadHotRanges, req, + if (i == 0) { + // Use the actual begin key instead of the shard begin + partBegin = keys.begin; + } else { + partBegin = locations[i].first.begin; + } + if (i == nLocs - 1) { + // Use the actual end key instead of the shard end + partEnd = keys.end; + } else { + partEnd = locations[i].first.end; + } + ReadHotSubRangeRequest req(KeyRangeRef(partBegin, partEnd)); + fReplies[i] = loadBalance(locations[i].second, &StorageServerInterface::getReadHotRanges, req, TaskPriority::DataDistribution); } @@ -4114,8 +4141,21 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key try { state int nLocs = locations.size(); state vector> fReplies(nLocs); + KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - SplitRangeRequest req(locations[i].first, chunkSize); + if (i == 0) { + // Use the actual begin key instead of the shard begin + partBegin = keys.begin; + } else { + partBegin = locations[i].first.begin; + } + if (i == nLocs - 1) { + // Use the actual end key instead of the shard end + partEnd = keys.end; + } else { + partEnd = locations[i].first.end; + } + SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize); fReplies[i] = loadBalance(locations[i].second, &StorageServerInterface::getRangeSplitPoints, req, TaskPriority::DataDistribution); } @@ -4123,6 +4163,7 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key wait(waitForAll(fReplies)); Standalone> results; + results.push_back_deep(results.arena(), keys.begin); for (int i = 0; i < nLocs; i++) { if (i > 0) { results.push_back_deep(results.arena(), locations[i].first.begin); // Need this shard boundary @@ -4130,6 +4171,9 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key results.append_deep(results.arena(), fReplies[i].get().splitPoints.begin(), fReplies[i].get().splitPoints.size()); } + if (results.back() != keys.end) { + results.push_back_deep(results.arena(), keys.end); + } return results; } catch (Error& e) { diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index a026b4b5db..20d645aea9 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -264,6 +264,7 @@ public: Future>> getReadHotRanges(KeyRange const& keys); // Try to split the given range into equally sized chunks based on estimated size. + // The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end] Future>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize); // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true ); diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h index 9f77d7aed3..0ac479bcd1 100644 --- a/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/StorageMetrics.actor.h @@ -527,6 +527,70 @@ private: } }; +TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/simple") { + + int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; + StorageServerMetrics ssm; + + ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit); + + vector t = ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 2000 * sampleUnit); + + ASSERT(t.size() == 1 && t[0] == LiteralStringRef("Bah")); + + return Void(); +} + +TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/multipleReturnedPoints") { + + int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; + StorageServerMetrics ssm; + + ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit); + + vector t = ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 600 * sampleUnit); + + ASSERT(t.size() == 3 && t[0] == LiteralStringRef("Absolute") && t[1] == LiteralStringRef("Apple") && + t[2] == LiteralStringRef("Bah")); + + return Void(); +} + +TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/chunkTooLarge") { + + int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; + StorageServerMetrics ssm; + + ssm.byteSample.sample.insert(LiteralStringRef("A"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 80 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 10 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("But"), 10 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 30 * sampleUnit); + + vector t = ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 1000 * sampleUnit); + + ASSERT(t.size() == 0); + + return Void(); +} + TEST_CASE("/fdbserver/StorageMetricSample/readHotDetect/simple") { int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; diff --git a/tests/StorageMetricsSampleTests.txt b/tests/StorageMetricsSampleTests.txt index 6cd7823565..2207e0a5d3 100644 --- a/tests/StorageMetricsSampleTests.txt +++ b/tests/StorageMetricsSampleTests.txt @@ -3,4 +3,4 @@ testName=UnitTests startDelay=0 useDB=false maxTestCases=0 -testsMatching=/fdbserver/StorageMetricSample/readHotDetect/ \ No newline at end of file +testsMatching=/fdbserver/StorageMetricSample \ No newline at end of file From 00e9f8b9bf3dcb5779c34ff22d2343abe7bf05ec Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Tue, 30 Jun 2020 14:28:15 -0700 Subject: [PATCH 006/458] Added bindings tests; Protected new SSI endpoints under new 7.0 ProtocolVersion --- bindings/bindingtester/tests/api.py | 18 +++++++++++++ bindings/flow/tester/Tester.actor.cpp | 27 +++++++++++++++++++ bindings/go/go.sum | 2 ++ bindings/go/src/_stacktester/stacktester.go | 11 ++++++++ .../apple/foundationdb/KeyArrayResult.java | 2 +- .../foundationdb/test/AsyncStackTester.java | 7 +++++ .../foundationdb/test/StackOperation.java | 1 + .../apple/foundationdb/test/StackTester.java | 6 +++++ bindings/python/tests/tester.py | 4 +++ bindings/ruby/tests/tester.rb | 3 +++ fdbclient/NativeAPI.actor.cpp | 4 +-- fdbclient/StorageServerInterface.h | 7 +++-- fdbserver/StorageMetrics.actor.h | 26 ++++++++++++++++-- fdbserver/worker.actor.cpp | 3 +++ flow/ProtocolVersion.h | 1 + tests/StorageMetricsSampleTests.txt | 2 +- 16 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 bindings/go/go.sum diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 5e8d2d66a2..934baa0798 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -157,6 +157,7 @@ class ApiTest(Test): read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] txn_sizes = ['GET_APPROXIMATE_SIZE'] + # storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] storage_metrics = ['GET_ESTIMATED_RANGE_SIZE'] op_choices += reads @@ -553,6 +554,23 @@ class ApiTest(Test): instructions.push_args(key1, key2) instructions.append(op) self.add_strings(1) + elif op == 'GET_RANGE_SPLIT_POINTS': + # Protect against inverted range and identical keys + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + while key1 == key2: + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + if key1 > key2: + key1, key2 = key2, key1 + + # TODO: randomize chunkSize but should not exceed 100M(shard limit) + chunkSize = 10000000 # 10M + instructions.push_args(key1, key2, chunkSize) + instructions.append(op) + self.add_strings(1) else: assert False, 'Unknown operation: ' + op diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index 578f159f8c..0cbe6480fc 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -661,6 +661,33 @@ struct GetEstimatedRangeSize : InstructionFunc { const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE"; REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize); +struct GetRangeSplitPoints : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + state std::vector items = data->stack.pop(3); + if (items.size() != 3) + return Void(); + + Standalone s1 = wait(items[0].value); + state Standalone beginKey = Tuple::unpack(s1).getString(0); + + Standalone s2 = wait(items[1].value); + state Standalone endKey = Tuple::unpack(s2).getString(0); + + Standalone s3 = wait(items[2].value); + state int64_t chunkSize = Tuple::unpack(s3).getInt(0); + + Future>> fsplitPoints = instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize); + FDBStandalone> splitPoints = wait(fsplitPoints); + data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS")); + + return Void(); + } +}; +const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS"; +REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints); + struct GetKeyFunc : InstructionFunc { static const char* name; diff --git a/bindings/go/go.sum b/bindings/go/go.sum new file mode 100644 index 0000000000..3ab73eafae --- /dev/null +++ b/bindings/go/go.sum @@ -0,0 +1,2 @@ +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 9737391569..d986ddec53 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -579,6 +579,17 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { if e != nil { panic(e) } + case op == "GET_RANGE_SPLIT_POINTS": + r := sm.popKeyRange() + chunkSize := sm.waitAndPop().item.(int64) + _, e := rt.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) { + _ = rtr.GetRangeSplitPoints(r, chunkSize).MustGet() + sm.store(idx, []byte("GOT_RANGE_SPLIT_POINTS")) + return nil, nil + }) + if e != nil { + panic(e) + } case op == "COMMIT": sm.store(idx, sm.currentTransaction().Commit()) case op == "RESET": diff --git a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java index 244435eb00..f63fc16d62 100644 --- a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java +++ b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java @@ -23,7 +23,7 @@ package com.apple.foundationdb; import java.util.ArrayList; import java.util.List; -class KeyArrayResult { +public class KeyArrayResult { final List keys; KeyArrayResult(byte[] keyBytes, int[] keyLengths) { diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index 97defab88f..f584f452a9 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -38,6 +38,7 @@ import com.apple.foundationdb.FDB; import com.apple.foundationdb.FDBException; import com.apple.foundationdb.KeySelector; import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.KeyArrayResult; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; import com.apple.foundationdb.StreamingMode; @@ -229,6 +230,12 @@ public class AsyncStackTester { inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); }, FDB.DEFAULT_EXECUTOR); } + else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) { + List params = inst.popParams(3).join(); + return inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).thenAcceptAsync(splitPoints -> { + inst.push("GOT_RANGE_SPLIT_POINTS".getBytes()); + }, FDB.DEFAULT_EXECUTOR); + } else if(op == StackOperation.GET_RANGE) { return inst.popParams(5).thenComposeAsync(params -> { int limit = StackUtils.getInt(params.get(2)); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index 634a217c7f..bece744605 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -57,6 +57,7 @@ enum StackOperation { GET_APPROXIMATE_SIZE, GET_VERSIONSTAMP, GET_ESTIMATED_RANGE_SIZE, + GET_RANGE_SPLIT_POINTS, SET_READ_VERSION, ON_ERROR, SUB, diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index f196301865..0490e2a5fb 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.FDB; import com.apple.foundationdb.FDBException; import com.apple.foundationdb.KeySelector; import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.KeyArrayResult; import com.apple.foundationdb.LocalityUtil; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; @@ -211,6 +212,11 @@ public class StackTester { Long size = inst.readTr.getEstimatedRangeSizeBytes((byte[])params.get(0), (byte[])params.get(1)).join(); inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); } + else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) { + List params = inst.popParams(3).join(); + KeyArrayResult splitPoints = inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).join(); + inst.push("GOT_RANGE_SPLIT_POINTS".getBytes()); + } else if(op == StackOperation.GET_RANGE) { List params = inst.popParams(5).join(); diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index f6eab9c207..6aa41dea4a 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -393,6 +393,10 @@ class Tester: begin, end = inst.pop(2) estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() inst.push(b"GOT_ESTIMATED_RANGE_SIZE") + elif inst.op == six.u("GET_RANGE_SPLIT_POINTS"): + begin, end, chunkSize = inst.pop(3) + estimatedSize = obj.get_range_split_points(begin, end, chunkSize).wait() + inst.push(b"GOT_RANGE_SPLIT_POINTS") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) diff --git a/bindings/ruby/tests/tester.rb b/bindings/ruby/tests/tester.rb index 3860e7d190..e653bdaf93 100755 --- a/bindings/ruby/tests/tester.rb +++ b/bindings/ruby/tests/tester.rb @@ -320,6 +320,9 @@ class Tester when "GET_ESTIMATED_RANGE_SIZE" inst.tr.get_estimated_range_size_bytes(inst.wait_and_pop, inst.wait_and_pop).to_i inst.push("GOT_ESTIMATED_RANGE_SIZE") + when "GET_RANGE_SPLIT_POINTS" + inst.tr.get_range_split_points(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop).length() + inst.push("GOT_RANGE_SPLIT_POINTS") when "GET_KEY" selector = FDB::KeySelector.new(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop) prefix = inst.wait_and_pop diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index a3a5680803..747700491b 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -4018,7 +4018,7 @@ ACTOR Future>> getReadHotRanges(Database cx, K partEnd = locations[i].first.end; } ReadHotSubRangeRequest req(KeyRangeRef(partBegin, partEnd)); - fReplies[i] = loadBalance(locations[i].second, &StorageServerInterface::getReadHotRanges, req, + fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getReadHotRanges, req, TaskPriority::DataDistribution); } @@ -4156,7 +4156,7 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key partEnd = locations[i].first.end; } SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize); - fReplies[i] = loadBalance(locations[i].second, &StorageServerInterface::getRangeSplitPoints, req, + fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getRangeSplitPoints, req, TaskPriority::DataDistribution); } diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index a89018ce98..41dcf4b7de 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -100,8 +100,10 @@ struct StorageServerInterface { getKeyValueStoreType = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(9) ); watchValue = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(10) ); getReadHotRanges = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(11) ); - getRangeSplitPoints = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); + if(ar.protocolVersion().hasRangeSplit()) { + getRangeSplitPoints = + RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); + } } } else { ASSERT(Ar::isDeserializing); @@ -129,6 +131,7 @@ struct StorageServerInterface { streams.push_back(getKeyValueStoreType.getReceiver()); streams.push_back(watchValue.getReceiver()); streams.push_back(getReadHotRanges.getReceiver()); + streams.push_back(getRangeSplitPoints.getReceiver()); FlowTransport::transport().addEndpoints(streams); } }; diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h index 0ac479bcd1..f44c2b1d7a 100644 --- a/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/StorageMetrics.actor.h @@ -474,7 +474,7 @@ struct StorageServerMetrics { std::vector v = getReadHotRanges(req.keys, SERVER_KNOBS->SHARD_MAX_READ_DENSITY_RATIO, SERVER_KNOBS->READ_HOT_SUB_RANGE_CHUNK_SIZE, SERVER_KNOBS->SHARD_READ_HOT_BANDWITH_MIN_PER_KSECONDS); - reply.readHotRanges = VectorRef(v.data(), v.size()); + reply.readHotRanges.append_deep(reply.readHotRanges.arena(), v.data(), v.size()); req.reply.send(reply); } @@ -503,7 +503,7 @@ struct StorageServerMetrics { SplitRangeReply reply; std::vector points = getSplitPoints(req.keys, req.chunkSize); - reply.splitPoints = VectorRef(points.data(), points.size()); + reply.splitPoints.append_deep(reply.splitPoints.arena(), points.data(), points.size()); req.reply.send(reply); } @@ -570,6 +570,28 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/multipleReturnedPoint return Void(); } +TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/noneSplitable") { + + int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; + StorageServerMetrics ssm; + + ssm.byteSample.sample.insert(LiteralStringRef("A"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Absolute"), 800 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Apple"), 1000 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bah"), 20 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Banana"), 80 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Bob"), 200 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("But"), 100 * sampleUnit); + ssm.byteSample.sample.insert(LiteralStringRef("Cat"), 300 * sampleUnit); + + vector t = ssm.getSplitPoints(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("C")), 10000 * sampleUnit); + + ASSERT(t.size() == 0); + + return Void(); +} + + TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/chunkTooLarge") { int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE; diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index a7307df52c..ea7888992c 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -679,6 +679,7 @@ ACTOR Future storageServerRollbackRebooter( Future prevStorageServer DUMPTOKEN(recruited.waitMetrics); DUMPTOKEN(recruited.splitMetrics); DUMPTOKEN(recruited.getReadHotRanges); + DUMPTOKEN(recruited.getRangeSplitPoints); DUMPTOKEN(recruited.getStorageMetrics); DUMPTOKEN(recruited.waitFailure); DUMPTOKEN(recruited.getQueuingMetrics); @@ -1008,6 +1009,7 @@ ACTOR Future workerServer( DUMPTOKEN(recruited.waitMetrics); DUMPTOKEN(recruited.splitMetrics); DUMPTOKEN(recruited.getReadHotRanges); + DUMPTOKEN(recruited.getRangeSplitPoints); DUMPTOKEN(recruited.getStorageMetrics); DUMPTOKEN(recruited.waitFailure); DUMPTOKEN(recruited.getQueuingMetrics); @@ -1318,6 +1320,7 @@ ACTOR Future workerServer( DUMPTOKEN(recruited.waitMetrics); DUMPTOKEN(recruited.splitMetrics); DUMPTOKEN(recruited.getReadHotRanges); + DUMPTOKEN(recruited.getRangeSplitPoints); DUMPTOKEN(recruited.getStorageMetrics); DUMPTOKEN(recruited.waitFailure); DUMPTOKEN(recruited.getQueuingMetrics); diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index c2bb47f78c..785f8c24c6 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -128,6 +128,7 @@ public: // introduced features PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, ReportConflictingKeys); PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, SmallEndpoints); PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, CacheRole); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070000000LL, RangeSplit); }; // These impact both communications and the deserialization of certain database and IKeyValueStore keys. diff --git a/tests/StorageMetricsSampleTests.txt b/tests/StorageMetricsSampleTests.txt index 2207e0a5d3..0b0af55a65 100644 --- a/tests/StorageMetricsSampleTests.txt +++ b/tests/StorageMetricsSampleTests.txt @@ -3,4 +3,4 @@ testName=UnitTests startDelay=0 useDB=false maxTestCases=0 -testsMatching=/fdbserver/StorageMetricSample \ No newline at end of file +testsMatching=/fdbserver/StorageMetricSample/rangeSplitPoints \ No newline at end of file From 547f13d49db76247c4a6dec9c5561db55a230c40 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Tue, 30 Jun 2020 14:33:56 -0700 Subject: [PATCH 007/458] Delete go.sum Remove junk file --- bindings/go/go.sum | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 bindings/go/go.sum diff --git a/bindings/go/go.sum b/bindings/go/go.sum deleted file mode 100644 index 3ab73eafae..0000000000 --- a/bindings/go/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From b0e9d321cccce8f176f43a678cee518d1effd5dd Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Tue, 30 Jun 2020 14:44:28 -0700 Subject: [PATCH 008/458] Apply suggestions from code review --- bindings/c/foundationdb/fdb_c.h | 4 ++-- bindings/go/go.mod | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 8eeebd70db..4327a4a52a 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -147,11 +147,11 @@ extern "C" { #if FDB_API_VERSION >= 14 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t - fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_key_array, + fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_kv, int* out_count, fdb_bool_t* out_more ); #endif DLLEXPORT WARN_UNUSED_RESULT fdb_error_t - fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_k, + fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count); DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f, diff --git a/bindings/go/go.mod b/bindings/go/go.mod index 65d3ee7383..0700d7cf9f 100644 --- a/bindings/go/go.mod +++ b/bindings/go/go.mod @@ -3,4 +3,4 @@ module github.com/apple/foundationdb/bindings/go // The FoundationDB go bindings currently have no external golang dependencies outside of // the go standard library. -require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 +require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543// indirect From d689d346343ff173850b045a7be7e6bb213a1d8c Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Thu, 2 Jul 2020 10:24:26 -0700 Subject: [PATCH 009/458] Enable range split bindings tests --- bindings/bindingtester/tests/api.py | 3 +-- fdbclient/NativeAPI.actor.cpp | 42 +++++------------------------ 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 934baa0798..df90adf890 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -157,8 +157,7 @@ class ApiTest(Test): read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] txn_sizes = ['GET_APPROXIMATE_SIZE'] - # storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] - storage_metrics = ['GET_ESTIMATED_RANGE_SIZE'] + storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] op_choices += reads op_choices += mutations diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 747700491b..d68a7a7d6a 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -3899,18 +3899,8 @@ ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRang state StorageMetrics total; KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - if (i == 0) { - // Use the actual begin key instead of the shard begin - partBegin = keys.begin; - } else { - partBegin = locations[i].first.begin; - } - if (i == nLocs - 1) { - // Use the actual end key instead of the shard end - partEnd = keys.end; - } else { - partEnd = locations[i].first.end; - } + partBegin = (i == 0) ? keys.begin : locations[i].first.begin; + partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end; fx[i] = doGetStorageMetrics(cx, KeyRangeRef(partBegin, partEnd), locations[i].second); } wait(waitForAll(fx)); @@ -4005,18 +3995,8 @@ ACTOR Future>> getReadHotRanges(Database cx, K state vector> fReplies(nLocs); KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - if (i == 0) { - // Use the actual begin key instead of the shard begin - partBegin = keys.begin; - } else { - partBegin = locations[i].first.begin; - } - if (i == nLocs - 1) { - // Use the actual end key instead of the shard end - partEnd = keys.end; - } else { - partEnd = locations[i].first.end; - } + partBegin = (i == 0) ? keys.begin : locations[i].first.begin; + partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end; ReadHotSubRangeRequest req(KeyRangeRef(partBegin, partEnd)); fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getReadHotRanges, req, TaskPriority::DataDistribution); @@ -4143,18 +4123,8 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key state vector> fReplies(nLocs); KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - if (i == 0) { - // Use the actual begin key instead of the shard begin - partBegin = keys.begin; - } else { - partBegin = locations[i].first.begin; - } - if (i == nLocs - 1) { - // Use the actual end key instead of the shard end - partEnd = keys.end; - } else { - partEnd = locations[i].first.end; - } + partBegin = (i == 0) ? keys.begin : locations[i].first.begin; + partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end; SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize); fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getRangeSplitPoints, req, TaskPriority::DataDistribution); From 7c98cac754838127758f6040107b0a0086db4cdb Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Thu, 2 Jul 2020 16:25:43 -0700 Subject: [PATCH 010/458] Fix a Go binding error --- bindings/c/fdb_c.cpp | 4 ++-- bindings/c/foundationdb/fdb_c.h | 2 +- bindings/go/src/fdb/futures.go | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index ed6f681037..538b66c51e 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -638,14 +638,14 @@ fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const } -extern "C" DLLEXPORT +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) { KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); } -extern "C" DLLEXPORT +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize) { KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 4327a4a52a..e0f6b450a7 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -266,7 +266,7 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); - + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize); diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 31211679a8..e51d5eaa8d 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -306,8 +306,8 @@ func (f *futureKeyValueArray) Get() ([]KeyValue, bool, error) { return ret, (more != 0), nil } -// FutureKeyArray represents the asynchronous result of a function -// that returns an array of keys. FutureKeyArray is a lightweight object +// FutureKeyArray represents the asynchronous result of a function +// that returns an array of keys. FutureKeyArray is a lightweight object // that may be efficiently copied, and is safe for concurrent use by multiple goroutines. type FutureKeyArray interface { @@ -334,14 +334,14 @@ func (f *futureKeyArray) Get() ([]Key, error) { var ks *C.FDBKey var count C.int - if err:= C.fdb_future_get_key_array(f.ptr, &ks, &count); err != 0 { + if err := C.fdb_future_get_key_array(f.ptr, &ks, &count); err != 0 { return nil, Error{int(err)} } ret := make([]Key, int(count)) - for i:= 0; i Date: Fri, 7 Aug 2020 14:03:42 -0700 Subject: [PATCH 011/458] Fix the build error. --- fdbclient/NativeAPI.actor.cpp | 3 ++- fdbclient/StorageServerInterface.h | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index afb5e3bbe6..e5482a4853 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -4350,10 +4350,11 @@ Future>> Transaction::getReadHotRanges(KeyRang } ACTOR Future>> getRangeSplitPoints(Database cx, KeyRange keys, int64_t chunkSize) { + state Span span("NAPI:GetRangeSplitPoints"_loc); loop { state vector>> locations = wait(getKeyRangeLocations(cx, keys, 100, false, &StorageServerInterface::getRangeSplitPoints, - TransactionInfo(TaskPriority::DataDistribution))); + TransactionInfo(TaskPriority::DataDistribution, span.context))); try { state int nLocs = locations.size(); state vector> fReplies(nLocs); diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 24582fb2ac..23c3b776a1 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -466,7 +466,7 @@ struct ReadHotSubRangeRequest { }; struct SplitRangeReply { - constexpr static FileIdentifier file_identifier = 21813134; + constexpr static FileIdentifier file_identifier = 11813134; // If the given range can be divided, contains the split points. // If the given range cannot be divided(for exmaple its total size is smaller than the chunk size), this would be // empty @@ -478,7 +478,7 @@ struct SplitRangeReply { } }; struct SplitRangeRequest { - constexpr static FileIdentifier file_identifier = 30725174; + constexpr static FileIdentifier file_identifier = 10725174; Arena arena; KeyRangeRef keys; int64_t chunkSize; From b9474055a5d8dcb60523386f76a069538def9b95 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 14 Aug 2020 01:01:08 -0700 Subject: [PATCH 012/458] Add \xff\xff/configuration/class to read process class --- fdbclient/NativeAPI.actor.cpp | 4 + fdbclient/SpecialKeySpace.actor.cpp | 123 +++++++++++++++++++++++++++- fdbclient/SpecialKeySpace.actor.h | 22 ++++- 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index d87905f675..2198f1b6a1 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -921,6 +921,10 @@ DatabaseContext::DatabaseContext(Reference( KeyRangeRef(LiteralStringRef("inProgressExclusion/"), LiteralStringRef("inProgressExclusion0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))); + registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); } if (apiVersionAtLeast(630)) { registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY, diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 79a18cfa1e..efbae9cd12 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -36,7 +36,9 @@ std::unordered_map SpecialKeySpace::moduleToB KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) }, { SpecialKeySpace::MODULE::MANAGEMENT, KeyRangeRef(LiteralStringRef("\xff\xff/management/"), LiteralStringRef("\xff\xff/management0")) }, - { SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) } + { SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) }, + { SpecialKeySpace::MODULE::CONFIGURATION, + KeyRangeRef(LiteralStringRef("\xff\xff/configuration/"), LiteralStringRef("\xff\xff/configuration0")) } }; std::unordered_map SpecialKeySpace::managementApiCommandToRange = { @@ -587,6 +589,46 @@ Future> ManagementCommandsOptionsImpl::commit(ReadYourWrit return Optional(); } +Standalone rywModuleGetRange(ReadYourWritesTransaction* ryw, Standalone* res, + KeyRangeRef kr) { + // res is read from database, if ryw enabled, we update it with writes in the transaction + if (ryw->readYourWritesDisabled()) { + return *res; + } else { + Standalone result; + result.arena().dependsOn(res->arena()); + RangeMap>, KeyRangeRef>::Ranges ranges = + ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); + RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); + int index = 0; + while (iter != ranges.end()) { + // add all previous entries into result + KeyRef rk = (*res)[index].key; + while (index < res->size() && rk < iter->begin()) { + result.push_back(result.arena(), KeyValueRef(rk, (*res)[index].value)); + ++index; + } + std::pair> entry = iter->value(); + if (entry.first) { + // add the writen entries if exists + if (entry.second.present()) { + result.push_back(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + } + // move index to skip all entries in the iter->range + while (index < res->size() && iter->range().contains((*res)[index].key)) ++index; + } + ++iter; + } + // add all remaining entries into result + while (index < res->size()) { + const KeyValueRef& kv = (*res)[index]; + result.push_back(result.arena(), KeyValueRef(kv.key, kv.value)); + ++index; + } + return result; + } +} + // read from rwModule ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) { @@ -671,7 +713,7 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, bool failed, std:: while (iter != ranges.end()) { auto entry = iter->value(); // only check for exclude(set) operation, include(clear) are not checked - TraceEvent(SevInfo, "ParseNetworkAddress") + TraceEvent(SevInfo, "ParseNetworkAddress") // TODO : change to SevDebug .detail("Valid", entry.first) .detail("Set", entry.second.present()) .detail("Key", iter->begin().toString()); @@ -959,3 +1001,80 @@ Future> ExclusionInProgressRangeImpl::getRange(ReadYo KeyRangeRef kr) const { return ExclusionInProgressActor(ryw, getKeyRange().begin, kr); } + +ACTOR Future> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, + KeyRangeRef kr) { + state Future> processClasses = ryw->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); + state Future> processData = ryw->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); + + wait(success(processClasses) && success(processData)); + ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); + + std::map>, ProcessClass> id_class; + for (int i = 0; i < processClasses.get().size(); i++) { + id_class[decodeProcessClassKey(processClasses.get()[i].key)] = + decodeProcessClassValue(processClasses.get()[i].value); + } + + Standalone result; + + for (int i = 0; i < processData.get().size(); i++) { + ProcessData data = decodeWorkerListValue(processData.get()[i].value); + ProcessClass processClass = id_class[data.locality.processId()]; + + if (processClass.classSource() == ProcessClass::DBSource || + data.processClass.classType() == ProcessClass::UnsetClass) + data.processClass = processClass; + + if (data.processClass.classType() != ProcessClass::TesterClass) { + result.push_back_deep(result.arena(), KeyValueRef(prefix.withSuffix(data.address.toString()), + Value(data.processClass.toString()))); + } + } + return rywModuleGetRange(ryw, &result, kr); +} + +ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { + state Optional result; + auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); + auto iter = ranges.begin(); + while (iter != ranges.end()) { + auto entry = iter->value(); + // only check for setclass(set) operation, (clear) are not checked + if (entry.first && entry.second.present()) { + // validate network address + Key address = iter->begin().removePrefix(range.begin); + auto a = AddressExclusion::parse(address); + if (!a.isValid()) { + std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n"; + if (address.toString().find(":tls") != std::string::npos) + error += " Do not include the `:tls' suffix when naming a process\n"; + result = ManagementAPIError::toJsonString(false, "setclass", error); + return result; + } + // validate class type + ValueRef processClassType = entry.second.get(); + ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource); + if (processClass.classType() == ProcessClass::InvalidClass && + processClassType != LiteralStringRef("default")) { + std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n"; + result = ManagementAPIError::toJsonString(false, "setclass", error); + return result; + } + } + ++iter; + } + return result; +} + +ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + return getProcessClassActor(ryw, getKeyRange().begin, kr); +} + +Future> ProcessClassRangeImpl::commit(ReadYourWritesTransaction* ryw) { + return processClassCommitActor(ryw, getKeyRange()); +} \ No newline at end of file diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 4cbc9c5002..59259f0a4a 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -67,9 +67,15 @@ private: class SpecialKeyRangeRWImpl : public SpecialKeyRangeReadImpl { public: - virtual void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) = 0; - virtual void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) = 0; - virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) = 0; + virtual void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); + } + virtual void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { + ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); + } + virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); + } virtual Future> commit( ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space // Given the special key to write, return the real key that needs to be modified @@ -125,6 +131,7 @@ class SpecialKeySpace { public: enum class MODULE { CLUSTERFILEPATH, + CONFIGURATION, // Configuration of the cluster CONNECTIONSTRING, ERRORMSG, // A single key space contains a json string which describes the last error in special-key-space MANAGEMENT, // Management-API @@ -273,5 +280,14 @@ public: Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; }; +class ProcessClassRangeImpl : public SpecialKeyRangeRWImpl { +public: + explicit ProcessClassRangeImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Key decode(const KeyRef& key) const override { return Key(); } + Key encode(const KeyRef& key) const override { return Key(); } + Future> commit(ReadYourWritesTransaction* ryw) override; +}; + #include "flow/unactorcompiler.h" #endif From 5660de9c0918e5c9eb2c5e47df1ab19bccd28158 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Wed, 19 Aug 2020 17:54:38 -0700 Subject: [PATCH 013/458] Add \xff\xff/configuration/class to change process class --- fdbclient/SpecialKeySpace.actor.cpp | 41 ++++++++-- fdbclient/SpecialKeySpace.actor.h | 2 + .../SpecialKeySpaceCorrectness.actor.cpp | 76 +++++++++++++------ 3 files changed, 91 insertions(+), 28 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index efbae9cd12..b8fc0cd4c9 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1036,7 +1036,15 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran } ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { - state Optional result; + // enable related options + ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + ryw->setOption(FDBTransactionOptions::LOCK_AWARE); + ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); + vector workers = wait( + getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit() + + Optional result; auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); auto iter = ranges.begin(); while (iter != ranges.end()) { @@ -1045,13 +1053,13 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa if (entry.first && entry.second.present()) { // validate network address Key address = iter->begin().removePrefix(range.begin); - auto a = AddressExclusion::parse(address); - if (!a.isValid()) { + AddressExclusion addr = AddressExclusion::parse(address); + if (!addr.isValid()) { std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n"; if (address.toString().find(":tls") != std::string::npos) error += " Do not include the `:tls' suffix when naming a process\n"; result = ManagementAPIError::toJsonString(false, "setclass", error); - return result; + break; } // validate class type ValueRef processClassType = entry.second.get(); @@ -1060,8 +1068,23 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa processClassType != LiteralStringRef("default")) { std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n"; result = ManagementAPIError::toJsonString(false, "setclass", error); - return result; + break; } + // write to transaction + // make sure we use the Transaction object to avoid used_during_commit() + bool foundChange = false; + for (int i = 0; i < workers.size(); i++) { + if (addr.excludes(workers[i].address)) { + if (processClass.classType() != ProcessClass::InvalidClass) + ryw->getTransaction().set(processClassKeyFor(workers[i].locality.processId().get()), + processClassValue(processClass)); + else + ryw->getTransaction().clear(processClassKeyFor(workers[i].locality.processId().get())); + foundChange = true; + } + } + if (foundChange) + ryw->getTransaction().set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString()); } ++iter; } @@ -1077,4 +1100,12 @@ Future> ProcessClassRangeImpl::getRange(ReadYourWrite Future> ProcessClassRangeImpl::commit(ReadYourWritesTransaction* ryw) { return processClassCommitActor(ryw, getKeyRange()); +} + +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override { + throw special_keys_api_failure(); +} + +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override { + throw special_keys_api_failure(); } \ No newline at end of file diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 59259f0a4a..c252731993 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -287,6 +287,8 @@ public: Key decode(const KeyRef& key) const override { return Key(); } Key encode(const KeyRef& key) const override { return Key(); } Future> commit(ReadYourWritesTransaction* ryw) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; }; #include "flow/unactorcompiler.h" diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index ab81f1f7bf..032305fae1 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -112,9 +112,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ACTOR Future _start(Database cx, SpecialKeySpaceCorrectnessWorkload* self) { testRywLifetime(cx); wait(timeout(self->testSpecialKeySpaceErrors(cx, self) && self->getRangeCallActor(cx, self) && - testConflictRanges(cx, /*read*/ true, self) && testConflictRanges(cx, /*read*/ false, self) && - self->managementApiCorrectnessActor(cx, self), + testConflictRanges(cx, /*read*/ true, self) && testConflictRanges(cx, /*read*/ false, self), self->testDuration, Void())); + // Only use one client to avoid potential conflicts on changing cluster configuration + if (self->clientId == 0) wait(self->managementApiCorrectnessActor(cx, self)); return Void(); } @@ -383,27 +384,6 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ASSERT(e.code() == error_code_special_keys_cross_module_clear); tx->reset(); } - // Management api error, and error message shema check - try { - tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tx->set(LiteralStringRef("Invalid_Network_Address") - .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")), - ValueRef()); - wait(tx->commit()); - ASSERT(false); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) throw; - ASSERT(e.code() == error_code_special_keys_api_failure); - Optional errorMsg = - wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); - ASSERT(errorMsg.present()); - std::string errorStr; - auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); - auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); - // special_key_space_management_api_error_msg schema validation - ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); - tx->reset(); - } return Void(); } @@ -564,6 +544,56 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { for (int i = 0; i < res.size() - 1; ++i) ASSERT(res[i].key < res[i + 1].key); tx->reset(); } + // "exclude" error message shema check + try { + tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tx->set(LiteralStringRef("Invalid_Network_Address") + .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")), + ValueRef()); + wait(tx->commit()); + ASSERT(false); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; + ASSERT(e.code() == error_code_special_keys_api_failure); + Optional errorMsg = + wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); + ASSERT(errorMsg.present()); + std::string errorStr; + auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); + auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); + // special_key_space_management_api_error_msg schema validation + ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + tx->reset(); + } + // "setclass" + { + try { + tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + vector workers = wait(getWorkers(&tx->getTransaction())); + auto worker = deterministicRandom()->randomChoice(workers); + std::string addr = worker.address.toString(); + std::string suffix = ":tls"; + // remove :tls suffix if needed + if ((addr.size() >= suffix.size()) && (addr.rfind(suffix) == addr.size() - suffix.size())) + addr = addr.substr(0, addr.size() - suffix.size()); + tx->set(Key("class/" + addr) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + LiteralStringRef("InvalidProcessType")); + wait(tx->commit()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; + ASSERT(e.code() == error_code_special_keys_api_failure); + Optional errorMsg = + wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); + ASSERT(errorMsg.present()); + std::string errorStr; + auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); + auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); + // special_key_space_management_api_error_msg schema validation + ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + tx->reset(); + } + } return Void(); } }; From 707b88583aae1bfd30acb3f152b9c851fa094e76 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 13:50:35 -0700 Subject: [PATCH 014/458] Add default encode, decode methods. Add test for setclass special keys --- fdbclient/SpecialKeySpace.actor.cpp | 127 +++++++----------- fdbclient/SpecialKeySpace.actor.h | 16 ++- .../SpecialKeySpaceCorrectness.actor.cpp | 68 ++++++++-- 3 files changed, 114 insertions(+), 97 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index b8fc0cd4c9..e0633f3c36 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -572,63 +572,11 @@ void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const } } -Key ManagementCommandsOptionsImpl::decode(const KeyRef& key) const { - // Should never be used - ASSERT(false); - return key; -} - -Key ManagementCommandsOptionsImpl::encode(const KeyRef& key) const { - // Should never be used - ASSERT(false); - return key; -} - Future> ManagementCommandsOptionsImpl::commit(ReadYourWritesTransaction* ryw) { // Nothing to do, keys should be used by other impls' commit callback return Optional(); } -Standalone rywModuleGetRange(ReadYourWritesTransaction* ryw, Standalone* res, - KeyRangeRef kr) { - // res is read from database, if ryw enabled, we update it with writes in the transaction - if (ryw->readYourWritesDisabled()) { - return *res; - } else { - Standalone result; - result.arena().dependsOn(res->arena()); - RangeMap>, KeyRangeRef>::Ranges ranges = - ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); - RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); - int index = 0; - while (iter != ranges.end()) { - // add all previous entries into result - KeyRef rk = (*res)[index].key; - while (index < res->size() && rk < iter->begin()) { - result.push_back(result.arena(), KeyValueRef(rk, (*res)[index].value)); - ++index; - } - std::pair> entry = iter->value(); - if (entry.first) { - // add the writen entries if exists - if (entry.second.present()) { - result.push_back(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); - } - // move index to skip all entries in the iter->range - while (index < res->size() && iter->range().contains((*res)[index].key)) ++index; - } - ++iter; - } - // add all remaining entries into result - while (index < res->size()) { - const KeyValueRef& kv = (*res)[index]; - result.push_back(result.arena(), KeyValueRef(kv.key, kv.value)); - ++index; - } - return result; - } -} - // read from rwModule ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) { @@ -1002,37 +950,60 @@ Future> ExclusionInProgressRangeImpl::getRange(ReadYo return ExclusionInProgressActor(ryw, getKeyRange().begin, kr); } +Standalone rywModuleGetRange(ReadYourWritesTransaction* ryw, Standalone res, + KeyRangeRef kr) { + // res is read from database, if ryw enabled, we update it with writes in the transaction + Standalone result; + RangeMap>, KeyRangeRef>::Ranges ranges = + ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); + RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); + int index = 0; + while (iter != ranges.end()) { + // add all previous entries into result + while (index < res.size() && res[index].key < iter->begin()) { + result.push_back(result.arena(), KeyValueRef(res[index].key, res[index].value)); + result.arena().dependsOn(res.arena()); + ++index; + } + std::pair> entry = iter->value(); + if (entry.first) { + // add the writen entries if exists + if (entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + } + // move index to skip all entries in the iter->range + while (index < res.size() && iter->range().contains(res[index].key)) ++index; + } + ++iter; + } + // add all remaining entries into result + while (index < res.size()) { + const KeyValueRef& kv = res[index]; + result.push_back(result.arena(), KeyValueRef(kv.key, kv.value)); + result.arena().dependsOn(res.arena()); + ++index; + } + return result; +} + ACTOR Future> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) { - state Future> processClasses = ryw->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); - state Future> processData = ryw->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); - - wait(success(processClasses) && success(processData)); - ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); - ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); - - std::map>, ProcessClass> id_class; - for (int i = 0; i < processClasses.get().size(); i++) { - id_class[decodeProcessClassKey(processClasses.get()[i].key)] = - decodeProcessClassValue(processClasses.get()[i].value); - } - + vector _workers = wait(getWorkers(&ryw->getTransaction())); + auto workers = _workers; // strip const + std::sort(workers.begin(), workers.end(), ProcessData::sort_by_address()); Standalone result; - - for (int i = 0; i < processData.get().size(); i++) { - ProcessData data = decodeWorkerListValue(processData.get()[i].value); - ProcessClass processClass = id_class[data.locality.processId()]; - - if (processClass.classSource() == ProcessClass::DBSource || - data.processClass.classType() == ProcessClass::UnsetClass) - data.processClass = processClass; - - if (data.processClass.classType() != ProcessClass::TesterClass) { - result.push_back_deep(result.arena(), KeyValueRef(prefix.withSuffix(data.address.toString()), - Value(data.processClass.toString()))); + for (auto& w : workers) { + // exclude :tls in keys even the network addresss is TLS + Key k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port))); + if (kr.contains(k)) { + Value v(w.processClass.toString()); + result.push_back(result.arena(), KeyValueRef(k, v)); + result.arena().dependsOn(k.arena()); + result.arena().dependsOn(v.arena()); } } - return rywModuleGetRange(ryw, &result, kr); + if (ryw->readYourWritesDisabled()) return result; + return rywModuleGetRange(ryw, result, kr); } ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index c252731993..fa46af8c55 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -79,9 +79,17 @@ public: virtual Future> commit( ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space // Given the special key to write, return the real key that needs to be modified - virtual Key decode(const KeyRef& key) const = 0; + virtual Key decode(const KeyRef& key) const { + // Default implementation should never be used + ASSERT(false); + return key; + } // Given the read key, return the corresponding special key - virtual Key encode(const KeyRef& key) const = 0; + virtual Key encode(const KeyRef& key) const { + // Default implementation should never be used + ASSERT(false); + return key; + }; explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} @@ -245,8 +253,6 @@ public: void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; - Key decode(const KeyRef& key) const override; - Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; }; @@ -284,8 +290,6 @@ class ProcessClassRangeImpl : public SpecialKeyRangeRWImpl { public: explicit ProcessClassRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; - Key decode(const KeyRef& key) const override { return Key(); } - Key encode(const KeyRef& key) const override { return Key(); } Future> commit(ReadYourWritesTransaction* ryw) override; void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 032305fae1..a922054ecc 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -36,7 +36,7 @@ public: // all keys are written to RYW, since GRV is set, the read should happen locally ASSERT(resultFuture.isReady()); auto result = resultFuture.getValue(); - ASSERT(!result.more); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); // To make the test more complext, instead of simply returning the k-v pairs, we reverse all the value strings auto kvs = resultFuture.getValue(); for (int i = 0; i < kvs.size(); ++i) { @@ -523,6 +523,19 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { return Void(); } + bool getRangeResultInOrder(const Standalone& result) { + for (int i = 0; i < result.size() - 1; ++i) { + if (result[i].key >= result[i + 1].key) { + TraceEvent(SevDebug, "GetRangeResultNotInOrder") + .detail("Index", i) + .detail("Key1", result[i].key) + .detail("Key2", result[i + 1].key); + return false; + } + } + return true; + } + ACTOR Future managementApiCorrectnessActor(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) { // All management api related tests Database cx = cx_->clone(); @@ -536,12 +549,13 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { .withSuffix(option), ValueRef()); } - Standalone res = wait(tx->getRange( + Standalone result = wait(tx->getRange( KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin), CLIENT_KNOBS->TOO_MANY)); - ASSERT(res.size() == SpecialKeySpace::getManagementApiOptionsSet().size()); - for (int i = 0; i < res.size() - 1; ++i) ASSERT(res[i].key < res[i + 1].key); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(result.size() == SpecialKeySpace::getManagementApiOptionsSet().size()); + ASSERT(self->getRangeResultInOrder(result)); tx->reset(); } // "exclude" error message shema check @@ -569,17 +583,45 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { { try { tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + // test getRange + state Standalone result = wait(tx->getRange( + KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + CLIENT_KNOBS->TOO_MANY)); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(self->getRangeResultInOrder(result)); + // check correctness of classType of each process vector workers = wait(getWorkers(&tx->getTransaction())); - auto worker = deterministicRandom()->randomChoice(workers); - std::string addr = worker.address.toString(); - std::string suffix = ":tls"; - // remove :tls suffix if needed - if ((addr.size() >= suffix.size()) && (addr.rfind(suffix) == addr.size() - suffix.size())) - addr = addr.substr(0, addr.size() - suffix.size()); - tx->set(Key("class/" + addr) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), - LiteralStringRef("InvalidProcessType")); + for (const auto& worker : workers ) { + // TODO : test here + // ASSERT(!worker.address.isTLS()); + Key addr = Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + bool found = false; + for (const auto& kv : result) { + if (kv.key == addr) { + ASSERT(kv.value.toString() == worker.processClass.toString()); + found = true; + break; + } + } + // Each process should find its corresponding element + ASSERT(found); + } + state ProcessData worker = deterministicRandom()->randomChoice(workers); + state Key addr = Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + tx->set(addr, LiteralStringRef("InvalidProcessType")); + // test ryw + Optional processType = wait(tx->get(addr)); + ASSERT(processType.present() && processType.get() == LiteralStringRef("InvalidProcessType")); + // test ryw disabled + tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE); + Optional originalProcessType = wait(tx->get(addr)); + ASSERT(originalProcessType.present() && originalProcessType.get() == worker.processClass.toString()); + // test error handling (invalid value type) wait(tx->commit()); + ASSERT(false); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; ASSERT(e.code() == error_code_special_keys_api_failure); From 1815cc24a3224422413bcd8652778d5870ecb1af Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 14:26:41 -0700 Subject: [PATCH 015/458] Use default set and clear implementations for exclude and failed --- fdbclient/SpecialKeySpace.actor.cpp | 24 ------------------------ fdbclient/SpecialKeySpace.actor.h | 6 ------ 2 files changed, 30 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index e0633f3c36..890810cd42 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -630,18 +630,6 @@ Future> ExcludeServersRangeImpl::getRange(ReadYourWri return rwModuleGetRangeActor(ryw, this, kr); } -void ExcludeServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); -} - -void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); -} - -void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { - ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); -} - Key ExcludeServersRangeImpl::decode(const KeyRef& key) const { return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) .withPrefix(LiteralStringRef("\xff/conf/")); @@ -867,18 +855,6 @@ Future> FailedServersRangeImpl::getRange(ReadYourWrit return rwModuleGetRangeActor(ryw, this, kr); } -void FailedServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); -} - -void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); -} - -void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { - ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); -} - Key FailedServersRangeImpl::decode(const KeyRef& key) const { return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) .withPrefix(LiteralStringRef("\xff/conf/")); diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index fa46af8c55..c9fad65cad 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -260,9 +260,6 @@ class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl { public: explicit ExcludeServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; - void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; @@ -272,9 +269,6 @@ class FailedServersRangeImpl : public SpecialKeyRangeRWImpl { public: explicit FailedServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; - void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; From 4a8a356ffbdc78f8274c141f932cc60a85b5889d Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 14:38:07 -0700 Subject: [PATCH 016/458] throw special_keys_api_failure if clear called on \xff\xff/configuration/class --- fdbclient/SpecialKeySpace.actor.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 890810cd42..68d495c77d 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1049,10 +1049,17 @@ Future> ProcessClassRangeImpl::commit(ReadYourWritesTransa return processClassCommitActor(ryw, getKeyRange()); } -void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override { +void throwNotAllowedError(ReadYourWritesTransaction* ryw) { + auto msg = ManagementAPIError::toJsonString( + false, "setclass", "Clear operation is meaningless thus forbidden for setclass"); + ryw->setSpecialKeySpaceErrorMsg(msg); throw special_keys_api_failure(); } +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override { + return throwNotAllowedError(ryw); +} + void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override { - throw special_keys_api_failure(); + return throwNotAllowedError(ryw); } \ No newline at end of file From e54f728d426fe438f86a9cb3fb63962d98438524 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 14:41:17 -0700 Subject: [PATCH 017/458] update comments --- fdbclient/SpecialKeySpace.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 68d495c77d..44383bb3f9 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -649,7 +649,7 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, bool failed, std:: while (iter != ranges.end()) { auto entry = iter->value(); // only check for exclude(set) operation, include(clear) are not checked - TraceEvent(SevInfo, "ParseNetworkAddress") // TODO : change to SevDebug + TraceEvent(SevDebug, "ParseNetworkAddress") .detail("Valid", entry.first) .detail("Set", entry.second.present()) .detail("Key", iter->begin().toString()); @@ -996,7 +996,7 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa auto iter = ranges.begin(); while (iter != ranges.end()) { auto entry = iter->value(); - // only check for setclass(set) operation, (clear) are not checked + // only check for setclass(set) operation, (clear) are forbidden thus not exist if (entry.first && entry.second.present()) { // validate network address Key address = iter->begin().removePrefix(range.begin); From 802125e01a23816faf0116cad1f88659089749c2 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 14:50:30 -0700 Subject: [PATCH 018/458] clang-format --- .../workloads/SpecialKeySpaceCorrectness.actor.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index a922054ecc..d5ab0cdaad 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -592,11 +592,12 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ASSERT(self->getRangeResultInOrder(result)); // check correctness of classType of each process vector workers = wait(getWorkers(&tx->getTransaction())); - for (const auto& worker : workers ) { + for (const auto& worker : workers) { // TODO : test here // ASSERT(!worker.address.isTLS()); - Key addr = Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + Key addr = + Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); bool found = false; for (const auto& kv : result) { if (kv.key == addr) { @@ -609,8 +610,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ASSERT(found); } state ProcessData worker = deterministicRandom()->randomChoice(workers); - state Key addr = Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + state Key addr = + Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); tx->set(addr, LiteralStringRef("InvalidProcessType")); // test ryw Optional processType = wait(tx->get(addr)); From 556b239057ae4c11607f066dcfe2f7d8e79e90ad Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 20 Aug 2020 14:51:41 -0700 Subject: [PATCH 019/458] clang-format --- fdbclient/NativeAPI.actor.cpp | 9 +++++---- fdbclient/SpecialKeySpace.actor.cpp | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 4c0c95ef56..375e37c5c4 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -921,10 +921,11 @@ DatabaseContext::DatabaseContext(Reference( KeyRangeRef(LiteralStringRef("inProgressExclusion/"), LiteralStringRef("inProgressExclusion0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))); - registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READWRITE, - std::make_unique( - KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); } if (apiVersionAtLeast(630)) { registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY, diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 44383bb3f9..c85c85a3e6 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1050,8 +1050,8 @@ Future> ProcessClassRangeImpl::commit(ReadYourWritesTransa } void throwNotAllowedError(ReadYourWritesTransaction* ryw) { - auto msg = ManagementAPIError::toJsonString( - false, "setclass", "Clear operation is meaningless thus forbidden for setclass"); + auto msg = ManagementAPIError::toJsonString(false, "setclass", + "Clear operation is meaningless thus forbidden for setclass"); ryw->setSpecialKeySpaceErrorMsg(msg); throw special_keys_api_failure(); } @@ -1062,4 +1062,4 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRange void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override { return throwNotAllowedError(ryw); -} \ No newline at end of file +} From cf19c5dac9de5f07e6fa092eb85820e4410955fa Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 21 Aug 2020 00:56:12 -0700 Subject: [PATCH 020/458] Move validation of network address and class type before get worker list --- fdbclient/SpecialKeySpace.actor.cpp | 62 +++++++++++++++++++---------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index c85c85a3e6..7e6b41f917 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -910,7 +910,7 @@ ACTOR Future> ExclusionInProgressActor(ReadYourWrites } for (auto const& address : inProgressExclusion) { - Key addrKey = prefix.withSuffix(address.toString()); + Key addrKey = prefix.withSuffix(address.toString()); // TODO : sort and remove :tls if (kr.contains(addrKey)) { result.push_back(result.arena(), KeyValueRef(addrKey, ValueRef())); result.arena().dependsOn(addrKey.arena()); @@ -966,7 +966,10 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran KeyRangeRef kr) { vector _workers = wait(getWorkers(&ryw->getTransaction())); auto workers = _workers; // strip const - std::sort(workers.begin(), workers.end(), ProcessData::sort_by_address()); + // TODO : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 + std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { + return formatIpPort(lhs.address.ip, lhs.address.port) < formatIpPort(rhs.address.ip, rhs.address.port); + }); Standalone result; for (auto& w : workers) { // exclude :tls in keys even the network addresss is TLS @@ -991,34 +994,19 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa vector workers = wait( getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit() - Optional result; auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); auto iter = ranges.begin(); while (iter != ranges.end()) { auto entry = iter->value(); - // only check for setclass(set) operation, (clear) are forbidden thus not exist + // only loop through (set) operation, (clear) not exist if (entry.first && entry.second.present()) { - // validate network address + // parse network address Key address = iter->begin().removePrefix(range.begin); AddressExclusion addr = AddressExclusion::parse(address); - if (!addr.isValid()) { - std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n"; - if (address.toString().find(":tls") != std::string::npos) - error += " Do not include the `:tls' suffix when naming a process\n"; - result = ManagementAPIError::toJsonString(false, "setclass", error); - break; - } - // validate class type + // parse class type ValueRef processClassType = entry.second.get(); ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource); - if (processClass.classType() == ProcessClass::InvalidClass && - processClassType != LiteralStringRef("default")) { - std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n"; - result = ManagementAPIError::toJsonString(false, "setclass", error); - break; - } - // write to transaction - // make sure we use the Transaction object to avoid used_during_commit() + // make sure we use the underlying Transaction object to avoid used_during_commit() bool foundChange = false; for (int i = 0; i < workers.size(); i++) { if (addr.excludes(workers[i].address)) { @@ -1035,7 +1023,7 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa } ++iter; } - return result; + return Optional(); } ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} @@ -1046,6 +1034,36 @@ Future> ProcessClassRangeImpl::getRange(ReadYourWrite } Future> ProcessClassRangeImpl::commit(ReadYourWritesTransaction* ryw) { + // Validate network address and process class type + Optional errorMsg; + auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(getKeyRange()); + auto iter = ranges.begin(); + while (iter != ranges.end()) { + auto entry = iter->value(); + // only check for setclass(set) operation, (clear) are forbidden thus not exist + if (entry.first && entry.second.present()) { + // validate network address + Key address = iter->begin().removePrefix(range.begin); + AddressExclusion addr = AddressExclusion::parse(address); + if (!addr.isValid()) { + std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n"; + if (address.toString().find(":tls") != std::string::npos) + error += " Do not include the `:tls' suffix when naming a process\n"; + errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); + return errorMsg; + } + // validate class type + ValueRef processClassType = entry.second.get(); + ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource); + if (processClass.classType() == ProcessClass::InvalidClass && + processClassType != LiteralStringRef("default")) { + std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n"; + errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); + return errorMsg; + } + } + ++iter; + } return processClassCommitActor(ryw, getKeyRange()); } From 951699af4f2b45d4f95a2bac3637493c370d757d Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 21 Aug 2020 00:57:04 -0700 Subject: [PATCH 021/458] Refine tests for management special keys --- .../SpecialKeySpaceCorrectness.actor.cpp | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index d5ab0cdaad..bb51ff095a 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -568,15 +568,20 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ASSERT(false); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; - ASSERT(e.code() == error_code_special_keys_api_failure); - Optional errorMsg = - wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); - ASSERT(errorMsg.present()); - std::string errorStr; - auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); - auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); - // special_key_space_management_api_error_msg schema validation - ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + if (e.code() == error_code_special_keys_api_failure) { + Optional errorMsg = + wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); + ASSERT(errorMsg.present()); + std::string errorStr; + auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); + auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); + // special_key_space_management_api_error_msg schema validation + ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + ASSERT(valueObj["command"].get_str() == "exclude" && !valueObj["retriable"].get_bool()); + } else { + TraceEvent(SevDebug, "UnexpectedError").detail("Command", "Exclude").error(e); + wait(tx->onError(e)); + } tx->reset(); } // "setclass" @@ -593,8 +598,6 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { // check correctness of classType of each process vector workers = wait(getWorkers(&tx->getTransaction())); for (const auto& worker : workers) { - // TODO : test here - // ASSERT(!worker.address.isTLS()); Key addr = Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); @@ -626,15 +629,20 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ASSERT(false); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; - ASSERT(e.code() == error_code_special_keys_api_failure); - Optional errorMsg = - wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); - ASSERT(errorMsg.present()); - std::string errorStr; - auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); - auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); - // special_key_space_management_api_error_msg schema validation - ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + if (e.code() == error_code_special_keys_api_failure) { + Optional errorMsg = + wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); + ASSERT(errorMsg.present()); + std::string errorStr; + auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); + auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); + // special_key_space_management_api_error_msg schema validation + ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + ASSERT(valueObj["command"].get_str() == "setclass" && !valueObj["retriable"].get_bool()); + } else { + TraceEvent(SevDebug, "UnexpectedError").detail("Command", "Setclass").error(e); + wait(tx->onError(e)); + } tx->reset(); } } From b8475ebb4b5ca4a6174ff35a224fed6ce5576feb Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 21 Aug 2020 01:20:14 -0700 Subject: [PATCH 022/458] Fix network address order bug in ExclusionInProgressActor --- fdbclient/SpecialKeySpace.actor.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 7e6b41f917..dde4010062 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -909,8 +909,14 @@ ACTOR Future> ExclusionInProgressActor(ReadYourWrites } } + // sort and remove :tls + std::set inProgressAddresses; for (auto const& address : inProgressExclusion) { - Key addrKey = prefix.withSuffix(address.toString()); // TODO : sort and remove :tls + inProgressAddresses.insert(formatIpPort(address.ip, address.port)); + } + + for (auto const& address : inProgressAddresses) { + Key addrKey = prefix.withSuffix(address); if (kr.contains(addrKey)) { result.push_back(result.arena(), KeyValueRef(addrKey, ValueRef())); result.arena().dependsOn(addrKey.arena()); From 62a678a281d0376e6ab89991939bd9ff44d3a31a Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 21 Aug 2020 10:49:13 -0700 Subject: [PATCH 023/458] Disable buggify in SpecialKeySpaceCorrectnessTest to avoid false positives --- tests/fast/SpecialKeySpaceCorrectness.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/fast/SpecialKeySpaceCorrectness.toml b/tests/fast/SpecialKeySpaceCorrectness.toml index d88477d29d..eb0dcb38ee 100644 --- a/tests/fast/SpecialKeySpaceCorrectness.toml +++ b/tests/fast/SpecialKeySpaceCorrectness.toml @@ -1,3 +1,5 @@ +buggify = false + [[test]] testTitle = 'SpecialKeySpaceCorrectnessTest' From 4e08aca32ef4ac5a8eebc95a4b5fea850962052d Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 25 Aug 2020 14:58:47 -0700 Subject: [PATCH 024/458] Change SevInfo to SevError --- fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index bb51ff095a..e0e5b0b916 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -526,7 +526,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { bool getRangeResultInOrder(const Standalone& result) { for (int i = 0; i < result.size() - 1; ++i) { if (result[i].key >= result[i + 1].key) { - TraceEvent(SevDebug, "GetRangeResultNotInOrder") + TraceEvent(SevError, "TestFailure") + .detail("Reason", "GetRangeResultNotInOrder") .detail("Index", i) .detail("Key1", result[i].key) .detail("Key2", result[i + 1].key); @@ -570,7 +571,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_special_keys_api_failure) { Optional errorMsg = - wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); + wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin)); ASSERT(errorMsg.present()); std::string errorStr; auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); From 6d17e996fb3defcc87d606a9a58ff78a7260c4f4 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 25 Aug 2020 14:59:43 -0700 Subject: [PATCH 025/458] Remove override --- fdbclient/SpecialKeySpace.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index dde4010062..46ab97b5d9 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1080,10 +1080,10 @@ void throwNotAllowedError(ReadYourWritesTransaction* ryw) { throw special_keys_api_failure(); } -void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override { +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { return throwNotAllowedError(ryw); } -void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override { +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { return throwNotAllowedError(ryw); } From a07b9f234d6098fa1222598d1b9ce6646209de4d Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 25 Aug 2020 18:18:32 -0700 Subject: [PATCH 026/458] Add readonly range \xff\xff/configuration/process/class_source, and change \xff\xff/configuration/class/ to \xff\xff/configuration/process/class_type/ --- fdbclient/NativeAPI.actor.cpp | 7 ++++- fdbclient/SpecialKeySpace.actor.cpp | 31 ++++++++++++++++++- fdbclient/SpecialKeySpace.actor.h | 6 ++++ .../SpecialKeySpaceCorrectness.actor.cpp | 6 ++-- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 375e37c5c4..3c57a12ece 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -924,7 +924,12 @@ DatabaseContext::DatabaseContext(Reference( - KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) + KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READONLY, + std::make_unique( + KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); } if (apiVersionAtLeast(630)) { diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 46ab97b5d9..f1bad698e6 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -972,7 +972,7 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran KeyRangeRef kr) { vector _workers = wait(getWorkers(&ryw->getTransaction())); auto workers = _workers; // strip const - // TODO : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 + // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { return formatIpPort(lhs.address.ip, lhs.address.port) < formatIpPort(rhs.address.ip, rhs.address.port); }); @@ -1087,3 +1087,32 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRange void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { return throwNotAllowedError(ryw); } + +ACTOR Future> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, + KeyRangeRef kr) { + vector _workers = wait(getWorkers(&ryw->getTransaction())); + auto workers = _workers; // strip const + // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 + std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { + return formatIpPort(lhs.address.ip, lhs.address.port) < formatIpPort(rhs.address.ip, rhs.address.port); + }); + Standalone result; + for (auto& w : workers) { + // exclude :tls in keys even the network addresss is TLS + Key k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port))); + if (kr.contains(k)) { + Value v(w.processClass.sourceString()); + result.push_back(result.arena(), KeyValueRef(k, v)); + result.arena().dependsOn(k.arena()); + result.arena().dependsOn(v.arena()); + } + } + return result; +} + +ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} + +Future> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + return getProcessClassSourceActor(ryw, getKeyRange().begin, kr); +} \ No newline at end of file diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index c9fad65cad..39f868d312 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -289,5 +289,11 @@ public: void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; }; +class ProcessClassSourceRangeImpl : public SpecialKeyRangeReadImpl { +public: + explicit ProcessClassSourceRangeImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; +}; + #include "flow/unactorcompiler.h" #endif diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index e0e5b0b916..7f81f765ab 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -591,7 +591,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); // test getRange state Standalone result = wait(tx->getRange( - KeyRangeRef(LiteralStringRef("class/"), LiteralStringRef("class0")) + KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), CLIENT_KNOBS->TOO_MANY)); ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); @@ -600,7 +600,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { vector workers = wait(getWorkers(&tx->getTransaction())); for (const auto& worker : workers) { Key addr = - Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + Key("process/class_type/" + formatIpPort(worker.address.ip, worker.address.port)) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); bool found = false; for (const auto& kv : result) { @@ -615,7 +615,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { } state ProcessData worker = deterministicRandom()->randomChoice(workers); state Key addr = - Key("class/" + formatIpPort(worker.address.ip, worker.address.port)) + Key("process/class_type/" + formatIpPort(worker.address.ip, worker.address.port)) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); tx->set(addr, LiteralStringRef("InvalidProcessType")); // test ryw From 6c981096ec5f93bb263548b5ba83aea51ce24641 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 25 Aug 2020 19:46:13 -0700 Subject: [PATCH 027/458] Add test for \xff\xff/configuration/process/class_source/ --- .../SpecialKeySpaceCorrectness.actor.cpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 7f81f765ab..25cfe89674 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -647,6 +647,51 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { tx->reset(); } } + // read class_source + { + try { + // test getRange + state Standalone class_source_result = wait(tx->getRange( + KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + CLIENT_KNOBS->TOO_MANY)); + ASSERT(!class_source_result.more && class_source_result.size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(self->getRangeResultInOrder(class_source_result)); + // check correctness of classType of each process + vector workers = wait(getWorkers(&tx->getTransaction())); + for (const auto& worker : workers) { + Key addr = + Key("process/class_source/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + bool found = false; + for (const auto& kv : class_source_result) { + if (kv.key == addr) { + ASSERT(kv.value.toString() == worker.processClass.sourceString()); + // Default source string is command_line + ASSERT(kv.value == LiteralStringRef("command_line")); + found = true; + break; + } + } + // Each process should find its corresponding element + ASSERT(found); + } + ProcessData worker = deterministicRandom()->randomChoice(workers); + state std::string address = formatIpPort(worker.address.ip, worker.address.port); + tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tx->set(Key("process/class_type/" + address) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + LiteralStringRef("unset")); + wait(tx->commit()); + Optional class_source = wait(tx->get(Key("process/class_source/" + address) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + ASSERT(class_source.present() && class_source.get() == LiteralStringRef("set_class")); + tx->reset(); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; + tx->onError(e); + } + } return Void(); } }; From 919c78d7edb6b37c67d36852444a7582a0937406 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Wed, 26 Aug 2020 11:02:22 -0700 Subject: [PATCH 028/458] Add wait to onError --- fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 25cfe89674..a41d60a2ed 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -689,7 +689,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { tx->reset(); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; - tx->onError(e); + wait(tx->onError(e)); } } return Void(); From 94221f1e919b9383502529cdf1652588a08714b5 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 27 Aug 2020 11:53:30 -0700 Subject: [PATCH 029/458] Disable support for pattern match of a whole machine address --- fdbclient/SpecialKeySpace.actor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index f1bad698e6..301ec8b751 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1057,6 +1057,12 @@ Future> ProcessClassRangeImpl::commit(ReadYourWritesTransa error += " Do not include the `:tls' suffix when naming a process\n"; errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); return errorMsg; + } else if (addr.isWholeMachine()) { + std::string error = "ERROR: \'" + address.toString() + + "\' is a whole machine address which we do not support. Please apply the change on " + "each process individually\n"; + errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); + return errorMsg; } // validate class type ValueRef processClassType = entry.second.get(); From caeeea365e6bbbe7fb321657068e659d3e2e176f Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Thu, 27 Aug 2020 23:07:22 -0700 Subject: [PATCH 030/458] Revert "Disable support for pattern match of a whole machine address" This reverts commit 94221f1e919b9383502529cdf1652588a08714b5. --- fdbclient/SpecialKeySpace.actor.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 301ec8b751..f1bad698e6 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1057,12 +1057,6 @@ Future> ProcessClassRangeImpl::commit(ReadYourWritesTransa error += " Do not include the `:tls' suffix when naming a process\n"; errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); return errorMsg; - } else if (addr.isWholeMachine()) { - std::string error = "ERROR: \'" + address.toString() + - "\' is a whole machine address which we do not support. Please apply the change on " - "each process individually\n"; - errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); - return errorMsg; } // validate class type ValueRef processClassType = entry.second.get(); From 0dc5736f5417136c1765dc730abe7df05a4bbd0f Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 28 Aug 2020 01:01:37 -0700 Subject: [PATCH 031/458] Allow directly read \xff\xff/management/* without setting option --- fdbclient/SpecialKeySpace.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index f1bad698e6..d19fee97f4 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -582,7 +582,7 @@ ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTra const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) { state KeyRangeRef range = impl->getKeyRange(); Standalone resultWithoutPrefix = - wait(ryw->getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY)); + wait(ryw->getTransaction().getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY)); ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); Standalone result; if (ryw->readYourWritesDisabled()) { From fd7198d874ba0550c45e75c75e57a04fa65cd7b1 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 29 Aug 2020 19:53:04 -0700 Subject: [PATCH 032/458] Extend backup container interface to support query restorable files set by key ranges --- fdbbackup/backup.actor.cpp | 138 +++++++++++++++++++++++++++- fdbclient/BackupContainer.actor.cpp | 80 +++++++++++----- fdbclient/BackupContainer.h | 8 +- fdbclient/FDBTypes.h | 9 ++ fdbclient/NativeAPI.actor.cpp | 6 ++ 5 files changed, 208 insertions(+), 33 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 0eac955518..9d2a204bdc 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "flow/Arena.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -81,7 +82,22 @@ enum enumProgramExe { }; enum enumBackupType { - BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP + BACKUP_UNDEFINED = 0, + BACKUP_START, + BACKUP_MODIFY, + BACKUP_STATUS, + BACKUP_ABORT, + BACKUP_WAIT, + BACKUP_DISCONTINUE, + BACKUP_PAUSE, + BACKUP_RESUME, + BACKUP_EXPIRE, + BACKUP_DELETE, + BACKUP_DESCRIBE, + BACKUP_LIST, + BACKUP_QUERY, + BACKUP_DUMP, + BACKUP_CLEANUP }; enum enumDBType { @@ -585,6 +601,38 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = { SO_END_OF_OPTIONS }; +CSimpleOpt::SOption g_rgBackupQueryOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, +#ifndef TLS_DISABLED + TLS_OPTION_FLAGS +#endif + SO_END_OF_OPTIONS +}; + // g_rgRestoreOptions is used by fdbrestore and fastrestore_tool CSimpleOpt::SOption g_rgRestoreOptions[] = { #ifdef _WIN32 @@ -918,13 +966,16 @@ void printBackupContainerInfo() { static void printBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str()); + printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | " + "list | query | cleanup) [OPTIONS]\n\n", + exeBackup.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" " FoundationDB cluster. The default is first the value of the\n" " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); printf(" -d, --destcontainer URL\n" - " The Backup container URL for start, modify, describe, expire, and delete operations.\n"); + " The Backup container URL for start, modify, describe, query, expire, and delete " + "operations.\n"); printBackupContainerInfo(); printf(" -b, --base_url BASEURL\n" " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n"); @@ -956,8 +1007,8 @@ static void printBackupUsage(bool devhelp) { " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + printf(" -k KEYS List of key ranges to backup or to filter the backup.\n" + " If not specified, the entire database will be backed up or no filter will be applied.\n"); printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" @@ -1273,6 +1324,7 @@ enumBackupType getBackupType(std::string backupType) values["delete"] = BACKUP_DELETE; values["describe"] = BACKUP_DESCRIBE; values["list"] = BACKUP_LIST; + values["query"] = BACKUP_QUERY; values["dump"] = BACKUP_DUMP; values["modify"] = BACKUP_MODIFY; } @@ -2400,6 +2452,73 @@ ACTOR Future describeBackup(const char *name, std::string destinationConta return Void(); } +// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for +// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version +// resolved to that timestamp. +ACTOR Future queryBackup(const char* name, std::string destinationContainer, + Standalone> keyRangesFilter, Version restoreVersion, + std::string originalClusterFile, std::string restoreTimestamp) { + // Resolve restoreTimestamp if given + if (!restoreTimestamp.empty()) { + if (originalClusterFile.empty()) { + printf("Error: an original cluster file must be given in order to resolve restore target timestamp '%s'\n", + restoreTimestamp.c_str()); + return Void(); + } + + if (!fileExists(originalClusterFile)) { + printf("Error: original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + return Void(); + } + + Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST); + Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); + printf("Timestamp '%s' resolves to version %" PRId64 "\n", restoreTimestamp.c_str(), v); + restoreVersion = v; + } + + try { + state Reference bc = openBackupContainer(name, destinationContainer); + if (restoreVersion == invalidVersion) { + printf("Using the maximum restorable version for the specified key ranges.\n"); + BackupDescription desc = wait(bc->describeBackup()); + if (!desc.maxRestorableVersion.present()) { + printf("Error: the specified backup is not restorable to any version.\n"); + return Void(); + } + restoreVersion = desc.maxRestorableVersion.get(); + } else if (restoreVersion == latestVersion) { + printf("Using the minimum restorable version for the specified key ranges.\n"); + } else if (restoreVersion < 0) { + printf("Error: the specified restorable version is not valid."); + } + Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); + if (fileSet.present()) { + printf("Key ranges filter: %s\n", keyRangesFilter.empty() ? "empty" : printable(keyRangesFilter).c_str()); + printf("Restoring to version: %" PRId64 "\n", fileSet.get().targetVersion); + printf("Range Files (file_name; file_size; key_range; version): \n"); + for (const auto& rangeFile : fileSet.get().ranges) { + ASSERT(fileSet.get().keyRanges.count(rangeFile.fileName)); + printf(" %s; %" PRId64 ", %s; %" PRId64 "\n", rangeFile.fileName.c_str(), rangeFile.fileSize, + fileSet.get().keyRanges.at(rangeFile.fileName).toString().c_str(), rangeFile.version); + } + printf("Log Files (file_name; file_size; begin_version; end_version): \n"); + for (const auto& log : fileSet.get().logs) { + printf(" %s; %" PRId64 "; %" PRId64 "; %" PRId64 "\n", log.fileName.c_str(), log.fileSize, + log.beginVersion, log.endVersion); + } + } else { + printf("No restorable files set found for specified key ranges.\n"); + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } + + return Void(); +} + ACTOR Future listBackup(std::string baseUrl) { try { std::vector containers = wait(IBackupContainer::listContainers(baseUrl)); @@ -2769,6 +2888,9 @@ int main(int argc, char* argv[]) { case BACKUP_LIST: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT); break; + case BACKUP_QUERY: + args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT); + break; case BACKUP_MODIFY: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT); break; @@ -3661,6 +3783,12 @@ int main(int argc, char* argv[]) { f = stopAfter( listBackup(baseUrl) ); break; + case BACKUP_QUERY: + initTraceFile(); + f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeys, restoreVersion, + restoreClusterFileOrig, restoreTimestamp)); + break; + case BACKUP_DUMP: initTraceFile(); f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) ); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a76f01b991..2edeb89b69 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -22,6 +22,7 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/JsonBuilder.h" +#include "flow/Arena.h" #include "flow/Trace.h" #include "flow/UnitTest.h" #include "flow/Hash3.h" @@ -1364,24 +1365,54 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } - ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion) { - // Find the most recent keyrange snapshot to end at or before targetVersion - state Optional snapshot; - std::vector snapshots = wait(bc->listKeyspaceSnapshots()); - for(auto const &s : snapshots) { - if(s.endVersion <= targetVersion) - snapshot = s; - } + ACTOR static Future> getRestoreSet_impl(Reference bc, + Version targetVersion, + VectorRef keyRangesFilter) { + // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. + state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); + state int i = snapshots.size() - 1; + for (; i >= 0; i--) { + state KeyspaceSnapshotFile snapshot = snapshots[i]; + // The smallest version of filtered range files >= snapshot beginVersion > targetVersion + if (targetVersion >= 0 && snapshot.beginVersion > targetVersion) { + break; + } - if(snapshot.present()) { state RestorableFileSet restorable; - restorable.snapshot = snapshot.get(); - restorable.targetVersion = targetVersion; + state Version minKeyRangeVersion = MAX_VERSION; + state Version maxKeyRangeVersion = -1; std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = std::move(results.first); - restorable.keyRanges = std::move(results.second); + wait(bc->readKeyspaceSnapshot(snapshot)); + + // Filter by keyRangesFilter. + if (keyRangesFilter.empty()) { + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + minKeyRangeVersion = snapshot.beginVersion; + maxKeyRangeVersion = snapshot.endVersion; + } else { + for (const auto& rangeFile : results.first) { + const auto& keyRange = results.second.at(rangeFile.fileName); + if (keyRange.intersects(keyRangesFilter)) { + restorable.ranges.push_back(rangeFile); + restorable.keyRanges[rangeFile.fileName] = keyRange; + minKeyRangeVersion = std::min(minKeyRangeVersion, rangeFile.version); + maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version); + } + } + // No range file match 'keyRangesFilter'. + if (restorable.ranges.empty()) { + continue; + } + } + if (targetVersion >= 0 && targetVersion < maxKeyRangeVersion) continue; + // 'latestVersion' represents using the minimum restorable version in a snapshot. + if (targetVersion == latestVersion) { + targetVersion = maxKeyRangeVersion; + } + restorable.targetVersion = targetVersion; + restorable.snapshot = snapshot; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { // Sanity check key ranges @@ -1395,9 +1426,8 @@ public: } } - // No logs needed if there is a complete key space snapshot at the target version. - if (snapshot.get().beginVersion == snapshot.get().endVersion && - snapshot.get().endVersion == targetVersion) { + // No logs needed if there is a complete filtered key space snapshot at the target version. + if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; return Optional(restorable); } @@ -1405,8 +1435,8 @@ public: // FIXME: check if there are tagged logs. for each tag, there is no version gap. state std::vector logs; state std::vector plogs; - wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + wait(store(logs, bc->listLogFiles(minKeyRangeVersion, targetVersion, false)) && + store(plogs, bc->listLogFiles(minKeyRangeVersion, targetVersion, true))); if (plogs.size() > 0) { logs.swap(plogs); @@ -1418,12 +1448,11 @@ public: // Remove duplicated log files that can happen for old epochs. std::vector filtered = filterDuplicates(logs); - restorable.logs.swap(filtered); // sort by version order again for continuous analysis std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { - restorable.continuousBeginVersion = snapshot.get().beginVersion; + if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, targetVersion)) { + restorable.continuousBeginVersion = minKeyRangeVersion; restorable.continuousEndVersion = targetVersion + 1; // not inclusive return Optional(restorable); } @@ -1434,7 +1463,7 @@ public: std::sort(logs.begin(), logs.end()); // If there are logs and the first one starts at or before the snapshot begin version then proceed - if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { + if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { Version end = logs.begin()->endVersion; computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); if (end >= targetVersion) { @@ -1448,8 +1477,9 @@ public: return Optional(); } - Future> getRestoreSet(Version targetVersion) final { - return getRestoreSet_impl(Reference::addRef(this), targetVersion); + Future> getRestoreSet(Version targetVersion, + VectorRef keyRangesFilter) final { + return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); } private: diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 8ac79937dd..fdce885329 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -280,9 +280,11 @@ public: virtual Future dumpFileList(Version begin = 0, Version end = std::numeric_limits::max()) = 0; - // Get exactly the files necessary to restore to targetVersion. Returns non-present if - // restore to given version is not possible. - virtual Future> getRestoreSet(Version targetVersion) = 0; + // Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion. + // If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. Returns non-present if + // restoring to the given version is not possible. + virtual Future> getRestoreSet(Version targetVersion, + VectorRef keyRangesFilter = {}) = 0; // Get an IBackupContainer based on a container spec string static Reference openContainer(std::string url); diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 117414923c..c859408f37 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -230,6 +230,7 @@ std::string describe( std::set const& items, int max_items = -1 ) { std::string printable( const StringRef& val ); std::string printable( const std::string& val ); std::string printable( const KeyRangeRef& range ); +std::string printable(const VectorRef& val); std::string printable( const VectorRef& val ); std::string printable( const VectorRef& val ); std::string printable( const KeyValueRef& val ); @@ -261,6 +262,14 @@ struct KeyRangeRef { bool contains( const KeyRef& key ) const { return begin <= key && key < end; } bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; } bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; } + bool intersects(const VectorRef& keysVec) const { + for (const auto& keys : keysVec) { + if (intersects(keys)) { + return true; + } + } + return false; + } bool empty() const { return begin == end; } bool singleKeyRange() const { return equalsKeyAfter(begin, end); } diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 715c93b6af..cbd9c42be3 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -148,6 +148,12 @@ std::string printable( const KeyRangeRef& range ) { return printable(range.begin) + " - " + printable(range.end); } +std::string printable(const VectorRef& val) { + std::string s; + for (int i = 0; i < val.size(); i++) s = s + printable(val[i]) + " "; + return s; +} + int unhex( char c ) { if (c >= '0' && c <= '9') return c-'0'; From 33aa10b4617da51ebc25de63063e93c93d6a6a34 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 29 Aug 2020 20:10:45 -0700 Subject: [PATCH 033/458] Minor optimizations --- fdbbackup/backup.actor.cpp | 1 + fdbclient/BackupContainer.actor.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 9d2a204bdc..f3dfbe9eb4 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2491,6 +2491,7 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine printf("Using the minimum restorable version for the specified key ranges.\n"); } else if (restoreVersion < 0) { printf("Error: the specified restorable version is not valid."); + return Void(); } Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); if (fileSet.present()) { diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 2edeb89b69..a7a3ad8b07 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1401,7 +1401,7 @@ public: maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version); } } - // No range file match 'keyRangesFilter'. + // No range file matches 'keyRangesFilter'. if (restorable.ranges.empty()) { continue; } From 30e27ba27b0d455ddd76694be30508a92ff74553 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sun, 30 Aug 2020 00:44:17 -0700 Subject: [PATCH 034/458] Add support for keys in CLI --- fdbbackup/backup.actor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index f3dfbe9eb4..52cc6921c4 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -610,6 +610,8 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, From b6c0299d0972e3f09e70db04027046cc507c37cb Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 09:31:57 -0700 Subject: [PATCH 035/458] Add help message in backup CLI for added options --- fdbbackup/backup.actor.cpp | 15 ++++++++++++--- fdbclient/BackupContainer.actor.cpp | 11 +++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 52cc6921c4..74b6c0dd7c 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -605,11 +605,11 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { #ifdef _WIN32 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_RESTORE_TIMESTAMP, "--query_restore_timestamp", SO_REQ_SEP }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-rv", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "--restore_version", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP }, { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, @@ -991,6 +991,15 @@ static void printBackupUsage(bool devhelp) { printf(" --delete_before_days NUM_DAYS\n" " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); + printf(" -qrv --query_restore_version VERSION\n" + " For query operations, set target version for restoring a backup. Set -1 for maximum " + "restorable version and -2 for minimum restorable version.\n"); + printf( + " --query_restore_timestamp\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a7a3ad8b07..bffa45f3fc 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1372,9 +1372,8 @@ public: state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; for (; i >= 0; i--) { - state KeyspaceSnapshotFile snapshot = snapshots[i]; // The smallest version of filtered range files >= snapshot beginVersion > targetVersion - if (targetVersion >= 0 && snapshot.beginVersion > targetVersion) { + if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { break; } @@ -1383,14 +1382,14 @@ public: state Version maxKeyRangeVersion = -1; std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot)); + wait(bc->readKeyspaceSnapshot(snapshots[i])); // Filter by keyRangesFilter. if (keyRangesFilter.empty()) { restorable.ranges = std::move(results.first); restorable.keyRanges = std::move(results.second); - minKeyRangeVersion = snapshot.beginVersion; - maxKeyRangeVersion = snapshot.endVersion; + minKeyRangeVersion = snapshots[i].beginVersion; + maxKeyRangeVersion = snapshots[i].endVersion; } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1412,7 +1411,7 @@ public: targetVersion = maxKeyRangeVersion; } restorable.targetVersion = targetVersion; - restorable.snapshot = snapshot; + restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { // Sanity check key ranges From e9d1f1c9c8fbc64fee7d3c4b55c92d1cb6372daf Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 09:43:11 -0700 Subject: [PATCH 036/458] change formatting --- fdbbackup/backup.actor.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 74b6c0dd7c..2a9f12fcfd 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -993,13 +993,10 @@ static void printBackupUsage(bool devhelp) { " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); printf(" -qrv --query_restore_version VERSION\n" " For query operations, set target version for restoring a backup. Set -1 for maximum " - "restorable version and -2 for minimum restorable version.\n"); - printf( - " --query_restore_timestamp\n" - " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", - BackupAgentBase::timeFormat().c_str()); - printf( - " and it will be converted to a version from that time using metadata in the cluster file.\n"); + " restorable version and -2 for minimum restorable version.\n"); + printf(" --query_restore_timestamp DATETIME\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); + printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" @@ -1018,7 +1015,7 @@ static void printBackupUsage(bool devhelp) { " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup or to filter the backup.\n" + printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" " If not specified, the entire database will be backed up or no filter will be applied.\n"); printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); From 79424f6d7a49c5c2eddb3b78f77a0e8e2a4a53ea Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Mon, 31 Aug 2020 19:38:37 -0400 Subject: [PATCH 037/458] Optionally kill existing servers Moved wait amounts to variables Made server address to a random ip address on local host --- contrib/Joshua/scripts/bindingTest.sh | 3 ++- contrib/Joshua/scripts/localClusterStart.sh | 29 ++++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh index 8e2fde1f7d..28b16715fb 100755 --- a/contrib/Joshua/scripts/bindingTest.sh +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -1,6 +1,7 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pkill fdbserver +KILLSERVERS="${KILLSERVERS:-0}" +if [ "${KILLSERVERS}" -gt 0 ]; then pkill fdbserver; fi ulimit -S -c unlimited unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 3ba4cb9dcb..ed693d52ac 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -6,6 +6,8 @@ LOGDIR="${WORKDIR}/log" ETCDIR="${WORKDIR}/etc" BINDIR="${BINDIR:-${SCRIPTDIR}}" FDBSERVERPORT="${FDBSERVERPORT:-4500}" +SERVERCHECKS="${SERVERCHECKS:-10}" +CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" @@ -13,6 +15,12 @@ LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" status=0 messagetime=0 messagecount=0 +let index2="${RANDOM} % 256" +let index3="${RANDOM} % 256" +let index4="(${RANDOM} % 255) + 1" +# Define a random ip address on localhost +IPADDRESS="127.${index2}.${index3}.${index4}" + function log { @@ -98,23 +106,23 @@ function createDirectories { then echo 'Failed to display user message' let status="${status} + 1" - + elif ! mkdir -p "${LOGDIR}" "${ETCDIR}" then log "Failed to create directories" let status="${status} + 1" - + # Display user message elif ! displayMessage "Setting file permissions" then log 'Failed to display user message' let status="${status} + 1" - + elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli" then log "Failed to set file permissions" let status="${status} + 1" - + else while read filepath do @@ -148,7 +156,10 @@ function createClusterFile { else description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) - echo "$description:$random_str@127.0.0.1:${FDBSERVERPORT}" > "${FDBCONF}" + let index2="${RANDOM} % 256" + let index3="${RANDOM} % 256" + let index4="(${RANDOM} % 255) + 1" + echo "${description}:${random_str}@${IPADDRESS}:${FDBSERVERPORT}" > "${FDBCONF}" fi if [ "${status}" -ne 0 ]; then @@ -170,7 +181,7 @@ function startFdbServer { log 'Failed to display user message' let status="${status} + 1" - elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "auto:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/$$" &> "${LOGDIR}/fdbserver.log" & + elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" # Maybe the server is already running @@ -226,7 +237,7 @@ function verifyAvailable { # Determine if status json says the database is available. else - avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 10 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` + avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout "${SERVERCHECKS}" 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` log "Avail value: ${avail}" "${DEBUGLEVEL}" if [[ -n "${avail}" ]] ; then return 0 @@ -262,7 +273,7 @@ function createDatabase { # Configure the database. else - "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout 240 --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" + "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout "${CONFIGUREWAIT}" --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" if ! displayMessage "Checking if config succeeded" then @@ -270,7 +281,7 @@ function createDatabase { fi iteration=0 - while [[ "${iteration}" -lt 10 ]] && ! verifyAvailable + while [[ "${iteration}" -lt "${SERVERCHECKS}" ]] && ! verifyAvailable do log "Database not created (iteration ${iteration})." let iteration="${iteration} + 1" From 738248dff25742e3ee99350da62e40da51602a6c Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 09:33:37 -0400 Subject: [PATCH 038/458] Added support for a global audit log to help debugging efforts --- contrib/Joshua/scripts/localClusterStart.sh | 52 +++++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index ed693d52ac..7bf3c8fd3f 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -10,6 +10,8 @@ SERVERCHECKS="${SERVERCHECKS:-10}" CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" +AUDITCLUSTER="${AUDITCLUSTER:-1}" +AUDITLOG="${AUDITLOG:-/tmp/audit-cluster.log}" # Initialize the variables status=0 @@ -172,8 +174,34 @@ function createClusterFile { return ${status} } +# Stop the Cluster from running. +function stopCluster { + # Add an audit entree, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Stopping Fdbserver (%6s)\n' "$(date +'%Y-%m-%d %H:%M:%S (%s)')" "${$}" "${FDBSERVERID}" >> "${AUDITLOG}" + fi + if [ -z "${FDBSERVERID}" ]; then + log 'FDB Server process is not defined' + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}"; then + log "Failed to locate FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to kill FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + else + log "Killed FDB Server process (${FDBSERVERID})" + fi + return "${status}" +} + # Start the server running. function startFdbServer { + # Add an audit entree, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Starting Fdbserver\n' "$(date +'%Y-%m-%d %H:%M:%S (%s)')" "${$}" >> "${AUDITLOG}" + fi + if [ "${status}" -ne 0 ]; then : elif ! displayMessage "Starting Fdb Server" @@ -185,14 +213,17 @@ function startFdbServer { then log "Failed to start FDB Server" # Maybe the server is already running - FDBSERVERID="$(pidof fdbserver)" + #FDBSERVERID="$(pidof fdbserver)" let status="${status} + 1" else FDBSERVERID="${!}" fi - if ! kill -0 ${FDBSERVERID} ; then - log "FDB Server start failed." + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server start failed because no process" + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}" ; then + log "FDB Server start failed because no perms" let status="${status} + 1" fi @@ -221,30 +252,31 @@ function getStatus { # Verify that the cluster is available. function verifyAvailable { + local status=0 + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server process is not defined." + let status="${status} + 1" # Verify that the server is running. - if ! kill -0 "${FDBSERVERID}" + elif ! kill -0 "${FDBSERVERID}" then log "FDB server process (${FDBSERVERID}) is not running" let status="${status} + 1" - return 1 - # Display user message. elif ! displayMessage "Checking cluster availability" then log 'Failed to display user message' let status="${status} + 1" - return 1 - # Determine if status json says the database is available. else avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout "${SERVERCHECKS}" 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` log "Avail value: ${avail}" "${DEBUGLEVEL}" if [[ -n "${avail}" ]] ; then - return 0 + : else - return 1 + let status="${status} + 1" fi fi + return "${status}" } # Configure the database on the server. From e5a8bf659a4febf4c9b301e60561ca65f6b6361a Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 09:34:46 -0400 Subject: [PATCH 039/458] Ensure that the cluster is stopped when exiting --- contrib/Joshua/scripts/bindingTest.sh | 2 -- contrib/Joshua/scripts/bindingTestScript.sh | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh index 28b16715fb..3e926140e0 100755 --- a/contrib/Joshua/scripts/bindingTest.sh +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -1,7 +1,5 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -KILLSERVERS="${KILLSERVERS:-0}" -if [ "${KILLSERVERS}" -gt 0 ]; then pkill fdbserver; fi ulimit -S -c unlimited unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 9ef19ab1a6..135a4ada86 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -7,7 +7,7 @@ SCRIPTID="${$}" SAVEONERROR="${SAVEONERROR:-1}" PYTHONDIR="${BINDIR}/tests/python" testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh" -VERSION="1.6" +VERSION="1.7" source ${SCRIPTDIR}/localClusterStart.sh @@ -36,6 +36,9 @@ fi # Begin the cluster using the logic in localClusterStart.sh. startCluster +# Stop the cluster on exit +trap "stopCluster" EXIT + # Display user message if [ "${status}" -ne 0 ]; then : From 8fa599a4351858e5f878d4548a8ea8b108a60f22 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 09:53:03 -0400 Subject: [PATCH 040/458] Removed total seconds from audit --- contrib/Joshua/scripts/localClusterStart.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 7bf3c8fd3f..90852c9f7d 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -178,7 +178,7 @@ function createClusterFile { function stopCluster { # Add an audit entree, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then - printf '%-15s (%6s) Stopping Fdbserver (%6s)\n' "$(date +'%Y-%m-%d %H:%M:%S (%s)')" "${$}" "${FDBSERVERID}" >> "${AUDITLOG}" + printf '%-15s (%6s) Stopping Fdbserver (%6s)\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBSERVERID}" >> "${AUDITLOG}" fi if [ -z "${FDBSERVERID}" ]; then log 'FDB Server process is not defined' @@ -199,7 +199,7 @@ function stopCluster { function startFdbServer { # Add an audit entree, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then - printf '%-15s (%6s) Starting Fdbserver\n' "$(date +'%Y-%m-%d %H:%M:%S (%s)')" "${$}" >> "${AUDITLOG}" + printf '%-15s (%6s) Starting Fdbserver\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" >> "${AUDITLOG}" fi if [ "${status}" -ne 0 ]; then From d349efba575a4b0453b6118b65800830d2efb4cd Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 10:08:52 -0400 Subject: [PATCH 041/458] Added support for killing the cluster gracefully --- contrib/Joshua/scripts/bindingTestScript.sh | 5 +++++ contrib/Joshua/scripts/localClusterStart.sh | 13 +++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 135a4ada86..971039f3c8 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -80,4 +80,9 @@ if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then env > "${LOGDIR}/env.log" fi +# Stop the cluster +if stopCluster; then + unset FDBSERVERID +fi + exit "${status}" diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 90852c9f7d..cbdbb9216e 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -22,6 +22,7 @@ let index3="${RANDOM} % 256" let index4="(${RANDOM} % 255) + 1" # Define a random ip address on localhost IPADDRESS="127.${index2}.${index3}.${index4}" +CLUSTERSTRING="${IPADDRESS}:${FDBSERVERPORT}" function log @@ -178,7 +179,7 @@ function createClusterFile { function stopCluster { # Add an audit entree, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then - printf '%-15s (%6s) Stopping Fdbserver (%6s)\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBSERVERID}" >> "${AUDITLOG}" + printf '%-15s (%6s) Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" "${FDBSERVERID}" >> "${AUDITLOG}" fi if [ -z "${FDBSERVERID}" ]; then log 'FDB Server process is not defined' @@ -186,11 +187,15 @@ function stopCluster { elif ! kill -0 "${FDBSERVERID}"; then log "Failed to locate FDB Server process (${FDBSERVERID})" let status="${status} + 1" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'kill all' --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + then + log "Killed cluster (${FDBSERVERID}) via cli" + elif ! kill -9 "${FDBSERVERID}"; then - log "Failed to kill FDB Server process (${FDBSERVERID})" + log "Failed to forcibly kill FDB Server process (${FDBSERVERID})" let status="${status} + 1" else - log "Killed FDB Server process (${FDBSERVERID})" + log "Forcibly killed FDB Server process (${FDBSERVERID})" fi return "${status}" } @@ -199,7 +204,7 @@ function stopCluster { function startFdbServer { # Add an audit entree, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then - printf '%-15s (%6s) Starting Fdbserver\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" >> "${AUDITLOG}" + printf '%-15s (%6s) Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" >> "${AUDITLOG}" fi if [ "${status}" -ne 0 ]; then From 6dddac5af21213e60ede9923f33270cf813b38f1 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 10:25:03 -0400 Subject: [PATCH 042/458] Fixed the kill call to the coordinator --- contrib/Joshua/scripts/localClusterStart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index cbdbb9216e..aa13c4da8b 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -187,7 +187,7 @@ function stopCluster { elif ! kill -0 "${FDBSERVERID}"; then log "Failed to locate FDB Server process (${FDBSERVERID})" let status="${status} + 1" - elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'kill all' --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill ${CLUSTERSTRING}" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" then log "Killed cluster (${FDBSERVERID}) via cli" From d40bdff7e86e465fdb3c35be658bb95f8884d801 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 11:12:57 -0400 Subject: [PATCH 043/458] Called the kill command before killing cluster --- contrib/Joshua/scripts/localClusterStart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index aa13c4da8b..cd55bde7e6 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -187,7 +187,7 @@ function stopCluster { elif ! kill -0 "${FDBSERVERID}"; then log "Failed to locate FDB Server process (${FDBSERVERID})" let status="${status} + 1" - elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill ${CLUSTERSTRING}" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${CLUSTERSTRING}" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" then log "Killed cluster (${FDBSERVERID}) via cli" From a88a41d07e4bc3e4cf00aa6c4d5f19d34d4a95d8 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 13:38:52 -0400 Subject: [PATCH 044/458] Disabled kernel AIO for bindingtester to allow ramdisk to be supported Added sleep to kill command to allow time for server to receive the command --- contrib/Joshua/scripts/localClusterStart.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index cd55bde7e6..b0c2cee6ea 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -187,7 +187,7 @@ function stopCluster { elif ! kill -0 "${FDBSERVERID}"; then log "Failed to locate FDB Server process (${FDBSERVERID})" let status="${status} + 1" - elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${CLUSTERSTRING}" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${CLUSTERSTRING}; sleep 3" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" then log "Killed cluster (${FDBSERVERID}) via cli" @@ -214,7 +214,7 @@ function startFdbServer { log 'Failed to display user message' let status="${status} + 1" - elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & + elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$} --knob_disable_posix_kernel_aio=1" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" # Maybe the server is already running From 9ee4e38ca4b91ddf297c2b92cba61042c59ad5e7 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 13:59:00 -0400 Subject: [PATCH 045/458] Fixed location of the knob --- contrib/Joshua/scripts/localClusterStart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index b0c2cee6ea..bd9df0247c 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -214,7 +214,7 @@ function startFdbServer { log 'Failed to display user message' let status="${status} + 1" - elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$} --knob_disable_posix_kernel_aio=1" &> "${LOGDIR}/fdbserver.log" & + elif ! "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" # Maybe the server is already running From 335bf882cda7cb32a9ee6684154133a3249d44d1 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 15:12:31 -0400 Subject: [PATCH 046/458] Disabled audit by default --- contrib/Joshua/scripts/localClusterStart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index bd9df0247c..46c9886b5c 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -10,7 +10,7 @@ SERVERCHECKS="${SERVERCHECKS:-10}" CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" -AUDITCLUSTER="${AUDITCLUSTER:-1}" +AUDITCLUSTER="${AUDITCLUSTER:-0}" AUDITLOG="${AUDITLOG:-/tmp/audit-cluster.log}" # Initialize the variables From 06527ffb70a5ead5e25a429b941417e962475dfb Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 1 Sep 2020 15:12:58 -0400 Subject: [PATCH 047/458] Capture any errors along with standard output --- contrib/Joshua/scripts/bindingTestScript.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 971039f3c8..898fd39aeb 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -61,8 +61,8 @@ fi # Display directory and log information, if an error occurred if [ "${status}" -ne 0 ] then - ls "${WORKDIR}" > "${LOGDIR}/dir.log" - ps -eafw > "${LOGDIR}/process-preclean.log" + ls "${WORKDIR}" &> "${LOGDIR}/dir.log" + ps -eafwH &> "${LOGDIR}/process-preclean.log" if [ -f "${FDBCONF}" ]; then cp -f "${FDBCONF}" "${LOGDIR}/" fi @@ -74,10 +74,10 @@ fi # Save debug information files, environment, and log information, if an error occurred if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then - ps -eafw > "${LOGDIR}/process-exit.log" - netstat -na > "${LOGDIR}/netstat.log" - df -h > "${LOGDIR}/disk.log" - env > "${LOGDIR}/env.log" + ps -eafwH &> "${LOGDIR}/process-exit.log" + netstat -na &> "${LOGDIR}/netstat.log" + df -h &> "${LOGDIR}/disk.log" + env &> "${LOGDIR}/env.log" fi # Stop the cluster From 9c130b5ea2f059b6cd4dfc053c132cd55fa610cd Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Wed, 2 Sep 2020 14:13:58 -0400 Subject: [PATCH 048/458] added new system keys --- fdbclient/SystemData.cpp | 3 +++ fdbclient/SystemData.h | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index a9bb73fae6..cb3b247b4c 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1060,3 +1060,6 @@ const KeyRangeRef testOnlyTxnStateStorePrefixRange( LiteralStringRef("\xff/TESTONLYtxnStateStore/"), LiteralStringRef("\xff/TESTONLYtxnStateStore0") ); + +const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery"); +const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 08bfb6ff88..4db3225fb5 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -396,6 +396,10 @@ std::pair decodeHealthyZoneValue( ValueRef const& ); // Used to create artifically large txnStateStore instances in testing. extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; +// Snapshot + Incremental Restore +extern const KeyRef writeRecoveryKey; +extern const KeyRef snapshotEndVersionKey; + #pragma clang diagnostic pop #endif From d334b6484edafb1e66f2c9353c95107dccf8b28e Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Wed, 2 Sep 2020 15:17:54 -0400 Subject: [PATCH 049/458] attempt to write to system keys with snapshot --- fdbclient/SystemData.cpp | 2 ++ fdbclient/SystemData.h | 1 + fdbserver/DataDistribution.actor.cpp | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index cb3b247b4c..0650f580f9 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1062,4 +1062,6 @@ const KeyRangeRef testOnlyTxnStateStorePrefixRange( ); const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery"); +const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1"); +const ValueRef writeRecoveryKeyFalse = LiteralStringRef("0"); const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 4db3225fb5..4006282708 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -398,6 +398,7 @@ extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; // Snapshot + Incremental Restore extern const KeyRef writeRecoveryKey; +extern const ValueRef writeRecoveryKeyTrue, writeRecoveryKeyFalse; extern const KeyRef snapshotEndVersionKey; #pragma clang diagnostic pop diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 094e2111de..94a49b8889 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -4728,6 +4728,22 @@ static std::set const& normalDataDistributorErrors() { ACTOR Future ddSnapCreateCore(DistributorSnapRequest snapReq, Reference> db ) { state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true); + state Reference tr(new ReadYourWritesTransaction(cx)); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + TraceEvent("SnapDataDistributor_WriteFlagAttempt") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); + tr->set(writeRecoveryKey, writeRecoveryKeyTrue); + wait(tr->commit()); + break; + } catch (Error& e) { + TraceEvent("SnapDataDistributor_WriteFlagError").error(e); + wait(tr->onError(e)); + } + } TraceEvent("SnapDataDistributor_SnapReqEnter") .detail("SnapPayload", snapReq.snapPayload) .detail("SnapUID", snapReq.snapUID); From 430921f25fa2ae6723f52269ef95d41cfd14385f Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Wed, 2 Sep 2020 13:38:24 -0700 Subject: [PATCH 050/458] Update local kv's contruction using result.arena() --- fdbclient/SpecialKeySpace.actor.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index d19fee97f4..f64550b77b 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -979,12 +979,10 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran Standalone result; for (auto& w : workers) { // exclude :tls in keys even the network addresss is TLS - Key k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port))); + KeyRef k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port), result.arena())); if (kr.contains(k)) { - Value v(w.processClass.toString()); + ValueRef v(result.arena(), w.processClass.toString()); result.push_back(result.arena(), KeyValueRef(k, v)); - result.arena().dependsOn(k.arena()); - result.arena().dependsOn(v.arena()); } } if (ryw->readYourWritesDisabled()) return result; From f9624b5a1a805d4715c36bc73f08daab674b5647 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Thu, 3 Sep 2020 13:56:23 -0700 Subject: [PATCH 051/458] Avoid unnecessary copy --- fdbclient/NativeAPI.actor.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index c186b61773..c1c541a51e 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -4429,8 +4429,11 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key if (i > 0) { results.push_back_deep(results.arena(), locations[i].first.begin); // Need this shard boundary } - results.append_deep(results.arena(), fReplies[i].get().splitPoints.begin(), - fReplies[i].get().splitPoints.size()); + if (fReplies[i].get().splitPoints.size() > 0) { + results.append(results.arena(), fReplies[i].get().splitPoints.begin(), + fReplies[i].get().splitPoints.size()); + results.arena().dependsOn(fReplies[i].get().splitPoints.arena()); + } } if (results.back() != keys.end) { results.push_back_deep(results.arena(), keys.end); From ee2ce6e7588ed8ab5626bfde761bfa786e8e3b83 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 4 Sep 2020 14:54:32 -0700 Subject: [PATCH 052/458] Refactor ryw function which updating read result with writes in the transaction --- fdbclient/SpecialKeySpace.actor.cpp | 138 ++++++++++++---------------- 1 file changed, 58 insertions(+), 80 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index f64550b77b..f40f245bc2 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -577,57 +577,72 @@ Future> ManagementCommandsOptionsImpl::commit(ReadYourWrit return Optional(); } -// read from rwModule -ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, - const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) { - state KeyRangeRef range = impl->getKeyRange(); +Standalone rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, + const Standalone& res) { + // "res" is the read result regardless of your writes, if ryw disabled, return immediately + if (ryw->readYourWritesDisabled()) return res; + // If ryw enabled, we update it with writes from the transaction + Standalone result; + RangeMap>, KeyRangeRef>::Ranges ranges = + ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); + RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); + auto iter2 = res.begin(); + result.arena().dependsOn(res.arena()); + while (iter != ranges.end() || iter2 != res.end()) { + if (iter == ranges.end()) { + result.push_back(result.arena(), KeyValueRef(iter2->key, iter2->value)); + ++iter2; + } else if (iter2 == res.end()) { + // insert if it is a set entry + std::pair> entry = iter->value(); + if (entry.first && entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + } + ++iter; + } else if (iter->range().contains(iter2->key)) { + std::pair> entry = iter->value(); + // if this is a valid range either for set or clear, move iter2 outside the range + if (entry.first) { + // insert if this is a set entry + if (entry.second.present()) + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + // move iter2 outside the range + while (iter2 != res.end() && iter->range().contains(iter2->key)) ++iter2; + } + ++iter; + } else if (iter->begin() > iter2->key) { + result.push_back(result.arena(), KeyValueRef(iter2->key, iter2->value)); + ++iter2; + } else if (iter->end() <= iter2->key) { + // insert if it is a set entry + std::pair> entry = iter->value(); + if (entry.first && entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + } + ++iter; + } + } + return result; +} + +// read from those readwrite modules in which special keys have one-to-one mapping with real persisted keys +ACTOR Future> rwModuleWithMappingGetRangeActor(ReadYourWritesTransaction* ryw, + const SpecialKeyRangeRWImpl* impl, + KeyRangeRef kr) { Standalone resultWithoutPrefix = wait(ryw->getTransaction().getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY)); ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); Standalone result; - if (ryw->readYourWritesDisabled()) { - for (const KeyValueRef& kv : resultWithoutPrefix) - result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); - } else { - RangeMap>, KeyRangeRef>::Ranges ranges = - ryw->getSpecialKeySpaceWriteMap().containedRanges(range); - RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); - int index = 0; - while (iter != ranges.end()) { - // add all previous entries into result - Key rk = impl->encode(resultWithoutPrefix[index].key); - while (index < resultWithoutPrefix.size() && rk < iter->begin()) { - result.push_back_deep(result.arena(), KeyValueRef(rk, resultWithoutPrefix[index].value)); - ++index; - } - std::pair> entry = iter->value(); - if (entry.first) { - // add the writen entries if exists - if (entry.second.present()) { - result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); - } - // move index to skip all entries in the iter->range - while (index < resultWithoutPrefix.size() && - iter->range().contains(impl->encode(resultWithoutPrefix[index].key))) - ++index; - } - ++iter; - } - // add all remaining entries into result - while (index < resultWithoutPrefix.size()) { - const KeyValueRef& kv = resultWithoutPrefix[index]; - result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); - ++index; - } - } - return result; + for (const KeyValueRef& kv : resultWithoutPrefix) + result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); + return rywGetRange(ryw, kr, result); } ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} Future> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { - return rwModuleGetRangeActor(ryw, this, kr); + return rwModuleWithMappingGetRangeActor(ryw, this, kr); } Key ExcludeServersRangeImpl::decode(const KeyRef& key) const { @@ -852,7 +867,7 @@ FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRange Future> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { - return rwModuleGetRangeActor(ryw, this, kr); + return rwModuleWithMappingGetRangeActor(ryw, this, kr); } Key FailedServersRangeImpl::decode(const KeyRef& key) const { @@ -932,42 +947,6 @@ Future> ExclusionInProgressRangeImpl::getRange(ReadYo return ExclusionInProgressActor(ryw, getKeyRange().begin, kr); } -Standalone rywModuleGetRange(ReadYourWritesTransaction* ryw, Standalone res, - KeyRangeRef kr) { - // res is read from database, if ryw enabled, we update it with writes in the transaction - Standalone result; - RangeMap>, KeyRangeRef>::Ranges ranges = - ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); - RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); - int index = 0; - while (iter != ranges.end()) { - // add all previous entries into result - while (index < res.size() && res[index].key < iter->begin()) { - result.push_back(result.arena(), KeyValueRef(res[index].key, res[index].value)); - result.arena().dependsOn(res.arena()); - ++index; - } - std::pair> entry = iter->value(); - if (entry.first) { - // add the writen entries if exists - if (entry.second.present()) { - result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); - } - // move index to skip all entries in the iter->range - while (index < res.size() && iter->range().contains(res[index].key)) ++index; - } - ++iter; - } - // add all remaining entries into result - while (index < res.size()) { - const KeyValueRef& kv = res[index]; - result.push_back(result.arena(), KeyValueRef(kv.key, kv.value)); - result.arena().dependsOn(res.arena()); - ++index; - } - return result; -} - ACTOR Future> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) { vector _workers = wait(getWorkers(&ryw->getTransaction())); @@ -985,8 +964,7 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran result.push_back(result.arena(), KeyValueRef(k, v)); } } - if (ryw->readYourWritesDisabled()) return result; - return rywModuleGetRange(ryw, result, kr); + return rywGetRange(ryw, kr, result); } ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { From 443e9a251d6a3b6c2900a372abd38e74d8fc3837 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 4 Sep 2020 15:21:47 -0700 Subject: [PATCH 053/458] Igore value in set in excluding(fail) implementation --- fdbclient/SpecialKeySpace.actor.cpp | 10 ++++++++++ fdbclient/SpecialKeySpace.actor.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index f40f245bc2..892d275f6a 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -645,6 +645,11 @@ Future> ExcludeServersRangeImpl::getRange(ReadYourWri return rwModuleWithMappingGetRangeActor(ryw, this, kr); } +void ExcludeServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { + // ignore value + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(ValueRef()))); +} + Key ExcludeServersRangeImpl::decode(const KeyRef& key) const { return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) .withPrefix(LiteralStringRef("\xff/conf/")); @@ -870,6 +875,11 @@ Future> FailedServersRangeImpl::getRange(ReadYourWrit return rwModuleWithMappingGetRangeActor(ryw, this, kr); } +void FailedServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { + // ignore value + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(ValueRef()))); +} + Key FailedServersRangeImpl::decode(const KeyRef& key) const { return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) .withPrefix(LiteralStringRef("\xff/conf/")); diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 39f868d312..91bb1cf872 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -260,6 +260,7 @@ class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl { public: explicit ExcludeServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; @@ -269,6 +270,7 @@ class FailedServersRangeImpl : public SpecialKeyRangeRWImpl { public: explicit FailedServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; From 1ca7fe1a05127a0248bd2a77afc2294a6de983b7 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 27 Aug 2020 15:11:16 -0700 Subject: [PATCH 054/458] Add span metadata message --- fdbclient/CommitTransaction.h | 2 + fdbclient/FDBTypes.h | 2 + fdbclient/MasterProxyInterface.h | 3 +- fdbclient/NativeAPI.actor.cpp | 2 +- fdbserver/ApplyMetadataMutation.cpp | 57 +++++++------ fdbserver/BackupWorker.actor.cpp | 3 +- fdbserver/CMakeLists.txt | 2 + fdbserver/LogSystem.h | 83 +++++++++++++++++-- fdbserver/MutationTracking.cpp | 5 ++ fdbserver/SpanContextMessage.h | 60 ++++++++++++++ fdbserver/StorageCache.actor.cpp | 8 ++ fdbserver/TLogServer.actor.cpp | 2 + fdbserver/storageserver.actor.cpp | 11 +++ .../workloads/ConfigureDatabase.actor.cpp | 2 +- 14 files changed, 205 insertions(+), 37 deletions(-) create mode 100644 fdbserver/SpanContextMessage.h diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index bc74941704..b2776a4dcd 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -49,6 +49,7 @@ static const char* typeString[] = { "SetValue", "MinV2", "AndV2", "CompareAndClear", + "Reserved_For_SpanContextMessage", "MAX_ATOMIC_OP" }; struct MutationRef { @@ -75,6 +76,7 @@ struct MutationRef { MinV2, AndV2, CompareAndClear, + Reserved_For_SpanContextMessage /* See fdbserver/SpanContextMessage.h */, MAX_ATOMIC_OP }; // This is stored this way for serialization purposes. diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 7e16dcd75f..80e8a8bb2c 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -129,9 +129,11 @@ enum { txsTagOld = -1, invalidTagOld = -100 }; struct TagsAndMessage { StringRef message; + // SpanID spanContext; VectorRef tags; TagsAndMessage() {} + // TagsAndMessage(SpanID spanContext) : spanContext(spanContext) {} TagsAndMessage(StringRef message, VectorRef tags) : message(message), tags(tags) {} // Loads tags and message from a serialized buffer. "rd" is checkpointed at diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h index 9e2b49037c..fbd98c6c2b 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/MasterProxyInterface.h @@ -165,7 +165,8 @@ struct CommitTransactionRequest : TimedRequest { Optional commitCostEstimation; Optional tagSet; - CommitTransactionRequest() : flags(0) {} + CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {} + CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {} template void serialize(Ar& ar) { diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index fba6fdf6f8..7099d6fbea 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -2678,7 +2678,7 @@ void debugAddTags(Transaction *tr) { Transaction::Transaction(Database const& cx) : cx(cx), info(cx->taskID, deterministicRandom()->randomUniqueID()), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), committedVersion(invalidVersion), versionstampPromise(Promise>()), options(cx), numErrors(0), - trLogInfo(createTrLogInfoProbabilistically(cx)), span(info.spanID, "Transaction"_loc) { + trLogInfo(createTrLogInfoProbabilistically(cx)), tr(info.spanID), span(info.spanID, "Transaction"_loc) { if (DatabaseContext::debugUseTags) { debugAddTags(this); } diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 23466ece9f..5872fdca0b 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -45,16 +45,21 @@ Reference getStorageInfo(UID id, std::map const& mutations, - IKeyValueStore* txnStateStore, LogPushData* toCommit, bool& confChange, - Reference logSystem, Version popVersion, - KeyRangeMap>* vecBackupKeys, KeyRangeMap* keyInfo, - KeyRangeMap* cacheInfo, std::map* uid_applyMutationsData, +void applyMetadataMutations(SpanID const& spanContext, UID const& dbgid, Arena& arena, + VectorRef const& mutations, IKeyValueStore* txnStateStore, + LogPushData* toCommit, bool& confChange, Reference logSystem, + Version popVersion, KeyRangeMap>* vecBackupKeys, + KeyRangeMap* keyInfo, KeyRangeMap* cacheInfo, + std::map* uid_applyMutationsData, RequestStream commit, Database cx, NotifiedVersion* commitVersion, std::map>* storageCache, std::map* tag_popped, bool initialCommit) { //std::map> cacheRangeInfo; std::map cachedRangeInfo; + if (toCommit) { + toCommit->addTransactionInfo(spanContext); + } + for (auto const& m : mutations) { //TraceEvent("MetadataMutation", dbgid).detail("M", m.toString()); @@ -102,7 +107,7 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefreadValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ).toString()); toCommit->addTag( decodeServerTagValue( txnStateStore->readValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ) ); - toCommit->addTypedMessage(privatized); + toCommit->writeTypedMessage(privatized); } } else if (m.param1.startsWith(serverTagPrefix)) { UID id = decodeServerTagKey(m.param1); @@ -114,9 +119,9 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefaddTag(tag); - toCommit->addTypedMessage(LogProtocolMessage()); + toCommit->writeTypedMessage(LogProtocolMessage(), true); toCommit->addTag(tag); - toCommit->addTypedMessage(privatized); + toCommit->writeTypedMessage(privatized); } if(!initialCommit) { txnStateStore->set(KeyValueRef(m.param1, m.param2)); @@ -168,7 +173,7 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefaddTag( cacheTag ); - toCommit->addTypedMessage(privatized); + toCommit->writeTypedMessage(privatized); } } else if (m.param1.startsWith(configKeysPrefix) || m.param1 == coordinatorsKey) { @@ -285,13 +290,13 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefaddTags(allTags); - toCommit->addTypedMessage(LogProtocolMessage()); + toCommit->writeTypedMessage(LogProtocolMessage(), true); } MutationRef privatized = m; privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena); toCommit->addTags(allTags); - toCommit->addTypedMessage(privatized); + toCommit->writeTypedMessage(privatized); } } else if (m.param1 == minRequiredCommitVersionKey) { @@ -347,7 +352,7 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefaddTag(decodeServerTagValue(kv.value)); - toCommit->addTypedMessage(privatized); + toCommit->writeTypedMessage(privatized); } } } @@ -538,37 +543,37 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefaddTags(allTags); - toCommit->addTypedMessage(mutationBegin); + toCommit->writeTypedMessage(mutationBegin); toCommit->addTags(allTags); - toCommit->addTypedMessage(mutationEnd); + toCommit->writeTypedMessage(mutationEnd); } } } -void applyMetadataMutations(ProxyCommitData& proxyCommitData, Arena& arena, Reference logSystem, - const VectorRef& mutations, LogPushData* toCommit, bool& confChange, - Version popVersion, bool initialCommit) { +void applyMetadataMutations(SpanID const& spanContext, ProxyCommitData& proxyCommitData, Arena& arena, + Reference logSystem, const VectorRef& mutations, + LogPushData* toCommit, bool& confChange, Version popVersion, bool initialCommit) { std::map* uid_applyMutationsData = nullptr; if (proxyCommitData.firstProxy) { uid_applyMutationsData = &proxyCommitData.uid_applyMutationsData; } - applyMetadataMutations(proxyCommitData.dbgid, arena, mutations, proxyCommitData.txnStateStore, toCommit, confChange, - logSystem, popVersion, &proxyCommitData.vecBackupKeys, &proxyCommitData.keyInfo, - &proxyCommitData.cacheInfo, uid_applyMutationsData, proxyCommitData.commit, - proxyCommitData.cx, &proxyCommitData.committedVersion, &proxyCommitData.storageCache, - &proxyCommitData.tag_popped, initialCommit); + applyMetadataMutations(spanContext, proxyCommitData.dbgid, arena, mutations, proxyCommitData.txnStateStore, toCommit, + confChange, logSystem, popVersion, &proxyCommitData.vecBackupKeys, &proxyCommitData.keyInfo, + &proxyCommitData.cacheInfo, uid_applyMutationsData, proxyCommitData.commit, + proxyCommitData.cx, &proxyCommitData.committedVersion, &proxyCommitData.storageCache, + &proxyCommitData.tag_popped, initialCommit); } -void applyMetadataMutations(const UID& dbgid, Arena& arena, const VectorRef& mutations, - IKeyValueStore* txnStateStore) { +void applyMetadataMutations(SpanID const& spanContext, const UID& dbgid, Arena& arena, + const VectorRef& mutations, IKeyValueStore* txnStateStore) { bool confChange; // Dummy variable, not used. - applyMetadataMutations(dbgid, arena, mutations, txnStateStore, /* toCommit= */ nullptr, confChange, + applyMetadataMutations(spanContext, dbgid, arena, mutations, txnStateStore, /* toCommit= */ nullptr, confChange, Reference(), /* popVersion= */ 0, /* vecBackupKeys= */ nullptr, /* keyInfo= */ nullptr, /* cacheInfo= */ nullptr, /* uid_applyMutationsData= */ nullptr, RequestStream(), Database(), /* commitVersion= */ nullptr, /* storageCache= */ nullptr, /* tag_popped= */ nullptr, /* initialCommit= */ false); -} \ No newline at end of file +} diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index 5860a6772a..6625ca6827 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -61,8 +61,9 @@ struct VersionedMessage { ArenaReader reader(arena, message, AssumeVersion(currentProtocolVersion)); - // Return false for LogProtocolMessage. + // Return false for metadata messages LogProtocolMessage and SpanContextMessage. if (LogProtocolMessage::isNextIn(reader)) return false; + if (SpanContextMessage::isNextIn(reader)) return false; reader >> *m; return normalKeys.contains(m->param1) || m->param1 == metadataVersionKey; diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 40b03e1aaa..965fd8ab0a 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -87,6 +87,7 @@ set(FDBSERVER_SRCS SimulatedCluster.actor.cpp SimulatedCluster.h SkipList.cpp + SpanContextMessage.h Status.actor.cpp Status.h StorageCache.actor.cpp @@ -127,6 +128,7 @@ set(FDBSERVER_SRCS workloads/BackupToDBUpgrade.actor.cpp workloads/BulkLoad.actor.cpp workloads/BulkSetup.actor.h + workloads/Basic.actor.cpp workloads/Cache.actor.cpp workloads/ChangeConfig.actor.cpp workloads/ClientTransactionProfileCorrectness.actor.cpp diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 2e2a5997e7..c50b1ea5a3 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -24,6 +24,7 @@ #include #include +#include "fdbserver/SpanContextMessage.h" #include "fdbserver/TLogInterface.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbclient/DatabaseConfiguration.h" @@ -828,6 +829,18 @@ struct CompareFirst { } }; +// Structure to store serialized mutations sent from the proxy to the +// transaction logs. The serialization repeats with the following format: +// +// +----------------------+ +----------------------+ +----------+ +----------------+ +----------------------+ +// | Message size | | Subsequence | | # of tags| | Tag | . . . . | Mutation | +// +----------------------+ +----------------------+ +----------+ +----------------+ +----------------------+ +// <------- 32 bits ------> <------- 32 bits ------> <- 16 bits-> <---- 24 bits ---> <---- variable bits ---> +// +// `Mutation` can be a serialized MutationRef or a special metadata message +// such as LogProtocolMessage or SpanContextMessage. The type of `Mutation` is +// uniquely identified by its first byte -- a value from MutationRef::Type. +// struct LogPushData : NonCopyable { // Log subsequences have to start at 1 (the MergedPeekCursor relies on this to make sure we never have !hasMessage() in the middle of data for a version @@ -859,7 +872,14 @@ struct LogPushData : NonCopyable { next_message_tags.insert(next_message_tags.end(), tags.begin(), tags.end()); } - void addMessage( StringRef rawMessageWithoutLength, bool usePreviousLocations ) { + // Add transaction info to be written before the first mutation in the transaction. + void addTransactionInfo(SpanID const& context) { + spanContext = context; + transactionSubseq = 0; + writtenLocations.clear(); + } + + void writeMessage( StringRef rawMessageWithoutLength, bool usePreviousLocations ) { if( !usePreviousLocations ) { prev_tags.clear(); if(logSystem->hasRemoteLogs()) { @@ -875,15 +895,16 @@ struct LogPushData : NonCopyable { uint32_t subseq = this->subsequence++; uint32_t msgsize = rawMessageWithoutLength.size() + sizeof(subseq) + sizeof(uint16_t) + sizeof(Tag)*prev_tags.size(); for(int loc : msg_locations) { - messagesWriter[loc] << msgsize << subseq << uint16_t(prev_tags.size()); + BinaryWriter& wr = messagesWriter[loc]; + wr << msgsize << subseq << uint16_t(prev_tags.size()); for(auto& tag : prev_tags) - messagesWriter[loc] << tag; - messagesWriter[loc].serializeBytes(rawMessageWithoutLength); + wr << tag; + wr.serializeBytes(rawMessageWithoutLength); } } template - void addTypedMessage(T const& item, bool allLocations = false) { + void writeTypedMessage(T const& item, bool metadataMessage = false, bool allLocations = false) { prev_tags.clear(); if(logSystem->hasRemoteLogs()) { prev_tags.push_back( logSystem->getRandomRouterTag() ); @@ -895,12 +916,31 @@ struct LogPushData : NonCopyable { logSystem->getPushLocations(prev_tags, msg_locations, allLocations); BinaryWriter bw(AssumeVersion(currentProtocolVersion)); + + // Metadata messages should be written before span information. If this + // isn't a metadata message, make sure all locations have had + // transaction info written to them. Mutations may have different sets + // of tags, so it is necessary to check all tag locations each time a + // mutation is written. + if (!metadataMessage) { + // If span information hasn't been written for this transaction yet, + // generate a subsequence value for the message. + if (!transactionSubseq) { + transactionSubseq = this->subsequence++; + } + + for (int loc : msg_locations) { + writeTransactionInfo(loc); + } + } + uint32_t subseq = this->subsequence++; bool first = true; int firstOffset=-1, firstLength=-1; for(int loc : msg_locations) { + BinaryWriter& wr = messagesWriter[loc]; + if (first) { - BinaryWriter& wr = messagesWriter[loc]; firstOffset = wr.getLength(); wr << uint32_t(0) << subseq << uint16_t(prev_tags.size()); for(auto& tag : prev_tags) @@ -911,7 +951,6 @@ struct LogPushData : NonCopyable { DEBUG_TAGS_AND_MESSAGE("ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength)).detail("PushLocations", msg_locations); first = false; } else { - BinaryWriter& wr = messagesWriter[loc]; BinaryWriter& from = messagesWriter[msg_locations[0]]; wr.serializeBytes( (uint8_t*)from.getData() + firstOffset, firstLength ); } @@ -929,7 +968,37 @@ private: std::vector prev_tags; std::vector messagesWriter; std::vector msg_locations; + // Stores message locations that have had span information written to them + // for the current transaction. Adding transaction info will reset this + // field. + std::unordered_set writtenLocations; uint32_t subsequence; + // Store transaction subsequence separately, as multiple mutations may need + // to write transaction info. This can happen if later mutations in a + // transaction need to write to a different location than earlier + // mutations. + uint32_t transactionSubseq; + SpanID spanContext; + + // Writes transaction info to the message stream for the given location if + // it has not already been written (for the current transaction). + void writeTransactionInfo(int location) { + if (writtenLocations.count(location) == 0) { + writtenLocations.insert(location); + + BinaryWriter& wr = messagesWriter[location]; + SpanContextMessage contextMessage(spanContext); + + int offset = wr.getLength(); + wr << uint32_t(0) << transactionSubseq << uint16_t(prev_tags.size()); + for(auto& tag : prev_tags) + wr << tag; + wr << contextMessage; + int length = wr.getLength() - offset; + *(uint32_t*)((uint8_t*)wr.getData() + offset) = length - sizeof(uint32_t); + } + } + }; #endif diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp index 9d090d59e3..f273057df7 100644 --- a/fdbserver/MutationTracking.cpp +++ b/fdbserver/MutationTracking.cpp @@ -21,6 +21,7 @@ #include #include "fdbserver/MutationTracking.h" #include "fdbserver/LogProtocolMessage.h" +#include "fdbserver/SpanContextMessage.h" #if defined(FDB_CLEAN_BUILD) && MUTATION_TRACKING_ENABLED #error "You cannot use mutation tracking in a clean/release build." @@ -71,6 +72,10 @@ TraceEvent debugTagsAndMessageEnabled( const char* context, Version version, Str LogProtocolMessage lpm; br >> lpm; rdr.setProtocolVersion(br.protocolVersion()); + } else if (SpanContextMessage::startsSpanContextMessage(mutationType)) { + BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); + SpanContextMessage scm; + br >> scm; } else { MutationRef m; BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); diff --git a/fdbserver/SpanContextMessage.h b/fdbserver/SpanContextMessage.h new file mode 100644 index 0000000000..da94fcc485 --- /dev/null +++ b/fdbserver/SpanContextMessage.h @@ -0,0 +1,60 @@ +/* + * SpanContextMessage.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FDBSERVER_SPANCONTEXTMESSAGE_H +#define FDBSERVER_SPANCONTEXTMESSAGE_H +#pragma once + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/CommitTransaction.h" + +struct SpanContextMessage { + // This message is pushed into the the transaction logs' memory to inform + // it what transaction subsequent mutations were a part of. This allows + // transaction logs and storage servers to associate mutations with a + // transaction identifier, called a span context. + // + // This message is similar to LogProtocolMessage. Storage servers read the + // first byte of this message to uniquely identify it, meaning it will + // never be mistaken for another message. See LogProtocolMessage.h for more + // information. + + SpanID spanContext; + + SpanContextMessage() {} + SpanContextMessage(SpanID const& spanContext) : spanContext(spanContext) {} + + std::string toString() const { + return format("code: %d, span context: %s", MutationRef::Reserved_For_SpanContextMessage, spanContext.toString().c_str()); + } + + template + void serialize(Ar& ar) { + uint8_t poly = MutationRef::Reserved_For_SpanContextMessage; + serializer(ar, poly, spanContext); + } + + static bool startsSpanContextMessage(uint8_t byte) { + return byte == MutationRef::Reserved_For_SpanContextMessage; + } + template static bool isNextIn(Ar& ar) { return startsSpanContextMessage(*(const uint8_t*)ar.peekBytes(1)); } +}; + +#endif diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index 44a3edfbfd..31ee4a4cb3 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -1763,6 +1763,10 @@ ACTOR Future pullAsyncData( StorageCacheData *data ) { dbgLastMessageWasProtocol = true; cloneCursor1->setProtocolVersion(cloneReader.protocolVersion()); } + else if (SpanContextMessage::isNextIn(cloneReader)) { + SpanContextMessage scm; + cloneReader >> scm; + } else { MutationRef msg; cloneReader >> msg; @@ -1835,6 +1839,10 @@ ACTOR Future pullAsyncData( StorageCacheData *data ) { data->logProtocol = reader.protocolVersion(); cloneCursor2->setProtocolVersion(data->logProtocol); } + else if (SpanContextMessage::isNextIn(reader)) { + SpanContextMessage scm; + reader >> scm; + } else { MutationRef msg; reader >> msg; diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 84c1655cac..ab015b1458 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -28,6 +28,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbserver/LogProtocolMessage.h" +#include "fdbserver/SpanContextMessage.h" #include "fdbserver/TLogInterface.h" #include "fdbserver/Knobs.h" #include "fdbserver/IKeyValueStore.h" @@ -1393,6 +1394,7 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req ACTOR Future> parseMessagesForTag( StringRef commitBlob, Tag tag, int logRouters ) { // See the comment in LogSystem.cpp for the binary format of commitBlob. state std::vector relevantMessages; + // TODO: Change to passed in protocol version state BinaryReader rd(commitBlob, AssumeVersion(currentProtocolVersion)); while (!rd.empty()) { TagsAndMessage tagsAndMessage; diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 561cbd85da..3d8822e1ae 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -43,6 +43,7 @@ #include "fdbserver/Knobs.h" #include "fdbserver/LatencyBandConfig.h" #include "fdbserver/LogProtocolMessage.h" +#include "fdbserver/SpanContextMessage.h" #include "fdbserver/LogSystem.h" #include "fdbserver/MoveKeys.actor.h" #include "fdbserver/MutationTracking.h" @@ -2847,6 +2848,11 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) dbgLastMessageWasProtocol = true; cloneCursor1->setProtocolVersion(cloneReader.protocolVersion()); } + else if (SpanContextMessage::isNextIn(cloneReader)) { + SpanContextMessage scm; + cloneReader >> scm; + // TODO: Set span context state here + } else { MutationRef msg; cloneReader >> msg; @@ -2940,6 +2946,11 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) data->storage.changeLogProtocol(ver, data->logProtocol); cloneCursor2->setProtocolVersion(rd.protocolVersion()); } + else if (SpanContextMessage::isNextIn(rd)) { + SpanContextMessage scm; + rd >> scm; + // TODO: Set span context state here + } else { MutationRef msg; rd >> msg; diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.actor.cpp index dca9994710..8536361eba 100644 --- a/fdbserver/workloads/ConfigureDatabase.actor.cpp +++ b/fdbserver/workloads/ConfigureDatabase.actor.cpp @@ -31,7 +31,7 @@ static const char* storeTypes[] = { "ssd", "ssd-1", "ssd-2", "memory", "memory-1 static const char* logTypes[] = { "log_engine:=1", "log_engine:=2", "log_spill:=1", "log_spill:=2", - "log_version:=2", "log_version:=3", "log_version:=4" + "log_version:=2", "log_version:=3", "log_version:=4", "log_version:=5" }; static const char* redundancies[] = { "single", "double", "triple" }; static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_enabled:=1" }; From 2a58e775d2f2c464d652b53dd085e2f3b89830af Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 27 Aug 2020 16:16:05 -0700 Subject: [PATCH 055/458] Add original changes --- fdbserver/ApplyMetadataMutation.h | 10 ++++---- fdbserver/LogSystem.h | 2 +- fdbserver/MasterProxyServer.actor.cpp | 27 +++++++++++++-------- fdbserver/TLogInterface.h | 7 +++--- fdbserver/TagPartitionedLogSystem.actor.cpp | 5 ++-- fdbserver/masterserver.actor.cpp | 2 +- 6 files changed, 31 insertions(+), 22 deletions(-) diff --git a/fdbserver/ApplyMetadataMutation.h b/fdbserver/ApplyMetadataMutation.h index 52355f2c19..5a06a31c40 100644 --- a/fdbserver/ApplyMetadataMutation.h +++ b/fdbserver/ApplyMetadataMutation.h @@ -39,10 +39,10 @@ inline bool isMetadataMutation(MutationRef const& m) { Reference getStorageInfo(UID id, std::map>* storageCache, IKeyValueStore* txnStateStore); -void applyMetadataMutations(ProxyCommitData& proxyCommitData, Arena& arena, Reference logSystem, - const VectorRef& mutations, LogPushData* pToCommit, bool& confChange, - Version popVersion, bool initialCommit); -void applyMetadataMutations(const UID& dbgid, Arena& arena, const VectorRef& mutations, - IKeyValueStore* txnStateStore); +void applyMetadataMutations(SpanID const& spanContext, ProxyCommitData& proxyCommitData, Arena& arena, + Reference logSystem, const VectorRef& mutations, + LogPushData* pToCommit, bool& confChange, Version popVersion, bool initialCommit); +void applyMetadataMutations(SpanID const& spanContext, const UID& dbgid, Arena& arena, + const VectorRef& mutations, IKeyValueStore* txnStateStore); #endif diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index c50b1ea5a3..b683646d81 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -679,7 +679,7 @@ struct ILogSystem { // Never returns normally, but throws an error if the subsystem stops working //Future push( UID bundle, int64_t seq, VectorRef messages ); - virtual Future push( Version prevVersion, Version version, Version knownCommittedVersion, Version minKnownCommittedVersion, struct LogPushData& data, Optional debugID = Optional() ) = 0; + virtual Future push( Version prevVersion, Version version, Version knownCommittedVersion, Version minKnownCommittedVersion, struct LogPushData& data, SpanID const& spanContext, Optional debugID = Optional() ) = 0; // Waits for the version number of the bundle (in this epoch) to be prevVersion (i.e. for all pushes ordered earlier) // Puts the given messages into the bundle, each with the given tags, and with message versions (version, 0) - (version, N) // Changes the version number of the bundle to be version (unblocking the next push) diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 7d8a36a66a..7a2ffea45d 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -296,6 +296,8 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::mapaddTransactionInfo(SpanID()); + // Serialize the log range mutations within the map for (; logRangeMutation != logRangeMutations->end(); ++logRangeMutation) { @@ -357,7 +359,7 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::maptagsForKey(backupMutation.param1); toCommit->addTags(tags); - toCommit->addTypedMessage(backupMutation); + toCommit->writeTypedMessage(backupMutation); // if (DEBUG_MUTATION("BackupProxyCommit", commitVersion, backupMutation)) { // TraceEvent("BackupProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("BackupMutation", backupMutation.toString()) @@ -396,7 +398,7 @@ struct CommitBatchContext { int batchOperations = 0; - Span span = Span("MP:commitBatch"_loc); + Span span; int64_t batchBytes = 0; @@ -476,7 +478,9 @@ CommitBatchContext::CommitBatchContext(ProxyCommitData* const pProxyCommitData_, localBatchNumber(++pProxyCommitData->localCommitBatchesStarted), toCommit(pProxyCommitData->logSystem), - committed(trs.size()) { + committed(trs.size()), + + span("MP:commitBatch"_loc) { evaluateBatchSize(); @@ -671,7 +675,7 @@ void applyMetadataEffect(CommitBatchContext* self) { for (int resolver = 0; resolver < self->resolution.size(); resolver++) committed = committed && self->resolution[resolver].stateMutations[versionIndex][transactionIndex].committed; if (committed) { - applyMetadataMutations(*self->pProxyCommitData, self->arena, self->pProxyCommitData->logSystem, + applyMetadataMutations(SpanID(), *self->pProxyCommitData, self->arena, self->pProxyCommitData->logSystem, self->resolution[0].stateMutations[versionIndex][transactionIndex].mutations, /* pToCommit= */ nullptr, self->forceRecovery, /* popVersion= */ 0, /* initialCommit */ false); @@ -754,7 +758,7 @@ ACTOR Future applyMetadataToCommittedTransactions(CommitBatchContext* self for (t = 0; t < trs.size() && !self->forceRecovery; t++) { if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) { self->commitCount++; - applyMetadataMutations(*pProxyCommitData, self->arena, pProxyCommitData->logSystem, + applyMetadataMutations(trs[t].spanContext, *pProxyCommitData, self->arena, pProxyCommitData->logSystem, trs[t].transaction.mutations, &self->toCommit, self->forceRecovery, self->commitVersion + 1, /* initialCommit= */ false); } @@ -803,6 +807,9 @@ ACTOR Future assignMutationsToStorageServers(CommitBatchContext* self) { state Optional* trCost = &trs[self->transactionNum].commitCostEstimation; state int mutationNum = 0; state VectorRef* pMutations = &trs[self->transactionNum].transaction.mutations; + + self->toCommit.addTransactionInfo(trs[self->transactionNum].spanContext); + for (; mutationNum < pMutations->size(); mutationNum++) { if(self->yieldBytes > SERVER_KNOBS->DESIRED_TOTAL_BYTES) { self->yieldBytes = 0; @@ -857,7 +864,7 @@ ACTOR Future assignMutationsToStorageServers(CommitBatchContext* self) { if(pProxyCommitData->cacheInfo[m.param1]) { self->toCommit.addTag(cacheTag); } - self->toCommit.addTypedMessage(m); + self->toCommit.writeTypedMessage(m); } else if (m.type == MutationRef::ClearRange) { KeyRangeRef clearRange(KeyRangeRef(m.param1, m.param2)); @@ -908,7 +915,7 @@ ACTOR Future assignMutationsToStorageServers(CommitBatchContext* self) { if(pProxyCommitData->needsCacheTag(clearRange)) { self->toCommit.addTag(cacheTag); } - self->toCommit.addTypedMessage(m); + self->toCommit.writeTypedMessage(m); } else { UNREACHABLE(); } @@ -1049,7 +1056,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { if(firstMessage) { self->toCommit.addTxsTag(); } - self->toCommit.addMessage(StringRef(m.begin(), m.size()), !firstMessage); + self->toCommit.writeMessage(StringRef(m.begin(), m.size()), !firstMessage); firstMessage = false; } @@ -1064,7 +1071,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { self->commitStartTime = now(); pProxyCommitData->lastStartCommit = self->commitStartTime; - self->loggingComplete = pProxyCommitData->logSystem->push( self->prevVersion, self->commitVersion, pProxyCommitData->committedVersion.get(), pProxyCommitData->minKnownCommittedVersion, self->toCommit, self->debugID ); + self->loggingComplete = pProxyCommitData->logSystem->push( self->prevVersion, self->commitVersion, pProxyCommitData->committedVersion.get(), pProxyCommitData->minKnownCommittedVersion, self->toCommit, self->span.context, self->debugID ); if (!self->forceRecovery) { ASSERT(pProxyCommitData->latestLocalCommitBatchLogging.get() == self->localBatchNumber-1); @@ -1806,7 +1813,7 @@ ACTOR Future masterProxyServerCore( Arena arena; bool confChanges; - applyMetadataMutations(commitData, arena, Reference(), mutations, + applyMetadataMutations(SpanID(), commitData, arena, Reference(), mutations, /* pToCommit= */ nullptr, confChanges, /* popVersion= */ 0, /* initialCommit= */ true); } diff --git a/fdbserver/TLogInterface.h b/fdbserver/TLogInterface.h index f5afa5df87..6fff6fb1f5 100644 --- a/fdbserver/TLogInterface.h +++ b/fdbserver/TLogInterface.h @@ -240,6 +240,7 @@ struct TLogCommitReply { struct TLogCommitRequest { constexpr static FileIdentifier file_identifier = 4022206; + SpanID spanContext; Arena arena; Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion; @@ -249,11 +250,11 @@ struct TLogCommitRequest { Optional debugID; TLogCommitRequest() {} - TLogCommitRequest( const Arena& a, Version prevVersion, Version version, Version knownCommittedVersion, Version minKnownCommittedVersion, StringRef messages, Optional debugID ) - : arena(a), prevVersion(prevVersion), version(version), knownCommittedVersion(knownCommittedVersion), minKnownCommittedVersion(minKnownCommittedVersion), messages(messages), debugID(debugID) {} + TLogCommitRequest( const SpanID& context, const Arena& a, Version prevVersion, Version version, Version knownCommittedVersion, Version minKnownCommittedVersion, StringRef messages, Optional debugID ) + : spanContext(context), arena(a), prevVersion(prevVersion), version(version), knownCommittedVersion(knownCommittedVersion), minKnownCommittedVersion(minKnownCommittedVersion), messages(messages), debugID(debugID) {} template void serialize( Ar& ar ) { - serializer(ar, prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, messages, reply, arena, debugID); + serializer(ar, prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, messages, reply, arena, spanContext, debugID); } }; diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 529cf58ac9..09e90d6eb1 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -527,7 +527,8 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted push(Version prevVersion, Version version, Version knownCommittedVersion, - Version minKnownCommittedVersion, LogPushData& data, Optional debugID) final { + Version minKnownCommittedVersion, LogPushData& data, + SpanID const& spanContext, Optional debugID) final { // FIXME: Randomize request order as in LegacyLogSystem? vector> quorumResults; vector> allReplies; @@ -542,7 +543,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted> tLogCommitResults; for(int loc=0; loc< it->logServers.size(); loc++) { Standalone msg = data.getMessages(location); - allReplies.push_back( recordPushMetrics( it->connectionResetTrackers[loc], it->logServers[loc]->get().interf().address(), it->logServers[loc]->get().interf().commit.getReply( TLogCommitRequest( msg.arena(), prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, msg, debugID ), TaskPriority::ProxyTLogCommitReply ) ) ); + allReplies.push_back( recordPushMetrics( it->connectionResetTrackers[loc], it->logServers[loc]->get().interf().address(), it->logServers[loc]->get().interf().commit.getReply( TLogCommitRequest( spanContext, msg.arena(), prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, msg, debugID ), TaskPriority::ProxyTLogCommitReply ) ) ); Future commitSuccess = success(allReplies.back()); addActor.get().send(commitSuccess); tLogCommitResults.push_back(commitSuccess); diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index ce5c993d77..bc1168c58a 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1613,7 +1613,7 @@ ACTOR Future masterCore( Reference self ) { } } - applyMetadataMutations(self->dbgid, recoveryCommitRequest.arena, tr.mutations.slice(mmApplied, tr.mutations.size()), + applyMetadataMutations(SpanID(), self->dbgid, recoveryCommitRequest.arena, tr.mutations.slice(mmApplied, tr.mutations.size()), self->txnStateStore); mmApplied = tr.mutations.size(); From f896c6899603d52ee26230e55595d0bfea37ff53 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 27 Aug 2020 17:39:09 -0700 Subject: [PATCH 056/458] Cleanup --- fdbserver/ApplyMetadataMutation.cpp | 8 ++--- fdbserver/CMakeLists.txt | 1 - fdbserver/LogSystem.h | 1 - fdbserver/workloads/Basic.actor.cpp | 50 +++++++++++++++++++++++++++++ tests/AsyncFileCorrectness.txt | 12 ------- 5 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 fdbserver/workloads/Basic.actor.cpp delete mode 100644 tests/AsyncFileCorrectness.txt diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 5872fdca0b..e20c17bbdc 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -560,10 +560,10 @@ void applyMetadataMutations(SpanID const& spanContext, ProxyCommitData& proxyCom } applyMetadataMutations(spanContext, proxyCommitData.dbgid, arena, mutations, proxyCommitData.txnStateStore, toCommit, - confChange, logSystem, popVersion, &proxyCommitData.vecBackupKeys, &proxyCommitData.keyInfo, - &proxyCommitData.cacheInfo, uid_applyMutationsData, proxyCommitData.commit, - proxyCommitData.cx, &proxyCommitData.committedVersion, &proxyCommitData.storageCache, - &proxyCommitData.tag_popped, initialCommit); + confChange, logSystem, popVersion, &proxyCommitData.vecBackupKeys, &proxyCommitData.keyInfo, + &proxyCommitData.cacheInfo, uid_applyMutationsData, proxyCommitData.commit, + proxyCommitData.cx, &proxyCommitData.committedVersion, &proxyCommitData.storageCache, + &proxyCommitData.tag_popped, initialCommit); } void applyMetadataMutations(SpanID const& spanContext, const UID& dbgid, Arena& arena, diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 965fd8ab0a..e13bec6be4 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -128,7 +128,6 @@ set(FDBSERVER_SRCS workloads/BackupToDBUpgrade.actor.cpp workloads/BulkLoad.actor.cpp workloads/BulkSetup.actor.h - workloads/Basic.actor.cpp workloads/Cache.actor.cpp workloads/ChangeConfig.actor.cpp workloads/ClientTransactionProfileCorrectness.actor.cpp diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index b683646d81..692d7bc727 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -998,7 +998,6 @@ private: *(uint32_t*)((uint8_t*)wr.getData() + offset) = length - sizeof(uint32_t); } } - }; #endif diff --git a/fdbserver/workloads/Basic.actor.cpp b/fdbserver/workloads/Basic.actor.cpp new file mode 100644 index 0000000000..2c1686144f --- /dev/null +++ b/fdbserver/workloads/Basic.actor.cpp @@ -0,0 +1,50 @@ +#include "fdbserver/IKeyValueStore.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +// Basic workload which runs a single transaction in simulation. +struct BasicWorkload : TestWorkload { + BasicWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {} + + std::string description() override { + return "Basic"; + } + + Future start(Database const& cx) override { + if (clientId == 0) { + TraceEvent("ABC_start"); + return testKVStore(cx->clone()); + } + + return Void(); + } + + Future check(Database const& cx) override { + return true; + } + + void getMetrics(vector& m) override {} + + ACTOR Future testKVStore(Database cx) { + state Reference tr = + Reference(new ReadYourWritesTransaction(cx)); + tr->set(KeyRef("foo2"), ValueRef("bar")); + wait(tr->commit()); + // auto ver = tr->getCommittedVersion(); + + /* + UID id = deterministicRandom()->randomUniqueID(); + std::string fn = id.toString(); + state IKeyValueStore* store = keyValueStoreMemory(fn, id, 500e6); + + wait(store->init()); + + KeyValueRef kv = KeyValueRef(StringRef("foo"), StringRef("bar")); + store->set(kv); + */ + + return Void(); + } +}; + +WorkloadFactory BasicWorkloadFactory("Basic"); diff --git a/tests/AsyncFileCorrectness.txt b/tests/AsyncFileCorrectness.txt deleted file mode 100644 index 161d746428..0000000000 --- a/tests/AsyncFileCorrectness.txt +++ /dev/null @@ -1,12 +0,0 @@ -testTitle=AsyncFileCorrectnessTest -useDB=false -runSetup=true -clearAfterTest=false - - testName=AsyncFileCorrectness - testDuration=10.0 - unbufferedIO=true - ;fileName=/home/ajb/testfilecorrectness - targetFileSize=327680 - maxOperationSize=8192 - numSimultaneousOperations=10 From 9398025f6ab12375fc0820eada3e603696b25395 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 27 Aug 2020 17:43:22 -0700 Subject: [PATCH 057/458] Remove test --- fdbserver/workloads/Basic.actor.cpp | 50 ----------------------------- 1 file changed, 50 deletions(-) delete mode 100644 fdbserver/workloads/Basic.actor.cpp diff --git a/fdbserver/workloads/Basic.actor.cpp b/fdbserver/workloads/Basic.actor.cpp deleted file mode 100644 index 2c1686144f..0000000000 --- a/fdbserver/workloads/Basic.actor.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "fdbserver/IKeyValueStore.h" -#include "fdbserver/workloads/workloads.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -// Basic workload which runs a single transaction in simulation. -struct BasicWorkload : TestWorkload { - BasicWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {} - - std::string description() override { - return "Basic"; - } - - Future start(Database const& cx) override { - if (clientId == 0) { - TraceEvent("ABC_start"); - return testKVStore(cx->clone()); - } - - return Void(); - } - - Future check(Database const& cx) override { - return true; - } - - void getMetrics(vector& m) override {} - - ACTOR Future testKVStore(Database cx) { - state Reference tr = - Reference(new ReadYourWritesTransaction(cx)); - tr->set(KeyRef("foo2"), ValueRef("bar")); - wait(tr->commit()); - // auto ver = tr->getCommittedVersion(); - - /* - UID id = deterministicRandom()->randomUniqueID(); - std::string fn = id.toString(); - state IKeyValueStore* store = keyValueStoreMemory(fn, id, 500e6); - - wait(store->init()); - - KeyValueRef kv = KeyValueRef(StringRef("foo"), StringRef("bar")); - store->set(kv); - */ - - return Void(); - } -}; - -WorkloadFactory BasicWorkloadFactory("Basic"); From 53b7721d6c0223053a21ad7cd8b70ee00cca76fe Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 28 Aug 2020 12:02:51 -0700 Subject: [PATCH 058/458] Add additional trace information --- fdbclient/FDBTypes.h | 2 -- fdbserver/MasterProxyServer.actor.cpp | 13 +++++++++---- fdbserver/Resolver.actor.cpp | 1 + fdbserver/TLogServer.actor.cpp | 1 + fdbserver/TagPartitionedLogSystem.actor.cpp | 1 + fdbserver/storageserver.actor.cpp | 4 ++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 80e8a8bb2c..7e16dcd75f 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -129,11 +129,9 @@ enum { txsTagOld = -1, invalidTagOld = -100 }; struct TagsAndMessage { StringRef message; - // SpanID spanContext; VectorRef tags; TagsAndMessage() {} - // TagsAndMessage(SpanID spanContext) : spanContext(spanContext) {} TagsAndMessage(StringRef message, VectorRef tags) : message(message), tags(tags) {} // Loads tags and message from a serialized buffer. "rd" is checkpointed at diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 7a2ffea45d..7648289f27 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -539,6 +539,7 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { state const int64_t localBatchNumber = self->localBatchNumber; state const int latencyBucket = self->latencyBucket; state const Optional& debugID = self->debugID; + state Span span("MP:preresolutionProcessing"_loc, self->span.context); // Pre-resolution the commits TEST(pProxyCommitData->latestLocalCommitBatchResolving.get() < localBatchNumber - 1); @@ -556,7 +557,7 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { ); } - GetCommitVersionRequest req(self->span.context, pProxyCommitData->commitVersionRequestNumber++, + GetCommitVersionRequest req(span.context, pProxyCommitData->commitVersionRequestNumber++, pProxyCommitData->mostRecentProcessedRequestNumber, pProxyCommitData->dbgid); GetCommitVersionReply versionReply = wait(brokenPromiseToNever( pProxyCommitData->master.getCommitVersion.getReply( @@ -595,13 +596,14 @@ ACTOR Future getResolution(CommitBatchContext* self) { // resolution processing but is still using CPU ProxyCommitData* pProxyCommitData = self->pProxyCommitData; std::vector& trs = self->trs; + state Span span("MP:getResolution"_loc, self->span.context); ResolutionRequestBuilder requests( pProxyCommitData, self->commitVersion, self->prevVersion, pProxyCommitData->version, - self->span + span ); int conflictRangeCount = 0; self->maxTransactionBytes = 0; @@ -969,6 +971,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { state std::vector& trs = self->trs; state const int64_t localBatchNumber = self->localBatchNumber; state const Optional& debugID = self->debugID; + state Span span("MP:postResolution"_loc, self->span.context); TEST(pProxyCommitData->latestLocalCommitBatchLogging.get() < localBatchNumber - 1); // Queuing post-resolution commit processing wait(pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1)); @@ -1021,7 +1024,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { // This should be *extremely* rare in the real world, but knob buggification should make it happen in simulation TEST(true); // Semi-committed pipeline limited by MVCC window //TraceEvent("ProxyWaitingForCommitted", pProxyCommitData->dbgid).detail("CommittedVersion", pProxyCommitData->committedVersion.get()).detail("NeedToCommit", commitVersion); - waitVersionSpan = Span(deterministicRandom()->randomUniqueID(), "MP:overMaxReadTransactionLifeVersions"_loc, {self->span.context}); + waitVersionSpan = Span(deterministicRandom()->randomUniqueID(), "MP:overMaxReadTransactionLifeVersions"_loc, {span.context}); choose{ when(wait(pProxyCommitData->committedVersion.whenAtLeast(self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS))) { wait(yield()); @@ -1071,7 +1074,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { self->commitStartTime = now(); pProxyCommitData->lastStartCommit = self->commitStartTime; - self->loggingComplete = pProxyCommitData->logSystem->push( self->prevVersion, self->commitVersion, pProxyCommitData->committedVersion.get(), pProxyCommitData->minKnownCommittedVersion, self->toCommit, self->span.context, self->debugID ); + self->loggingComplete = pProxyCommitData->logSystem->push( self->prevVersion, self->commitVersion, pProxyCommitData->committedVersion.get(), pProxyCommitData->minKnownCommittedVersion, self->toCommit, span.context, self->debugID ); if (!self->forceRecovery) { ASSERT(pProxyCommitData->latestLocalCommitBatchLogging.get() == self->localBatchNumber-1); @@ -1093,6 +1096,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { ACTOR Future transactionLogging(CommitBatchContext* self) { state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData; + state Span span("MP:transactionLogging"_loc, self->span.context); try { choose { @@ -1128,6 +1132,7 @@ ACTOR Future transactionLogging(CommitBatchContext* self) { ACTOR Future reply(CommitBatchContext* self) { state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData; + state Span span("MP:reply"_loc, self->span.context); const Optional& debugID = self->debugID; diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp index 8a2cac8171..05046b0766 100644 --- a/fdbserver/Resolver.actor.cpp +++ b/fdbserver/Resolver.actor.cpp @@ -104,6 +104,7 @@ ACTOR Future resolveBatch( ResolveTransactionBatchRequest req) { state Optional debugID; + state Span span("R:resolveBatch"_loc, req.spanContext); // The first request (prevVersion < 0) comes from the master state NetworkAddress proxyAddress = req.prevVersion >= 0 ? req.reply.getEndpoint().getPrimaryAddress() : NetworkAddress(); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index ab015b1458..beaa945270 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -1851,6 +1851,7 @@ ACTOR Future tLogCommit( TLogCommitRequest req, Reference logData, PromiseStream warningCollectorInput ) { + state Span span("TLog:tLogCommit"_loc, req.spanContext); state Optional tlogDebugID; if(req.debugID.present()) { diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 09e90d6eb1..6687a5d26f 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -533,6 +533,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted> quorumResults; vector> allReplies; int location = 0; + Span span("TPLS:push"_loc, spanContext); for(auto& it : tLogs) { if(it->isLocal && it->logServers.size()) { if(it->connectionResetTrackers.size() == 0) { diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 3d8822e1ae..af6fd39796 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -2777,6 +2777,7 @@ private: ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) { state double start; + state Span span("SS:update"_loc); try { // If we are disk bound and durableVersion is very old, we need to block updates or we could run out of memory // This is often referred to as the storage server e-brake (emergency brake) @@ -2851,7 +2852,6 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) else if (SpanContextMessage::isNextIn(cloneReader)) { SpanContextMessage scm; cloneReader >> scm; - // TODO: Set span context state here } else { MutationRef msg; @@ -2949,7 +2949,7 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) else if (SpanContextMessage::isNextIn(rd)) { SpanContextMessage scm; rd >> scm; - // TODO: Set span context state here + span.addParent(scm.spanContext); } else { MutationRef msg; From 00d3aa3acc2d7cfbfc2a7aa1e145ec2d6c5ab8eb Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 28 Aug 2020 15:16:54 -0700 Subject: [PATCH 059/458] Update formatting --- fdbserver/MasterProxyServer.actor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 7648289f27..cf4df617b6 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -295,7 +295,6 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::mapLOG_RANGE_BLOCK_SIZE; state int yieldBytes = 0; state BinaryWriter valueWriter(Unversioned()); - toCommit->addTransactionInfo(SpanID()); // Serialize the log range mutations within the map From b96dbc45cbb26de0c5582daf22af27289a5637a9 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Mon, 31 Aug 2020 10:39:07 -0700 Subject: [PATCH 060/458] Update formatting --- fdbserver/MasterProxyServer.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index cf4df617b6..7648289f27 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -295,6 +295,7 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::mapLOG_RANGE_BLOCK_SIZE; state int yieldBytes = 0; state BinaryWriter valueWriter(Unversioned()); + toCommit->addTransactionInfo(SpanID()); // Serialize the log range mutations within the map From 7dc55fdffd7a403242f1f38ab5d3c6b7d73cf153 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Mon, 31 Aug 2020 14:46:41 -0700 Subject: [PATCH 061/458] Revert state --- fdbserver/TLogServer.actor.cpp | 1 - tests/AsyncFileCorrectness.txt | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/AsyncFileCorrectness.txt diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index beaa945270..912c8fe0bb 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -1394,7 +1394,6 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req ACTOR Future> parseMessagesForTag( StringRef commitBlob, Tag tag, int logRouters ) { // See the comment in LogSystem.cpp for the binary format of commitBlob. state std::vector relevantMessages; - // TODO: Change to passed in protocol version state BinaryReader rd(commitBlob, AssumeVersion(currentProtocolVersion)); while (!rd.empty()) { TagsAndMessage tagsAndMessage; diff --git a/tests/AsyncFileCorrectness.txt b/tests/AsyncFileCorrectness.txt new file mode 100644 index 0000000000..161d746428 --- /dev/null +++ b/tests/AsyncFileCorrectness.txt @@ -0,0 +1,12 @@ +testTitle=AsyncFileCorrectnessTest +useDB=false +runSetup=true +clearAfterTest=false + + testName=AsyncFileCorrectness + testDuration=10.0 + unbufferedIO=true + ;fileName=/home/ajb/testfilecorrectness + targetFileSize=327680 + maxOperationSize=8192 + numSimultaneousOperations=10 From 783e6a170e9fc3d395843c71d23998c92bf67077 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 4 Sep 2020 17:36:56 -0700 Subject: [PATCH 062/458] Add code coverage --- fdbserver/LogSystem.h | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 692d7bc727..3e21c85619 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -874,6 +874,7 @@ struct LogPushData : NonCopyable { // Add transaction info to be written before the first mutation in the transaction. void addTransactionInfo(SpanID const& context) { + TEST(!spanContext.isValid()); // addTransactionInfo with invalid SpanID spanContext = context; transactionSubseq = 0; writtenLocations.clear(); From efde86340a55d3498ee24cf8504cc30e341dde52 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 4 Sep 2020 17:37:34 -0700 Subject: [PATCH 063/458] Add knob to disable span serialization --- fdbserver/LogSystem.h | 4 ++++ flow/Knobs.cpp | 2 ++ flow/Knobs.h | 2 ++ 3 files changed, 8 insertions(+) diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 3e21c85619..22418224ab 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -30,6 +30,7 @@ #include "fdbclient/DatabaseConfiguration.h" #include "fdbserver/MutationTracking.h" #include "flow/IndexedSet.h" +#include "flow/Knobs.h" #include "fdbrpc/ReplicationPolicy.h" #include "fdbrpc/Locality.h" #include "fdbrpc/Replication.h" @@ -984,6 +985,9 @@ private: // Writes transaction info to the message stream for the given location if // it has not already been written (for the current transaction). void writeTransactionInfo(int location) { + if (!FLOW_KNOBS->WRITE_TRACING_ENABLED) { + return; + } if (writtenLocations.count(location) == 0) { writtenLocations.insert(location); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 5019923e8c..9c06722744 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -61,6 +61,8 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( HUGE_ARENA_LOGGING_BYTES, 100e6 ); init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 ); + init( WRITE_TRACING_ENABLED, true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false; + //connectionMonitor init( CONNECTION_MONITOR_LOOP_TIME, isSimulated ? 0.75 : 1.0 ); if( randomize && BUGGIFY ) CONNECTION_MONITOR_LOOP_TIME = 6.0; init( CONNECTION_MONITOR_TIMEOUT, isSimulated ? 1.50 : 2.0 ); if( randomize && BUGGIFY ) CONNECTION_MONITOR_TIMEOUT = 6.0; diff --git a/flow/Knobs.h b/flow/Knobs.h index bb719c2686..7f97df82fa 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -69,6 +69,8 @@ public: double HUGE_ARENA_LOGGING_BYTES; double HUGE_ARENA_LOGGING_INTERVAL; + bool WRITE_TRACING_ENABLED; + //run loop profiling double RUN_LOOP_PROFILING_INTERVAL; double SLOWTASK_PROFILING_LOG_INTERVAL; From 1ad5e174585f9b02d1b0df97a11d36e4655bbdec Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 5 Sep 2020 11:14:59 -0700 Subject: [PATCH 064/458] add support for comparing original and current impls --- fdbclient/BackupContainer.actor.cpp | 166 ++++++++++++++++-- ...kupAndParallelRestoreCorrectness.actor.cpp | 1 + flow/error_definitions.h | 1 + 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index bffa45f3fc..a5365d7a35 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1365,16 +1365,106 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } + ACTOR static Future> getRestoreSet_impl_original( + Reference bc, Version targetVersion) { + // Find the most recent keyrange snapshot to end at or before targetVersion + state Optional snapshot; + std::vector snapshots = wait(bc->listKeyspaceSnapshots()); + // printf("old h1 %ld %ld\n", targetVersion, snapshots.size()); + for (auto const& s : snapshots) { + if (s.endVersion <= targetVersion) snapshot = s; + } + + if (snapshot.present()) { + // printf("h2 %ld %ld\n", snapshot.get().beginVersion, snapshot.get().endVersion); + state RestorableFileSet restorable; + restorable.snapshot = snapshot.get(); + restorable.targetVersion = targetVersion; + + std::pair, std::map> results = + wait(bc->readKeyspaceSnapshot(snapshot.get())); + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + // TODO: Reenable the sanity check after TooManyFiles error is resolved + if (false && g_network->isSimulated()) { + // Sanity check key ranges + state std::map::iterator rit; + for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { + auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), + [file = rit->first](const RangeFile f) { return f.fileName == file; }); + ASSERT(it != restorable.ranges.end()); + KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); + ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end); + } + } + + // No logs needed if there is a complete key space snapshot at the target version. + if (snapshot.get().beginVersion == snapshot.get().endVersion && + snapshot.get().endVersion == targetVersion) { + restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; + return Optional(restorable); + } + + // FIXME: check if there are tagged logs. for each tag, there is no version gap. + state std::vector logs; + state std::vector plogs; + wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && + store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + + if (plogs.size() > 0) { + logs.swap(plogs); + // sort by tag ID so that filterDuplicates works. + std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) { + return std::tie(a.tagId, a.beginVersion, a.endVersion) < + std::tie(b.tagId, b.beginVersion, b.endVersion); + }); + + // Remove duplicated log files that can happen for old epochs. + std::vector filtered = filterDuplicates(logs); + + restorable.logs.swap(filtered); + // sort by version order again for continuous analysis + std::sort(restorable.logs.begin(), restorable.logs.end()); + if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { + restorable.continuousBeginVersion = snapshot.get().beginVersion; + restorable.continuousEndVersion = targetVersion + 1; // not inclusive + return Optional(restorable); + } + return Optional(); + } + + // List logs in version order so log continuity can be analyzed + std::sort(logs.begin(), logs.end()); + + // printf("old backup used log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); + // If there are logs and the first one starts at or before the snapshot begin version then proceed + if (!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { + Version end = logs.begin()->endVersion; + computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); + if (end >= targetVersion) { + restorable.continuousBeginVersion = logs.begin()->beginVersion; + restorable.continuousEndVersion = end; + return Optional(restorable); + } + } + } + + return Optional(); + } + + // might have problem working with old backup ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion, VectorRef keyRangesFilter) { // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; + // printf("new h1 %ld %ld %ld\n", targetVersion, keyRangesFilter.size(), snapshots.size()); for (; i >= 0; i--) { + // printf("h2 %ld %ld\n", snapshots[i].beginVersion, snapshots[i].endVersion); // The smallest version of filtered range files >= snapshot beginVersion > targetVersion if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { - break; + continue; } state RestorableFileSet restorable; @@ -1384,12 +1474,18 @@ public: std::pair, std::map> results = wait(bc->readKeyspaceSnapshot(snapshots[i])); + // Old backup does not have metadata about key ranges and can not be filtered with key ranges. + if (keyRangesFilter.size() && results.second.empty() && !results.first.empty()) { + throw backup_not_filterable_with_key_ranges(); + } + // Filter by keyRangesFilter. if (keyRangesFilter.empty()) { restorable.ranges = std::move(results.first); restorable.keyRanges = std::move(results.second); minKeyRangeVersion = snapshots[i].beginVersion; maxKeyRangeVersion = snapshots[i].endVersion; + // printf("h3 %ld %ld %ld\n", minKeyRangeVersion, maxKeyRangeVersion, restorable.keyRanges.size()); } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1405,12 +1501,14 @@ public: continue; } } - if (targetVersion >= 0 && targetVersion < maxKeyRangeVersion) continue; // 'latestVersion' represents using the minimum restorable version in a snapshot. - if (targetVersion == latestVersion) { - targetVersion = maxKeyRangeVersion; - } - restorable.targetVersion = targetVersion; + // if (targetVersion == latestVersion) { + // restorable.targetVersion = maxKeyRangeVersion; + // } else + restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; + + if (restorable.targetVersion < maxKeyRangeVersion) continue; + // restorable.targetVersion = targetVersion; restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { @@ -1426,7 +1524,7 @@ public: } // No logs needed if there is a complete filtered key space snapshot at the target version. - if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == targetVersion) { + if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; return Optional(restorable); } @@ -1434,8 +1532,8 @@ public: // FIXME: check if there are tagged logs. for each tag, there is no version gap. state std::vector logs; state std::vector plogs; - wait(store(logs, bc->listLogFiles(minKeyRangeVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(minKeyRangeVersion, targetVersion, true))); + wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) && + store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true))); if (plogs.size() > 0) { logs.swap(plogs); @@ -1450,9 +1548,9 @@ public: restorable.logs.swap(filtered); // sort by version order again for continuous analysis std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, targetVersion)) { + if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) { restorable.continuousBeginVersion = minKeyRangeVersion; - restorable.continuousEndVersion = targetVersion + 1; // not inclusive + restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive return Optional(restorable); } return Optional(); @@ -1461,11 +1559,17 @@ public: // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); + // printf("6.2 backup used: log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); // If there are logs and the first one starts at or before the snapshot begin version then proceed if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { + Version end = logs.begin()->endVersion; - computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); - if (end >= targetVersion) { + computeRestoreEndVersion(logs, &restorable.logs, &end, restorable.targetVersion); + + // printf("6.2 backup used: end_version: %ld target: %ld\n", end, restorable.targetVersion); + + if (end >= restorable.targetVersion) { + // printf("6.2 backup used finish\n"); restorable.continuousBeginVersion = logs.begin()->beginVersion; restorable.continuousEndVersion = end; return Optional(restorable); @@ -1476,9 +1580,43 @@ public: return Optional(); } + static void printRestorableSet(const RestorableFileSet& restorable) { + printf("restorable: begin %ld, end %ld, target %ld, ranges size %ld, logs size %ld, filename %s\n", + restorable.continuousBeginVersion, restorable.continuousEndVersion, restorable.targetVersion, + restorable.ranges.size(), restorable.logs.size(), restorable.snapshot.fileName.c_str()); + } + + ACTOR static Future> getRestoreSet_compare(Reference bc, + Version targetVersion, + VectorRef keyRangesFilter) { + + state Optional newResult = wait(getRestoreSet_impl(bc, targetVersion, keyRangesFilter)); + if (keyRangesFilter.empty() && targetVersion != latestVersion) { + state Optional oldResult = wait(getRestoreSet_impl_original(bc, targetVersion)); + // printf("comparing\n"); + // if (oldResult.present()) { + // // printf("old\n"); + // printRestorableSet(oldResult.get()); + // } + ASSERT(newResult.present() == oldResult.present()); + if (newResult.present()) { + // printf("new\n"); + // printRestorableSet(newResult.get()); + ASSERT(oldResult.get().continuousBeginVersion == oldResult.get().continuousBeginVersion); + ASSERT(oldResult.get().continuousEndVersion == oldResult.get().continuousEndVersion); + ASSERT(oldResult.get().targetVersion == oldResult.get().targetVersion); + ASSERT(oldResult.get().ranges.size() == oldResult.get().ranges.size()); + ASSERT(oldResult.get().keyRanges.size() == oldResult.get().keyRanges.size()); + ASSERT(oldResult.get().snapshot.fileName == oldResult.get().snapshot.fileName); + } + } + return newResult; + } + Future> getRestoreSet(Version targetVersion, VectorRef keyRangesFilter) final { - return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); + return getRestoreSet_compare(Reference::addRef(this), targetVersion, + keyRangesFilter); } private: diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp index af983a13ee..dfa20c95ec 100644 --- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp @@ -468,6 +468,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { .detail("LastBackupContainer", lastBackupContainer->getURL()) .detail("RestoreAfter", self->restoreAfter) .detail("BackupTag", printable(self->backupTag)); + // start restoring auto container = IBackupContainer::openContainer(lastBackupContainer->getURL()); BackupDescription desc = wait(container->describeBackup()); diff --git a/flow/error_definitions.h b/flow/error_definitions.h index ca8460548d..3baf2aaa2c 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -197,6 +197,7 @@ ERROR( backup_cannot_expire, 2316, "Cannot expire requested data from backup wit ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL") ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs") ERROR( backup_does_not_exist, 2319, "Backup does not exist") +ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges") ERROR( restore_invalid_version, 2361, "Invalid restore version") ERROR( restore_corrupted_data, 2362, "Corrupted backup data") ERROR( restore_missing_data, 2363, "Missing backup data") From 0e49542d59b0c6d5e0cee2ef52b6b444c7fba972 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Sat, 5 Sep 2020 13:58:22 -0700 Subject: [PATCH 065/458] Fix data race described in #3749 --- flow/ThreadHelper.actor.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index ed6a9cdc7d..b695e46d2b 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -312,12 +312,12 @@ public: } virtual void cancel() { - // Cancels the action and decrements the reference count by 1 - // The if statement is just an optimization. It's ok if we take the wrong path due to a race - if(isReadyUnsafe()) - delref(); - else - onMainThreadVoid( [this](){ this->cancelFuture.cancel(); this->delref(); }, NULL ); + onMainThreadVoid( + [this]() { + this->cancelFuture.cancel(); + this->delref(); + }, + nullptr); } void releaseMemory() { From 3728ed03ddfbab5093910f726d727880f2ec2cdb Mon Sep 17 00:00:00 2001 From: Young Liu Date: Sat, 5 Sep 2020 18:55:09 -0700 Subject: [PATCH 066/458] Resolve comments --- fdbbackup/backup.actor.cpp | 128 +++++++++++++++++++------ fdbclient/BackupContainer.actor.cpp | 143 +--------------------------- flow/error_definitions.h | 1 + 3 files changed, 101 insertions(+), 171 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 2a9f12fcfd..efd88087c6 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -18,7 +18,9 @@ * limitations under the License. */ +#include "fdbclient/JsonBuilder.h" #include "flow/Arena.h" +#include "flow/Trace.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -120,7 +122,7 @@ enum { OPT_USE_PARTITIONED_LOG, // Backup and Restore constants - OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, + OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, // Backup Modify OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID, @@ -610,8 +612,8 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, { OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP }, { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, @@ -992,8 +994,8 @@ static void printBackupUsage(bool devhelp) { " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); printf(" -qrv --query_restore_version VERSION\n" - " For query operations, set target version for restoring a backup. Set -1 for maximum " - " restorable version and -2 for minimum restorable version.\n"); + " For query operations, set target version for restoring a backup. Set -1 for maximum\n" + " restorable version (default) and -2 for minimum restorable version.\n"); printf(" --query_restore_timestamp DATETIME\n" " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); @@ -2460,71 +2462,125 @@ ACTOR Future describeBackup(const char *name, std::string destinationConta return Void(); } +static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) { + result["error"] = errorMessage; + printf("%s\n", result.getJson().c_str()); + TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage); +} + // If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for // selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version // resolved to that timestamp. ACTOR Future queryBackup(const char* name, std::string destinationContainer, Standalone> keyRangesFilter, Version restoreVersion, - std::string originalClusterFile, std::string restoreTimestamp) { + std::string originalClusterFile, std::string restoreTimestamp, bool verbose) { + state UID operationId = deterministicRandom()->randomUniqueID(); + state JsonBuilderObject result; + state std::string errorMessage; + result["key_ranges_filter"] = printable(keyRangesFilter); + result["destination_container"] = destinationContainer; + + TraceEvent("BackupQueryStart") + .detail("OperationId", operationId) + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("SpecifiedRestoreVersion", restoreVersion) + .detail("RestoreTimestamp", restoreTimestamp) + .detail("BackupClusterFile", originalClusterFile); + // Resolve restoreTimestamp if given if (!restoreTimestamp.empty()) { if (originalClusterFile.empty()) { - printf("Error: an original cluster file must be given in order to resolve restore target timestamp '%s'\n", - restoreTimestamp.c_str()); + reportBackupQueryError( + operationId, result, + format("an original cluster file must be given in order to resolve restore target timestamp '%s'", + restoreTimestamp.c_str())); return Void(); } if (!fileExists(originalClusterFile)) { - printf("Error: original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + reportBackupQueryError(operationId, result, + format("The specified original source database cluster file '%s' does not exist\n", + originalClusterFile.c_str())); return Void(); } Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST); Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); - printf("Timestamp '%s' resolves to version %" PRId64 "\n", restoreTimestamp.c_str(), v); + result["restore_timestamp"] = restoreTimestamp; + result["restore_timestamp_resolved_version"] = v; restoreVersion = v; } try { state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { - printf("Using the maximum restorable version for the specified key ranges.\n"); BackupDescription desc = wait(bc->describeBackup()); if (!desc.maxRestorableVersion.present()) { - printf("Error: the specified backup is not restorable to any version.\n"); + reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); return Void(); } restoreVersion = desc.maxRestorableVersion.get(); - } else if (restoreVersion == latestVersion) { - printf("Using the minimum restorable version for the specified key ranges.\n"); - } else if (restoreVersion < 0) { - printf("Error: the specified restorable version is not valid."); + } else if (restoreVersion < 0 && restoreVersion != latestVersion) { + reportBackupQueryError(operationId, result, + errorMessage = + format("the specified restorable version %ld is not valid", restoreVersion)); return Void(); } Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); if (fileSet.present()) { - printf("Key ranges filter: %s\n", keyRangesFilter.empty() ? "empty" : printable(keyRangesFilter).c_str()); - printf("Restoring to version: %" PRId64 "\n", fileSet.get().targetVersion); - printf("Range Files (file_name; file_size; key_range; version): \n"); + int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0; + result["restore_version"] = fileSet.get().targetVersion; + JsonBuilderArray rangeFilesJson; + JsonBuilderArray logFilesJson; for (const auto& rangeFile : fileSet.get().ranges) { - ASSERT(fileSet.get().keyRanges.count(rangeFile.fileName)); - printf(" %s; %" PRId64 ", %s; %" PRId64 "\n", rangeFile.fileName.c_str(), rangeFile.fileSize, - fileSet.get().keyRanges.at(rangeFile.fileName).toString().c_str(), rangeFile.version); + JsonBuilderObject object; + object["file_name"] = rangeFile.fileName; + object["file_size"] = rangeFile.fileSize; + object["version"] = rangeFile.version; + object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0 + ? "none" + : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); + rangeFilesJson.push_back(object); + totalRangeFilesSize += rangeFile.fileSize; } - printf("Log Files (file_name; file_size; begin_version; end_version): \n"); for (const auto& log : fileSet.get().logs) { - printf(" %s; %" PRId64 "; %" PRId64 "; %" PRId64 "\n", log.fileName.c_str(), log.fileSize, - log.beginVersion, log.endVersion); + JsonBuilderObject object; + object["file_name"] = log.fileName; + object["file_size"] = log.fileSize; + object["begin_version"] = log.beginVersion; + object["end_version"] = log.endVersion; + logFilesJson.push_back(object); + totalLogFilesSize += log.fileSize; } + + result["total_range_files_size"] = totalRangeFilesSize; + result["total_log_files_size"] = totalLogFilesSize; + + if (verbose) { + result["ranges"] = rangeFilesJson; + result["logs"] = logFilesJson; + } + + TraceEvent("BackupQueryReceivedRestorableFilesSet") + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("ActualRestoreVersion", fileSet.get().targetVersion) + .detail("NumRangeFiles", fileSet.get().ranges.size()) + .detail("NumLogFiles", fileSet.get().logs.size()) + .detail("RangeFilesBytes", totalRangeFilesSize) + .detail("LogFilesBytes", totalLogFilesSize); } else { - printf("No restorable files set found for specified key ranges.\n"); + reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges"); + return Void(); } + } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; + reportBackupQueryError(operationId, result, e.what()); + return Void(); } + printf("%s\n", result.getJson().c_str()); return Void(); } @@ -3039,6 +3095,7 @@ int main(int argc, char* argv[]) { std::string addPrefix; std::string removePrefix; Standalone> backupKeys; + Standalone> backupKeysFilter; int maxErrors = 20; Version restoreVersion = invalidVersion; std::string restoreTimestamp; @@ -3259,6 +3316,15 @@ int main(int argc, char* argv[]) { return FDB_EXIT_ERROR; } break; + case OPT_BACKUPKEYS_FILTER: + try { + addKeyRange(args->OptionArg(), backupKeysFilter); + } + catch (Error &) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + break; case OPT_DESTCONTAINER: destinationContainer = args->OptionArg(); // If the url starts with '/' then prepend "file://" for backwards compatibility @@ -3794,8 +3860,8 @@ int main(int argc, char* argv[]) { case BACKUP_QUERY: initTraceFile(); - f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeys, restoreVersion, - restoreClusterFileOrig, restoreTimestamp)); + f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion, + restoreClusterFileOrig, restoreTimestamp, !quietDisplay)); break; case BACKUP_DUMP: diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a5365d7a35..f5f5bce2ae 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1365,103 +1365,13 @@ public: return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); } - ACTOR static Future> getRestoreSet_impl_original( - Reference bc, Version targetVersion) { - // Find the most recent keyrange snapshot to end at or before targetVersion - state Optional snapshot; - std::vector snapshots = wait(bc->listKeyspaceSnapshots()); - // printf("old h1 %ld %ld\n", targetVersion, snapshots.size()); - for (auto const& s : snapshots) { - if (s.endVersion <= targetVersion) snapshot = s; - } - - if (snapshot.present()) { - // printf("h2 %ld %ld\n", snapshot.get().beginVersion, snapshot.get().endVersion); - state RestorableFileSet restorable; - restorable.snapshot = snapshot.get(); - restorable.targetVersion = targetVersion; - - std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = std::move(results.first); - restorable.keyRanges = std::move(results.second); - // TODO: Reenable the sanity check after TooManyFiles error is resolved - if (false && g_network->isSimulated()) { - // Sanity check key ranges - state std::map::iterator rit; - for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { - auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), - [file = rit->first](const RangeFile f) { return f.fileName == file; }); - ASSERT(it != restorable.ranges.end()); - KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); - ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end); - } - } - - // No logs needed if there is a complete key space snapshot at the target version. - if (snapshot.get().beginVersion == snapshot.get().endVersion && - snapshot.get().endVersion == targetVersion) { - restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; - return Optional(restorable); - } - - // FIXME: check if there are tagged logs. for each tag, there is no version gap. - state std::vector logs; - state std::vector plogs; - wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); - - if (plogs.size() > 0) { - logs.swap(plogs); - // sort by tag ID so that filterDuplicates works. - std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) { - return std::tie(a.tagId, a.beginVersion, a.endVersion) < - std::tie(b.tagId, b.beginVersion, b.endVersion); - }); - - // Remove duplicated log files that can happen for old epochs. - std::vector filtered = filterDuplicates(logs); - - restorable.logs.swap(filtered); - // sort by version order again for continuous analysis - std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { - restorable.continuousBeginVersion = snapshot.get().beginVersion; - restorable.continuousEndVersion = targetVersion + 1; // not inclusive - return Optional(restorable); - } - return Optional(); - } - - // List logs in version order so log continuity can be analyzed - std::sort(logs.begin(), logs.end()); - - // printf("old backup used log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); - // If there are logs and the first one starts at or before the snapshot begin version then proceed - if (!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { - Version end = logs.begin()->endVersion; - computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); - if (end >= targetVersion) { - restorable.continuousBeginVersion = logs.begin()->beginVersion; - restorable.continuousEndVersion = end; - return Optional(restorable); - } - } - } - - return Optional(); - } - - // might have problem working with old backup ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion, VectorRef keyRangesFilter) { // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); state int i = snapshots.size() - 1; - // printf("new h1 %ld %ld %ld\n", targetVersion, keyRangesFilter.size(), snapshots.size()); for (; i >= 0; i--) { - // printf("h2 %ld %ld\n", snapshots[i].beginVersion, snapshots[i].endVersion); // The smallest version of filtered range files >= snapshot beginVersion > targetVersion if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { continue; @@ -1485,7 +1395,6 @@ public: restorable.keyRanges = std::move(results.second); minKeyRangeVersion = snapshots[i].beginVersion; maxKeyRangeVersion = snapshots[i].endVersion; - // printf("h3 %ld %ld %ld\n", minKeyRangeVersion, maxKeyRangeVersion, restorable.keyRanges.size()); } else { for (const auto& rangeFile : results.first) { const auto& keyRange = results.second.at(rangeFile.fileName); @@ -1498,17 +1407,13 @@ public: } // No range file matches 'keyRangesFilter'. if (restorable.ranges.empty()) { - continue; + throw backup_not_overlapped_with_keys_filter(); } } // 'latestVersion' represents using the minimum restorable version in a snapshot. - // if (targetVersion == latestVersion) { - // restorable.targetVersion = maxKeyRangeVersion; - // } else restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; - if (restorable.targetVersion < maxKeyRangeVersion) continue; - // restorable.targetVersion = targetVersion; + restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { @@ -1558,65 +1463,23 @@ public: // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); - - // printf("6.2 backup used: log begin version: %ld\n", logs.empty() ? -1 : logs.front().beginVersion); // If there are logs and the first one starts at or before the snapshot begin version then proceed if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { - Version end = logs.begin()->endVersion; computeRestoreEndVersion(logs, &restorable.logs, &end, restorable.targetVersion); - - // printf("6.2 backup used: end_version: %ld target: %ld\n", end, restorable.targetVersion); - if (end >= restorable.targetVersion) { - // printf("6.2 backup used finish\n"); restorable.continuousBeginVersion = logs.begin()->beginVersion; restorable.continuousEndVersion = end; return Optional(restorable); } } } - return Optional(); } - static void printRestorableSet(const RestorableFileSet& restorable) { - printf("restorable: begin %ld, end %ld, target %ld, ranges size %ld, logs size %ld, filename %s\n", - restorable.continuousBeginVersion, restorable.continuousEndVersion, restorable.targetVersion, - restorable.ranges.size(), restorable.logs.size(), restorable.snapshot.fileName.c_str()); - } - - ACTOR static Future> getRestoreSet_compare(Reference bc, - Version targetVersion, - VectorRef keyRangesFilter) { - - state Optional newResult = wait(getRestoreSet_impl(bc, targetVersion, keyRangesFilter)); - if (keyRangesFilter.empty() && targetVersion != latestVersion) { - state Optional oldResult = wait(getRestoreSet_impl_original(bc, targetVersion)); - // printf("comparing\n"); - // if (oldResult.present()) { - // // printf("old\n"); - // printRestorableSet(oldResult.get()); - // } - ASSERT(newResult.present() == oldResult.present()); - if (newResult.present()) { - // printf("new\n"); - // printRestorableSet(newResult.get()); - ASSERT(oldResult.get().continuousBeginVersion == oldResult.get().continuousBeginVersion); - ASSERT(oldResult.get().continuousEndVersion == oldResult.get().continuousEndVersion); - ASSERT(oldResult.get().targetVersion == oldResult.get().targetVersion); - ASSERT(oldResult.get().ranges.size() == oldResult.get().ranges.size()); - ASSERT(oldResult.get().keyRanges.size() == oldResult.get().keyRanges.size()); - ASSERT(oldResult.get().snapshot.fileName == oldResult.get().snapshot.fileName); - } - } - return newResult; - } - Future> getRestoreSet(Version targetVersion, VectorRef keyRangesFilter) final { - return getRestoreSet_compare(Reference::addRef(this), targetVersion, - keyRangesFilter); + return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter); } private: diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 3baf2aaa2c..6d9d803cc8 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -198,6 +198,7 @@ ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs") ERROR( backup_does_not_exist, 2319, "Backup does not exist") ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges") +ERROR( backup_not_overlapped_with_keys_filter, 2321, "Backup key ranges doesn't overlap with key ranges filter") ERROR( restore_invalid_version, 2361, "Invalid restore version") ERROR( restore_corrupted_data, 2362, "Corrupted backup data") ERROR( restore_missing_data, 2363, "Missing backup data") From 63a5f26a236d978babd7c2b0e055eb578e19dca5 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 8 Sep 2020 10:46:19 -0700 Subject: [PATCH 067/458] Remove implicitly setting read_system_keys when set SPECIAL_KEY_SPACE_ENABLE_WRITES, we can achieve read/set system keys through transaction object --- fdbclient/ReadYourWrites.actor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 5693a48ea9..267c2fc744 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -2053,9 +2053,6 @@ void ReadYourWritesTransaction::setOptionImpl( FDBTransactionOptions::Option opt case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES: validateOptionValue(value, false); options.specialKeySpaceChangeConfiguration = true; - // By default, it allows to read system keys - // More options will be implicitly enabled if needed when doing set or clear - options.readSystemKeys = true; break; default: break; From 43e3e320e317a065baa30b0d7ff690adbeb23ed5 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 8 Sep 2020 11:08:48 -0700 Subject: [PATCH 068/458] Refactor getrange for read-write module and add a test to make sure we have consistent results --- fdbclient/SpecialKeySpace.actor.cpp | 21 ++++ fdbclient/SpecialKeySpace.actor.h | 8 ++ .../SpecialKeySpaceCorrectness.actor.cpp | 109 +++++++++++------- 3 files changed, 99 insertions(+), 39 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 892d275f6a..aeb3bc7717 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -50,6 +50,9 @@ std::unordered_map SpecialKeySpace::managementApiCommandT std::set SpecialKeySpace::options = { "excluded/force", "failed/force" }; +Standalone rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, + const Standalone& res); + // This function will move the given KeySelector as far as possible to the standard form: // orEqual == false && offset == 1 (Standard form) // If the corresponding key is not in the underlying key range, it will move over the range @@ -458,6 +461,24 @@ Future SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) { return commitActor(this, ryw); } +SKSCTestImpl::SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { + ASSERT(range.contains(kr)); + auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY); + // all keys are written to RYW, since GRV is set, the read should happen locally + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); + auto kvs = resultFuture.getValue(); + return rywGetRange(ryw, kr, kvs); +} + +Future> SKSCTestImpl::commit(ReadYourWritesTransaction* ryw) { + ASSERT(false); + return Optional(); +} + ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} ACTOR static Future> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) { diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 91bb1cf872..e824958ded 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -216,6 +216,14 @@ private: void modulesBoundaryInit(); }; +// Used for SpecialKeySpaceCorrectnessWorkload +class SKSCTestImpl : public SpecialKeyRangeRWImpl { +public: + explicit SKSCTestImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future> commit(ReadYourWritesTransaction* ryw) override; +}; + // Use special key prefix "\xff\xff/transaction/conflicting_keys/", // to retrieve keys which caused latest not_committed(conflicting with another transaction) error. // The returned key value pairs are interpretted as : diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index a41d60a2ed..91f59e31ae 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -27,27 +27,6 @@ #include "fdbserver/workloads/workloads.actor.h" #include "flow/actorcompiler.h" -class SKSCTestImpl : public SpecialKeyRangeReadImpl { -public: - explicit SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} - virtual Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { - ASSERT(range.contains(kr)); - auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY); - // all keys are written to RYW, since GRV is set, the read should happen locally - ASSERT(resultFuture.isReady()); - auto result = resultFuture.getValue(); - ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); - // To make the test more complext, instead of simply returning the k-v pairs, we reverse all the value strings - auto kvs = resultFuture.getValue(); - for (int i = 0; i < kvs.size(); ++i) { - std::string valStr(kvs[i].value.toString()); - std::reverse(valStr.begin(), valStr.end()); - kvs[i].value = ValueRef(kvs.arena(), valStr); - } - return kvs; - } -}; - struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { int actorCount, minKeysPerRange, maxKeysPerRange, rangeCount, keyBytes, valBytes, conflictRangeSizeFactor; @@ -86,6 +65,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { cx->specialKeySpace = std::make_unique(); self->ryw = Reference(new ReadYourWritesTransaction(cx)); self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED); + self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); self->ryw->setVersion(100); self->ryw->clear(normalKeys); // generate key ranges @@ -97,7 +77,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { self->impls.push_back(std::make_shared(KeyRangeRef(startKey, endKey))); // Although there are already ranges registered, the testing range will replace them cx->specialKeySpace->registerKeyRange(SpecialKeySpace::MODULE::TESTONLY, - SpecialKeySpace::IMPLTYPE::READONLY, self->keys.back(), + SpecialKeySpace::IMPLTYPE::READWRITE, self->keys.back(), self->impls.back().get()); // generate keys in each key range int keysInRange = deterministicRandom()->randomInt(self->minKeysPerRange, self->maxKeysPerRange + 1); @@ -157,6 +137,47 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { .detail("Reverse", reverse); ++self->wrongResults; } + + // check ryw result consistency + KeyRange rkr = self->randomKeyRange(); + KeyRef rkey1 = rkr.begin; + KeyRef rkey2 = rkr.end; + // randomly set/clear two keys or clear a key range + if (deterministicRandom()->coinflip()) { + Value rvalue1 = self->randomValue(); + cx->specialKeySpace->set(self->ryw.getPtr(), rkey1, rvalue1); + self->ryw->set(rkey1, rvalue1); + Value rvalue2 = self->randomValue(); + cx->specialKeySpace->set(self->ryw.getPtr(), rkey2, rvalue2); + self->ryw->set(rkey2, rvalue2); + } else if (deterministicRandom()->coinflip()) { + cx->specialKeySpace->clear(self->ryw.getPtr(), rkey1); + self->ryw->clear(rkey1); + cx->specialKeySpace->clear(self->ryw.getPtr(), rkey2); + self->ryw->clear(rkey2); + } else { + cx->specialKeySpace->clear(self->ryw.getPtr(), rkr); + self->ryw->clear(rkr); + } + // use the same key selectors again to test consistency of ryw + auto correctRywResultFuture = self->ryw->getRange(begin, end, limit, false, reverse); + ASSERT(correctRywResultFuture.isReady()); + auto correctRywResult = correctRywResultFuture.getValue(); + auto testRywResultFuture = cx->specialKeySpace->getRange(self->ryw.getPtr(), begin, end, limit, reverse); + ASSERT(testRywResultFuture.isReady()); + auto testRywResult = testRywResultFuture.getValue(); + + // check the consistency of results + if (!self->compareRangeResult(correctRywResult, testRywResult)) { + TraceEvent(SevError, "TestFailure") + .detail("Reason", "Results from getRange(ryw) are inconsistent") + .detail("Begin", begin.toString()) + .detail("End", end.toString()) + .detail("LimitRows", limit.rows) + .detail("LimitBytes", limit.bytes) + .detail("Reverse", reverse); + ++self->wrongResults; + } } } @@ -189,13 +210,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { .detail("TestKey", printable(res2[i].key)); return false; } - // Value strings should be reversed pairs - std::string valStr(res2[i].value.toString()); - std::reverse(valStr.begin(), valStr.end()); - Value valReversed(valStr); - if (res1[i].value != valReversed) { + if (res1[i].value != res2[i].value) { TraceEvent(SevError, "TestFailure") - .detail("Reason", "Values are inconsistent, CorrectValue should be the reverse of the TestValue") + .detail("Reason", "Values are inconsistent") .detail("Index", i) .detail("CorrectValue", printable(res1[i].value)) .detail("TestValue", printable(res2[i].value)); @@ -206,7 +223,14 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { return true; } - KeySelector randomKeySelector() { + KeyRange randomKeyRange() { + Key prefix = keys[deterministicRandom()->randomInt(0, rangeCount)].begin; + Key rkey1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))).withPrefix(prefix); + Key rkey2 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))).withPrefix(prefix); + return rkey1 <= rkey2 ? KeyRangeRef(rkey1, rkey2) : KeyRangeRef(rkey2, rkey1); + } + + Key randomKey() { Key randomKey; if (deterministicRandom()->random01() < absoluteRandomProb) { Key prefix; @@ -224,9 +248,15 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { KeyRangeRef randomKeyRangeRef = keys[deterministicRandom()->randomInt(0, keys.size())]; randomKey = deterministicRandom()->coinflip() ? randomKeyRangeRef.begin : randomKeyRangeRef.end; } + return randomKey; + } + + Value randomValue() { return Value(deterministicRandom()->randomAlphaNumeric(valBytes)); } + + KeySelector randomKeySelector() { // covers corner cases where offset points outside the key space int offset = deterministicRandom()->randomInt(-keysCount.getValue() - 1, keysCount.getValue() + 2); - return KeySelectorRef(randomKey, deterministicRandom()->coinflip(), offset); + return KeySelectorRef(randomKey(), deterministicRandom()->coinflip(), offset); } GetRangeLimits randomLimits() { @@ -652,17 +682,17 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { try { // test getRange state Standalone class_source_result = wait(tx->getRange( - KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0")) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), - CLIENT_KNOBS->TOO_MANY)); + KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + CLIENT_KNOBS->TOO_MANY)); ASSERT(!class_source_result.more && class_source_result.size() < CLIENT_KNOBS->TOO_MANY); ASSERT(self->getRangeResultInOrder(class_source_result)); // check correctness of classType of each process vector workers = wait(getWorkers(&tx->getTransaction())); for (const auto& worker : workers) { Key addr = - Key("process/class_source/" + formatIpPort(worker.address.ip, worker.address.port)) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); + Key("process/class_source/" + formatIpPort(worker.address.ip, worker.address.port)) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin); bool found = false; for (const auto& kv : class_source_result) { if (kv.key == addr) { @@ -680,11 +710,12 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { state std::string address = formatIpPort(worker.address.ip, worker.address.port); tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); tx->set(Key("process/class_type/" + address) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), - LiteralStringRef("unset")); + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), + LiteralStringRef("unset")); wait(tx->commit()); - Optional class_source = wait(tx->get(Key("process/class_source/" + address) - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + Optional class_source = wait(tx->get( + Key("process/class_source/" + address) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); ASSERT(class_source.present() && class_source.get() == LiteralStringRef("set_class")); tx->reset(); } catch (Error& e) { From cda9e93fe047539af1b5efa20471ccd07d108fc1 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Tue, 8 Sep 2020 11:37:27 -0700 Subject: [PATCH 069/458] Clang-format --- fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 91f59e31ae..8097fd0ee1 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -225,8 +225,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { KeyRange randomKeyRange() { Key prefix = keys[deterministicRandom()->randomInt(0, rangeCount)].begin; - Key rkey1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))).withPrefix(prefix); - Key rkey2 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))).withPrefix(prefix); + Key rkey1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))) + .withPrefix(prefix); + Key rkey2 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes))) + .withPrefix(prefix); return rkey1 <= rkey2 ? KeyRangeRef(rkey1, rkey2) : KeyRangeRef(rkey2, rkey1); } From 23e1ff694c6c30178e6696944e714604128464b7 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 07:49:59 -0700 Subject: [PATCH 070/458] Report missing old tlogs in recovery between accepting commits and storage recovered --- fdbclient/Schemas.cpp | 13 +++++++++++++ fdbserver/Status.actor.cpp | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index c6d931ec9c..8b9e891630 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -278,6 +278,18 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "address":"1.2.3.4:1234" } ], + "epoch": { + "epoch": 1, + "epoch_begin": 23, + "epoch_end": 112315141 + }, + "missing_logs": [ + { + "id":"6f8d623d0cb9966f", + "healthy":false, + "address":"1.2.3.5:1234" + } + ], "log_replication_factor":3, "log_write_anti_quorum":0, "log_fault_tolerance":2, @@ -288,6 +300,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "satellite_log_fault_tolerance":2 } ], + "possibly_losing_old_logs_data": true, "fault_tolerance":{ "max_zone_failures_without_losing_availability":0, "max_zone_failures_without_losing_data":0 diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 30194ad945..8018c2f514 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1910,11 +1910,11 @@ ACTOR static Future clusterSummaryStatisticsFetcher(WorkerEve static JsonBuilderArray oldTlogFetcher(int* oldLogFaultTolerance, Reference> db, std::unordered_map const& address_workers) { JsonBuilderArray oldTlogsArray; - if(db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) { for(auto it : db->get().logSystemConfig.oldTLogs) { JsonBuilderObject statusObj; JsonBuilderArray logsObj; + JsonBuilderArray failedLogsObj; Optional sat_log_replication_factor, sat_log_write_anti_quorum, sat_log_fault_tolerance, log_replication_factor, log_write_anti_quorum, log_fault_tolerance, remote_log_replication_factor, remote_log_fault_tolerance; int maxFaultTolerance = 0; @@ -1932,6 +1932,7 @@ static JsonBuilderArray oldTlogFetcher(int* oldLogFaultTolerance, Reference clusterGetStatus( statusObj["old_logs"] = oldTlogFetcher(&oldLogFaultTolerance, db, address_workers); } + // Used as a signal that storage servers may not be able to catch up certain log generations + statusObj["possibly_losing_old_logs_data"] = oldLogFaultTolerance < 0; + if(configuration.present()) { int extraTlogEligibleZones = getExtraTLogEligibleZones(workers, configuration.get()); statusObj["fault_tolerance"] = faultToleranceStatusFetcher(configuration.get(), coordinators, workers, extraTlogEligibleZones, minReplicasRemaining, loadResult.present() && loadResult.get().healthyZone.present()); From 69c417073b80b683b1fdd76fe50ab3a5237ae5f0 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Mon, 31 Aug 2020 10:17:11 -0700 Subject: [PATCH 071/458] add release notes --- .../sphinx/source/release-notes/release-notes-630.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index db2ee09d0b..ad0344e3e8 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -7,6 +7,13 @@ Release Notes 6.3.5 ===== +* Fix an issue where ``fdbcli --exec 'exclude no_wait ...'`` would incorrectly report that processes can safely be removed from the cluster. `(PR #3566) `_ +* When a configuration key is changed, it will always be included in ``status json`` output, even the value is reverted back to the default value. `(PR #3610) `_ +* Report missing old tlogs information when in recovery before storage servers are fully recovered. `(PR #3706) `_ + +6.3.4 +===== + Features -------- From 4363dd0f25ee3562122e52a5453a5c6e1545f0fc Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Tue, 8 Sep 2020 14:26:01 -0700 Subject: [PATCH 072/458] This resolves issue #3739 by exposing time since last full recovery. --- fdbclient/Schemas.cpp | 1 + fdbserver/Status.actor.cpp | 22 +++++++++++++++++++--- fdbserver/masterserver.actor.cpp | 2 ++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 2e9403dc66..908069a567 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -482,6 +482,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( )statusSchema" R"statusSchema( "recovery_state":{ + "time_since_last_fully_recovered_seconds":1, "required_resolvers":1, "required_proxies":1, "required_grv_proxies":1, diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 2c56833e9c..57f3202b31 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1028,8 +1028,13 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails state JsonBuilderObject message; try { - state Future activeGens = timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0); - TraceEventFields md = wait( timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0) ); + std::vector> futures; + futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0)); + futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryFullyRecovered") ) ), 1.0)); + futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0)); + std::vector msgs = wait(getAll(futures)); + + const TraceEventFields& md = msgs[2]; int mStatusCode = md.getInt("StatusCode"); if (mStatusCode < 0 || mStatusCode >= RecoveryStatus::END) throw attribute_not_found(); @@ -1037,6 +1042,17 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]); *statusCode = mStatusCode; + const TraceEventFields& mLastRecoveryMsg = msgs[1]; + std::string lastFullyRecoveredTimeS; + if (mLastRecoveryMsg.tryGetValue("Time", lastFullyRecoveredTimeS)) { + double lastFullyRecoveredTime = atof(lastFullyRecoveredTimeS.c_str()); + // `lastFullyRecoveredTime` is the timestamp taken on master so the time interval calculated below may not + // be accurate due to the clock skew across the network, but it's good enough for the purpose it's used. + message["time_since_last_fully_recovered_seconds"] = now() - lastFullyRecoveredTime; + } else { + message["time_since_last_fully_recovered_seconds"] = -1; + } + // Add additional metadata for certain statuses if (mStatusCode == RecoveryStatus::recruiting_transaction_servers) { int requiredLogs = atoi( md.getValue("RequiredTLogs").c_str() ); @@ -1056,7 +1072,7 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails // TODO: time_in_recovery: 0.5 // time_in_state: 0.1 - TraceEventFields mdActiveGens = wait(activeGens); + const TraceEventFields& mdActiveGens = msgs[0]; if(mdActiveGens.size()) { int activeGenerations = mdActiveGens.getInt("ActiveGenerations"); message["active_generations"] = activeGenerations; diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index ce5c993d77..4803930bfc 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1276,6 +1276,8 @@ ACTOR Future trackTlogRecovery( Reference self, Referencedbgid) .detail("ActiveGenerations", 1) .trackLatest("MasterRecoveryGenerations"); From 90984467c85c2306a8d6ae94d3785b39c1dcec5c Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Tue, 8 Sep 2020 14:57:39 -0700 Subject: [PATCH 073/458] Add comments for WATCH_OVERHEAD_BYTES --- fdbserver/storageserver.actor.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index c8e7118fef..3faed41df3 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -1058,7 +1058,12 @@ ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) } ++data->numWatches; - data->watchBytes += ( req.key.expectedSize() + req.value.expectedSize() + 1000 ); + + // Pessimistic estimate the number of overhead bytes used by each + // watch. Watch key references are stored in an AsyncMap, and actors + // must be kept alive until the watch is finished. + state size_t WATCH_OVERHEAD_BYTES = 1000; + data->watchBytes += (req.key.expectedSize() + req.value.expectedSize() + WATCH_OVERHEAD_BYTES); try { if(latest < minVersion) { // If the version we read is less than minVersion, then we may fail to be notified of any changes that occur up to or including minVersion @@ -1071,10 +1076,10 @@ ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) } wait(watchFuture); --data->numWatches; - data->watchBytes -= ( req.key.expectedSize() + req.value.expectedSize() + 1000 ); + data->watchBytes -= (req.key.expectedSize() + req.value.expectedSize() + WATCH_OVERHEAD_BYTES); } catch( Error &e ) { --data->numWatches; - data->watchBytes -= ( req.key.expectedSize() + req.value.expectedSize() + 1000 ); + data->watchBytes -= (req.key.expectedSize() + req.value.expectedSize() + WATCH_OVERHEAD_BYTES); throw; } } catch( Error &e ) { From 2ce81a3c4440517898e9653be195b6880387e97b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 9 Sep 2020 10:35:42 -0700 Subject: [PATCH 074/458] Make WATCH_OVERHEAD_BYTES constexpr --- fdbserver/storageserver.actor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 3faed41df3..336381761a 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -1011,6 +1011,11 @@ ACTOR Future getValueQ( StorageServer* data, GetValueRequest req ) { return Void(); }; +// Pessimistic estimate the number of overhead bytes used by each +// watch. Watch key references are stored in an AsyncMap, and actors +// must be kept alive until the watch is finished. +static constexpr size_t WATCH_OVERHEAD_BYTES = 1000; + ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) { try { ++data->counters.watchQueries; @@ -1058,11 +1063,6 @@ ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) } ++data->numWatches; - - // Pessimistic estimate the number of overhead bytes used by each - // watch. Watch key references are stored in an AsyncMap, and actors - // must be kept alive until the watch is finished. - state size_t WATCH_OVERHEAD_BYTES = 1000; data->watchBytes += (req.key.expectedSize() + req.value.expectedSize() + WATCH_OVERHEAD_BYTES); try { if(latest < minVersion) { From 5e06f41c5949356f08fea63673b593b58d239660 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Wed, 9 Sep 2020 13:42:59 -0400 Subject: [PATCH 075/458] Corrected spelling Removed unused lines from file --- contrib/Joshua/scripts/localClusterStart.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 46c9886b5c..656de162d9 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -177,7 +177,7 @@ function createClusterFile { # Stop the Cluster from running. function stopCluster { - # Add an audit entree, if enabled + # Add an audit entry, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then printf '%-15s (%6s) Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" "${FDBSERVERID}" >> "${AUDITLOG}" fi @@ -202,7 +202,7 @@ function stopCluster { # Start the server running. function startFdbServer { - # Add an audit entree, if enabled + # Add an audit entry, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then printf '%-15s (%6s) Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" >> "${AUDITLOG}" fi @@ -217,8 +217,6 @@ function startFdbServer { elif ! "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" - # Maybe the server is already running - #FDBSERVERID="$(pidof fdbserver)" let status="${status} + 1" else FDBSERVERID="${!}" From 1155d015c9a24bccff422267230ba6d3f09f04c7 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Wed, 9 Sep 2020 11:54:58 -0700 Subject: [PATCH 076/458] fetch current log generation as well --- .../release-notes/release-notes-630.rst | 5 - fdbcli/fdbcli.actor.cpp | 56 +++++- fdbclient/ManagementAPI.actor.cpp | 1 + fdbclient/Schemas.cpp | 22 +- fdbserver/Status.actor.cpp | 188 ++++++++++-------- 5 files changed, 162 insertions(+), 110 deletions(-) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index ad0344e3e8..ece1be18c2 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -7,13 +7,8 @@ Release Notes 6.3.5 ===== -* Fix an issue where ``fdbcli --exec 'exclude no_wait ...'`` would incorrectly report that processes can safely be removed from the cluster. `(PR #3566) `_ -* When a configuration key is changed, it will always be included in ``status json`` output, even the value is reverted back to the default value. `(PR #3610) `_ * Report missing old tlogs information when in recovery before storage servers are fully recovered. `(PR #3706) `_ -6.3.4 -===== - Features -------- diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 98d5de5a72..3157173196 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -20,6 +20,7 @@ #include "boost/lexical_cast.hpp" #include "fdbclient/NativeAPI.actor.h" +#include "fdbclient/FDBTypes.h" #include "fdbclient/Status.h" #include "fdbclient/StatusClient.h" #include "fdbclient/DatabaseContext.h" @@ -1207,14 +1208,61 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, int minLoss = std::min(availLoss, dataLoss); const char *faultDomain = machinesAreZones ? "machine" : "zone"; - if (minLoss == 1) - outputString += format("1 %s", faultDomain); - else - outputString += format("%d %ss", minLoss, faultDomain); + outputString += format("%d %ss", minLoss, faultDomain); if (dataLoss > availLoss){ outputString += format(" (%d without data loss)", dataLoss); } + + // We may have data loss between accepting_commits and storage_recovered (exclusive). + if (dataLoss == -1) { + ASSERT(availLoss == -1); + outputString += format("\nThe database may have data loss and availability loss"); + StatusObjectReader logs; + std::string missingLogs; + // StatusObjectReader recoveryState; + // std::string recoveryStage; + // if (statusObjCluster.get("recovery_state", recoveryState)) { + // recoveryState.get("name", recoveryStage); + // } + + if (statusObjCluster.get("logs", logs)) { + for (auto logsObj : logs.obj()) { + StatusObjectReader logEpoch(logsObj.second); + bool possiblyLosingData; + if (logEpoch.get("possibly_losing_data", possiblyLosingData) && + !possiblyLosingData) { + continue; + } + int64_t epoch, beginVersion = invalidVersion, endVersion = invalidVersion; + bool current; + logEpoch.get("epoch", epoch); + logEpoch.get("begin_version", beginVersion); + logEpoch.get("end_version", endVersion); + logEpoch.get("current", current); + missingLogs += format("\nLog epoch: %ld current: %s begin: %ld end: %ld, missing " + "log interfaces(id,address):\n", + epoch, current ? "true" : "false", beginVersion, endVersion); + for (auto logEpochObj : logEpoch.obj()) { + StatusObjectReader logInterface(logEpochObj.second); + bool healthy; + std::string address, id; + if (logInterface.get("healthy", healthy) && !healthy && + logInterface.has("address")) { + logInterface.get("id", address); + logInterface.get("address", address); + missingLogs += format("%s,%s ", address.c_str()); + } + } + } + } + + if (!missingLogs.empty()) { + outputString += "\nPlease restart following tlog interfaces, otherwise storage " + "servers may never be able to catch up:"; + outputString += missingLogs; + } + } } } diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index e683a21d56..354f64e7ee 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -28,6 +28,7 @@ #include "fdbclient/DatabaseContext.h" #include "fdbrpc/simulator.h" #include "fdbclient/StatusClient.h" +#include "flow/Trace.h" #include "flow/UnitTest.h" #include "fdbrpc/ReplicationPolicy.h" #include "fdbrpc/Replication.h" diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 8b9e891630..805820bf61 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -269,27 +269,20 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "run_loop_busy":0.2 } }, - "old_logs":[ + "logs":[ { - "logs":[ + "log_interfaces":[ { "id":"7f8d623d0cb9966e", "healthy":true, "address":"1.2.3.4:1234" } ], - "epoch": { - "epoch": 1, - "epoch_begin": 23, - "epoch_end": 112315141 - }, - "missing_logs": [ - { - "id":"6f8d623d0cb9966f", - "healthy":false, - "address":"1.2.3.5:1234" - } - ], + "epoch":1, + "current":false, + "begin_version":23, + "end_version":112315141, + "possibly_losing_data":true, "log_replication_factor":3, "log_write_anti_quorum":0, "log_fault_tolerance":2, @@ -300,7 +293,6 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "satellite_log_fault_tolerance":2 } ], - "possibly_losing_old_logs_data": true, "fault_tolerance":{ "max_zone_failures_without_losing_availability":0, "max_zone_failures_without_losing_data":0 diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 8018c2f514..c0d2c8e578 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1908,91 +1908,104 @@ ACTOR static Future clusterSummaryStatisticsFetcher(WorkerEve return statusObj; } -static JsonBuilderArray oldTlogFetcher(int* oldLogFaultTolerance, Reference> db, std::unordered_map const& address_workers) { - JsonBuilderArray oldTlogsArray; - if(db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) { - for(auto it : db->get().logSystemConfig.oldTLogs) { - JsonBuilderObject statusObj; - JsonBuilderArray logsObj; - JsonBuilderArray failedLogsObj; - Optional sat_log_replication_factor, sat_log_write_anti_quorum, sat_log_fault_tolerance, log_replication_factor, log_write_anti_quorum, log_fault_tolerance, remote_log_replication_factor, remote_log_fault_tolerance; +static JsonBuilderObject tlogFetcher(int* logFaultTolerance, const std::vector& tLogs, + std::unordered_map const& address_workers) { + JsonBuilderObject statusObj; + JsonBuilderArray logsObj; + Optional sat_log_replication_factor, sat_log_write_anti_quorum, sat_log_fault_tolerance, + log_replication_factor, log_write_anti_quorum, log_fault_tolerance, remote_log_replication_factor, + remote_log_fault_tolerance; - int maxFaultTolerance = 0; + int maxFaultTolerance = 0; - for(int i = 0; i < it.tLogs.size(); i++) { - int failedLogs = 0; - for(auto& log : it.tLogs[i].tLogs) { - JsonBuilderObject logObj; - bool failed = !log.present() || !address_workers.count(log.interf().address()); - logObj["id"] = log.id().shortString(); - logObj["healthy"] = !failed; - if(log.present()) { - logObj["address"] = log.interf().address().toString(); - } - logsObj.push_back(logObj); - if(failed) { - failedLogs++; - failedLogsObj.push_back(logObj); - } - } - maxFaultTolerance = std::max(maxFaultTolerance, it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs); - if(it.tLogs[i].isLocal && it.tLogs[i].locality == tagLocalitySatellite) { - sat_log_replication_factor = it.tLogs[i].tLogReplicationFactor; - sat_log_write_anti_quorum = it.tLogs[i].tLogWriteAntiQuorum; - sat_log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs; - } - else if(it.tLogs[i].isLocal) { - log_replication_factor = it.tLogs[i].tLogReplicationFactor; - log_write_anti_quorum = it.tLogs[i].tLogWriteAntiQuorum; - log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs; - } - else { - remote_log_replication_factor = it.tLogs[i].tLogReplicationFactor; - remote_log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - failedLogs; - } + for (int i = 0; i < tLogs.size(); i++) { + int failedLogs = 0; + for (auto& log : tLogs[i].tLogs) { + JsonBuilderObject logObj; + bool failed = !log.present() || !address_workers.count(log.interf().address()); + logObj["id"] = log.id().shortString(); + logObj["healthy"] = !failed; + if (log.present()) { + logObj["address"] = log.interf().address().toString(); } - *oldLogFaultTolerance = std::min(*oldLogFaultTolerance, maxFaultTolerance); - statusObj["logs"] = logsObj; - - JsonBuilderObject epochInfo; - epochInfo["epoch"] = it.epoch; - epochInfo["epoch_begin"] = it.epochBegin; - epochInfo["epoch_end"] = it.epochEnd; - statusObj["epoch"] = epochInfo; - - // We may lose logs in this log generation, storage servers may never be able to catch up this log - // generation. - if (maxFaultTolerance < 0) { - statusObj["missing_logs"] = failedLogsObj; + logsObj.push_back(logObj); + if (failed) { + failedLogs++; } - - if (sat_log_replication_factor.present()) - statusObj["satellite_log_replication_factor"] = sat_log_replication_factor.get(); - if (sat_log_write_anti_quorum.present()) - statusObj["satellite_log_write_anti_quorum"] = sat_log_write_anti_quorum.get(); - if (sat_log_fault_tolerance.present()) - statusObj["satellite_log_fault_tolerance"] = sat_log_fault_tolerance.get(); - - if (log_replication_factor.present()) - statusObj["log_replication_factor"] = log_replication_factor.get(); - if (log_write_anti_quorum.present()) - statusObj["log_write_anti_quorum"] = log_write_anti_quorum.get(); - if (log_fault_tolerance.present()) - statusObj["log_fault_tolerance"] = log_fault_tolerance.get(); - - if (remote_log_replication_factor.present()) - statusObj["remote_log_replication_factor"] = remote_log_replication_factor.get(); - if (remote_log_fault_tolerance.present()) - statusObj["remote_log_fault_tolerance"] = remote_log_fault_tolerance.get(); - - oldTlogsArray.push_back(statusObj); + } + // The log generation's fault tolerance is the maximum tlog fault tolerance of each region. + maxFaultTolerance = + std::max(maxFaultTolerance, tLogs[i].tLogReplicationFactor - 1 - tLogs[i].tLogWriteAntiQuorum - failedLogs); + if (tLogs[i].isLocal && tLogs[i].locality == tagLocalitySatellite) { + sat_log_replication_factor = tLogs[i].tLogReplicationFactor; + sat_log_write_anti_quorum = tLogs[i].tLogWriteAntiQuorum; + sat_log_fault_tolerance = tLogs[i].tLogReplicationFactor - 1 - tLogs[i].tLogWriteAntiQuorum - failedLogs; + } else if (tLogs[i].isLocal) { + log_replication_factor = tLogs[i].tLogReplicationFactor; + log_write_anti_quorum = tLogs[i].tLogWriteAntiQuorum; + log_fault_tolerance = tLogs[i].tLogReplicationFactor - 1 - tLogs[i].tLogWriteAntiQuorum - failedLogs; + } else { + remote_log_replication_factor = tLogs[i].tLogReplicationFactor; + remote_log_fault_tolerance = tLogs[i].tLogReplicationFactor - 1 - failedLogs; } } + *logFaultTolerance = std::min(*logFaultTolerance, maxFaultTolerance); + statusObj["log_interfaces"] = logsObj; + // We may lose logs in this log generation, storage servers may never be able to catch up this log + // generation. + statusObj["possibly_losing_data"] = maxFaultTolerance < 0; - return oldTlogsArray; + if (sat_log_replication_factor.present()) + statusObj["satellite_log_replication_factor"] = sat_log_replication_factor.get(); + if (sat_log_write_anti_quorum.present()) + statusObj["satellite_log_write_anti_quorum"] = sat_log_write_anti_quorum.get(); + if (sat_log_fault_tolerance.present()) statusObj["satellite_log_fault_tolerance"] = sat_log_fault_tolerance.get(); + + if (log_replication_factor.present()) statusObj["log_replication_factor"] = log_replication_factor.get(); + if (log_write_anti_quorum.present()) statusObj["log_write_anti_quorum"] = log_write_anti_quorum.get(); + if (log_fault_tolerance.present()) statusObj["log_fault_tolerance"] = log_fault_tolerance.get(); + + if (remote_log_replication_factor.present()) + statusObj["remote_log_replication_factor"] = remote_log_replication_factor.get(); + if (remote_log_fault_tolerance.present()) + statusObj["remote_log_fault_tolerance"] = remote_log_fault_tolerance.get(); + + return statusObj; } -static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector& workers, int extraTlogEligibleZones, int minReplicasRemaining, bool underMaintenance) { +static JsonBuilderArray tlogFetcher(int* logFaultTolerance, Reference> db, + std::unordered_map const& address_workers) { + JsonBuilderArray tlogsArray; + if (db->get().recoveryState >= RecoveryState::ALL_LOGS_RECRUITED) { + JsonBuilderObject tlogsStatus; + tlogsStatus = tlogFetcher(logFaultTolerance, db->get().logSystemConfig.tLogs, address_workers); + tlogsStatus["epoch"] = db->get().logSystemConfig.epoch; + tlogsStatus["current"] = true; + if (db->get().logSystemConfig.recoveredAt.present()) { + tlogsStatus["begin_version"] = db->get().logSystemConfig.recoveredAt.get(); + } + tlogsArray.push_back(tlogsStatus); + } + + if (db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && + db->get().recoveryState < RecoveryState::STORAGE_RECOVERED) { + for (auto it : db->get().logSystemConfig.oldTLogs) { + JsonBuilderObject oldTlogsStatus = tlogFetcher(logFaultTolerance, it.tLogs, address_workers); + oldTlogsStatus["epoch"] = it.epoch; + oldTlogsStatus["current"] = false; + oldTlogsStatus["begin_version"] = it.epochBegin; + oldTlogsStatus["end_version"] = it.epochEnd; + tlogsArray.push_back(oldTlogsStatus); + } + } + return tlogsArray; +} + +static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, + ServerCoordinators coordinators, + std::vector& workers, int extraTlogEligibleZones, + int minReplicasRemaining, int oldLogFaultTolerance, + bool underMaintenance) { JsonBuilderObject statusObj; // without losing data @@ -2024,17 +2037,21 @@ static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration confi } maxCoordinatorZoneFailures += 1; } - + // max zone failures that we can tolerate to not lose data int zoneFailuresWithoutLosingData = std::min(maxZoneFailures, maxCoordinatorZoneFailures); if (minReplicasRemaining >= 0){ zoneFailuresWithoutLosingData = std::min(zoneFailuresWithoutLosingData, minReplicasRemaining - 1); } - statusObj["max_zone_failures_without_losing_data"] = std::max(zoneFailuresWithoutLosingData, 0); + // oldLogFaultTolerance means max failures we can tolerate to lose logs data. + zoneFailuresWithoutLosingData = std::min(zoneFailuresWithoutLosingData, oldLogFaultTolerance); + statusObj["max_zone_failures_without_losing_data"] = + zoneFailuresWithoutLosingData < 0 ? -1 : zoneFailuresWithoutLosingData; // without losing availablity - statusObj["max_zone_failures_without_losing_availability"] = std::max(std::min(extraTlogEligibleZones, zoneFailuresWithoutLosingData), 0); + statusObj["max_zone_failures_without_losing_availability"] = + std::min(extraTlogEligibleZones, zoneFailuresWithoutLosingData); return statusObj; } @@ -2427,17 +2444,16 @@ ACTOR Future clusterGetStatus( futures2.push_back(clusterSummaryStatisticsFetcher(pMetrics, storageServerFuture, tLogFuture, &status_incomplete_reasons)); state std::vector workerStatuses = wait(getAll(futures2)); - int oldLogFaultTolerance = 100; - if(db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && db->get().logSystemConfig.oldTLogs.size() > 0) { - statusObj["old_logs"] = oldTlogFetcher(&oldLogFaultTolerance, db, address_workers); + int logFaultTolerance = 100; + if (db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) { + statusObj["logs"] = tlogFetcher(&logFaultTolerance, db, address_workers); } - // Used as a signal that storage servers may not be able to catch up certain log generations - statusObj["possibly_losing_old_logs_data"] = oldLogFaultTolerance < 0; - if(configuration.present()) { int extraTlogEligibleZones = getExtraTLogEligibleZones(workers, configuration.get()); - statusObj["fault_tolerance"] = faultToleranceStatusFetcher(configuration.get(), coordinators, workers, extraTlogEligibleZones, minReplicasRemaining, loadResult.present() && loadResult.get().healthyZone.present()); + statusObj["fault_tolerance"] = faultToleranceStatusFetcher( + configuration.get(), coordinators, workers, extraTlogEligibleZones, minReplicasRemaining, + logFaultTolerance, loadResult.present() && loadResult.get().healthyZone.present()); } state JsonBuilderObject configObj = From 1615bd1a1d338bb0a4e42e8c698829e1e6bd28ae Mon Sep 17 00:00:00 2001 From: Young Liu Date: Wed, 9 Sep 2020 13:57:26 -0700 Subject: [PATCH 077/458] minor improvement --- fdbcli/fdbcli.actor.cpp | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 3157173196..0392315cd1 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -1220,12 +1220,6 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, outputString += format("\nThe database may have data loss and availability loss"); StatusObjectReader logs; std::string missingLogs; - // StatusObjectReader recoveryState; - // std::string recoveryStage; - // if (statusObjCluster.get("recovery_state", recoveryState)) { - // recoveryState.get("name", recoveryStage); - // } - if (statusObjCluster.get("logs", logs)) { for (auto logsObj : logs.obj()) { StatusObjectReader logEpoch(logsObj.second); @@ -1234,24 +1228,25 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, !possiblyLosingData) { continue; } - int64_t epoch, beginVersion = invalidVersion, endVersion = invalidVersion; + // Current epoch doesn't have an end version. + int64_t epoch, beginVersion, endVersion = invalidVersion; bool current; logEpoch.get("epoch", epoch); logEpoch.get("begin_version", beginVersion); logEpoch.get("end_version", endVersion); logEpoch.get("current", current); - missingLogs += format("\nLog epoch: %ld current: %s begin: %ld end: %ld, missing " + missingLogs += format("\n%s Log epoch: %ld begin: %ld end: %ld%s, missing " "log interfaces(id,address):\n", - epoch, current ? "true" : "false", beginVersion, endVersion); + current ? "Current" : "Old", epoch, beginVersion, endVersion, + endVersion == invalidVersion ? "(unknown)" : ""); for (auto logEpochObj : logEpoch.obj()) { StatusObjectReader logInterface(logEpochObj.second); bool healthy; std::string address, id; - if (logInterface.get("healthy", healthy) && !healthy && - logInterface.has("address")) { - logInterface.get("id", address); + if (logInterface.get("healthy", healthy) && !healthy) { + logInterface.get("id", id); logInterface.get("address", address); - missingLogs += format("%s,%s ", address.c_str()); + missingLogs += format("%s,%s ", id.c_str(), address.c_str()); } } } From 2803e6be529e390ac88cfb224dbe55cbb3992ef5 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Wed, 9 Sep 2020 16:16:11 -0700 Subject: [PATCH 078/458] Remove enabling ACCESS_SYSTEM_KEYS --- fdbclient/SpecialKeySpace.actor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index aeb3bc7717..b8600f2a14 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -829,7 +829,6 @@ ACTOR Future checkExclusion(Database db, std::vector* ad } void includeServers(ReadYourWritesTransaction* ryw) { - ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); ryw->setOption(FDBTransactionOptions::LOCK_AWARE); ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); @@ -1000,7 +999,6 @@ ACTOR Future> getProcessClassActor(ReadYourWritesTran ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { // enable related options - ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); ryw->setOption(FDBTransactionOptions::LOCK_AWARE); ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); From 2c224de2f8bf975784d08469179678ac1c6865d4 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Wed, 9 Sep 2020 16:19:55 -0700 Subject: [PATCH 079/458] Update test of setclass --- fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 8097fd0ee1..3066126269 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -713,7 +713,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); tx->set(Key("process/class_type/" + address) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin), - LiteralStringRef("unset")); + Value(worker.processClass.toString())); // Set it as the same class type as before, thus only + // class source will be changed wait(tx->commit()); Optional class_source = wait(tx->get( Key("process/class_source/" + address) From 1867ee1f5f8f2f59a34138b78447c934e14cfdd2 Mon Sep 17 00:00:00 2001 From: Young Liu Date: Wed, 9 Sep 2020 22:34:36 -0700 Subject: [PATCH 080/458] Change cli output format --- fdbcli/fdbcli.actor.cpp | 48 ++++++++++++++++++-------------------- fdbserver/Status.actor.cpp | 45 ++++++++++++++--------------------- 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 0392315cd1..0c2e11e1a4 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -1214,15 +1214,15 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, outputString += format(" (%d without data loss)", dataLoss); } - // We may have data loss between accepting_commits and storage_recovered (exclusive). if (dataLoss == -1) { - ASSERT(availLoss == -1); - outputString += format("\nThe database may have data loss and availability loss"); + ASSERT_WE_THINK(availLoss == -1); + outputString += format( + "\n\n Warning: the database may have data loss and availability loss. Please restart " + "following tlog interfaces, otherwise storage servers may never be able to catch " + "up.\n"); StatusObjectReader logs; - std::string missingLogs; - if (statusObjCluster.get("logs", logs)) { - for (auto logsObj : logs.obj()) { - StatusObjectReader logEpoch(logsObj.second); + if (statusObjCluster.has("logs")) { + for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) { bool possiblyLosingData; if (logEpoch.get("possibly_losing_data", possiblyLosingData) && !possiblyLosingData) { @@ -1235,28 +1235,26 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, logEpoch.get("begin_version", beginVersion); logEpoch.get("end_version", endVersion); logEpoch.get("current", current); - missingLogs += format("\n%s Log epoch: %ld begin: %ld end: %ld%s, missing " - "log interfaces(id,address):\n", - current ? "Current" : "Old", epoch, beginVersion, endVersion, - endVersion == invalidVersion ? "(unknown)" : ""); - for (auto logEpochObj : logEpoch.obj()) { - StatusObjectReader logInterface(logEpochObj.second); - bool healthy; - std::string address, id; - if (logInterface.get("healthy", healthy) && !healthy) { - logInterface.get("id", id); - logInterface.get("address", address); - missingLogs += format("%s,%s ", id.c_str(), address.c_str()); + std::string missing_log_interfaces; + if (logEpoch.has("log_interfaces")) { + for (StatusObjectReader logInterface : logEpoch.last().get_array()) { + bool healthy; + std::string address, id; + if (logInterface.get("healthy", healthy) && !healthy) { + logInterface.get("id", id); + logInterface.get("address", address); + missing_log_interfaces += format("%s,%s ", id.c_str(), address.c_str()); + } } } + outputString += format( + " %s log epoch: %ld begin: %ld end: %s, missing " + "log interfaces(id,address): %s\n", + current ? "Current" : "Old", epoch, beginVersion, + endVersion == invalidVersion ? "(unknown)" : format("%ld", endVersion).c_str(), + missing_log_interfaces.c_str()); } } - - if (!missingLogs.empty()) { - outputString += "\nPlease restart following tlog interfaces, otherwise storage " - "servers may never be able to catch up:"; - outputString += missingLogs; - } } } } diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index c0d2c8e578..3b45e7a56f 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1976,27 +1976,21 @@ static JsonBuilderObject tlogFetcher(int* logFaultTolerance, const std::vector> db, std::unordered_map const& address_workers) { JsonBuilderArray tlogsArray; - if (db->get().recoveryState >= RecoveryState::ALL_LOGS_RECRUITED) { - JsonBuilderObject tlogsStatus; - tlogsStatus = tlogFetcher(logFaultTolerance, db->get().logSystemConfig.tLogs, address_workers); - tlogsStatus["epoch"] = db->get().logSystemConfig.epoch; - tlogsStatus["current"] = true; - if (db->get().logSystemConfig.recoveredAt.present()) { - tlogsStatus["begin_version"] = db->get().logSystemConfig.recoveredAt.get(); - } - tlogsArray.push_back(tlogsStatus); + JsonBuilderObject tlogsStatus; + tlogsStatus = tlogFetcher(logFaultTolerance, db->get().logSystemConfig.tLogs, address_workers); + tlogsStatus["epoch"] = db->get().logSystemConfig.epoch; + tlogsStatus["current"] = true; + if (db->get().logSystemConfig.recoveredAt.present()) { + tlogsStatus["begin_version"] = db->get().logSystemConfig.recoveredAt.get(); } - - if (db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && - db->get().recoveryState < RecoveryState::STORAGE_RECOVERED) { - for (auto it : db->get().logSystemConfig.oldTLogs) { - JsonBuilderObject oldTlogsStatus = tlogFetcher(logFaultTolerance, it.tLogs, address_workers); - oldTlogsStatus["epoch"] = it.epoch; - oldTlogsStatus["current"] = false; - oldTlogsStatus["begin_version"] = it.epochBegin; - oldTlogsStatus["end_version"] = it.epochEnd; - tlogsArray.push_back(oldTlogsStatus); - } + tlogsArray.push_back(tlogsStatus); + for (auto it : db->get().logSystemConfig.oldTLogs) { + JsonBuilderObject oldTlogsStatus = tlogFetcher(logFaultTolerance, it.tLogs, address_workers); + oldTlogsStatus["epoch"] = it.epoch; + oldTlogsStatus["current"] = false; + oldTlogsStatus["begin_version"] = it.epochBegin; + oldTlogsStatus["end_version"] = it.epochEnd; + tlogsArray.push_back(oldTlogsStatus); } return tlogsArray; } @@ -2044,14 +2038,11 @@ static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration confi zoneFailuresWithoutLosingData = std::min(zoneFailuresWithoutLosingData, minReplicasRemaining - 1); } - // oldLogFaultTolerance means max failures we can tolerate to lose logs data. - zoneFailuresWithoutLosingData = std::min(zoneFailuresWithoutLosingData, oldLogFaultTolerance); - statusObj["max_zone_failures_without_losing_data"] = - zoneFailuresWithoutLosingData < 0 ? -1 : zoneFailuresWithoutLosingData; - - // without losing availablity + // oldLogFaultTolerance means max failures we can tolerate to lose logs data. -1 means we lose data or availability. + zoneFailuresWithoutLosingData = std::max(std::min(zoneFailuresWithoutLosingData, oldLogFaultTolerance), -1); + statusObj["max_zone_failures_without_losing_data"] = zoneFailuresWithoutLosingData; statusObj["max_zone_failures_without_losing_availability"] = - std::min(extraTlogEligibleZones, zoneFailuresWithoutLosingData); + std::max(std::min(extraTlogEligibleZones, zoneFailuresWithoutLosingData), -1); return statusObj; } From 81ac8211d1f1125d1759f0032a86399bb1142acf Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Thu, 10 Sep 2020 08:32:52 -0700 Subject: [PATCH 081/458] Add comment --- flow/ThreadHelper.actor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index b695e46d2b..e034ccf049 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -331,6 +331,7 @@ private: int32_t valueReferenceCount; protected: + // The caller of any of these *Unsafe functions should be holding |mutex| bool isReadyUnsafe() const { return status >= Set; } bool isErrorUnsafe() const { return status == ErrorSet; } bool canBeSetUnsafe() const { return status == Unset; } From 35bef73a1c9fd871dbed982ba78296f7309b588a Mon Sep 17 00:00:00 2001 From: Young Liu Date: Thu, 10 Sep 2020 17:44:15 -0700 Subject: [PATCH 082/458] Rename proxy to commit proxy --- contrib/commit_debug.py | 16 +- design/backup_v2_partitioned_logs.md | 8 +- .../sphinx/source/api-error-codes.rst | 2 +- .../sphinx/source/disk-snapshot-backup.rst | 2 +- .../source/mr-status-json-schemas.rst.inc | 10 +- fdbcli/fdbcli.actor.cpp | 63 +++++--- fdbclient/CMakeLists.txt | 2 +- fdbclient/ClientWorkerInterface.h | 2 +- fdbclient/ClusterInterface.h | 2 +- ...roxyInterface.h => CommitProxyInterface.h} | 19 +-- fdbclient/CoordinationInterface.h | 2 +- fdbclient/DatabaseConfiguration.cpp | 69 ++++----- fdbclient/DatabaseConfiguration.h | 11 +- fdbclient/DatabaseContext.h | 10 +- fdbclient/GrvProxyInterface.h | 2 + fdbclient/Knobs.cpp | 4 +- fdbclient/Knobs.h | 4 +- fdbclient/ManagementAPI.actor.cpp | 29 ++-- fdbclient/ManagementAPI.actor.h | 15 +- fdbclient/MonitorLeader.actor.cpp | 45 +++--- fdbclient/MonitorLeader.h | 7 +- fdbclient/MutationList.h | 3 +- fdbclient/NativeAPI.actor.cpp | 97 ++++++------ fdbclient/NativeAPI.actor.h | 2 +- fdbclient/Schemas.cpp | 14 +- fdbclient/TagThrottle.actor.cpp | 2 +- fdbrpc/Locality.cpp | 8 +- fdbrpc/Locality.h | 13 +- fdbrpc/simulator.h | 3 +- fdbserver/BackupWorker.actor.cpp | 2 +- fdbserver/CMakeLists.txt | 2 +- fdbserver/ClusterController.actor.cpp | 85 +++++------ ....actor.cpp => CommitProxyServer.actor.cpp} | 142 ++++++++---------- fdbserver/GrvProxyServer.actor.cpp | 6 +- fdbserver/Knobs.cpp | 4 +- fdbserver/Knobs.h | 7 +- fdbserver/Ratekeeper.actor.cpp | 68 +++++---- fdbserver/Resolver.actor.cpp | 4 +- fdbserver/SimulatedCluster.actor.cpp | 4 +- fdbserver/Status.actor.cpp | 84 ++++++----- fdbserver/WorkerInterface.actor.h | 29 ++-- fdbserver/fdbserver.actor.cpp | 9 +- fdbserver/masterserver.actor.cpp | 109 +++++++------- fdbserver/storageserver.actor.cpp | 10 +- fdbserver/worker.actor.cpp | 23 +-- .../workloads/ConsistencyCheck.actor.cpp | 39 +++-- fdbserver/workloads/Rollback.actor.cpp | 10 +- fdbserver/workloads/TargetedKill.actor.cpp | 20 +-- flow/error_definitions.h | 6 +- tests/status/invalid_proc_addresses.json | 6 +- .../local_6_machine_no_replicas_remain.json | 6 +- .../separate_2_of_3_coordinators_remain.json | 4 +- .../separate_cannot_write_cluster_file.json | 6 +- tests/status/separate_idle.json | 2 +- tests/status/separate_initializing.json | 2 +- tests/status/separate_no_database.json | 2 +- tests/status/separate_not_enough_servers.json | 4 +- ...single_process_too_many_config_params.json | 2 +- 58 files changed, 598 insertions(+), 565 deletions(-) rename fdbclient/{MasterProxyInterface.h => CommitProxyInterface.h} (96%) rename fdbserver/{MasterProxyServer.actor.cpp => CommitProxyServer.actor.cpp} (95%) diff --git a/contrib/commit_debug.py b/contrib/commit_debug.py index 7f6de3ff91..b37b5260d0 100755 --- a/contrib/commit_debug.py +++ b/contrib/commit_debug.py @@ -24,22 +24,22 @@ def parse_args(): # (e)nd of a span with a better given name locationToPhase = { "NativeAPI.commit.Before": [], - "MasterProxyServer.batcher": [("b", "Commit")], - "MasterProxyServer.commitBatch.Before": [], - "MasterProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")], - "MasterProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")], + "CommitProxyServer.batcher": [("b", "Commit")], + "CommitProxyServer.commitBatch.Before": [], + "CommitProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")], + "CommitProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")], "Resolver.resolveBatch.Before": [("b", "Resolver.PipelineWait")], "Resolver.resolveBatch.AfterQueueSizeCheck": [], "Resolver.resolveBatch.AfterOrderer": [("e", "Resolver.PipelineWait"), ("b", "Resolver.Conflicts")], "Resolver.resolveBatch.After": [("e", "Resolver.Conflicts")], - "MasterProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")], - "MasterProxyServer.commitBatch.ProcessingMutations": [], - "MasterProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")], + "CommitProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")], + "CommitProxyServer.commitBatch.ProcessingMutations": [], + "CommitProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")], "TLog.tLogCommit.BeforeWaitForVersion": [("b", "TLog.PipelineWait")], "TLog.tLogCommit.Before": [("e", "TLog.PipelineWait")], "TLog.tLogCommit.AfterTLogCommit": [("b", "TLog.FSync")], "TLog.tLogCommit.After": [("e", "TLog.FSync")], - "MasterProxyServer.commitBatch.AfterLogPush": [("e", "Commit")], + "CommitProxyServer.commitBatch.AfterLogPush": [("e", "Commit")], "NativeAPI.commit.After": [], } diff --git a/design/backup_v2_partitioned_logs.md b/design/backup_v2_partitioned_logs.md index 18369cdd6f..97526f5f89 100644 --- a/design/backup_v2_partitioned_logs.md +++ b/design/backup_v2_partitioned_logs.md @@ -16,7 +16,7 @@ As an essential component of a database system, backup and restore is commonly u ## Background -FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at Proxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of Proxy CPU time. +FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at CommitProxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of CommitProxy CPU time. The design of old backup system is [here](https://github.com/apple/foundationdb/blob/master/design/backup.md), and the data format of range files and mutations files is [here](https://github.com/apple/foundationdb/blob/master/design/backup-dataFormat.md). The technical overview of FDB is [here](https://github.com/apple/foundationdb/wiki/Technical-Overview-of-the-Database). The FDB recovery is described in this [doc](https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md). @@ -37,7 +37,7 @@ The design of old backup system is [here](https://github.com/apple/foundationdb/ Feature priorities: Feature 1, 2, 3, 4, 5 are must-have; Feature 6 is better to have. -1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the Proxy, thus reduce TLog write bandwidth usage by half and significantly improve Proxy CPU usage; +1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the CommitProxy, thus reduce TLog write bandwidth usage by half and significantly improve CommitProxy CPU usage; 2. **Correctness**: The restored database must be consistent: each *restored* state (i.e., key-value pair) at a version `v` must match the original state at version `v`. 3. **Performance**: The backup system should be performant, mostly measured as a small CPU overhead on transaction logs and backup workers. The version lag on backup workers is an indicator of performance. 4. **Fault-tolerant**: The backup system should be fault-tolerant to node failures in the FDB cluster. @@ -153,9 +153,9 @@ The requirement of the new backup system raises several design challenges: **Master**: The master is responsible for coordinating the transition of the FDB transaction sub-system from one generation to the next. In particular, the master recruits backup workers during the recovery. -**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs. +**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the commit proxy in version order, and only respond to the commit proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs. -**Proxy**: The proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, Proxies are responsible to group mutations into backup mutations and write them to the database. +**CommitProxy**: The commit proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, Proxies are responsible to group mutations into backup mutations and write them to the database. **GrvProxy**: The GRV proxies are responsible for providing read versions. ## System overview diff --git a/documentation/sphinx/source/api-error-codes.rst b/documentation/sphinx/source/api-error-codes.rst index f013f4aabd..48c1c215a6 100644 --- a/documentation/sphinx/source/api-error-codes.rst +++ b/documentation/sphinx/source/api-error-codes.rst @@ -40,7 +40,7 @@ FoundationDB may return the following error codes from API functions. If you nee +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | external_client_already_loaded | 1040| External client has already been loaded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ -| proxy_memory_limit_exceeded | 1042| Proxy commit memory limit exceeded | +| proxy_memory_limit_exceeded | 1042| CommitProxy commit memory limit exceeded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | batch_transaction_throttled | 1051| Batch GRV request rate limit exceeded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ diff --git a/documentation/sphinx/source/disk-snapshot-backup.rst b/documentation/sphinx/source/disk-snapshot-backup.rst index e5eccd8051..33b97b8c09 100644 --- a/documentation/sphinx/source/disk-snapshot-backup.rst +++ b/documentation/sphinx/source/disk-snapshot-backup.rst @@ -104,7 +104,7 @@ Field Name Description ``Name for the snapshot file`` recommended name for the disk snapshot cluster-name:ip-addr:port:UID ================================ ======================================================== ======================================================== -``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or MasterProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally. +``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or CommitProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally. Management of disk snapshots ---------------------------- diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 0f4b6a9aa9..d7af4a0885 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -27,7 +27,7 @@ "storage", "transaction", "resolution", - "proxy", + "commit_proxy", "grv_proxy", "master", "test", @@ -61,7 +61,7 @@ "role":{ "$enum":[ "master", - "proxy", + "commit_proxy", "grv_proxy", "log", "storage", @@ -447,7 +447,7 @@ ], "recovery_state":{ "required_resolvers":1, - "required_proxies":1, + "required_commit_proxies":1, "required_grv_proxies":1, "name":{ // "fully_recovered" is the healthy state; other states are normal to transition through but not to persist in "$enum":[ @@ -633,11 +633,11 @@ "address":"10.0.4.1" } ], - "auto_proxies":3, + "auto_commit_proxies":3, "auto_resolvers":1, "auto_logs":3, "backup_worker_enabled":1, - "proxies":5 // this field will be absent if a value has not been explicitly set + "commit_proxies":5 // this field will be absent if a value has not been explicitly set }, "data":{ "least_operating_space_bytes_log_server":0, diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 6351219341..a304daa2ad 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -470,7 +470,8 @@ void initHelp() { "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); helpMap["configure"] = CommandHelp( "configure [new] " - "|grv_" + "|grv_" "proxies=|logs=|resolvers=>*", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing " @@ -479,10 +480,13 @@ void initHelp() { "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " - "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set " - "to -1 which restores the number of proxies to the default value.\n\ngrv_proxies=: Sets the " + "datasets.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. Must " + "be at least 1, or set " + "to -1 which restores the number of commit proxies to the default value.\n\ngrv_proxies=: Sets " + "the " "desired number of GRV proxies in the cluster. Must be at least 1, or set to -1 which restores the number of " - "proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must be " + "GRV proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must " + "be " "at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: " "Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the " "number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); @@ -871,12 +875,13 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, fatalRecoveryState = true; if (name == "recruiting_transaction_servers") { - description += format("\nNeed at least %d log servers across unique zones, %d proxies, " - "%d GRV proxies and %d resolvers.", - recoveryState["required_logs"].get_int(), - recoveryState["required_proxies"].get_int(), - recoveryState["required_grv_proxies"].get_int(), - recoveryState["required_resolvers"].get_int()); + description += + format("\nNeed at least %d log servers across unique zones, %d commit proxies, " + "%d GRV proxies and %d resolvers.", + recoveryState["required_logs"].get_int(), + recoveryState["required_commit_proxies"].get_int(), + recoveryState["required_grv_proxies"].get_int(), + recoveryState["required_resolvers"].get_int()); if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) { auto numOfNonExcludedProcessesAndZones = getNumOfNonExcludedProcessAndZones(statusObjCluster); description += format("\nHave %d non-excluded processes on %d machines across %d zones.", numOfNonExcludedProcessesAndZones.first, getNumofNonExcludedMachines(statusObjCluster), numOfNonExcludedProcessesAndZones.second); @@ -1026,8 +1031,8 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, outputString += format("\n Exclusions - %d (type `exclude' for details)", excludedServersArr.size()); } - if (statusObjConfig.get("proxies", intVal)) - outputString += format("\n Desired Proxies - %d", intVal); + if (statusObjConfig.get("commit_proxies", intVal)) + outputString += format("\n Desired Commit Proxies - %d", intVal); if (statusObjConfig.get("grv_proxies", intVal)) outputString += format("\n Desired GRV Proxies - %d", intVal); @@ -1790,14 +1795,14 @@ ACTOR Future configure( Database db, std::vector tokens, Refere bool noChanges = conf.get().old_replication == conf.get().auto_replication && conf.get().old_logs == conf.get().auto_logs && - conf.get().old_proxies == conf.get().auto_proxies && + conf.get().old_commit_proxies == conf.get().auto_commit_proxies && conf.get().old_grv_proxies == conf.get().auto_grv_proxies && conf.get().old_resolvers == conf.get().auto_resolvers && conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && - conf.get().old_proxies == conf.get().desired_proxies && + conf.get().old_commit_proxies == conf.get().desired_commit_proxies && conf.get().old_grv_proxies == conf.get().desired_grv_proxies && conf.get().old_resolvers == conf.get().desired_resolvers; @@ -1816,8 +1821,11 @@ ACTOR Future configure( Database db, std::vector tokens, Refere outputString += format("| replication | %16s | %16s |\n", conf.get().old_replication.c_str(), conf.get().auto_replication.c_str()); outputString += format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); outputString += conf.get().auto_logs != conf.get().desired_logs ? format(" (manually set; would be %d)\n", conf.get().desired_logs) : "\n"; - outputString += format("| proxies | %16d | %16d |", conf.get().old_proxies, conf.get().auto_proxies); - outputString += conf.get().auto_proxies != conf.get().desired_proxies ? format(" (manually set; would be %d)\n", conf.get().desired_proxies) : "\n"; + outputString += format("| commit_proxies | %16d | %16d |", conf.get().old_commit_proxies, + conf.get().auto_commit_proxies); + outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) + : "\n"; outputString += format("| grv_proxies | %16d | %16d |", conf.get().old_grv_proxies, conf.get().auto_grv_proxies); outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies @@ -2531,11 +2539,24 @@ void onOffGenerator(const char* text, const char *line, std::vector } void configureGenerator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = { - "new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", - "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "memory-radixtree-beta", "proxies=", - "grv_proxies=", "logs=", "resolvers=", nullptr - }; + const char* opts[] = { "new", + "single", + "double", + "triple", + "three_data_hall", + "three_datacenter", + "ssd", + "ssd-1", + "ssd-2", + "memory", + "memory-1", + "memory-2", + "memory-radixtree-beta", + "commit_proxies=", + "grv_proxies=", + "logs=", + "resolvers=", + nullptr }; arrayGenerator(text, line, opts, lc); } diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 43f9343b28..3f7333b632 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -33,7 +33,7 @@ set(FDBCLIENT_SRCS Knobs.h ManagementAPI.actor.cpp ManagementAPI.actor.h - MasterProxyInterface.h + CommitProxyInterface.h MetricLogger.actor.cpp MetricLogger.h MonitorLeader.actor.cpp diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index 4b4f822fc9..c4bdb2bc1b 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/FailureMonitor.h" #include "fdbclient/Status.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" // Streams from WorkerInterface that are safe and useful to call from a client. // A ClientWorkerInterface is embedded as the first element of a WorkerInterface. diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h index c957ae8633..2570666b12 100644 --- a/fdbclient/ClusterInterface.h +++ b/fdbclient/ClusterInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/FailureMonitor.h" #include "fdbclient/Status.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ClientWorkerInterface.h" struct ClusterInterface { diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/CommitProxyInterface.h similarity index 96% rename from fdbclient/MasterProxyInterface.h rename to fdbclient/CommitProxyInterface.h index 9e2b49037c..c6b12dd7f2 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -1,6 +1,6 @@ /* - * MasterProxyInterface.h + * CommitProxyInterface.h * * This source file is part of the FoundationDB open source project * @@ -19,8 +19,8 @@ * limitations under the License. */ -#ifndef FDBCLIENT_MASTERPROXYINTERFACE_H -#define FDBCLIENT_MASTERPROXYINTERFACE_H +#ifndef FDBCLIENT_COMMITPROXYINTERFACE_H +#define FDBCLIENT_COMMITPROXYINTERFACE_H #pragma once #include @@ -36,7 +36,7 @@ #include "fdbrpc/TimedRequest.h" #include "GrvProxyInterface.h" -struct MasterProxyInterface { +struct CommitProxyInterface { constexpr static FileIdentifier file_identifier = 8954922; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 1 }; @@ -59,8 +59,8 @@ struct MasterProxyInterface { UID id() const { return commit.getEndpoint().token; } std::string toString() const { return id().shortString(); } - bool operator == (MasterProxyInterface const& r) const { return id() == r.id(); } - bool operator != (MasterProxyInterface const& r) const { return id() != r.id(); } + bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); } + bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); } template @@ -101,8 +101,9 @@ struct ClientDBInfo { constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes vector< GrvProxyInterface > grvProxies; - vector< MasterProxyInterface > masterProxies; - Optional firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk + vector commitProxies; + Optional + firstCommitProxy; // not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk double clientTxnInfoSampleRate; int64_t clientTxnInfoSizeLimit; Optional forward; @@ -122,7 +123,7 @@ struct ClientDBInfo { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, grvProxies, masterProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, + serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, transactionTagSampleRate, transactionTagSampleCost); } }; diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h index 0dc2970ca1..95423bf6ca 100644 --- a/fdbclient/CoordinationInterface.h +++ b/fdbclient/CoordinationInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/fdbrpc.h" #include "fdbrpc/Locality.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ClusterInterface.h" const int MAX_CLUSTER_FILE_BYTES = 60000; diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index b1580205a0..f70fc4275c 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -29,12 +29,12 @@ DatabaseConfiguration::DatabaseConfiguration() void DatabaseConfiguration::resetInternal() { // does NOT reset rawConfiguration initialized = false; - proxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = + commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1; tLogVersion = TLogVersion::DEFAULT; tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END; tLogSpillType = TLogSpillType::DEFAULT; - autoProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES; + autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES; autoGrvProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES; autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS; autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS; @@ -164,38 +164,21 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() { } bool DatabaseConfiguration::isValid() const { - if( !(initialized && - tLogWriteAntiQuorum >= 0 && - tLogWriteAntiQuorum <= tLogReplicationFactor/2 && - tLogReplicationFactor >= 1 && - storageTeamSize >= 1 && - getDesiredProxies() >= 1 && - getDesiredGrvProxies() >= 1 && - getDesiredLogs() >= 1 && - getDesiredResolvers() >= 1 && - tLogVersion != TLogVersion::UNSET && - tLogVersion >= TLogVersion::MIN_RECRUITABLE && - tLogVersion <= TLogVersion::MAX_SUPPORTED && - tLogDataStoreType != KeyValueStoreType::END && - tLogSpillType != TLogSpillType::UNSET && - !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && - storageServerStoreType != KeyValueStoreType::END && - autoProxyCount >= 1 && - autoGrvProxyCount >= 1 && - autoResolverCount >= 1 && - autoDesiredTLogCount >= 1 && - storagePolicy && - tLogPolicy && - getDesiredRemoteLogs() >= 1 && - remoteTLogReplicationFactor >= 0 && - repopulateRegionAntiQuorum >= 0 && - repopulateRegionAntiQuorum <= 1 && - usableRegions >= 1 && - usableRegions <= 2 && - regions.size() <= 2 && - ( usableRegions == 1 || regions.size() == 2 ) && - ( regions.size() == 0 || regions[0].priority >= 0 ) && - ( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication + if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 && + tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredCommitProxies() >= 1 && + getDesiredGrvProxies() >= 1 && getDesiredLogs() >= 1 && getDesiredResolvers() >= 1 && + tLogVersion != TLogVersion::UNSET && tLogVersion >= TLogVersion::MIN_RECRUITABLE && + tLogVersion <= TLogVersion::MAX_SUPPORTED && tLogDataStoreType != KeyValueStoreType::END && + tLogSpillType != TLogSpillType::UNSET && + !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && + storageServerStoreType != KeyValueStoreType::END && autoCommitProxyCount >= 1 && autoGrvProxyCount >= 1 && + autoResolverCount >= 1 && autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy && + getDesiredRemoteLogs() >= 1 && remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 && + repopulateRegionAntiQuorum <= 1 && usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 && + (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) && + (regions.size() == 0 || + tLogPolicy->info() != + "dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication return false; } @@ -318,11 +301,11 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { if (desiredTLogCount != -1 || isOverridden("logs")) { result["logs"] = desiredTLogCount; } - if (proxyCount != -1 || isOverridden("proxies")) { - result["proxies"] = proxyCount; + if (commitProxyCount != -1 || isOverridden("commit_proxies")) { + result["commit_proxies"] = commitProxyCount; } if (grvProxyCount != -1 || isOverridden("grv_proxies")) { - result["grv_proxies"] = proxyCount; + result["grv_proxies"] = commitProxyCount; } if (resolverCount != -1 || isOverridden("resolvers")) { result["resolvers"] = resolverCount; @@ -336,8 +319,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { if (repopulateRegionAntiQuorum != 0 || isOverridden("repopulate_anti_quorum")) { result["repopulate_anti_quorum"] = repopulateRegionAntiQuorum; } - if (autoProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_PROXIES || isOverridden("auto_proxies")) { - result["auto_proxies"] = autoProxyCount; + if (autoCommitProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES || isOverridden("auto_commit_proxies")) { + result["auto_commit_proxies"] = autoCommitProxyCount; } if (autoGrvProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES || isOverridden("auto_grv_proxies")) { result["auto_grv_proxies"] = autoGrvProxyCount; @@ -419,8 +402,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { if (ck == LiteralStringRef("initialized")) { initialized = true; - } else if (ck == LiteralStringRef("proxies")) { - parse(&proxyCount, value); + } else if (ck == LiteralStringRef("commit_proxies")) { + parse(&commitProxyCount, value); } else if (ck == LiteralStringRef("grv_proxies")) { parse(&grvProxyCount, value); } else if (ck == LiteralStringRef("resolvers")) { @@ -459,8 +442,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { } else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; - } else if (ck == LiteralStringRef("auto_proxies")) { - parse(&autoProxyCount, value); + } else if (ck == LiteralStringRef("auto_commit_proxies")) { + parse(&autoCommitProxyCount, value); } else if (ck == LiteralStringRef("auto_grv_proxies")) { parse(&autoGrvProxyCount, value); } else if (ck == LiteralStringRef("auto_resolvers")) { diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 4a045200e8..5f3a852ed9 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -149,9 +149,9 @@ struct DatabaseConfiguration { return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1); } - // Proxy Servers - int32_t proxyCount; - int32_t autoProxyCount; + // CommitProxy Servers + int32_t commitProxyCount; + int32_t autoCommitProxyCount; int32_t grvProxyCount; int32_t autoGrvProxyCount; @@ -192,7 +192,10 @@ struct DatabaseConfiguration { bool isExcludedServer( NetworkAddressList ) const; std::set getExcludedServers() const; - int32_t getDesiredProxies() const { if(proxyCount == -1) return autoProxyCount; return proxyCount; } + int32_t getDesiredCommitProxies() const { + if (commitProxyCount == -1) return autoCommitProxyCount; + return commitProxyCount; + } int32_t getDesiredGrvProxies() const { if (grvProxyCount == -1) return autoGrvProxyCount; return grvProxyCount; diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 0f86d41f9e..f9367482e5 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -29,7 +29,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/KeyRangeMap.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/SpecialKeySpace.actor.h" #include "fdbrpc/QueueModel.h" #include "fdbrpc/MultiInterface.h" @@ -68,7 +68,7 @@ struct LocationInfo : MultiInterface } }; -using ProxyInfo = ModelInterface; +using CommitProxyInfo = ModelInterface; using GrvProxyInfo = ModelInterface; class ClientTagThrottleData : NonCopyable { @@ -165,8 +165,8 @@ public: bool sampleOnCost(uint64_t cost) const; void updateProxies(); - Reference getMasterProxies(bool useProvisionalProxies); - Future> getMasterProxiesFuture(bool useProvisionalProxies); + Reference getCommitProxies(bool useProvisionalProxies); + Future> getCommitProxiesFuture(bool useProvisionalProxies); Reference getGrvProxies(bool useProvisionalProxies); Future onProxiesChanged(); Future getHealthMetrics(bool detailed); @@ -219,7 +219,7 @@ public: Reference>> connectionFile; AsyncTrigger proxiesChangeTrigger; Future monitorProxiesInfoChange; - Reference masterProxies; + Reference commitProxies; Reference grvProxies; bool proxyProvisional; UID proxiesLastChange; diff --git a/fdbclient/GrvProxyInterface.h b/fdbclient/GrvProxyInterface.h index 06d4b7e946..94820a175f 100644 --- a/fdbclient/GrvProxyInterface.h +++ b/fdbclient/GrvProxyInterface.h @@ -27,6 +27,8 @@ // with RateKeeper to gather health information of the cluster. struct GrvProxyInterface { constexpr static FileIdentifier file_identifier = 8743216; + enum { LocationAwareLoadBalance = 1 }; + enum { AlwaysFresh = 1 }; Optional processId; bool provisional; diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index c2e99f63fb..d1ec7a4f5f 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -52,7 +52,7 @@ void ClientKnobs::initialize(bool randomize) { init( COORDINATOR_RECONNECTION_DELAY, 1.0 ); init( CLIENT_EXAMPLE_AMOUNT, 20 ); init( MAX_CLIENT_STATUS_AGE, 1.0 ); - init( MAX_MASTER_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_MASTER_PROXY_CONNECTIONS = 1; + init( MAX_COMMIT_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_COMMIT_PROXY_CONNECTIONS = 1; init( MAX_GRV_PROXY_CONNECTIONS, 3 ); if( randomize && BUGGIFY ) MAX_GRV_PROXY_CONNECTIONS = 1; init( STATUS_IDLE_TIMEOUT, 120.0 ); @@ -171,7 +171,7 @@ void ClientKnobs::initialize(bool randomize) { init( MIN_CLEANUP_SECONDS, 3600.0 ); // Configuration - init( DEFAULT_AUTO_PROXIES, 3 ); + init( DEFAULT_AUTO_COMMIT_PROXIES, 3 ); init( DEFAULT_AUTO_GRV_PROXIES, 1 ); init( DEFAULT_AUTO_RESOLVERS, 1 ); init( DEFAULT_AUTO_LOGS, 3 ); diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index 30e7e7f687..7edaf18e7d 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -46,7 +46,7 @@ public: double COORDINATOR_RECONNECTION_DELAY; int CLIENT_EXAMPLE_AMOUNT; double MAX_CLIENT_STATUS_AGE; - int MAX_MASTER_PROXY_CONNECTIONS; + int MAX_COMMIT_PROXY_CONNECTIONS; int MAX_GRV_PROXY_CONNECTIONS; double STATUS_IDLE_TIMEOUT; @@ -167,7 +167,7 @@ public: double MIN_CLEANUP_SECONDS; // Configuration - int32_t DEFAULT_AUTO_PROXIES; + int32_t DEFAULT_AUTO_COMMIT_PROXIES; int32_t DEFAULT_AUTO_GRV_PROXIES; int32_t DEFAULT_AUTO_RESOLVERS; int32_t DEFAULT_AUTO_LOGS; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index a05fce601e..e4a5183b95 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -78,8 +78,9 @@ std::map configForToken( std::string const& mode ) { std::string key = mode.substr(0, pos); std::string value = mode.substr(pos+1); - if ((key == "logs" || key == "proxies" || key == "grv_proxies" || key == "resolvers" || key == "remote_logs" || - key == "log_routers" || key == "usable_regions" || key == "repopulate_anti_quorum") && + if ((key == "logs" || key == "commit_proxies" || key == "grv_proxies" || key == "resolvers" || + key == "remote_logs" || key == "log_routers" || key == "usable_regions" || + key == "repopulate_anti_quorum") && isInteger(value)) { out[p+key] = value; } @@ -656,7 +657,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { } if (processClass.classType() == ProcessClass::TransactionClass || - processClass.classType() == ProcessClass::ProxyClass || + processClass.classType() == ProcessClass::CommitProxyClass || processClass.classType() == ProcessClass::GrvProxyClass || processClass.classType() == ProcessClass::ResolutionClass || processClass.classType() == ProcessClass::StatelessClass || @@ -701,7 +702,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { if (proc.second == ProcessClass::StatelessClass) { existingStatelessCount++; } - if(proc.second == ProcessClass::ProxyClass) { + if (proc.second == ProcessClass::CommitProxyClass) { existingProxyCount++; } if (proc.second == ProcessClass::GrvProxyClass) { @@ -734,16 +735,16 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { resolverCount = result.old_resolvers; } - result.desired_proxies = std::max(std::min(12, processCount / 15), 1); + result.desired_commit_proxies = std::max(std::min(12, processCount / 15), 1); int proxyCount; - if (!statusObjConfig.get("proxies", result.old_proxies)) { - result.old_proxies = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES; - statusObjConfig.get("auto_proxies", result.old_proxies); - result.auto_proxies = result.desired_proxies; - proxyCount = result.auto_proxies; + if (!statusObjConfig.get("commit_proxies", result.old_commit_proxies)) { + result.old_commit_proxies = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES; + statusObjConfig.get("auto_commit_proxies", result.old_commit_proxies); + result.auto_commit_proxies = result.desired_commit_proxies; + proxyCount = result.auto_commit_proxies; } else { - result.auto_proxies = result.old_proxies; - proxyCount = result.old_proxies; + result.auto_commit_proxies = result.old_commit_proxies; + proxyCount = result.old_commit_proxies; } // Need to configure a good number. @@ -857,8 +858,8 @@ ACTOR Future autoConfig( Database cx, ConfigureAutoRe if (conf.auto_logs != conf.old_logs) tr.set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs)); - if(conf.auto_proxies != conf.old_proxies) - tr.set(configKeysPrefix.toString() + "auto_proxies", format("%d", conf.auto_proxies)); + if (conf.auto_commit_proxies != conf.old_commit_proxies) + tr.set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies)); if (conf.auto_grv_proxies != conf.old_grv_proxies) tr.set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies)); diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h index 20b2a447d9..e87f9aedd2 100644 --- a/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/ManagementAPI.actor.h @@ -86,7 +86,7 @@ struct ConfigureAutoResult { int32_t machines; std::string old_replication; - int32_t old_proxies; + int32_t old_commit_proxies; int32_t old_grv_proxies; int32_t old_resolvers; int32_t old_logs; @@ -94,23 +94,24 @@ struct ConfigureAutoResult { int32_t old_machines_with_transaction; std::string auto_replication; - int32_t auto_proxies; + int32_t auto_commit_proxies; int32_t auto_grv_proxies; int32_t auto_resolvers; int32_t auto_logs; int32_t auto_processes_with_transaction; int32_t auto_machines_with_transaction; - int32_t desired_proxies; + int32_t desired_commit_proxies; int32_t desired_grv_proxies; int32_t desired_resolvers; int32_t desired_logs; ConfigureAutoResult() - : processes(-1), machines(-1), old_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1), - old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_proxies(-1), auto_grv_proxies(-1), - auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1), - desired_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1), desired_logs(-1) {} + : processes(-1), machines(-1), old_commit_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1), + old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_commit_proxies(-1), + auto_grv_proxies(-1), auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), + auto_machines_with_transaction(-1), desired_commit_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1), + desired_logs(-1) {} bool isValid() const { return processes != -1; } }; diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index 1e13b18560..e3ac757840 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -624,8 +624,8 @@ ACTOR Future getClientInfoFromLeader( Referenceget().get().clientInterface.openDatabase.getReply( req ) ) ) ) { TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()) - .detail("MasterProxy0", ni.masterProxies.size() ? ni.masterProxies[0].id() : UID()) - .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) + .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID()) + .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) .detail("ClientID", ni.id); clientData->clientInfo->set(CachedSerialization(ni)); } @@ -681,24 +681,25 @@ ACTOR Future monitorLeaderForProxies( Key clusterKey, vector& lastMasterProxyUIDs, std::vector& lastMasterProxies, - std::vector& lastGrvProxyUIDs, std::vector& lastGrvProxies) { - if(ni.masterProxies.size() > CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS) { - std::vector masterProxyUIDs; - for(auto& masterProxy : ni.masterProxies) { - masterProxyUIDs.push_back(masterProxy.id()); +void shrinkProxyList(ClientDBInfo& ni, std::vector& lastCommitProxyUIDs, + std::vector& lastCommitProxies, std::vector& lastGrvProxyUIDs, + std::vector& lastGrvProxies) { + if (ni.commitProxies.size() > CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS) { + std::vector commitProxyUIDs; + for (auto& commitProxy : ni.commitProxies) { + commitProxyUIDs.push_back(commitProxy.id()); } - if(masterProxyUIDs != lastMasterProxyUIDs) { - lastMasterProxyUIDs.swap(masterProxyUIDs); - lastMasterProxies = ni.masterProxies; - deterministicRandom()->randomShuffle(lastMasterProxies); - lastMasterProxies.resize(CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS); - for(int i = 0; i < lastMasterProxies.size(); i++) { - TraceEvent("ConnectedMasterProxy").detail("MasterProxy", lastMasterProxies[i].id()); + if (commitProxyUIDs != lastCommitProxyUIDs) { + lastCommitProxyUIDs.swap(commitProxyUIDs); + lastCommitProxies = ni.commitProxies; + deterministicRandom()->randomShuffle(lastCommitProxies); + lastCommitProxies.resize(CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS); + for (int i = 0; i < lastCommitProxies.size(); i++) { + TraceEvent("ConnectedCommitProxy").detail("CommitProxy", lastCommitProxies[i].id()); } } - ni.firstProxy = ni.masterProxies[0]; - ni.masterProxies = lastMasterProxies; + ni.firstCommitProxy = ni.commitProxies[0]; + ni.commitProxies = lastCommitProxies; } if(ni.grvProxies.size() > CLIENT_KNOBS->MAX_GRV_PROXY_CONNECTIONS) { std::vector grvProxyUIDs; @@ -719,14 +720,16 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector& lastMasterProxyUIDs, s } // Leader is the process that will be elected by coordinators as the cluster controller -ACTOR Future monitorProxiesOneGeneration( Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, Reference>>> supportedVersions, Key traceLogGroup) { +ACTOR Future monitorProxiesOneGeneration( + Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, + Reference>>> supportedVersions, Key traceLogGroup) { state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString(); state vector addrs = cs.coordinators(); state int idx = 0; state int successIdx = 0; state Optional incorrectTime; - state std::vector lastProxyUIDs; - state std::vector lastProxies; + state std::vector lastCommitProxyUIDs; + state std::vector lastCommitProxies; state std::vector lastGrvProxyUIDs; state std::vector lastGrvProxies; @@ -780,7 +783,7 @@ ACTOR Future monitorProxiesOneGeneration( ReferencenotifyConnected(); auto& ni = rep.get().mutate(); - shrinkProxyList(ni, lastProxyUIDs, lastProxies, lastGrvProxyUIDs, lastGrvProxies); + shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies); clientInfo->set( ni ); successIdx = idx; } else { diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index 58f1fd3bbd..643cf361c7 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #define CLUSTER_FILE_ENV_VAR_NAME "FDB_CLUSTER_FILE" @@ -67,8 +67,9 @@ Future monitorLeaderForProxies( Value const& key, vector c Future monitorProxies( Reference>> const& connFile, Reference> const& clientInfo, Reference>>> const& supportedVersions, Key const& traceLogGroup ); -void shrinkProxyList( ClientDBInfo& ni, std::vector& lastMasterProxyUIDs, std::vector& lastMasterProxies, - std::vector& lastGrvProxyUIDs, std::vector& lastGrvProxies); +void shrinkProxyList(ClientDBInfo& ni, std::vector& lastCommitProxyUIDs, + std::vector& lastCommitProxies, std::vector& lastGrvProxyUIDs, + std::vector& lastGrvProxies); #ifndef __INTEL_COMPILER #pragma region Implementation diff --git a/fdbclient/MutationList.h b/fdbclient/MutationList.h index bcc9b0db76..57aba3614c 100644 --- a/fdbclient/MutationList.h +++ b/fdbclient/MutationList.h @@ -151,7 +151,8 @@ public: } } - //FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also done there + // FIXME: this is re-implemented on the commit proxy to include a yield, any changes to this function should also + // done there template void serialize_save( Ar& ar ) const { serializer(ar, totalBytes); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index fba6fdf6f8..19683a536b 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -40,7 +40,7 @@ #include "fdbclient/KeyRangeMap.h" #include "fdbclient/Knobs.h" #include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/MutationList.h" #include "fdbclient/ReadYourWrites.h" @@ -484,15 +484,15 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext *cx) { } ACTOR static Future monitorProxiesChange(Reference> clientDBInfo, AsyncTrigger *triggerVar) { - state vector< MasterProxyInterface > curProxies; + state vector curCommitProxies; state vector< GrvProxyInterface > curGrvProxies; - curProxies = clientDBInfo->get().masterProxies; + curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; loop{ wait(clientDBInfo->onChange()); - if (clientDBInfo->get().masterProxies != curProxies || clientDBInfo->get().grvProxies != curGrvProxies) { - curProxies = clientDBInfo->get().masterProxies; + if (clientDBInfo->get().commitProxies != curCommitProxies || clientDBInfo->get().grvProxies != curGrvProxies) { + curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; triggerVar->trigger(); } @@ -881,7 +881,7 @@ DatabaseContext::DatabaseContext(Reference(specialKeys.begin, specialKeys.end, /* test */ false)) { dbId = deterministicRandom()->randomUniqueID(); - connected = (clientInfo->get().masterProxies.size() && clientInfo->get().grvProxies.size()) + connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) ? Void() : clientInfo->onChange(); @@ -1164,9 +1164,9 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional(value.get()) : Optional>(), clientLocality.machineId(), clientLocality.dcId() ); - if( clientInfo->get().masterProxies.size() ) - masterProxies = Reference( new ProxyInfo( clientInfo->get().masterProxies) ); - if( clientInfo->get().grvProxies.size() ) + if (clientInfo->get().commitProxies.size()) + commitProxies = Reference(new CommitProxyInfo(clientInfo->get().commitProxies)); + if( clientInfo->get().grvProxies.size() ) grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies ) ); server_interf.clear(); locationCache.insert( allKeys, Reference() ); @@ -1176,9 +1176,9 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional(value.get()) : Optional>()); - if( clientInfo->get().masterProxies.size() ) - masterProxies = Reference( new ProxyInfo( clientInfo->get().masterProxies)); - if( clientInfo->get().grvProxies.size() ) + if (clientInfo->get().commitProxies.size()) + commitProxies = Reference(new CommitProxyInfo(clientInfo->get().commitProxies)); + if( clientInfo->get().grvProxies.size() ) grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies )); server_interf.clear(); locationCache.insert( allKeys, Reference() ); @@ -1220,13 +1220,13 @@ ACTOR static Future switchConnectionFileImpl(ReferencegetConnectionString().toString()); // Reset state from former cluster. - self->masterProxies.clear(); + self->commitProxies.clear(); self->grvProxies.clear(); self->minAcceptableReadVersion = std::numeric_limits::max(); self->invalidateCache(allKeys); auto clearedClientInfo = self->clientInfo->get(); - clearedClientInfo.masterProxies.clear(); + clearedClientInfo.commitProxies.clear(); clearedClientInfo.grvProxies.clear(); clearedClientInfo.id = deterministicRandom()->randomUniqueID(); self->clientInfo->set(clearedClientInfo); @@ -1561,29 +1561,29 @@ void stopNetwork() { void DatabaseContext::updateProxies() { if (proxiesLastChange == clientInfo->get().id) return; proxiesLastChange = clientInfo->get().id; - masterProxies.clear(); + commitProxies.clear(); grvProxies.clear(); - bool masterProxyProvisional = false, grvProxyProvisional = false; - if (clientInfo->get().masterProxies.size()) { - masterProxies = Reference(new ProxyInfo(clientInfo->get().masterProxies)); - masterProxyProvisional = clientInfo->get().masterProxies[0].provisional; + bool commitProxyProvisional = false, grvProxyProvisional = false; + if (clientInfo->get().commitProxies.size()) { + commitProxies = Reference(new CommitProxyInfo(clientInfo->get().commitProxies)); + commitProxyProvisional = clientInfo->get().commitProxies[0].provisional; } if (clientInfo->get().grvProxies.size()) { grvProxies = Reference(new GrvProxyInfo(clientInfo->get().grvProxies)); grvProxyProvisional = clientInfo->get().grvProxies[0].provisional; } - if (clientInfo->get().masterProxies.size() && clientInfo->get().grvProxies.size()) { - ASSERT(masterProxyProvisional == grvProxyProvisional); - proxyProvisional = masterProxyProvisional; + if (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) { + ASSERT(commitProxyProvisional == grvProxyProvisional); + proxyProvisional = commitProxyProvisional; } } -Reference DatabaseContext::getMasterProxies(bool useProvisionalProxies) { +Reference DatabaseContext::getCommitProxies(bool useProvisionalProxies) { updateProxies(); if (proxyProvisional && !useProvisionalProxies) { - return Reference(); + return Reference(); } - return masterProxies; + return commitProxies; } Reference DatabaseContext::getGrvProxies(bool useProvisionalProxies) { @@ -1594,19 +1594,19 @@ Reference DatabaseContext::getGrvProxies(bool useProvisionalProxie return grvProxies; } -//Actor which will wait until the MultiInterface returned by the DatabaseContext cx is not NULL -ACTOR Future> getMasterProxiesFuture(DatabaseContext *cx, bool useProvisionalProxies) { +// Actor which will wait until the MultiInterface returned by the DatabaseContext cx is not NULL +ACTOR Future> getCommitProxiesFuture(DatabaseContext* cx, bool useProvisionalProxies) { loop{ - Reference proxies = cx->getMasterProxies(useProvisionalProxies); + Reference proxies = cx->getCommitProxies(useProvisionalProxies); if (proxies) return proxies; wait( cx->onProxiesChanged() ); } } -//Returns a future which will not be set until the ProxyInfo of this DatabaseContext is not NULL -Future> DatabaseContext::getMasterProxiesFuture(bool useProvisionalProxies) { - return ::getMasterProxiesFuture(this, useProvisionalProxies); +// Returns a future which will not be set until the CommitProxyInfo of this DatabaseContext is not NULL +Future> DatabaseContext::getCommitProxiesFuture(bool useProvisionalProxies) { + return ::getCommitProxiesFuture(this, useProvisionalProxies); } void GetRangeLimits::decrement( VectorRef const& data ) { @@ -1733,8 +1733,8 @@ ACTOR Future>> getKeyLocation_internal(Da ++cx->transactionKeyServerLocationRequests; choose { when (wait(cx->onProxiesChanged())) {} - when (GetKeyServerLocationsReply rep = wait(basicLoadBalance( - cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, + when(GetKeyServerLocationsReply rep = wait(basicLoadBalance( + cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(span.context, key, Optional(), 100, isBackward, key.arena()), TaskPriority::DefaultPromiseEndpoint))) { ++cx->transactionKeyServerLocationRequestsCompleted; @@ -1782,8 +1782,8 @@ ACTOR Future>>> getKeyRangeLocatio ++cx->transactionKeyServerLocationRequests; choose { when ( wait( cx->onProxiesChanged() ) ) {} - when ( GetKeyServerLocationsReply _rep = wait(basicLoadBalance( - cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, + when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance( + cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(span.context, keys.begin, keys.end, limit, reverse, keys.arena()), TaskPriority::DefaultPromiseEndpoint))) { ++cx->transactionKeyServerLocationRequestsCompleted; @@ -3450,14 +3450,16 @@ ACTOR static Future tryCommit( Database cx, Reference req.debugID = commitID; state Future reply; if (options.commitOnFirstProxy) { - if(cx->clientInfo->get().firstProxy.present()) { - reply = throwErrorOr ( brokenPromiseToMaybeDelivered ( cx->clientInfo->get().firstProxy.get().commit.tryGetReply(req) ) ); + if (cx->clientInfo->get().firstCommitProxy.present()) { + reply = throwErrorOr(brokenPromiseToMaybeDelivered( + cx->clientInfo->get().firstCommitProxy.get().commit.tryGetReply(req))); } else { - const std::vector& proxies = cx->clientInfo->get().masterProxies; + const std::vector& proxies = cx->clientInfo->get().commitProxies; reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never(); } } else { - reply = basicLoadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, true ); + reply = basicLoadBalance(cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::commit, + req, TaskPriority::DefaultPromiseEndpoint, true); } choose { @@ -3531,8 +3533,9 @@ ACTOR static Future tryCommit( Database cx, Reference // We don't know if the commit happened, and it might even still be in flight. if (!options.causalWriteRisky) { - // Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the version we submitted with is dead, or by committing a conflicting transaction successfully - //if ( cx->getMasterProxies()->masterGeneration <= originalMasterGeneration ) + // Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the + // version we submitted with is dead, or by committing a conflicting transaction successfully + // if ( cx->getCommitProxies()->masterGeneration <= originalMasterGeneration ) // To ensure the original request is not in flight, we need a key range which intersects its read conflict ranges // We pick a key range which also intersects its write conflict ranges, since that avoids potentially creating conflicts where there otherwise would be none @@ -4433,8 +4436,8 @@ ACTOR Future>> waitDataDistributionMetricsLis choose { when(wait(cx->onProxiesChanged())) {} when(ErrorOr rep = - wait(errorOr(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::getDDMetrics, - GetDDMetricsRequest(keys, shardLimit))))) { + wait(errorOr(basicLoadBalance(cx->getCommitProxies(false), &CommitProxyInterface::getDDMetrics, + GetDDMetricsRequest(keys, shardLimit))))) { if (rep.isError()) { throw rep.getError(); } @@ -4539,7 +4542,9 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn loop { choose { when(wait(cx->onProxiesChanged())) {} - when(wait(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::proxySnapReq, ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, true /*atmostOnce*/ ))) { + when(wait(basicLoadBalance(cx->getCommitProxies(false), &CommitProxyInterface::proxySnapReq, + ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, + true /*atmostOnce*/))) { TraceEvent("SnapCreateExit") .detail("SnapCmd", snapCmd.toString()) .detail("UID", snapUID); @@ -4567,8 +4572,8 @@ ACTOR Future checkSafeExclusions(Database cx, vector exc choose { when(wait(cx->onProxiesChanged())) {} when(ExclusionSafetyCheckReply _ddCheck = - wait(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::exclusionSafetyCheckReq, - req, cx->taskID))) { + wait(basicLoadBalance(cx->getCommitProxies(false), + &CommitProxyInterface::exclusionSafetyCheckReq, req, cx->taskID))) { ddCheck = _ddCheck.safe; break; } diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 2d35022a4a..35338b3c93 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -30,7 +30,7 @@ #include "flow/flow.h" #include "flow/TDMetric.actor.h" #include "fdbclient/FDBTypes.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 333887d1f3..6c20adc96e 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -47,7 +47,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "storage", "transaction", "resolution", - "proxy", + "commit_proxy", "grv_proxy", "master", "test", @@ -84,7 +84,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "role":{ "$enum":[ "master", - "proxy", + "commit_proxy", "grv_proxy", "log", "storage", @@ -486,7 +486,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( R"statusSchema( "recovery_state":{ "required_resolvers":1, - "required_proxies":1, + "required_commit_proxies":1, "required_grv_proxies":1, "name":{ "$enum":[ @@ -675,11 +675,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "address":"10.0.4.1" } ], - "auto_proxies":3, + "auto_commit_proxies":3, "auto_grv_proxies":1, "auto_resolvers":1, "auto_logs":3, - "proxies":5, + "commit_proxies":5, "grv_proxies":1, "backup_worker_enabled":1 }, @@ -879,11 +879,11 @@ const KeyRef JSONSchemas::clusterConfigurationSchema = LiteralStringRef(R"config "ssd-2", "memory" ]}, - "auto_proxies":3, + "auto_commit_proxies":3, "auto_grv_proxies":1, "auto_resolvers":1, "auto_logs":3, - "proxies":5 + "commit_proxies":5 "grv_proxies":1 })configSchema"); diff --git a/fdbclient/TagThrottle.actor.cpp b/fdbclient/TagThrottle.actor.cpp index a566b2fbfa..ebf0157d1c 100644 --- a/fdbclient/TagThrottle.actor.cpp +++ b/fdbclient/TagThrottle.actor.cpp @@ -19,7 +19,7 @@ */ #include "fdbclient/TagThrottle.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/DatabaseContext.h" #include "flow/actorcompiler.h" // has to be last include diff --git a/fdbrpc/Locality.cpp b/fdbrpc/Locality.cpp index 5409abdedf..1a1f64708d 100644 --- a/fdbrpc/Locality.cpp +++ b/fdbrpc/Locality.cpp @@ -63,9 +63,9 @@ ProcessClass::Fitness ProcessClass::machineClassFitness( ClusterRole role ) cons default: return ProcessClass::NeverAssign; } - case ProcessClass::Proxy: + case ProcessClass::CommitProxy: switch( _class ) { - case ProcessClass::ProxyClass: + case ProcessClass::CommitProxyClass: return ProcessClass::BestFit; case ProcessClass::StatelessClass: return ProcessClass::GoodFit; @@ -92,7 +92,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness( ClusterRole role ) cons return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::ProxyClass: + case ProcessClass::CommitProxyClass: return ProcessClass::OkayFit; case ProcessClass::ResolutionClass: return ProcessClass::OkayFit; @@ -192,7 +192,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness( ClusterRole role ) cons return ProcessClass::OkayFit; case ProcessClass::TransactionClass: return ProcessClass::OkayFit; - case ProcessClass::ProxyClass: + case ProcessClass::CommitProxyClass: return ProcessClass::OkayFit; case ProcessClass::GrvProxyClass: return ProcessClass::OkayFit; diff --git a/fdbrpc/Locality.h b/fdbrpc/Locality.h index da89dfc3cb..8f9be25818 100644 --- a/fdbrpc/Locality.h +++ b/fdbrpc/Locality.h @@ -33,7 +33,7 @@ struct ProcessClass { TransactionClass, ResolutionClass, TesterClass, - ProxyClass, // Process class of CommitProxy + CommitProxyClass, GrvProxyClass, MasterClass, StatelessClass, @@ -53,7 +53,7 @@ struct ProcessClass { enum ClusterRole { Storage, TLog, - Proxy, + CommitProxy, GrvProxy, Master, Resolver, @@ -77,7 +77,8 @@ public: if (s=="storage") _class = StorageClass; else if (s=="transaction") _class = TransactionClass; else if (s=="resolution") _class = ResolutionClass; - else if (s=="proxy") _class = ProxyClass; +// else if (s=="proxy") _class = CommitProxyClass; + else if (s=="commit_proxy") _class = CommitProxyClass; else if (s=="grv_proxy") _class = GrvProxyClass; else if (s=="master") _class = MasterClass; else if (s=="test") _class = TesterClass; @@ -99,7 +100,8 @@ public: if (classStr=="storage") _class = StorageClass; else if (classStr=="transaction") _class = TransactionClass; else if (classStr=="resolution") _class = ResolutionClass; - else if (classStr=="proxy") _class = ProxyClass; +// else if (classStr=="proxy") _class = CommitProxyClass; + else if (classStr=="commit_proxy") _class = CommitProxyClass; else if (classStr=="grv_proxy") _class = GrvProxyClass; else if (classStr=="master") _class = MasterClass; else if (classStr=="test") _class = TesterClass; @@ -137,7 +139,7 @@ public: case StorageClass: return "storage"; case TransactionClass: return "transaction"; case ResolutionClass: return "resolution"; - case ProxyClass: return "proxy"; + case CommitProxyClass: return "commit_proxy"; case GrvProxyClass: return "grv_proxy"; case MasterClass: return "master"; case TesterClass: return "test"; @@ -342,6 +344,7 @@ struct LBLocalityData { template struct LBLocalityData::type> { enum { Present = 1 }; + // TODO: figure out why some interfaces don't have locality. static LocalityData getLocality( Interface const& i ) { return i.locality; } static NetworkAddress getAddress( Interface const& i ) { return i.address(); } static bool alwaysFresh() { return Interface::AlwaysFresh; } diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 8f01cad30a..e27f12a744 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -97,7 +97,8 @@ public: case ProcessClass::StorageClass: return true; case ProcessClass::TransactionClass: return true; case ProcessClass::ResolutionClass: return false; - case ProcessClass::ProxyClass: return false; + case ProcessClass::CommitProxyClass: + return false; case ProcessClass::GrvProxyClass: return false; case ProcessClass::MasterClass: diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index 5860a6772a..2f7ff8fbca 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -21,7 +21,7 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" #include "fdbclient/DatabaseContext.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/SystemData.h" #include "fdbserver/BackupInterface.h" #include "fdbserver/BackupProgress.actor.h" diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 3404df2547..823150dfe9 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -46,7 +46,7 @@ set(FDBSERVER_SRCS LogSystemDiskQueueAdapter.h LogSystemPeekCursor.actor.cpp MasterInterface.h - MasterProxyServer.actor.cpp + CommitProxyServer.actor.cpp masterserver.actor.cpp MutationTracking.h MutationTracking.cpp diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index c51351b96e..5c4b1c8215 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -753,20 +753,21 @@ public: } } - auto first_proxy = getWorkerForRoleInDatacenter(dcId, ProcessClass::Proxy, ProcessClass::ExcludeFit, - req.configuration, id_used); + auto first_commit_proxy = getWorkerForRoleInDatacenter(dcId, ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, req.configuration, id_used); auto first_grv_proxy = getWorkerForRoleInDatacenter(dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, id_used); auto first_resolver = getWorkerForRoleInDatacenter(dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, id_used); - auto proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::Proxy, req.configuration.getDesiredProxies(), - req.configuration, id_used, first_proxy); + auto commit_proxies = + getWorkersForRoleInDatacenter(dcId, ProcessClass::CommitProxy, req.configuration.getDesiredCommitProxies(), + req.configuration, id_used, first_commit_proxy); auto grv_proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::GrvProxy, req.configuration.getDesiredGrvProxies(), req.configuration, id_used, first_grv_proxy); auto resolvers = getWorkersForRoleInDatacenter( dcId, ProcessClass::Resolver, req.configuration.getDesiredResolvers(), req.configuration, id_used, first_resolver ); - for (int i = 0; i < proxies.size(); i++) result.masterProxies.push_back(proxies[i].interf); + for (int i = 0; i < commit_proxies.size(); i++) result.commitProxies.push_back(commit_proxies[i].interf); for (int i = 0; i < grv_proxies.size(); i++) result.grvProxies.push_back(grv_proxies[i].interf); for(int i = 0; i < resolvers.size(); i++) result.resolvers.push_back(resolvers[i].interf); @@ -800,9 +801,9 @@ public: RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredSatelliteLogs(dcId), ProcessClass::TLog) .betterCount(RoleFitness(satelliteLogs, ProcessClass::TLog))) || - RoleFitness(SERVER_KNOBS->EXPECTED_PROXY_FITNESS, req.configuration.getDesiredProxies(), - ProcessClass::Proxy) - .betterCount(RoleFitness(proxies, ProcessClass::Proxy)) || + RoleFitness(SERVER_KNOBS->EXPECTED_PROXY_FITNESS, req.configuration.getDesiredCommitProxies(), + ProcessClass::CommitProxy) + .betterCount(RoleFitness(commit_proxies, ProcessClass::CommitProxy)) || RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, req.configuration.getDesiredGrvProxies(), ProcessClass::GrvProxy) .betterCount(RoleFitness(grv_proxies, ProcessClass::GrvProxy)) || @@ -911,22 +912,22 @@ public: try { //SOMEDAY: recruitment in other DCs besides the clusterControllerDcID will not account for the processes used by the master and cluster controller properly. auto used = id_used; - auto first_proxy = getWorkerForRoleInDatacenter(dcId, ProcessClass::Proxy, ProcessClass::ExcludeFit, - req.configuration, used); + auto first_commit_proxy = getWorkerForRoleInDatacenter( + dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, req.configuration, used); auto first_grv_proxy = getWorkerForRoleInDatacenter( dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, used); auto first_resolver = getWorkerForRoleInDatacenter( dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, used); - auto proxies = - getWorkersForRoleInDatacenter(dcId, ProcessClass::Proxy, req.configuration.getDesiredProxies(), - req.configuration, used, first_proxy); + auto commit_proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::CommitProxy, + req.configuration.getDesiredCommitProxies(), + req.configuration, used, first_commit_proxy); auto grv_proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::GrvProxy, req.configuration.getDesiredGrvProxies(), req.configuration, used, first_grv_proxy); auto resolvers = getWorkersForRoleInDatacenter( dcId, ProcessClass::Resolver, req.configuration.getDesiredResolvers(), req.configuration, used, first_resolver ); - RoleFitnessPair fitness(RoleFitness(proxies, ProcessClass::Proxy), + RoleFitnessPair fitness(RoleFitness(commit_proxies, ProcessClass::CommitProxy), RoleFitness(grv_proxies, ProcessClass::GrvProxy), RoleFitness(resolvers, ProcessClass::Resolver)); @@ -936,8 +937,8 @@ public: for (int i = 0; i < resolvers.size(); i++) { result.resolvers.push_back(resolvers[i].interf); } - for (int i = 0; i < proxies.size(); i++) { - result.masterProxies.push_back(proxies[i].interf); + for (int i = 0; i < commit_proxies.size(); i++) { + result.commitProxies.push_back(commit_proxies[i].interf); } for (int i = 0; i < grv_proxies.size(); i++) { result.grvProxies.push_back(grv_proxies[i].interf); @@ -982,8 +983,8 @@ public: .detail("Replication", req.configuration.tLogReplicationFactor) .detail("DesiredLogs", req.configuration.getDesiredLogs()) .detail("ActualLogs", result.tLogs.size()) - .detail("DesiredProxies", req.configuration.getDesiredProxies()) - .detail("ActualProxies", result.masterProxies.size()) + .detail("DesiredCommitProxies", req.configuration.getDesiredCommitProxies()) + .detail("ActualCommitProxies", result.commitProxies.size()) .detail("DesiredGrvProxies", req.configuration.getDesiredGrvProxies()) .detail("ActualGrvProxies", result.grvProxies.size()) .detail("DesiredResolvers", req.configuration.getDesiredResolvers()) @@ -993,8 +994,8 @@ public: (RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) .betterCount(RoleFitness(tlogs, ProcessClass::TLog)) || - RoleFitness(SERVER_KNOBS->EXPECTED_PROXY_FITNESS, req.configuration.getDesiredProxies(), - ProcessClass::Proxy) + RoleFitness(SERVER_KNOBS->EXPECTED_PROXY_FITNESS, req.configuration.getDesiredCommitProxies(), + ProcessClass::CommitProxy) .betterCount(bestFitness.proxy) || RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, req.configuration.getDesiredGrvProxies(), ProcessClass::GrvProxy) @@ -1028,7 +1029,8 @@ public: } getWorkerForRoleInDatacenter( regions[0].dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true ); - getWorkerForRoleInDatacenter( regions[0].dcId, ProcessClass::Proxy, ProcessClass::ExcludeFit, db.config, id_used, true ); + getWorkerForRoleInDatacenter(regions[0].dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, + db.config, id_used, true); getWorkerForRoleInDatacenter(regions[0].dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); @@ -1129,15 +1131,13 @@ public: } } - // Get proxy classes - std::vector proxyClasses; - for(auto& it : dbi.client.masterProxies) { - auto masterProxyWorker = id_worker.find(it.processId); - if ( masterProxyWorker == id_worker.end() ) - return false; - if ( masterProxyWorker->second.priorityInfo.isExcluded ) - return true; - proxyClasses.push_back(masterProxyWorker->second.details); + // Get commit proxy classes + std::vector commitProxyClasses; + for (auto& it : dbi.client.commitProxies) { + auto commitProxyWorker = id_worker.find(it.processId); + if (commitProxyWorker == id_worker.end()) return false; + if (commitProxyWorker->second.priorityInfo.isExcluded) return true; + commitProxyClasses.push_back(commitProxyWorker->second.details); } // Get grv proxy classes @@ -1285,25 +1285,25 @@ public: if(oldLogRoutersFit < newLogRoutersFit) return false; // Check proxy/grvProxy/resolver fitness - RoleFitnessPair oldInFit(RoleFitness(proxyClasses, ProcessClass::Proxy), + RoleFitnessPair oldInFit(RoleFitness(commitProxyClasses, ProcessClass::CommitProxy), RoleFitness(grvProxyClasses, ProcessClass::GrvProxy), RoleFitness(resolverClasses, ProcessClass::Resolver)); - auto first_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, ProcessClass::Proxy, - ProcessClass::ExcludeFit, db.config, id_used, true); + auto first_commit_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, db.config, id_used, true); auto first_grv_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); auto first_resolver = getWorkerForRoleInDatacenter(clusterControllerDcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true); - auto proxies = - getWorkersForRoleInDatacenter(clusterControllerDcId, ProcessClass::Proxy, db.config.getDesiredProxies(), - db.config, id_used, first_proxy, true); + auto commit_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, ProcessClass::CommitProxy, + db.config.getDesiredCommitProxies(), db.config, id_used, + first_commit_proxy, true); auto grv_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, ProcessClass::GrvProxy, db.config.getDesiredGrvProxies(), db.config, id_used, first_grv_proxy, true); auto resolvers = getWorkersForRoleInDatacenter( clusterControllerDcId, ProcessClass::Resolver, db.config.getDesiredResolvers(), db.config, id_used, first_resolver, true ); - RoleFitnessPair newInFit(RoleFitness(proxies, ProcessClass::Proxy), + RoleFitnessPair newInFit(RoleFitness(commit_proxies, ProcessClass::CommitProxy), RoleFitness(grv_proxies, ProcessClass::GrvProxy), RoleFitness(resolvers, ProcessClass::Resolver)); if (oldInFit.proxy.betterFitness(newInFit.proxy) || oldInFit.grvProxy.betterFitness(newInFit.grvProxy) || @@ -1358,7 +1358,7 @@ public: if (tlog.present() && tlog.interf().filteredLocality.processId() == processId) return true; } } - for (const MasterProxyInterface& interf : dbInfo.client.masterProxies) { + for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { if (interf.processId == processId) return true; } for (const GrvProxyInterface& interf : dbInfo.client.grvProxies) { @@ -1393,7 +1393,7 @@ public: } } } - for (const MasterProxyInterface& interf : dbInfo.client.masterProxies) { + for (const CommitProxyInterface& interf : dbInfo.client.commitProxies) { ASSERT(interf.processId.present()); idUsed[interf.processId]++; } @@ -1967,7 +1967,7 @@ void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest c .detail("Resolvers", req.resolvers.size()) .detail("RecoveryState", (int)req.recoveryState) .detail("RegistrationCount", req.registrationCount) - .detail("MasterProxies", req.masterProxies.size()) + .detail("CommitProxies", req.commitProxies.size()) .detail("GrvProxies", req.grvProxies.size()) .detail("RecoveryCount", req.recoveryCount) .detail("Stalled", req.recoveryStalled) @@ -2022,11 +2022,12 @@ void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest c } // Construct the client information - if (db->clientInfo->get().masterProxies != req.masterProxies || db->clientInfo->get().grvProxies != req.grvProxies) { + if (db->clientInfo->get().commitProxies != req.commitProxies || + db->clientInfo->get().grvProxies != req.grvProxies) { isChanged = true; ClientDBInfo clientInfo; clientInfo.id = deterministicRandom()->randomUniqueID(); - clientInfo.masterProxies = req.masterProxies; + clientInfo.commitProxies = req.commitProxies; clientInfo.grvProxies = req.grvProxies; clientInfo.clientTxnInfoSampleRate = db->clientInfo->get().clientTxnInfoSampleRate; clientInfo.clientTxnInfoSizeLimit = db->clientInfo->get().clientTxnInfoSizeLimit; diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp similarity index 95% rename from fdbserver/MasterProxyServer.actor.cpp rename to fdbserver/CommitProxyServer.actor.cpp index 7d8a36a66a..cb9a8c9486 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1,5 +1,5 @@ /* - * MasterProxyServer.actor.cpp + * CommitProxyServer.actor.cpp * * This source file is part of the FoundationDB open source project * @@ -25,7 +25,7 @@ #include "fdbclient/Atomic.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/Knobs.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" #include "fdbrpc/sim_validation.h" @@ -42,7 +42,6 @@ #include "fdbserver/ProxyCommitData.actor.h" #include "fdbserver/RatekeeperInterface.h" #include "fdbserver/RecoveryState.h" -#include "fdbserver/ServerDBInfo.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" #include "flow/ActorCollection.h" @@ -229,7 +228,7 @@ ACTOR Future commitBatcher(ProxyCommitData *commitData, PromiseStreamstats.txnCommitIn; if(req.debugID.present()) { - g_traceBatch.addEvent("CommitDebug", req.debugID.get().first(), "MasterProxyServer.batcher"); + g_traceBatch.addEvent("CommitDebug", req.debugID.get().first(), "CommitProxyServer.batcher"); } if(!batch.size()) { @@ -512,11 +511,7 @@ void CommitBatchContext::setupTraceBatch() { } if (debugID.present()) { - g_traceBatch.addEvent( - "CommitDebug", - debugID.get().first(), - "MasterProxyServer.commitBatch.Before" - ); + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.Before"); } } @@ -546,10 +541,8 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { ); if (debugID.present()) { - g_traceBatch.addEvent( - "CommitDebug", debugID.get().first(), - "MasterProxyServer.commitBatch.GettingCommitVersion" - ); + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), + "CommitProxyServer.commitBatch.GettingCommitVersion"); } GetCommitVersionRequest req(self->span.context, pProxyCommitData->commitVersionRequestNumber++, @@ -577,10 +570,7 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { //TraceEvent("ProxyGotVer", pProxyContext->dbgid).detail("Commit", commitVersion).detail("Prev", prevVersion); if (debugID.present()) { - g_traceBatch.addEvent( - "CommitDebug", debugID.get().first(), - "MasterProxyServer.commitBatch.GotCommitVersion" - ); + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.GotCommitVersion"); } return Void(); @@ -639,10 +629,8 @@ ACTOR Future getResolution(CommitBatchContext* self) { self->resolution.swap(*const_cast*>(&resolutionResp)); if (self->debugID.present()) { - g_traceBatch.addEvent( - "CommitDebug", self->debugID.get().first(), - "MasterProxyServer.commitBatch.AfterResolution" - ); + g_traceBatch.addEvent("CommitDebug", self->debugID.get().first(), + "CommitProxyServer.commitBatch.AfterResolution"); } return Void(); @@ -972,10 +960,8 @@ ACTOR Future postResolution(CommitBatchContext* self) { pProxyCommitData->stats.txnCommitResolved += trs.size(); if (debugID.present()) { - g_traceBatch.addEvent( - "CommitDebug", debugID.get().first(), - "MasterProxyServer.commitBatch.ProcessingMutations" - ); + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), + "CommitProxyServer.commitBatch.ProcessingMutations"); } self->isMyFirstBatch = !pProxyCommitData->version; @@ -1041,7 +1027,8 @@ ACTOR Future postResolution(CommitBatchContext* self) { self->msg = self->storeCommits.back().first.get(); if (self->debugID.present()) - g_traceBatch.addEvent("CommitDebug", self->debugID.get().first(), "MasterProxyServer.commitBatch.AfterStoreCommits"); + g_traceBatch.addEvent("CommitDebug", self->debugID.get().first(), + "CommitProxyServer.commitBatch.AfterStoreCommits"); // txnState (transaction subsystem state) tag: message extracted from log adapter bool firstMessage = true; @@ -1129,7 +1116,7 @@ ACTOR Future reply(CommitBatchContext* self) { //TraceEvent("ProxyPushed", pProxyCommitData->dbgid).detail("PrevVersion", prevVersion).detail("Version", commitVersion); if (debugID.present()) - g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.AfterLogPush"); + g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.AfterLogPush"); for (auto &p : self->storeCommits) { ASSERT(!p.second.isReady()); @@ -1328,7 +1315,8 @@ ACTOR static Future doKeyServerLocationRequest( GetKeyServerLocationsReque return Void(); } -ACTOR static Future readRequestServer( MasterProxyInterface proxy, PromiseStream> addActor, ProxyCommitData* commitData ) { +ACTOR static Future readRequestServer(CommitProxyInterface proxy, PromiseStream> addActor, + ProxyCommitData* commitData) { loop { GetKeyServerLocationsRequest req = waitNext(proxy.getKeyServersLocations.getFuture()); //WARNING: this code is run at a high priority, so it needs to do as little work as possible @@ -1344,7 +1332,7 @@ ACTOR static Future readRequestServer( MasterProxyInterface proxy, Promise } } -ACTOR static Future rejoinServer( MasterProxyInterface proxy, ProxyCommitData* commitData ) { +ACTOR static Future rejoinServer(CommitProxyInterface proxy, ProxyCommitData* commitData) { // We can't respond to these requests until we have valid txnStateStore wait(commitData->validState.getFuture()); @@ -1413,8 +1401,7 @@ ACTOR static Future rejoinServer( MasterProxyInterface proxy, ProxyCommitD } } -ACTOR Future ddMetricsRequestServer(MasterProxyInterface proxy, Reference> db) -{ +ACTOR Future ddMetricsRequestServer(CommitProxyInterface proxy, Reference> db) { loop { choose { when(state GetDDMetricsRequest req = waitNext(proxy.getDDMetrics.getFuture())) @@ -1496,17 +1483,17 @@ ACTOR Future monitorRemoteCommitted(ProxyCommitData* self) { } ACTOR Future proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* commitData) { - TraceEvent("SnapMasterProxy_SnapReqEnter") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID); + TraceEvent("SnapCommitProxy_SnapReqEnter") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); try { // whitelist check ExecCmdValueString execArg(snapReq.snapPayload); StringRef binPath = execArg.getBinaryPath(); if (!isWhitelisted(commitData->whitelistedBinPathVec, binPath)) { - TraceEvent("SnapMasterProxy_WhiteListCheckFailed") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID); + TraceEvent("SnapCommitProxy_WhiteListCheckFailed") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); throw snap_path_not_whitelisted(); } // db fully recovered check @@ -1516,9 +1503,9 @@ ACTOR Future proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co // Currently, snapshot of old tlog generation is not // supported and hence failing the snapshot request until // cluster is fully_recovered. - TraceEvent("SnapMasterProxy_ClusterNotFullyRecovered") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID); + TraceEvent("SnapCommitProxy_ClusterNotFullyRecovered") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); throw snap_not_fully_recovered_unsupported(); } @@ -1531,9 +1518,9 @@ ACTOR Future proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co // FIXME: logAntiQuorum not supported, remove it later, // In version2, we probably don't need this limtiation, but this needs to be tested. if (logAntiQuorum > 0) { - TraceEvent("SnapMasterProxy_LogAnitQuorumNotSupported") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID); + TraceEvent("SnapCommitProxy_LogAnitQuorumNotSupported") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); throw snap_log_anti_quorum_unsupported(); } @@ -1547,32 +1534,32 @@ ACTOR Future proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co try { wait(throwErrorOr(ddSnapReq)); } catch (Error& e) { - TraceEvent("SnapMasterProxy_DDSnapResponseError") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID) - .error(e, true /*includeCancelled*/ ); + TraceEvent("SnapCommitProxy_DDSnapResponseError") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID) + .error(e, true /*includeCancelled*/); throw e; } snapReq.reply.send(Void()); } catch (Error& e) { - TraceEvent("SnapMasterProxy_SnapReqError") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID) - .error(e, true /*includeCancelled*/); + TraceEvent("SnapCommitProxy_SnapReqError") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID) + .error(e, true /*includeCancelled*/); if (e.code() != error_code_operation_cancelled) { snapReq.reply.sendError(e); } else { throw e; } } - TraceEvent("SnapMasterProxy_SnapReqExit") - .detail("SnapPayload", snapReq.snapPayload) - .detail("SnapUID", snapReq.snapUID); + TraceEvent("SnapCommitProxy_SnapReqExit") + .detail("SnapPayload", snapReq.snapPayload) + .detail("SnapUID", snapReq.snapUID); return Void(); } ACTOR Future proxyCheckSafeExclusion(Reference> db, ExclusionSafetyCheckRequest req) { - TraceEvent("SafetyCheckMasterProxyBegin"); + TraceEvent("SafetyCheckCommitProxyBegin"); state ExclusionSafetyCheckReply reply(false); if (!db->get().distributor.present()) { TraceEvent(SevWarnAlways, "DataDistributorNotPresent").detail("Operation", "ExclusionSafetyCheck"); @@ -1586,7 +1573,7 @@ ACTOR Future proxyCheckSafeExclusion(Reference> db, DistributorExclusionSafetyCheckReply _reply = wait(throwErrorOr(safeFuture)); reply.safe = _reply.safe; } catch (Error& e) { - TraceEvent("SafetyCheckMasterProxyResponseError").error(e); + TraceEvent("SafetyCheckCommitProxyResponseError").error(e); if (e.code() != error_code_operation_cancelled) { req.reply.sendError(e); return Void(); @@ -1594,7 +1581,7 @@ ACTOR Future proxyCheckSafeExclusion(Reference> db, throw e; } } - TraceEvent("SafetyCheckMasterProxyFinish"); + TraceEvent("SafetyCheckCommitProxyFinish"); req.reply.send(reply); return Void(); } @@ -1631,15 +1618,10 @@ ACTOR Future reportTxnTagCommitCost(UID myID, Reference masterProxyServerCore( - MasterProxyInterface proxy, - MasterInterface master, - Reference> db, - LogEpoch epoch, - Version recoveryTransactionVersion, - bool firstProxy, - std::string whitelistBinPaths) -{ +ACTOR Future commitProxyServerCore(CommitProxyInterface proxy, MasterInterface master, + Reference> db, LogEpoch epoch, + Version recoveryTransactionVersion, bool firstProxy, + std::string whitelistBinPaths) { state ProxyCommitData commitData(proxy.id(), master, proxy.getConsistentReadVersion, recoveryTransactionVersion, proxy.commit, db, firstProxy); state Future sequenceFuture = (Sequence)0; @@ -1657,9 +1639,9 @@ ACTOR Future masterProxyServerCore( state GetHealthMetricsReply detailedHealthMetricsReply; addActor.send( waitFailureServer(proxy.waitFailure.getFuture()) ); - addActor.send( traceRole(Role::MASTER_PROXY, proxy.id()) ); + addActor.send(traceRole(Role::COMMIT_PROXY, proxy.id())); - //TraceEvent("ProxyInit1", proxy.id()); + //TraceEvent("CommitProxyInit1", proxy.id()); // Wait until we can load the "real" logsystem, since we don't support switching them currently while (!(commitData.db->get().master.id() == master.id() && commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) { @@ -1701,7 +1683,7 @@ ACTOR Future masterProxyServerCore( (int)std::min(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MAX, std::max(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MIN, SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE * - pow(commitData.db->get().client.masterProxies.size(), + pow(commitData.db->get().client.commitProxies.size(), SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER))); commitBatcherActor = commitBatcher(&commitData, batchedCommits, proxy.commit.getFuture(), commitBatchByteLimit, commitBatchesMemoryLimit); @@ -1723,7 +1705,7 @@ ACTOR Future masterProxyServerCore( //WARNING: this code is run at a high priority, so it needs to do as little work as possible const vector &trs = batchedRequests.first; int batchBytes = batchedRequests.second; - //TraceEvent("MasterProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount); + //TraceEvent("CommitProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount); if (trs.size() || (commitData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && now() - lastCommit >= SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL)) { lastCommit = now(); @@ -1824,27 +1806,27 @@ ACTOR Future masterProxyServerCore( } } -ACTOR Future checkRemoved(Reference> db, uint64_t recoveryCount, MasterProxyInterface myInterface) { +ACTOR Future checkRemoved(Reference> db, uint64_t recoveryCount, + CommitProxyInterface myInterface) { loop{ - if (db->get().recoveryCount >= recoveryCount && !std::count(db->get().client.masterProxies.begin(), db->get().client.masterProxies.end(), myInterface)) { + if (db->get().recoveryCount >= recoveryCount && + !std::count(db->get().client.commitProxies.begin(), db->get().client.commitProxies.end(), myInterface)) { throw worker_removed(); } wait(db->onChange()); } } -ACTOR Future masterProxyServer( - MasterProxyInterface proxy, - InitializeMasterProxyRequest req, - Reference> db, - std::string whitelistBinPaths) -{ +ACTOR Future commitProxyServer(CommitProxyInterface proxy, InitializeCommitProxyRequest req, + Reference> db, std::string whitelistBinPaths) { try { - state Future core = masterProxyServerCore(proxy, req.master, db, req.recoveryCount, req.recoveryTransactionVersion, req.firstProxy, whitelistBinPaths); + state Future core = + commitProxyServerCore(proxy, req.master, db, req.recoveryCount, req.recoveryTransactionVersion, + req.firstProxy, whitelistBinPaths); wait(core || checkRemoved(db, req.recoveryCount, proxy)); } catch (Error& e) { - TraceEvent("MasterProxyTerminated", proxy.id()).error(e, true); + TraceEvent("CommitProxyTerminated", proxy.id()).error(e, true); if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped && e.code() != error_code_master_tlog_failed && e.code() != error_code_coordinators_changed && diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp index 8e09d67dea..c169b9422b 100644 --- a/fdbserver/GrvProxyServer.actor.cpp +++ b/fdbserver/GrvProxyServer.actor.cpp @@ -21,7 +21,7 @@ #include "fdbclient/Notified.h" #include "fdbserver/LogSystem.h" #include "fdbserver/LogSystemDiskQueueAdapter.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/GrvProxyInterface.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" @@ -443,13 +443,13 @@ ACTOR Future sendGrvReplies(Future replyFuture, std:: TEST(true); // Auto TPS rate is unlimited } else { - TEST(true); // Proxy returning tag throttle + TEST(true); // GRV proxy returning tag throttle reply.tagThrottleInfo[tag.first] = tagItr->second; } } else { // This isn't required, but we might as well - TEST(true); // Proxy expiring tag throttle + TEST(true); // GRV proxy expiring tag throttle priorityThrottledTags.erase(tagItr); } } diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index 6122ca13ac..3f2ca37505 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -38,7 +38,7 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi init( MAX_VERSIONS_IN_FLIGHT_FORCED, 6e5 * VERSIONS_PER_SECOND ); //one week of versions init( MAX_READ_TRANSACTION_LIFE_VERSIONS, 5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS = VERSIONS_PER_SECOND; else if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS = std::max(1, 0.1 * VERSIONS_PER_SECOND); else if( randomize && BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = 10 * VERSIONS_PER_SECOND; init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS, 5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max(1, 1 * VERSIONS_PER_SECOND); - init( MAX_COMMIT_BATCH_INTERVAL, 2.0 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5; // Each master proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly + init( MAX_COMMIT_BATCH_INTERVAL, 2.0 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5; // Each commit proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly MAX_COMMIT_BATCH_INTERVAL = std::min(MAX_COMMIT_BATCH_INTERVAL, MAX_READ_TRANSACTION_LIFE_VERSIONS/double(2*VERSIONS_PER_SECOND)); // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough // TLogs @@ -328,7 +328,7 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi init( POLLING_FREQUENCY, 2.0 ); if( longLeaderElection ) POLLING_FREQUENCY = 8.0; init( HEARTBEAT_FREQUENCY, 0.5 ); if( longLeaderElection ) HEARTBEAT_FREQUENCY = 1.0; - // Master Proxy and GRV Proxy + // Commit CommitProxy and GRV CommitProxy init( START_TRANSACTION_BATCH_INTERVAL_MIN, 1e-6 ); init( START_TRANSACTION_BATCH_INTERVAL_MAX, 0.010 ); init( START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION, 0.5 ); diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h index e36de5f2eb..a2d58922fe 100644 --- a/fdbserver/Knobs.h +++ b/fdbserver/Knobs.h @@ -37,10 +37,11 @@ public: int64_t MAX_VERSIONS_IN_FLIGHT_FORCED; int64_t MAX_READ_TRANSACTION_LIFE_VERSIONS; int64_t MAX_WRITE_TRANSACTION_LIFE_VERSIONS; - double MAX_COMMIT_BATCH_INTERVAL; // Each master proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly + double MAX_COMMIT_BATCH_INTERVAL; // Each commit proxy generates a CommitTransactionBatchRequest at least this + // often, so that versions always advance smoothly // TLogs - double TLOG_TIMEOUT; // tlog OR master proxy failure - master's reaction time + double TLOG_TIMEOUT; // tlog OR commit proxy failure - master's reaction time double RECOVERY_TLOG_SMART_QUORUM_DELAY; // smaller might be better for bug amplification double TLOG_STORAGE_MIN_UPDATE_INTERVAL; double BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL; @@ -262,7 +263,7 @@ public: double POLLING_FREQUENCY; double HEARTBEAT_FREQUENCY; - // Master Proxy + // Commit CommitProxy double START_TRANSACTION_BATCH_INTERVAL_MIN; double START_TRANSACTION_BATCH_INTERVAL_MAX; double START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION; diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index b27610ceb1..980b2bd806 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -527,7 +527,7 @@ struct RatekeeperLimits { {} }; -struct ProxyInfo { +struct CommitProxyInfo { int64_t totalTransactions; int64_t batchTransactions; uint64_t lastThrottledTagChangeId; @@ -535,7 +535,9 @@ struct ProxyInfo { double lastUpdateTime; double lastTagPushTime; - ProxyInfo() : totalTransactions(0), batchTransactions(0), lastUpdateTime(0), lastThrottledTagChangeId(0), lastTagPushTime(0) {} + CommitProxyInfo() + : totalTransactions(0), batchTransactions(0), lastUpdateTime(0), lastThrottledTagChangeId(0), lastTagPushTime(0) { + } }; struct RatekeeperData { @@ -545,7 +547,7 @@ struct RatekeeperData { Map storageQueueInfo; Map tlogQueueInfo; - std::map proxyInfo; + std::map commitProxyInfo; Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes; HealthMetrics healthMetrics; DatabaseConfiguration configuration; @@ -1260,31 +1262,31 @@ void updateRate(RatekeeperData* self, RatekeeperLimits* limits) { if (deterministicRandom()->random01() < 0.1) { std::string name = "RkUpdate" + limits->context; TraceEvent(name.c_str(), self->id) - .detail("TPSLimit", limits->tpsLimit) - .detail("Reason", limitReason) - .detail("ReasonServerID", reasonID==UID() ? std::string() : Traceable::toString(reasonID)) - .detail("ReleasedTPS", self->smoothReleasedTransactions.smoothRate()) - .detail("ReleasedBatchTPS", self->smoothBatchReleasedTransactions.smoothRate()) - .detail("TPSBasis", actualTps) - .detail("StorageServers", sscount) - .detail("GrvProxies", self->proxyInfo.size()) - .detail("TLogs", tlcount) - .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer) - .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog) - .detail("WorstStorageServerQueue", worstStorageQueueStorageServer) - .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer) - .detail("WorstTLogQueue", worstStorageQueueTLog) - .detail("TotalDiskUsageBytes", totalDiskUsageBytes) - .detail("WorstStorageServerVersionLag", worstVersionLag) - .detail("LimitingStorageServerVersionLag", limitingVersionLag) - .detail("WorstStorageServerDurabilityLag", worstDurabilityLag) - .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag) - .detail("TagsAutoThrottled", self->throttledTags.autoThrottleCount()) - .detail("TagsAutoThrottledBusyRead", self->throttledTags.busyReadTagCount) - .detail("TagsAutoThrottledBusyWrite", self->throttledTags.busyWriteTagCount) - .detail("TagsManuallyThrottled", self->throttledTags.manualThrottleCount()) - .detail("AutoThrottlingEnabled", self->autoThrottlingEnabled) - .trackLatest(name); + .detail("TPSLimit", limits->tpsLimit) + .detail("Reason", limitReason) + .detail("ReasonServerID", reasonID == UID() ? std::string() : Traceable::toString(reasonID)) + .detail("ReleasedTPS", self->smoothReleasedTransactions.smoothRate()) + .detail("ReleasedBatchTPS", self->smoothBatchReleasedTransactions.smoothRate()) + .detail("TPSBasis", actualTps) + .detail("StorageServers", sscount) + .detail("GrvProxies", self->commitProxyInfo.size()) + .detail("TLogs", tlcount) + .detail("WorstFreeSpaceStorageServer", worstFreeSpaceStorageServer) + .detail("WorstFreeSpaceTLog", worstFreeSpaceTLog) + .detail("WorstStorageServerQueue", worstStorageQueueStorageServer) + .detail("LimitingStorageServerQueue", limitingStorageQueueStorageServer) + .detail("WorstTLogQueue", worstStorageQueueTLog) + .detail("TotalDiskUsageBytes", totalDiskUsageBytes) + .detail("WorstStorageServerVersionLag", worstVersionLag) + .detail("LimitingStorageServerVersionLag", limitingVersionLag) + .detail("WorstStorageServerDurabilityLag", worstDurabilityLag) + .detail("LimitingStorageServerDurabilityLag", limitingDurabilityLag) + .detail("TagsAutoThrottled", self->throttledTags.autoThrottleCount()) + .detail("TagsAutoThrottledBusyRead", self->throttledTags.busyReadTagCount) + .detail("TagsAutoThrottledBusyWrite", self->throttledTags.busyWriteTagCount) + .detail("TagsManuallyThrottled", self->throttledTags.manualThrottleCount()) + .detail("AutoThrottlingEnabled", self->autoThrottlingEnabled) + .trackLatest(name); } } @@ -1369,9 +1371,9 @@ ACTOR Future ratekeeper(RatekeeperInterface rkInterf, Reference SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit; double tooOld = now() - 1.0; - for(auto p=self.proxyInfo.begin(); p!=self.proxyInfo.end(); ) { + for (auto p = self.commitProxyInfo.begin(); p != self.commitProxyInfo.end();) { if (p->second.lastUpdateTime < tooOld) - p = self.proxyInfo.erase(p); + p = self.commitProxyInfo.erase(p); else ++p; } @@ -1380,7 +1382,7 @@ ACTOR Future ratekeeper(RatekeeperInterface rkInterf, Reference 0) { self.smoothReleasedTransactions.addDelta( req.totalReleasedTransactions - p.totalTransactions ); @@ -1397,8 +1399,8 @@ ACTOR Future ratekeeper(RatekeeperInterface rkInterf, ReferenceMETRIC_UPDATE_RATE; if(p.lastThrottledTagChangeId != self.throttledTagChangeId || now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) { diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp index 8a2cac8171..cdac445a40 100644 --- a/fdbserver/Resolver.actor.cpp +++ b/fdbserver/Resolver.actor.cpp @@ -243,7 +243,7 @@ ACTOR Future resolveBatch( // SOMEDAY: This is O(n) in number of proxies. O(log n) solution using appropriate data structure? Version oldestProxyVersion = req.version; for(auto itr = self->proxyInfoMap.begin(); itr != self->proxyInfoMap.end(); ++itr) { - //TraceEvent("ResolveBatchProxyVersion", self->dbgid).detail("Proxy", itr->first).detail("Version", itr->second.lastVersion); + //TraceEvent("ResolveBatchProxyVersion", self->dbgid).detail("CommitProxy", itr->first).detail("Version", itr->second.lastVersion); if(itr->first.isValid()) { // Don't consider the first master request oldestProxyVersion = std::min(itr->second.lastVersion, oldestProxyVersion); } @@ -311,7 +311,7 @@ ACTOR Future resolverCore( ResolverInterface resolver, InitializeResolverRequest initReq) { - state Reference self( new Resolver(resolver.id(), initReq.proxyCount, initReq.resolverCount) ); + state Reference self(new Resolver(resolver.id(), initReq.commitProxyCount, initReq.resolverCount)); state ActorCollection actors(false); state Future doPollMetrics = self->resolverCount > 1 ? Void() : Future(Never()); actors.add( waitFailureServer(resolver.waitFailure.getFuture()) ); diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 367514ef0a..8a718982bc 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -733,7 +733,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR bool generateFearless = simple ? false : (minimumRegions > 1 || deterministicRandom()->random01() < 0.5); datacenters = simple ? 1 : ( generateFearless ? ( minimumReplication > 0 || deterministicRandom()->random01() < 0.5 ? 4 : 6 ) : deterministicRandom()->randomInt( 1, 4 ) ); if (deterministicRandom()->random01() < 0.25) db.desiredTLogCount = deterministicRandom()->randomInt(1,7); - if (deterministicRandom()->random01() < 0.25) db.proxyCount = deterministicRandom()->randomInt(1, 7); + if (deterministicRandom()->random01() < 0.25) db.commitProxyCount = deterministicRandom()->randomInt(1, 7); if (deterministicRandom()->random01() < 0.25) db.grvProxyCount = deterministicRandom()->randomInt(1, 4); if (deterministicRandom()->random01() < 0.25) db.resolverCount = deterministicRandom()->randomInt(1,7); int storage_engine_type = deterministicRandom()->randomInt(0, 4); @@ -770,7 +770,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR // set_config("memory-radixtree-beta"); if(simple) { db.desiredTLogCount = 1; - db.proxyCount = 1; + db.commitProxyCount = 1; db.grvProxyCount = 1; db.resolverCount = 1; } diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 7759c193a4..6a2f674c97 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -574,7 +574,7 @@ struct RolesInfo { *pMetricVersion = metricVersion; return roles.insert( std::make_pair(iface.address(), obj ))->second; } - JsonBuilderObject& addRole(std::string const& role, MasterProxyInterface& iface, EventMap const& metrics) { + JsonBuilderObject& addRole(std::string const& role, CommitProxyInterface& iface, EventMap const& metrics) { JsonBuilderObject obj; obj["id"] = iface.id().shortString(); obj["role"] = role; @@ -646,11 +646,10 @@ ACTOR static Future processStatusFetcher( WorkerEvents mMetrics, WorkerEvents nMetrics, WorkerEvents errors, WorkerEvents traceFileOpenErrors, WorkerEvents programStarts, std::map> processIssues, vector> storageServers, - vector> tLogs, - vector> proxies, - vector> grvProxies, - ServerCoordinators coordinators, Database cx, Optional configuration, - Optional healthyZone, std::set* incomplete_reasons) { + vector> tLogs, vector> commitProxies, + vector> grvProxies, ServerCoordinators coordinators, Database cx, + Optional configuration, Optional healthyZone, + std::set* incomplete_reasons) { state JsonBuilderObject processMap; @@ -736,9 +735,9 @@ ACTOR static Future processStatusFetcher( roles.addCoordinatorRole(coordinator); } - state std::vector>::iterator proxy; - for(proxy = proxies.begin(); proxy != proxies.end(); ++proxy) { - roles.addRole( "proxy", proxy->first, proxy->second ); + state std::vector>::iterator commit_proxy; + for (commit_proxy = commitProxies.begin(); commit_proxy != commitProxies.end(); ++commit_proxy) { + roles.addRole("commit_proxy", commit_proxy->first, commit_proxy->second); wait(yield()); } @@ -1064,14 +1063,14 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails // Add additional metadata for certain statuses if (mStatusCode == RecoveryStatus::recruiting_transaction_servers) { int requiredLogs = atoi( md.getValue("RequiredTLogs").c_str() ); - int requiredProxies = atoi( md.getValue("RequiredProxies").c_str() ); + int requiredProxies = atoi(md.getValue("RequiredCommitProxies").c_str()); int requiredGrvProxies = atoi(md.getValue("RequiredGrvProxies").c_str()); int requiredResolvers = atoi( md.getValue("RequiredResolvers").c_str() ); //int requiredProcesses = std::max(requiredLogs, std::max(requiredResolvers, requiredProxies)); //int requiredMachines = std::max(requiredLogs, 1); message["required_logs"] = requiredLogs; - message["required_proxies"] = requiredProxies; + message["required_commit_proxies"] = requiredProxies; message["required_grv_proxies"] = requiredGrvProxies; message["required_resolvers"] = requiredResolvers; } else if (mStatusCode == RecoveryStatus::locking_old_transaction_servers) { @@ -1669,9 +1668,11 @@ ACTOR static Future>> getTLogsAndMetri return results; } -ACTOR static Future>> getProxiesAndMetrics(Reference> db, std::unordered_map address_workers) { - vector> results = wait(getServerMetrics( - db->get().client.masterProxies, address_workers, std::vector{ "CommitLatencyMetrics", "CommitLatencyBands" })); +ACTOR static Future>> getCommitProxiesAndMetrics( + Reference> db, std::unordered_map address_workers) { + vector> results = + wait(getServerMetrics(db->get().client.commitProxies, address_workers, + std::vector{ "CommitLatencyMetrics", "CommitLatencyBands" })); return results; } @@ -1755,16 +1756,18 @@ ACTOR static Future workloadStatusFetcher(Reference> proxyStatFutures; + state vector> commitProxyStatFutures; state vector> grvProxyStatFutures; std::map workersMap; for (auto const& w : workers) { workersMap[w.interf.address()] = w; } - for (auto &p : db->get().client.masterProxies) { + for (auto& p : db->get().client.commitProxies) { auto worker = getWorker(workersMap, p.address()); if (worker.present()) - proxyStatFutures.push_back(timeoutError(worker.get().interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("ProxyMetrics"))), 1.0)); + commitProxyStatFutures.push_back(timeoutError( + worker.get().interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("ProxyMetrics"))), + 1.0)); else throw all_alternatives_failed(); // We need data from all proxies for this result to be trustworthy } @@ -1775,7 +1778,7 @@ ACTOR static Future workloadStatusFetcher(Reference proxyStats = wait(getAll(proxyStatFutures)); + state vector commitProxyStats = wait(getAll(commitProxyStatFutures)); state vector grvProxyStats = wait(getAll(grvProxyStatFutures)); StatusCounter txnStartOut; @@ -1798,14 +1801,14 @@ ACTOR static Future workloadStatusFetcher(Reference clusterGetStatus( getProcessIssuesAsMessages(workerIssues); state vector> storageServers; state vector> tLogs; - state vector> proxies; + state vector> commit_proxies; state vector> grvProxies; state JsonBuilderObject qos; state JsonBuilderObject data_overlay; @@ -2504,7 +2507,8 @@ ACTOR Future clusterGetStatus( state Future>>> storageServerFuture = errorOr(getStorageServersAndMetrics(cx, address_workers, rkWorker)); state Future>>> tLogFuture = errorOr(getTLogsAndMetrics(db, address_workers)); - state Future>>> proxyFuture = errorOr(getProxiesAndMetrics(db, address_workers)); + state Future>>> commitProxyFuture = + errorOr(getCommitProxiesAndMetrics(db, address_workers)); state Future>>> grvProxyFuture = errorOr(getGrvProxiesAndMetrics(db, address_workers)); state int minReplicasRemaining = -1; @@ -2587,13 +2591,13 @@ ACTOR Future clusterGetStatus( messages.push_back(JsonBuilder::makeMessage("log_servers_error", "Timed out trying to retrieve log servers.")); } - // ...also proxies - ErrorOr>> _proxies = wait(proxyFuture); - if (_proxies.present()) { - proxies = _proxies.get(); - } - else { - messages.push_back(JsonBuilder::makeMessage("proxies_error", "Timed out trying to retrieve proxies.")); + // ...also commit proxies + ErrorOr>> _commit_proxies = wait(commitProxyFuture); + if (_commit_proxies.present()) { + commit_proxies = _commit_proxies.get(); + } else { + messages.push_back( + JsonBuilder::makeMessage("commit_proxies_error", "Timed out trying to retrieve commit proxies.")); } // ...also grv proxies @@ -2614,12 +2618,10 @@ ACTOR Future clusterGetStatus( statusObj["layers"] = layers; } - JsonBuilderObject processStatus = wait(processStatusFetcher(db, workers, pMetrics, mMetrics, networkMetrics, - latestError, traceFileOpenErrors, programStarts, - processIssues, storageServers, tLogs, proxies, - grvProxies, coordinators, cx, configuration, - loadResult.present() ? loadResult.get().healthyZone : Optional(), - &status_incomplete_reasons)); + JsonBuilderObject processStatus = wait(processStatusFetcher( + db, workers, pMetrics, mMetrics, networkMetrics, latestError, traceFileOpenErrors, programStarts, + processIssues, storageServers, tLogs, commit_proxies, grvProxies, coordinators, cx, configuration, + loadResult.present() ? loadResult.get().healthyZone : Optional(), &status_incomplete_reasons)); statusObj["processes"] = processStatus; statusObj["clients"] = clientStatusFetcher(clientStatus); diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index bf92f6afe5..74768844df 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -46,7 +46,7 @@ struct WorkerInterface { LocalityData locality; RequestStream< struct InitializeTLogRequest > tLog; RequestStream< struct RecruitMasterRequest > master; - RequestStream< struct InitializeMasterProxyRequest > masterProxy; + RequestStream commitProxy; RequestStream< struct InitializeGrvProxyRequest > grvProxy; RequestStream< struct InitializeDataDistributorRequest > dataDistributor; RequestStream< struct InitializeRatekeeperRequest > ratekeeper; @@ -81,7 +81,7 @@ struct WorkerInterface { clientInterface.initEndpoints(); tLog.getEndpoint( TaskPriority::Worker ); master.getEndpoint( TaskPriority::Worker ); - masterProxy.getEndpoint( TaskPriority::Worker ); + commitProxy.getEndpoint(TaskPriority::Worker); grvProxy.getEndpoint( TaskPriority::Worker ); resolver.getEndpoint( TaskPriority::Worker ); logRouter.getEndpoint( TaskPriority::Worker ); @@ -93,7 +93,10 @@ struct WorkerInterface { template void serialize(Ar& ar) { - serializer(ar, clientInterface, locality, tLog, master, masterProxy, grvProxy, dataDistributor, ratekeeper, resolver, storage, logRouter, debugPing, coordinationPing, waitFailure, setMetricsRate, eventLogRequest, traceBatchDumpRequest, testerInterface, diskStoreRequest, execReq, workerSnapReq, backup, updateServerDBInfo); + serializer(ar, clientInterface, locality, tLog, master, commitProxy, grvProxy, dataDistributor, ratekeeper, + resolver, storage, logRouter, debugPing, coordinationPing, waitFailure, setMetricsRate, + eventLogRequest, traceBatchDumpRequest, testerInterface, diskStoreRequest, execReq, workerSnapReq, + backup, updateServerDBInfo); } }; @@ -180,7 +183,7 @@ struct RegisterMasterRequest { UID id; LocalityData mi; LogSystemConfig logSystemConfig; - std::vector masterProxies; + std::vector commitProxies; std::vector grvProxies; std::vector resolvers; DBRecoveryCount recoveryCount; @@ -199,7 +202,7 @@ struct RegisterMasterRequest { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, id, mi, logSystemConfig, masterProxies, grvProxies, resolvers, recoveryCount, registrationCount, + serializer(ar, id, mi, logSystemConfig, commitProxies, grvProxies, resolvers, recoveryCount, registrationCount, configuration, priorCommittedLogServers, recoveryState, recoveryStalled, reply); } }; @@ -209,7 +212,7 @@ struct RecruitFromConfigurationReply { std::vector backupWorkers; std::vector tLogs; std::vector satelliteTLogs; - std::vector masterProxies; + std::vector commitProxies; std::vector grvProxies; std::vector resolvers; std::vector storageServers; @@ -221,7 +224,7 @@ struct RecruitFromConfigurationReply { template void serialize(Ar& ar) { - serializer(ar, tLogs, satelliteTLogs, masterProxies, grvProxies, resolvers, storageServers, oldLogRouters, dcId, + serializer(ar, tLogs, satelliteTLogs, commitProxies, grvProxies, resolvers, storageServers, oldLogRouters, dcId, satelliteFallback, backupWorkers); } }; @@ -433,13 +436,13 @@ struct RecruitMasterRequest { } }; -struct InitializeMasterProxyRequest { +struct InitializeCommitProxyRequest { constexpr static FileIdentifier file_identifier = 10344153; MasterInterface master; uint64_t recoveryCount; Version recoveryTransactionVersion; bool firstProxy; - ReplyPromise reply; + ReplyPromise reply; template void serialize(Ar& ar) { @@ -488,13 +491,13 @@ struct InitializeRatekeeperRequest { struct InitializeResolverRequest { constexpr static FileIdentifier file_identifier = 7413317; uint64_t recoveryCount; - int proxyCount; + int commitProxyCount; int resolverCount; ReplyPromise reply; template void serialize(Ar& ar) { - serializer(ar, recoveryCount, proxyCount, resolverCount, reply); + serializer(ar, recoveryCount, commitProxyCount, resolverCount, reply); } }; @@ -672,7 +675,7 @@ struct Role { static const Role STORAGE_SERVER; static const Role TRANSACTION_LOG; static const Role SHARED_TRANSACTION_LOG; - static const Role MASTER_PROXY; + static const Role COMMIT_PROXY; static const Role GRV_PROXY; static const Role MASTER; static const Role RESOLVER; @@ -735,7 +738,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, StorageServerIn Reference connFile ); // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID ACTOR Future masterServer(MasterInterface mi, Reference> db, Reference>> ccInterface, ServerCoordinators serverCoordinators, LifetimeToken lifetime, bool forceRecovery); -ACTOR Future masterProxyServer(MasterProxyInterface proxy, InitializeMasterProxyRequest req, +ACTOR Future commitProxyServer(CommitProxyInterface proxy, InitializeCommitProxyRequest req, Reference> db, std::string whitelistBinPaths); ACTOR Future grvProxyServer(GrvProxyInterface proxy, InitializeGrvProxyRequest req, Reference> db); ACTOR Future tLog(IKeyValueStore* persistentData, IDiskQueue* persistentQueue, diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 5c4bd54279..514f7e5122 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -542,9 +542,9 @@ static void printUsage( const char *name, bool devhelp ) { " The default value is 2GiB. When specified without a unit,\n" " MiB is assumed.\n"); printf(" -c CLASS, --class CLASS\n" - " Machine class (valid options are storage, transaction,\n" - " resolution, proxy, master, test, unset, stateless, log, router,\n" - " and cluster_controller).\n"); + " Machine class (valid options are storage, transaction,\n" + " resolution, grv_proxy, proxy, master, test, unset, stateless, log, router,\n" + " and cluster_controller).\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -2028,7 +2028,8 @@ int main(int argc, char* argv[]) { } static_assert( LBLocalityData::Present, "Storage server interface should be load balanced" ); - static_assert( LBLocalityData::Present, "Master proxy interface should be load balanced" ); + static_assert(LBLocalityData::Present, "Commit proxy interface should be load balanced"); + static_assert(LBLocalityData::Present, "GRV proxy interface should be load balanced"); static_assert( LBLocalityData::Present, "TLog interface should be load balanced" ); static_assert( !LBLocalityData::Present, "Master interface should not be load balanced" ); } diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index ce5c993d77..22b9495310 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -207,8 +207,8 @@ struct MasterData : NonCopyable, ReferenceCounted { return maxLocality + 1; } - std::vector masterProxies; - std::vector provisionalMasterProxies; + std::vector commitProxies; + std::vector provisionalCommitProxies; std::vector grvProxies; std::vector provisionalGrvProxies; std::vector resolvers; @@ -283,21 +283,24 @@ struct MasterData : NonCopyable, ReferenceCounted { ~MasterData() { if(txnStateStore) txnStateStore->close(); } }; -ACTOR Future newMasterProxies( Reference self, RecruitFromConfigurationReply recr ) { - vector> initializationReplies; - for( int i = 0; i < recr.masterProxies.size(); i++ ) { - InitializeMasterProxyRequest req; +ACTOR Future newCommitProxies(Reference self, RecruitFromConfigurationReply recr) { + vector> initializationReplies; + for (int i = 0; i < recr.commitProxies.size(); i++) { + InitializeCommitProxyRequest req; req.master = self->myInterface; req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; req.recoveryTransactionVersion = self->recoveryTransactionVersion; req.firstProxy = i == 0; - TraceEvent("MasterProxyReplies",self->dbgid).detail("WorkerID", recr.masterProxies[i].id()); - initializationReplies.push_back( transformErrors( throwErrorOr( recr.masterProxies[i].masterProxy.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) ); + TraceEvent("CommitProxyReplies", self->dbgid).detail("WorkerID", recr.commitProxies[i].id()); + initializationReplies.push_back( + transformErrors(throwErrorOr(recr.commitProxies[i].commitProxy.getReplyUnlessFailedFor( + req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + master_recovery_failed())); } - vector newRecruits = wait( getAll( initializationReplies ) ); - // It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->proxies[0] is the firstProxy. - self->masterProxies = newRecruits; + vector newRecruits = wait(getAll(initializationReplies)); + // It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->proxies[0] is the firstCommitProxy. + self->commitProxies = newRecruits; return Void(); } @@ -322,7 +325,7 @@ ACTOR Future newResolvers( Reference self, RecruitFromConfigur for( int i = 0; i < recr.resolvers.size(); i++ ) { InitializeResolverRequest req; req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; - req.proxyCount = recr.masterProxies.size(); + req.commitProxyCount = recr.commitProxies.size(); req.resolverCount = recr.resolvers.size(); TraceEvent("ResolverReplies",self->dbgid).detail("WorkerID", recr.resolvers[i].id()); initializationReplies.push_back( transformErrors( throwErrorOr( recr.resolvers[i].resolver.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) ); @@ -426,15 +429,15 @@ ACTOR Future newSeedServers( Reference self, RecruitFromConfig return Void(); } -Future waitProxyFailure( vector const& proxies ) { +Future waitCommitProxyFailure(vector const& commitProxies) { std::vector> failed; - for (auto proxy : proxies) { - failed.push_back(waitFailureClient(proxy.waitFailure, SERVER_KNOBS->TLOG_TIMEOUT, + for (auto commitProxy : commitProxies) { + failed.push_back(waitFailureClient(commitProxy.waitFailure, SERVER_KNOBS->TLOG_TIMEOUT, -SERVER_KNOBS->TLOG_TIMEOUT / SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, /*trace=*/true)); } ASSERT( failed.size() >= 1 ); - return tagError(quorum( failed, 1 ), master_proxy_failed()); + return tagError(quorum(failed, 1), commit_proxy_failed()); } Future waitGrvProxyFailure( vector const& grvProxies ) { @@ -499,14 +502,14 @@ ACTOR Future updateLogsValue( Reference self, Database cx ) { } Future sendMasterRegistration(MasterData* self, LogSystemConfig const& logSystemConfig, - vector proxies, vector grvProxies, + vector proxies, vector grvProxies, vector resolvers, DBRecoveryCount recoveryCount, vector priorCommittedLogServers) { RegisterMasterRequest masterReq; masterReq.id = self->myInterface.id(); masterReq.mi = self->myInterface.locality; masterReq.logSystemConfig = logSystemConfig; - masterReq.masterProxies = proxies; + masterReq.commitProxies = proxies; masterReq.grvProxies = grvProxies; masterReq.resolvers = resolvers; masterReq.recoveryCount = recoveryCount; @@ -536,14 +539,14 @@ ACTOR Future updateRegistration( Reference self, ReferencecstateUpdated.isSet()) { - wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->provisionalMasterProxies, + wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->provisionalCommitProxies, self->provisionalGrvProxies, self->resolvers, self->cstate.myDBState.recoveryCount, self->cstate.prevDBState.getPriorCommittedLogServers())); } else { updateLogsKey = updateLogsValue(self, cx); - wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->masterProxies, self->grvProxies, self->resolvers, - self->cstate.myDBState.recoveryCount, vector())); + wait(sendMasterRegistration(self.getPtr(), logSystemConfig, self->commitProxies, self->grvProxies, + self->resolvers, self->cstate.myDBState.recoveryCount, vector())); } } } @@ -551,14 +554,15 @@ ACTOR Future updateRegistration( Reference self, Reference> provisionalMaster( Reference parent, Future activate ) { wait(activate); - // Register a fake master proxy (to be provided right here) to make ourselves available to clients - parent->provisionalMasterProxies = vector(1); - parent->provisionalMasterProxies[0].provisional = true; - parent->provisionalMasterProxies[0].initEndpoints(); + // Register a fake commit proxy (to be provided right here) to make ourselves available to clients + parent->provisionalCommitProxies = vector(1); + parent->provisionalCommitProxies[0].provisional = true; + parent->provisionalCommitProxies[0].initEndpoints(); parent->provisionalGrvProxies = vector(1); parent->provisionalGrvProxies[0].provisional = true; parent->provisionalGrvProxies[0].initEndpoints(); - state Future waitMasterProxyFailure = waitFailureServer(parent->provisionalMasterProxies[0].waitFailure.getFuture()); + state Future waitCommitProxyFailure = + waitFailureServer(parent->provisionalCommitProxies[0].waitFailure.getFuture()); state Future waitGrvProxyFailure = waitFailureServer(parent->provisionalGrvProxies[0].waitFailure.getFuture()); parent->registrationTrigger.trigger(); @@ -567,8 +571,8 @@ ACTOR Future> provisionalMaster( Reference metadataVersion = parent->txnStateStore->readValue(metadataVersionKey).get(); - // We respond to a minimal subset of the master proxy protocol. Our sole purpose is to receive a single write-only transaction - // which might repair our configuration, and return it. + // We respond to a minimal subset of the commit proxy protocol. Our sole purpose is to receive a single write-only + // transaction which might repair our configuration, and return it. loop choose { when ( GetReadVersionRequest req = waitNext( parent->provisionalGrvProxies[0].getConsistentReadVersion.getFuture() ) ) { if ( req.flags & GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY && parent->lastEpochEnd ) { @@ -580,7 +584,7 @@ ACTOR Future> provisionalMaster( ReferenceprovisionalMasterProxies[0].commit.getFuture() ) ) { + when(CommitTransactionRequest req = waitNext(parent->provisionalCommitProxies[0].commit.getFuture())) { req.reply.send(Never()); // don't reply (clients always get commit_unknown_result) auto t = &req.transaction; if (t->read_snapshot == parent->lastEpochEnd && //< So no transactions can fall between the read snapshot and the recovery transaction this (might) be merged with @@ -600,10 +604,11 @@ ACTOR Future> provisionalMaster( ReferenceprovisionalMasterProxies[0].getKeyServersLocations.getFuture() ) ) { + when(GetKeyServerLocationsRequest req = + waitNext(parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture())) { req.reply.send(Never()); } - when ( wait( waitMasterProxyFailure ) ) { throw worker_removed(); } + when(wait(waitCommitProxyFailure)) { throw worker_removed(); } when ( wait( waitGrvProxyFailure ) ) { throw worker_removed(); } } } @@ -634,8 +639,8 @@ ACTOR Future>> recruitEverything( Refere .detail("Status", RecoveryStatus::names[RecoveryStatus::recruiting_transaction_servers]) .detail("RequiredTLogs", self->configuration.tLogReplicationFactor) .detail("DesiredTLogs", self->configuration.getDesiredLogs()) - .detail("RequiredProxies", 1) - .detail("DesiredProxies", self->configuration.getDesiredProxies()) + .detail("RequiredCommitProxies", 1) + .detail("DesiredCommitProxies", self->configuration.getDesiredCommitProxies()) .detail("RequiredGrvProxies", 1) .detail("DesiredGrvProxies", self->configuration.getDesiredGrvProxies()) .detail("RequiredResolvers", 1) @@ -664,20 +669,20 @@ ACTOR Future>> recruitEverything( Refere self->backupWorkers.swap(recruits.backupWorkers); TraceEvent("MasterRecoveryState", self->dbgid) - .detail("StatusCode", RecoveryStatus::initializing_transaction_servers) - .detail("Status", RecoveryStatus::names[RecoveryStatus::initializing_transaction_servers]) - .detail("MasterProxies", recruits.masterProxies.size()) - .detail("GrvProxies", recruits.grvProxies.size()) - .detail("TLogs", recruits.tLogs.size()) - .detail("Resolvers", recruits.resolvers.size()) - .detail("BackupWorkers", self->backupWorkers.size()) - .trackLatest("MasterRecoveryState"); + .detail("StatusCode", RecoveryStatus::initializing_transaction_servers) + .detail("Status", RecoveryStatus::names[RecoveryStatus::initializing_transaction_servers]) + .detail("CommitProxies", recruits.commitProxies.size()) + .detail("GrvProxies", recruits.grvProxies.size()) + .detail("TLogs", recruits.tLogs.size()) + .detail("Resolvers", recruits.resolvers.size()) + .detail("BackupWorkers", self->backupWorkers.size()) + .trackLatest("MasterRecoveryState"); // Actually, newSeedServers does both the recruiting and initialization of the seed servers; so if this is a brand new database we are sort of lying that we are // past the recruitment phase. In a perfect world we would split that up so that the recruitment part happens above (in parallel with recruiting the transaction servers?). wait( newSeedServers( self, recruits, seedServers ) ); state vector> confChanges; - wait(newMasterProxies(self, recruits) && newGrvProxies(self, recruits) && newResolvers(self, recruits) && + wait(newCommitProxies(self, recruits) && newGrvProxies(self, recruits) && newResolvers(self, recruits) && newTLogServers(self, recruits, oldLogSystem, &confChanges)); return confChanges; } @@ -803,7 +808,7 @@ ACTOR Future sendInitialCommitToResolvers( Reference self ) { state int64_t dataOutstanding = 0; state std::vector endpoints; - for(auto& it : self->masterProxies) { + for (auto& it : self->commitProxies) { endpoints.push_back(it.txnState.getEndpoint()); } @@ -1042,8 +1047,7 @@ ACTOR Future getVersion(Reference self, GetCommitVersionReques ACTOR Future provideVersions(Reference self) { state ActorCollection versionActors(false); - for (auto& p : self->masterProxies) - self->lastProxyVersionReplies[p.id()] = ProxyVersionReplies(); + for (auto& p : self->commitProxies) self->lastProxyVersionReplies[p.id()] = ProxyVersionReplies(); loop { choose { @@ -1183,8 +1187,7 @@ ACTOR Future resolutionBalancing(Reference self) { // TraceEvent("KeyResolver").detail("Range", it.range()).detail("Value", it.value()); self->resolverChangesVersion = self->version + 1; - for (auto& p : self->masterProxies) - self->resolverNeedingChanges.insert(p.id()); + for (auto& p : self->commitProxies) self->resolverNeedingChanges.insert(p.id()); self->resolverChanges.set(movedRanges); } catch( Error&e ) { if(e.code() != error_code_operation_failed) @@ -1199,7 +1202,7 @@ static std::set const& normalMasterErrors() { if (s.empty()) { s.insert( error_code_tlog_stopped ); s.insert( error_code_master_tlog_failed ); - s.insert( error_code_master_proxy_failed ); + s.insert(error_code_commit_proxy_failed); s.insert( error_code_grv_proxy_failed ); s.insert( error_code_master_resolver_failed ); s.insert( error_code_master_backup_worker_failed ); @@ -1544,8 +1547,8 @@ ACTOR Future masterCore( Reference self ) { recoverAndEndEpoch.cancel(); - ASSERT(self->masterProxies.size() <= self->configuration.getDesiredProxies()); - ASSERT(self->masterProxies.size() >= 1); + ASSERT(self->commitProxies.size() <= self->configuration.getDesiredCommitProxies()); + ASSERT(self->commitProxies.size() >= 1); ASSERT(self->grvProxies.size() <= self->configuration.getDesiredGrvProxies()); ASSERT(self->grvProxies.size() >= 1); ASSERT( self->resolvers.size() <= self->configuration.getDesiredResolvers() ); @@ -1620,10 +1623,10 @@ ACTOR Future masterCore( Reference self ) { tr.read_snapshot = self->recoveryTransactionVersion; // lastEpochEnd would make more sense, but isn't in the initial window of the resolver(s) TraceEvent("MasterRecoveryCommit", self->dbgid); - state Future> recoveryCommit = self->masterProxies[0].commit.tryGetReply(recoveryCommitRequest); + state Future> recoveryCommit = self->commitProxies[0].commit.tryGetReply(recoveryCommitRequest); self->addActor.send( self->logSystem->onError() ); self->addActor.send( waitResolverFailure( self->resolvers ) ); - self->addActor.send( waitProxyFailure( self->masterProxies) ); + self->addActor.send(waitCommitProxyFailure(self->commitProxies)); self->addActor.send( waitGrvProxyFailure( self->grvProxies ) ); self->addActor.send( provideVersions(self) ); self->addActor.send( serveLiveCommittedVersion(self) ); @@ -1758,7 +1761,7 @@ ACTOR Future masterServer( MasterInterface mi, Reference replaceInterface( StorageServer* self, StorageServerInterface loop { state Future infoChanged = self->db->onChange(); - state Reference proxies( new ProxyInfo(self->db->get().client.masterProxies) ); + state Reference proxies(new CommitProxyInfo(self->db->get().client.commitProxies)); choose { - when( GetStorageServerRejoinInfoReply _rep = wait( proxies->size() ? basicLoadBalance( proxies, &MasterProxyInterface::getStorageServerRejoinInfo, GetStorageServerRejoinInfoRequest(ssi.id(), ssi.locality.dcId()) ) : Never() ) ) { + when(GetStorageServerRejoinInfoReply _rep = + wait(proxies->size() + ? basicLoadBalance(proxies, &CommitProxyInterface::getStorageServerRejoinInfo, + GetStorageServerRejoinInfoRequest(ssi.id(), ssi.locality.dcId())) + : Never())) { state GetStorageServerRejoinInfoReply rep = _rep; try { tr.reset(); diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index 36c0a4d089..7ee360d825 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -114,13 +114,13 @@ ACTOR Future> broadcastDBInfoRequest(UpdateServerDBInfoReq } ACTOR static Future extractClientInfo( Reference> db, Reference> info ) { - state std::vector lastProxyUIDs; - state std::vector lastProxies; + state std::vector lastCommitProxyUIDs; + state std::vector lastCommitProxies; state std::vector lastGrvProxyUIDs; state std::vector lastGrvProxies; loop { ClientDBInfo ni = db->get().client; - shrinkProxyList(ni, lastProxyUIDs, lastProxies, lastGrvProxyUIDs, lastGrvProxies); + shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies); info->set( ni ); wait( db->onChange() ); } @@ -994,7 +994,7 @@ ACTOR Future workerServer( DUMPTOKEN(recruited.clientInterface.profiler); DUMPTOKEN(recruited.tLog); DUMPTOKEN(recruited.master); - DUMPTOKEN(recruited.masterProxy); + DUMPTOKEN(recruited.commitProxy); DUMPTOKEN(recruited.grvProxy); DUMPTOKEN(recruited.resolver); DUMPTOKEN(recruited.storage); @@ -1368,15 +1368,15 @@ ACTOR Future workerServer( } else forwardPromise( req.reply, storageCache.get( req.reqId ) ); } - when( InitializeMasterProxyRequest req = waitNext(interf.masterProxy.getFuture()) ) { - MasterProxyInterface recruited; + when(InitializeCommitProxyRequest req = waitNext(interf.commitProxy.getFuture())) { + CommitProxyInterface recruited; recruited.processId = locality.processId(); recruited.provisional = false; recruited.initEndpoints(); std::map details; details["ForMaster"] = req.master.id().shortString(); - startRole( Role::MASTER_PROXY, recruited.id(), interf.id(), details ); + startRole(Role::COMMIT_PROXY, recruited.id(), interf.id(), details); DUMPTOKEN(recruited.commit); DUMPTOKEN(recruited.getConsistentReadVersion); @@ -1385,9 +1385,10 @@ ACTOR Future workerServer( DUMPTOKEN(recruited.waitFailure); DUMPTOKEN(recruited.txnState); - //printf("Recruited as masterProxyServer\n"); - errorForwarders.add( zombie(recruited, forwardError( errors, Role::MASTER_PROXY, recruited.id(), - masterProxyServer( recruited, req, dbInfo, whitelistBinPaths ) ) ) ); + // printf("Recruited as commitProxyServer\n"); + errorForwarders.add( + zombie(recruited, forwardError(errors, Role::COMMIT_PROXY, recruited.id(), + commitProxyServer(recruited, req, dbInfo, whitelistBinPaths)))); req.reply.send(recruited); } when( InitializeGrvProxyRequest req = waitNext(interf.grvProxy.getFuture()) ) { @@ -1857,7 +1858,7 @@ const Role Role::WORKER("Worker", "WK", false); const Role Role::STORAGE_SERVER("StorageServer", "SS"); const Role Role::TRANSACTION_LOG("TLog", "TL"); const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL", false); -const Role Role::MASTER_PROXY("MasterProxyServer", "MP"); +const Role Role::COMMIT_PROXY("CommitProxyServer", "CP"); const Role Role::GRV_PROXY("GrvProxyServer", "GP"); const Role Role::MASTER("MasterServer", "MS"); const Role Role::RESOLVER("Resolver", "RV"); diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp index 1ed5f484b1..f1ab535116 100644 --- a/fdbserver/workloads/ConsistencyCheck.actor.cpp +++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp @@ -365,9 +365,9 @@ struct ConsistencyCheckWorkload : TestWorkload } } - //Get a list of storage servers from the master and compares them with the TLogs. - //If this is a quiescent check, then each master proxy needs to respond, otherwise only one needs to respond. - //Returns false if there is a failure (in this case, keyServersPromise will never be set) + // Get a list of storage servers from the master and compares them with the TLogs. + // If this is a quiescent check, then each commit proxy needs to respond, otherwise only one needs to respond. + // Returns false if there is a failure (in this case, keyServersPromise will never be set) ACTOR Future getKeyServers(Database cx, ConsistencyCheckWorkload *self, Promise>>> keyServersPromise) { state std::vector>> keyServers; @@ -380,13 +380,14 @@ struct ConsistencyCheckWorkload : TestWorkload state Span span(deterministicRandom()->randomUniqueID(), "WL:ConsistencyCheck"_loc); while (begin < end) { - state Reference proxyInfo = wait(cx->getMasterProxiesFuture(false)); + state Reference commitProxyInfo = wait(cx->getCommitProxiesFuture(false)); keyServerLocationFutures.clear(); - for (int i = 0; i < proxyInfo->size(); i++) + for (int i = 0; i < commitProxyInfo->size(); i++) keyServerLocationFutures.push_back( - proxyInfo->get(i, &MasterProxyInterface::getKeyServersLocations) + commitProxyInfo->get(i, &CommitProxyInterface::getKeyServersLocations) .getReplyUnlessFailedFor( - GetKeyServerLocationsRequest(span.context, begin, end, limitKeyServers, false, Arena()), 2, 0)); + GetKeyServerLocationsRequest(span.context, begin, end, limitKeyServers, false, Arena()), 2, + 0)); state bool keyServersInsertedForThisIteration = false; choose { @@ -399,8 +400,9 @@ struct ConsistencyCheckWorkload : TestWorkload //If performing quiescent check, then all master proxies should be reachable. Otherwise, only one needs to be reachable if (self->performQuiescentChecks && !shards.present()) { - TraceEvent("ConsistencyCheck_MasterProxyUnavailable").detail("MasterProxyID", proxyInfo->getId(i)); - self->testFailure("Master proxy unavailable"); + TraceEvent("ConsistencyCheck_CommitProxyUnavailable") + .detail("CommitProxyID", commitProxyInfo->getId(i)); + self->testFailure("Commit proxy unavailable"); return false; } @@ -1461,11 +1463,20 @@ struct ConsistencyCheckWorkload : TestWorkload return false; } - // Check proxy - ProcessClass::Fitness bestProxyFitness = getBestAvailableFitness(dcToNonExcludedClassTypes[masterDcId], ProcessClass::Proxy); - for (const auto& masterProxy : db.client.masterProxies) { - if (!nonExcludedWorkerProcessMap.count(masterProxy.address()) || nonExcludedWorkerProcessMap[masterProxy.address()].processClass.machineClassFitness(ProcessClass::Proxy) != bestProxyFitness) { - TraceEvent("ConsistencyCheck_ProxyNotBest").detail("BestProxyFitness", bestProxyFitness).detail("ExistingMasterProxyFitness", nonExcludedWorkerProcessMap.count(masterProxy.address()) ? nonExcludedWorkerProcessMap[masterProxy.address()].processClass.machineClassFitness(ProcessClass::Proxy) : -1); + // Check commit proxy + ProcessClass::Fitness bestCommitProxyFitness = + getBestAvailableFitness(dcToNonExcludedClassTypes[masterDcId], ProcessClass::CommitProxy); + for (const auto& commitProxy : db.client.commitProxies) { + if (!nonExcludedWorkerProcessMap.count(commitProxy.address()) || + nonExcludedWorkerProcessMap[commitProxy.address()].processClass.machineClassFitness( + ProcessClass::CommitProxy) != bestCommitProxyFitness) { + TraceEvent("ConsistencyCheck_CommitProxyNotBest") + .detail("BestCommitProxyFitness", bestCommitProxyFitness) + .detail("ExistingCommitProxyFitness", + nonExcludedWorkerProcessMap.count(commitProxy.address()) + ? nonExcludedWorkerProcessMap[commitProxy.address()].processClass.machineClassFitness( + ProcessClass::CommitProxy) + : -1); return false; } } diff --git a/fdbserver/workloads/Rollback.actor.cpp b/fdbserver/workloads/Rollback.actor.cpp index aab947efe6..2f4b7549ae 100644 --- a/fdbserver/workloads/Rollback.actor.cpp +++ b/fdbserver/workloads/Rollback.actor.cpp @@ -62,13 +62,13 @@ struct RollbackWorkload : TestWorkload { ACTOR Future simulateFailure( Database cx, RollbackWorkload* self ) { state ServerDBInfo system = self->dbInfo->get(); auto tlogs = system.logSystemConfig.allPresentLogs(); - - if( tlogs.empty() || system.client.masterProxies.empty() ) { + + if (tlogs.empty() || system.client.commitProxies.empty()) { TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "No tlogs in System Map"); return Void(); } - state MasterProxyInterface proxy = deterministicRandom()->randomChoice( system.client.masterProxies); + state CommitProxyInterface proxy = deterministicRandom()->randomChoice(system.client.commitProxies); int utIndex = deterministicRandom()->randomInt(0, tlogs.size()); state NetworkAddress uncloggedTLog = tlogs[utIndex].address(); @@ -81,8 +81,8 @@ struct RollbackWorkload : TestWorkload { } TraceEvent("AttemptingToTriggerRollback") - .detail("Proxy", proxy.address()) - .detail("UncloggedTLog", uncloggedTLog); + .detail("CommitProxy", proxy.address()) + .detail("UncloggedTLog", uncloggedTLog); for (int t = 0; t < tlogs.size(); t++) { if (t != utIndex) { diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.actor.cpp index 5eba5fd94f..de87ddec1c 100644 --- a/fdbserver/workloads/TargetedKill.actor.cpp +++ b/fdbserver/workloads/TargetedKill.actor.cpp @@ -87,19 +87,17 @@ struct TargetedKillWorkload : TestWorkload { NetworkAddress machine; if( self->machineToKill == "master" ) { machine = self->dbInfo->get().master.address(); - } - else if( self->machineToKill == "masterproxy" ) { - auto proxies = cx->getMasterProxies(false); + } else if (self->machineToKill == "commitproxy") { + auto proxies = cx->getCommitProxies(false); int o = deterministicRandom()->randomInt(0, proxies->size()); for( int i = 0; i < proxies->size(); i++) { - MasterProxyInterface mpi = proxies->getInterface(o); + CommitProxyInterface mpi = proxies->getInterface(o); machine = mpi.address(); if(machine != self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress()) break; o = ++o%proxies->size(); } - } - else if( self->machineToKill == "grvproxy" ) { + } else if (self->machineToKill == "grvproxy") { auto grvProxies = cx->getGrvProxies(false); int o = deterministicRandom()->randomInt(0, grvProxies->size()); for( int i = 0; i < grvProxies->size(); i++) { @@ -109,8 +107,7 @@ struct TargetedKillWorkload : TestWorkload { break; o = ++o%grvProxies->size(); } - } - else if( self->machineToKill == "tlog" ) { + } else if (self->machineToKill == "tlog") { auto tlogs = self->dbInfo->get().logSystemConfig.allPresentLogs(); int o = deterministicRandom()->randomInt(0, tlogs.size()); for( int i = 0; i < tlogs.size(); i++) { @@ -120,8 +117,8 @@ struct TargetedKillWorkload : TestWorkload { break; o = ++o%tlogs.size(); } - } - else if( self->machineToKill == "storage" || self->machineToKill == "ss" || self->machineToKill == "storageserver" ) { + } else if (self->machineToKill == "storage" || self->machineToKill == "ss" || + self->machineToKill == "storageserver") { int o = deterministicRandom()->randomInt(0,storageServers.size()); for( int i = 0; i < storageServers.size(); i++) { StorageServerInterface ssi = storageServers[o]; @@ -130,8 +127,7 @@ struct TargetedKillWorkload : TestWorkload { break; o = ++o%storageServers.size(); } - } - else if( self->machineToKill == "clustercontroller" || self->machineToKill == "cc" ) { + } else if (self->machineToKill == "clustercontroller" || self->machineToKill == "cc") { machine = self->dbInfo->get().clusterInterface.getWorkers.getEndpoint().getPrimaryAddress(); } diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 040f8d865c..e746d9c18c 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -65,7 +65,7 @@ ERROR( database_locked, 1038, "Database is locked" ) ERROR( cluster_version_changed, 1039, "The protocol version of the cluster has changed" ) ERROR( external_client_already_loaded, 1040, "External client has already been loaded" ) ERROR( lookup_failed, 1041, "DNS lookup failed" ) -ERROR( proxy_memory_limit_exceeded, 1042, "Proxy commit memory limit exceeded" ) +ERROR( proxy_memory_limit_exceeded, 1042, "CommitProxy commit memory limit exceeded" ) ERROR( shutdown_in_progress, 1043, "Operation no longer supported due to shutdown" ) ERROR( serialization_failed, 1044, "Failed to deserialize an object" ) ERROR( connection_unreferenced, 1048, "No peer references for connection" ) @@ -89,12 +89,12 @@ ERROR( master_tlog_failed, 1205, "Master terminating because a TLog failed" ) ERROR( worker_recovery_failed, 1206, "Recovery of a worker process failed" ) ERROR( please_reboot, 1207, "Reboot of server process requested" ) ERROR( please_reboot_delete, 1208, "Reboot of server process requested, with deletion of state" ) -ERROR( master_proxy_failed, 1209, "Master terminating because a Proxy failed" ) +ERROR( commit_proxy_failed, 1209, "Master terminating because a Commit CommitProxy failed" ) ERROR( master_resolver_failed, 1210, "Master terminating because a Resolver failed" ) ERROR( server_overloaded, 1211, "Server is under too much load and cannot respond" ) ERROR( master_backup_worker_failed, 1212, "Master terminating because a backup worker failed") ERROR( tag_throttled, 1213, "Transaction tag is being throttled" ) -ERROR( grv_proxy_failed, 1214, "Master terminating because a GRV Proxy failed" ) +ERROR( grv_proxy_failed, 1214, "Master terminating because a GRV CommitProxy failed" ) // 15xx Platform errors ERROR( platform_error, 1500, "Platform error" ) diff --git a/tests/status/invalid_proc_addresses.json b/tests/status/invalid_proc_addresses.json index 5be40ba744..752d3ab41f 100644 --- a/tests/status/invalid_proc_addresses.json +++ b/tests/status/invalid_proc_addresses.json @@ -223,7 +223,7 @@ "roles" : [ { "id" : "f29c4c66f293d1b1", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "44950eb0b3d862c0", @@ -264,7 +264,7 @@ "roles" : [ { "id" : "175f5bed1f306159", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "e583f98ea591c52a", @@ -342,7 +342,7 @@ "roles" : [ { "id" : "c97dc5f2e372921b", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "bbb368082d582712", diff --git a/tests/status/local_6_machine_no_replicas_remain.json b/tests/status/local_6_machine_no_replicas_remain.json index 7460096af4..bfd55b2cb0 100644 --- a/tests/status/local_6_machine_no_replicas_remain.json +++ b/tests/status/local_6_machine_no_replicas_remain.json @@ -172,7 +172,7 @@ }, { "id" : "066a9f0089483a5f", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "d0809246b42910f8", @@ -213,7 +213,7 @@ "roles" : [ { "id" : "3fc3c3d9c9e3349d", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "656697882cc0e76e", @@ -254,7 +254,7 @@ "roles" : [ { "id" : "586d54237f6bf4c7", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "09a94118dc82393a", diff --git a/tests/status/separate_2_of_3_coordinators_remain.json b/tests/status/separate_2_of_3_coordinators_remain.json index 5e4b8ecfd6..6c8f8caade 100644 --- a/tests/status/separate_2_of_3_coordinators_remain.json +++ b/tests/status/separate_2_of_3_coordinators_remain.json @@ -130,7 +130,7 @@ }, { "id" : "9159f5bae811936d", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "4ef3ec0982dab9fe", @@ -171,7 +171,7 @@ "roles" : [ { "id" : "9d158fb102da025f", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "02fe9302ba499227", diff --git a/tests/status/separate_cannot_write_cluster_file.json b/tests/status/separate_cannot_write_cluster_file.json index 654651d797..1394ca43cb 100644 --- a/tests/status/separate_cannot_write_cluster_file.json +++ b/tests/status/separate_cannot_write_cluster_file.json @@ -140,7 +140,7 @@ }, { "id" : "00e48601e43045c9", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "0df71fd71bbc14ee", @@ -181,7 +181,7 @@ }, { "id" : "07b3f5362cfec06b", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "bb25c74aca56ccf7", @@ -222,7 +222,7 @@ "roles" : [ { "id" : "361d515d63a595ad", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "4022c037e26868ae", diff --git a/tests/status/separate_idle.json b/tests/status/separate_idle.json index 636703aa0f..9ef918ae0a 100644 --- a/tests/status/separate_idle.json +++ b/tests/status/separate_idle.json @@ -118,7 +118,7 @@ }, { "id" : "4989d9993ee37183", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "009709c84d97df4d", diff --git a/tests/status/separate_initializing.json b/tests/status/separate_initializing.json index aa552f3fe4..a24b155f46 100644 --- a/tests/status/separate_initializing.json +++ b/tests/status/separate_initializing.json @@ -113,7 +113,7 @@ }, { "id" : "4989d9993ee37183", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "009709c84d97df4d", diff --git a/tests/status/separate_no_database.json b/tests/status/separate_no_database.json index 9966754a44..b7009bd89b 100644 --- a/tests/status/separate_no_database.json +++ b/tests/status/separate_no_database.json @@ -154,7 +154,7 @@ }, { "id" : "ae9fe51db979dfd1", - "role" : "proxy" + "role" : "commit_proxy" } ], "version" : "3.0.0-PRERELEASE" diff --git a/tests/status/separate_not_enough_servers.json b/tests/status/separate_not_enough_servers.json index 5e3589544b..9ae3a07d80 100644 --- a/tests/status/separate_not_enough_servers.json +++ b/tests/status/separate_not_enough_servers.json @@ -121,7 +121,7 @@ }, { "id" : "20beaadaa554dee3", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "f0a33233db8e5f67", @@ -143,7 +143,7 @@ "description" : "Recruiting new transaction servers.", "name" : "recruiting_transaction_servers", "required_logs" : 3, - "required_proxies" : 1, + "required_commit_proxies" : 1, "required_grv_proxies" : 1, "required_resolvers" : 1 }, diff --git a/tests/status/single_process_too_many_config_params.json b/tests/status/single_process_too_many_config_params.json index 304f58b59b..875b9e245a 100644 --- a/tests/status/single_process_too_many_config_params.json +++ b/tests/status/single_process_too_many_config_params.json @@ -123,7 +123,7 @@ }, { "id" : "242e27cd68b21c05", - "role" : "proxy" + "role" : "commit_proxy" }, { "id" : "faf07cf91f0ab29d", From 224f23b0f8bcdfe458fb0340fa704007b83d77b2 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 11:45:02 -0700 Subject: [PATCH 083/458] Rely on MasterRecoveryState message since we only care about the current generation. --- fdbserver/Status.actor.cpp | 6 ++---- fdbserver/masterserver.actor.cpp | 2 -- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 57f3202b31..e4584710df 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1030,11 +1030,10 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails try { std::vector> futures; futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0)); - futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryFullyRecovered") ) ), 1.0)); futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0)); std::vector msgs = wait(getAll(futures)); - const TraceEventFields& md = msgs[2]; + const TraceEventFields& md = msgs[1]; int mStatusCode = md.getInt("StatusCode"); if (mStatusCode < 0 || mStatusCode >= RecoveryStatus::END) throw attribute_not_found(); @@ -1042,9 +1041,8 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]); *statusCode = mStatusCode; - const TraceEventFields& mLastRecoveryMsg = msgs[1]; std::string lastFullyRecoveredTimeS; - if (mLastRecoveryMsg.tryGetValue("Time", lastFullyRecoveredTimeS)) { + if (mStatusCode == RecoveryStatus::fully_recovered && md.tryGetValue("Time", lastFullyRecoveredTimeS)) { double lastFullyRecoveredTime = atof(lastFullyRecoveredTimeS.c_str()); // `lastFullyRecoveredTime` is the timestamp taken on master so the time interval calculated below may not // be accurate due to the clock skew across the network, but it's good enough for the purpose it's used. diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index 4803930bfc..ce5c993d77 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1276,8 +1276,6 @@ ACTOR Future trackTlogRecovery( Reference self, Referencedbgid) .detail("ActiveGenerations", 1) .trackLatest("MasterRecoveryGenerations"); From 22996284c7ff50465a05fac5819c7932b874eae2 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Wed, 2 Sep 2020 15:44:55 -0400 Subject: [PATCH 084/458] added changes to allow writing of last epoch end version to special keys when performing recovery due to snapshot --- fdbclient/SystemData.cpp | 1 - fdbclient/SystemData.h | 2 +- fdbserver/ApplyMetadataMutation.cpp | 3 ++ fdbserver/masterserver.actor.cpp | 10 ++++++ fdbserver/workloads/SnapTest.actor.cpp | 46 ++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 2 deletions(-) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 0650f580f9..b402ad99a7 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1063,5 +1063,4 @@ const KeyRangeRef testOnlyTxnStateStorePrefixRange( const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery"); const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1"); -const ValueRef writeRecoveryKeyFalse = LiteralStringRef("0"); const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 4006282708..ac078768c3 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -398,7 +398,7 @@ extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; // Snapshot + Incremental Restore extern const KeyRef writeRecoveryKey; -extern const ValueRef writeRecoveryKeyTrue, writeRecoveryKeyFalse; +extern const ValueRef writeRecoveryKeyTrue; extern const KeyRef snapshotEndVersionKey; #pragma clang diagnostic pop diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 23466ece9f..61e9000250 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -300,6 +300,9 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRefset(KeyValueRef(m.param1, m.param2)); confChange = true; TEST(true); // Recovering at a higher version. + } else if (m.param1 == writeRecoveryKey) { + TraceEvent("WriteRecoveryKeySet", dbgid); + if (!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2)); } } else if (m.param2.size() && m.param2[0] == systemKeys.begin[0] && m.type == MutationRef::ClearRange) { diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index 4f21fa4f0c..35f18c8757 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1518,6 +1518,16 @@ ACTOR Future masterCore( Reference self ) { CommitTransactionRef &tr = recoveryCommitRequest.transaction; int mmApplied = 0; // The number of mutations in tr.mutations that have been applied to the txnStateStore so far if (self->lastEpochEnd != 0) { + Optional snapRecoveryFlag = self->txnStateStore->readValue(writeRecoveryKey).get(); + TraceEvent("MasterRecoverySnap") + .detail("SnapRecoveryFlag", snapRecoveryFlag.present() ? snapRecoveryFlag.get().toString() : "N/A"); + if (snapRecoveryFlag.present()) { + BinaryWriter bw(Unversioned()); + tr.set(recoveryCommitRequest.arena, snapshotEndVersionKey, (bw << self->lastEpochEnd).toValue()); + // Clear the key so multiple recoveries will not overwrite the first version recorded + self->txnStateStore->clear(singleKeyRange(writeRecoveryKey)); + tr.clear(recoveryCommitRequest.arena, singleKeyRange(writeRecoveryKey)); + } if(self->forceRecovery) { BinaryWriter bw(Unversioned()); tr.set(recoveryCommitRequest.arena, killStorageKey, (bw << self->safeLocality).toValue()); diff --git a/fdbserver/workloads/SnapTest.actor.cpp b/fdbserver/workloads/SnapTest.actor.cpp index 85c5fbbd09..d8c9891e57 100644 --- a/fdbserver/workloads/SnapTest.actor.cpp +++ b/fdbserver/workloads/SnapTest.actor.cpp @@ -2,6 +2,7 @@ #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" +#include "fdbclient/SystemData.h" #include "fdbrpc/ContinuousSample.h" #include "fdbmonitor/SimpleIni.h" #include "fdbserver/Status.h" @@ -194,6 +195,21 @@ public: // workload functions // create even keys before the snapshot wait(self->_create_keys(cx, "snapKey")); } else if (self->testID == 1) { + state Reference tr1(new ReadYourWritesTransaction(cx)); + loop { + try { + tr1->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr1->setOption(FDBTransactionOptions::LOCK_AWARE); + state Optional val = wait(tr1->get(writeRecoveryKey)); + state Optional val2 = wait(tr1->get(snapshotEndVersionKey)); + TraceEvent("CheckSpecialKey1") + .detail("WriteRecoveryValue", val.present() ? val.get().toString() : "N/A") + .detail("EndVersionValue", val2.present() ? val2.get().toString() : "N/A"); + break; + } catch (Error& e) { + wait(tr1->onError(e)); + } + } // create a snapshot state double toDelay = fmod(deterministicRandom()->randomUInt32(), self->maxSnapDelay); TraceEvent("ToDelay").detail("Value", toDelay); @@ -221,6 +237,21 @@ public: // workload functions } } } + state Reference tr2(new ReadYourWritesTransaction(cx)); + loop { + try { + tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr2->setOption(FDBTransactionOptions::LOCK_AWARE); + state Optional val3 = wait(tr2->get(writeRecoveryKey)); + state Optional val4 = wait(tr2->get(snapshotEndVersionKey)); + TraceEvent("CheckSpecialKey2") + .detail("WriteRecoveryValue", val3.present() ? val3.get().toString() : "N/A") + .detail("EndVersionValue", val4.present() ? val4.get().toString() : "N/A"); + break; + } catch (Error& e) { + wait(tr2->onError(e)); + } + } CSimpleIni ini; ini.SetUnicode(); ini.LoadFile(self->restartInfoLocation.c_str()); @@ -243,6 +274,21 @@ public: // workload functions TraceEvent(SevWarnAlways, "BackupFailedSkippingRestoreCheck"); return Void(); } + state Reference tr3(new ReadYourWritesTransaction(cx)); + loop { + try { + tr3->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr3->setOption(FDBTransactionOptions::LOCK_AWARE); + state Optional val5 = wait(tr3->get(writeRecoveryKey)); + state Optional val6 = wait(tr3->get(snapshotEndVersionKey)); + TraceEvent("CheckSpecialKey3") + .detail("WriteRecoveryValue", val5.present() ? val5.get().toString() : "N/A") + .detail("EndVersionValue", val6.present() ? val6.get().toString() : "N/A"); + break; + } catch (Error& e) { + wait(tr3->onError(e)); + } + } state KeySelector begin = firstGreaterOrEqual(normalKeys.begin); state KeySelector end = firstGreaterOrEqual(normalKeys.end); state int cnt = 0; From d5fba9a69beda3a568f1c2e065f832a85055090f Mon Sep 17 00:00:00 2001 From: XiaoxiWang Date: Fri, 11 Sep 2020 19:10:35 +0000 Subject: [PATCH 085/458] add write-tag throttling --- fdbserver/Ratekeeper.actor.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index 3f38adf48e..f07f7b44fe 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -924,18 +924,17 @@ void tryAutoThrottleTag(RatekeeperData* self, TransactionTag tag, double rate, d void tryAutoThrottleTag(RatekeeperData* self, StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) { - // TODO: reasonable criteria for write satuation should be investigated in experiment - // if (ss.busiestWriteTag.present() && storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES && - // storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) { - // // write-saturated - // tryAutoThrottleTag(self, ss.busiestWriteTag.get(), ss.busiestWriteTagRate, - //ss.busiestWriteTagFractionalBusyness); } else - if (ss.busiestReadTag.present() && - (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES || - storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS)) { - // read saturated - tryAutoThrottleTag(self, ss.busiestReadTag.get(), ss.busiestReadTagRate, ss.busiestReadTagFractionalBusyness, - TagThrottledReason::BUSY_READ); + // NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most of situation, this works. + // More indicators besides SQ and NDV could be investigated in the future + if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES || storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) { + if(ss.busiestWriteTag.present()) { + tryAutoThrottleTag(self, ss.busiestWriteTag.get(), ss.busiestWriteTagRate, + ss.busiestWriteTagFractionalBusyness, TagThrottledReason::BUSY_WRITE); + } + if(ss.busiestReadTag.present()) { + tryAutoThrottleTag(self, ss.busiestReadTag.get(), ss.busiestReadTagRate, + ss.busiestReadTagFractionalBusyness, TagThrottledReason::BUSY_READ); + } } } From 62c81e03c7e842e3d8bfd2c4fdda11e56d2647f5 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Fri, 11 Sep 2020 15:28:54 -0400 Subject: [PATCH 086/458] changed incremental backup workload to have an option to check system keys for version --- .../workloads/IncrementalBackup.actor.cpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.actor.cpp index 2537a84c90..aea0b8350d 100644 --- a/fdbserver/workloads/IncrementalBackup.actor.cpp +++ b/fdbserver/workloads/IncrementalBackup.actor.cpp @@ -24,6 +24,7 @@ #include "fdbclient/BackupContainer.h" #include "fdbserver/workloads/workloads.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/serialize.h" struct IncrementalBackupWorkload : TestWorkload { @@ -32,12 +33,14 @@ struct IncrementalBackupWorkload : TestWorkload { FileBackupAgent backupAgent; bool submitOnly; bool restoreOnly; + bool checkBeginVersion; IncrementalBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/")); tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default")); submitOnly = getOption(options, LiteralStringRef("submitOnly"), false); restoreOnly = getOption(options, LiteralStringRef("restoreOnly"), false); + checkBeginVersion = getOption(options, LiteralStringRef("checkBeginVersion"), false); } virtual std::string description() { return "IncrementalBackup"; } @@ -73,11 +76,28 @@ struct IncrementalBackupWorkload : TestWorkload { if (self->restoreOnly) { state Reference backupContainer; state UID backupUID; + state Version beginVersion = invalidVersion; TraceEvent("IBackupRestoreAttempt"); wait(success(self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID))); - // TODO: add testing scenario for atomics and beginVersion - wait(success(self->backupAgent.restore(cx, cx, Key(self->tag.toString()), Key(backupContainer->getURL()), - true, -1, true, normalKeys, Key(), Key(), true, true))); + if (self->checkBeginVersion) { + TraceEvent("IBackupReadSystemKeys"); + state Reference tr(new ReadYourWritesTransaction(cx)); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + Optional versionValue = wait(tr->get(snapshotEndVersionKey)); + beginVersion = BinaryReader::fromStringRef(versionValue.get(), Unversioned()); + break; + } catch (Error& e) { + TraceEvent("IBackupReadSystemKeysError").error(e); + wait(tr->onError(e)); + } + } + } + wait( + success(self->backupAgent.restore(cx, cx, Key(self->tag.toString()), Key(backupContainer->getURL()), + true, -1, true, normalKeys, Key(), Key(), true, true, beginVersion))); TraceEvent("IBackupRestoreSuccess"); } return Void(); From 2619e4d3df7fd51cdbc2e479fab638c9b3025f81 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 13:39:16 -0700 Subject: [PATCH 087/458] Use version clock to mitigate network clock skew. --- fdbserver/Status.actor.cpp | 15 +++++++-------- fdbserver/masterserver.actor.cpp | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index e4584710df..2370742b13 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1024,9 +1024,9 @@ static JsonBuilderObject clientStatusFetcher(std::map recoveryStateStatusFetcher(WorkerDetails mWorker, int workerCount, std::set *incomplete_reasons, int* statusCode) { +ACTOR static Future recoveryStateStatusFetcher(Database cx, WorkerDetails mWorker, int workerCount, std::set *incomplete_reasons, int* statusCode) { state JsonBuilderObject message; - + state Transaction tr(cx); try { std::vector> futures; futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0)); @@ -1041,12 +1041,11 @@ ACTOR static Future recoveryStateStatusFetcher(WorkerDetails message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]); *statusCode = mStatusCode; - std::string lastFullyRecoveredTimeS; - if (mStatusCode == RecoveryStatus::fully_recovered && md.tryGetValue("Time", lastFullyRecoveredTimeS)) { - double lastFullyRecoveredTime = atof(lastFullyRecoveredTimeS.c_str()); - // `lastFullyRecoveredTime` is the timestamp taken on master so the time interval calculated below may not - // be accurate due to the clock skew across the network, but it's good enough for the purpose it's used. - message["time_since_last_fully_recovered_seconds"] = now() - lastFullyRecoveredTime; + if (mStatusCode == RecoveryStatus::fully_recovered) { + Version rv = wait(tr.getReadVersion()); + int64_t fullyRecoveredAtVersion = md.getInt64("FullyRecoveredAtVersion"); + double lastFullyRecoveredSecondsAgo = std::max(0, rv - fullyRecoveredAtVersion) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND; + message["time_since_last_fully_recovered_seconds"] = lastFullyRecoveredSecondsAgo; } else { message["time_since_last_fully_recovered_seconds"] = -1; } diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index ce5c993d77..8427c8119b 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1274,6 +1274,7 @@ ACTOR Future trackTlogRecovery( Reference self, Referencedbgid) .detail("StatusCode", RecoveryStatus::fully_recovered) .detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered]) + .detail("FullyRecoveredAtVersion", self->version); .trackLatest("MasterRecoveryState"); TraceEvent("MasterRecoveryGenerations", self->dbgid) From f2f335156059d1299f86099594e6665fde562755 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 13:44:17 -0700 Subject: [PATCH 088/458] Only report if the field FullyRecoveredAtVersion exists. --- fdbserver/Status.actor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 2370742b13..db1fa0f459 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1041,7 +1041,8 @@ ACTOR static Future recoveryStateStatusFetcher(Database cx, W message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]); *statusCode = mStatusCode; - if (mStatusCode == RecoveryStatus::fully_recovered) { + std::string fullyRecoveredAtVersion; + if (mStatusCode == RecoveryStatus::fully_recovered && md.tryGetValue("FullyRecoveredAtVersion", fullyRecoveredAtVersion)) { Version rv = wait(tr.getReadVersion()); int64_t fullyRecoveredAtVersion = md.getInt64("FullyRecoveredAtVersion"); double lastFullyRecoveredSecondsAgo = std::max(0, rv - fullyRecoveredAtVersion) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND; From 52bd86ad42e63178b0dc1d33d27435f8862cb491 Mon Sep 17 00:00:00 2001 From: XiaoxiWang Date: Fri, 11 Sep 2020 20:50:19 +0000 Subject: [PATCH 089/458] update knob --- fdbserver/Knobs.cpp | 3 ++- fdbserver/Knobs.h | 3 ++- fdbserver/Ratekeeper.actor.cpp | 2 +- fdbserver/storageserver.actor.cpp | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index 6122ca13ac..e40c8edefe 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -566,7 +566,8 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi init( BEHIND_CHECK_COUNT, 2 ); init( BEHIND_CHECK_VERSIONS, 5 * VERSIONS_PER_SECOND ); init( WAIT_METRICS_WRONG_SHARD_CHANCE, isSimulated ? 1.0 : 0.1 ); - init( MIN_TAG_PAGES_RATE, 1.0e4 ); if( randomize && BUGGIFY ) MIN_TAG_PAGES_RATE = 0; + init( MIN_TAG_READ_PAGES_RATE, 1.0e4 ); if( randomize && BUGGIFY ) MIN_TAG_READ_PAGES_RATE = 0; + init( MIN_TAG_WRITE_PAGES_RATE, 3200 ); if( randomize && BUGGIFY ) MIN_TAG_WRITE_PAGES_RATE = 0; init( TAG_MEASUREMENT_INTERVAL, 30.0 ); if( randomize && BUGGIFY ) TAG_MEASUREMENT_INTERVAL = 1.0; init( READ_COST_BYTE_FACTOR, 16384 ); if( randomize && BUGGIFY ) READ_COST_BYTE_FACTOR = 4096; init( PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS, true ); if( randomize && BUGGIFY ) PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS = false; diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h index e36de5f2eb..0f2e0aefd1 100644 --- a/fdbserver/Knobs.h +++ b/fdbserver/Knobs.h @@ -495,7 +495,8 @@ public: int BEHIND_CHECK_COUNT; int64_t BEHIND_CHECK_VERSIONS; double WAIT_METRICS_WRONG_SHARD_CHANCE; - int64_t MIN_TAG_PAGES_RATE; + int64_t MIN_TAG_READ_PAGES_RATE; + int64_t MIN_TAG_WRITE_PAGES_RATE; double TAG_MEASUREMENT_INTERVAL; int64_t READ_COST_BYTE_FACTOR; bool PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS; diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index f07f7b44fe..af464b4cf5 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -877,7 +877,7 @@ Future refreshStorageServerCommitCost(RatekeeperData* self) { maxCost = cost; } } - if (maxRate > SERVER_KNOBS->MIN_TAG_PAGES_RATE) { + if (maxRate > SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE) { it->value.busiestWriteTag = busiestTag; // TraceEvent("RefreshSSCommitCost").detail("TotalWriteCost", it->value.totalWriteCost).detail("TotalWriteOps",it->value.totalWriteOps); ASSERT(it->value.totalWriteCosts > 0); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 561cbd85da..3c2bc266b7 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -500,7 +500,7 @@ public: previousBusiestTag.reset(); if (intervalStart > 0 && CLIENT_KNOBS->READ_TAG_SAMPLE_RATE > 0 && elapsed > 0) { double rate = busiestTagCount / CLIENT_KNOBS->READ_TAG_SAMPLE_RATE / elapsed; - if (rate > SERVER_KNOBS->MIN_TAG_PAGES_RATE) { + if (rate > SERVER_KNOBS->MIN_TAG_READ_PAGES_RATE) { previousBusiestTag = TagInfo(busiestTag, rate, (double)busiestTagCount / intervalTotalSampledCount); } From 3c7bd3549ac33e49d6c16918f35400e2e67303c3 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 14:23:27 -0700 Subject: [PATCH 090/458] Fix compile errors --- fdbserver/Status.actor.cpp | 19 ++++++++++--------- fdbserver/masterserver.actor.cpp | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index db1fa0f459..9b8acb6c96 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1028,12 +1028,13 @@ ACTOR static Future recoveryStateStatusFetcher(Database cx, W state JsonBuilderObject message; state Transaction tr(cx); try { - std::vector> futures; - futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0)); - futures.push_back(timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0)); - std::vector msgs = wait(getAll(futures)); + state Future mdActiveGensF = timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryGenerations") ) ), 1.0); + state Future mdF = timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0); + state Future rvF = timeoutError(tr.getReadVersion(), 1.0); - const TraceEventFields& md = msgs[1]; + wait(success(mdActiveGensF) && success(mdF) && success(rvF)); + + const TraceEventFields& md = mdF.get(); int mStatusCode = md.getInt("StatusCode"); if (mStatusCode < 0 || mStatusCode >= RecoveryStatus::END) throw attribute_not_found(); @@ -1041,11 +1042,11 @@ ACTOR static Future recoveryStateStatusFetcher(Database cx, W message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]); *statusCode = mStatusCode; + Version rv = rvF.get(); std::string fullyRecoveredAtVersion; if (mStatusCode == RecoveryStatus::fully_recovered && md.tryGetValue("FullyRecoveredAtVersion", fullyRecoveredAtVersion)) { - Version rv = wait(tr.getReadVersion()); int64_t fullyRecoveredAtVersion = md.getInt64("FullyRecoveredAtVersion"); - double lastFullyRecoveredSecondsAgo = std::max(0, rv - fullyRecoveredAtVersion) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND; + double lastFullyRecoveredSecondsAgo = std::max((int64_t)0, (int64_t)(rv - fullyRecoveredAtVersion)) / (double)SERVER_KNOBS->VERSIONS_PER_SECOND; message["time_since_last_fully_recovered_seconds"] = lastFullyRecoveredSecondsAgo; } else { message["time_since_last_fully_recovered_seconds"] = -1; @@ -1070,7 +1071,7 @@ ACTOR static Future recoveryStateStatusFetcher(Database cx, W // TODO: time_in_recovery: 0.5 // time_in_state: 0.1 - const TraceEventFields& mdActiveGens = msgs[0]; + const TraceEventFields& mdActiveGens = mdActiveGensF.get(); if(mdActiveGens.size()) { int activeGenerations = mdActiveGens.getInt("ActiveGenerations"); message["active_generations"] = activeGenerations; @@ -2396,7 +2397,7 @@ ACTOR Future clusterGetStatus( // construct status information for cluster subsections state int statusCode = (int) RecoveryStatus::END; - state JsonBuilderObject recoveryStateStatus = wait(recoveryStateStatusFetcher(mWorker, workers.size(), &status_incomplete_reasons, &statusCode)); + state JsonBuilderObject recoveryStateStatus = wait(recoveryStateStatusFetcher(cx, mWorker, workers.size(), &status_incomplete_reasons, &statusCode)); // machine metrics state WorkerEvents mMetrics = workerEventsVec[0].present() ? workerEventsVec[0].get().first : WorkerEvents(); diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index 8427c8119b..22f591dbfe 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -1274,7 +1274,7 @@ ACTOR Future trackTlogRecovery( Reference self, Referencedbgid) .detail("StatusCode", RecoveryStatus::fully_recovered) .detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered]) - .detail("FullyRecoveredAtVersion", self->version); + .detail("FullyRecoveredAtVersion", self->version) .trackLatest("MasterRecoveryState"); TraceEvent("MasterRecoveryGenerations", self->dbgid) From 1b923477f795420dca33485a772f5d366005b6d4 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 14:28:56 -0700 Subject: [PATCH 091/458] Apply suggestions from code review Co-authored-by: A.J. Beamon --- bindings/go/src/fdb/snapshot.go | 2 +- bindings/go/src/fdb/transaction.go | 2 +- .../java/src/main/com/apple/foundationdb/KeyArrayResult.java | 2 +- bindings/python/fdb/impl.py | 2 +- bindings/ruby/lib/fdbimpl.rb | 2 +- fdbclient/MultiVersionTransaction.actor.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bindings/go/src/fdb/snapshot.go b/bindings/go/src/fdb/snapshot.go index 2245fca6b9..09088ebefa 100644 --- a/bindings/go/src/fdb/snapshot.go +++ b/bindings/go/src/fdb/snapshot.go @@ -97,7 +97,7 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { ) } -// GetRangeSplitPoints will return a list of keys that can devide the given range into +// GetRangeSplitPoints will return a list of keys that can divide the given range into // chunks based on the chunk size provided. func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray { beginKey, endKey := r.FDBRangeKeys() diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index effe495d43..46a277647c 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -348,7 +348,7 @@ func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize in } } -// GetRangeSplitPoints will return a list of keys that can devide the given range into +// GetRangeSplitPoints will return a list of keys that can divide the given range into // chunks based on the chunk size provided. func (t Transaction) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray { beginKey, endKey := r.FDBRangeKeys() diff --git a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java index f63fc16d62..174bc89b19 100644 --- a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java +++ b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java @@ -3,7 +3,7 @@ * * This source file is part of the FoundationDB open source project * - * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index d68df5e3e5..5b9a414a2c 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -472,7 +472,7 @@ class TransactionRead(_FDBBase): end_key, len(end_key) )) - def get_range_split_points(self, begin_key, end_key, chunkSize): + def get_range_split_points(self, begin_key, end_key, chunk_size): if begin_key is None or end_key is None: raise Exception('Invalid begin key or end key') return FutureKeyArray(self.capi.fdb_transaction_get_range_split_points( diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 5564db4d0e..ea0f1bc347 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -848,7 +848,7 @@ module FDB Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize)) end - def get_range_split_points(begin_key, end_key, chunkSize) + def get_range_split_points(begin_key, end_key, chunk_size) bkey = FDB.key_to_bytes(begin_key) ekey = FDB.key_to_bytes(end_key) FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunkSize)) diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 12328b93c8..ec08c003d6 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -347,7 +347,7 @@ void DLApi::init() { loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); loadClientFunction(&api->futureGetStringArray, lib, fdbCPath, "fdb_future_get_string_array"); - loadClientFunction(&api->futureGetKeyArray, lib, fdbCPath, "fdb_future_get_key_array"); + loadClientFunction(&api->futureGetKeyArray, lib, fdbCPath, "fdb_future_get_key_array", headerVersion >= 700); loadClientFunction(&api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array"); loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback"); loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel"); From 813a2b3efe231d856edfe13465cbc60d4d0e19fc Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 14:30:41 -0700 Subject: [PATCH 092/458] Remove a function that have been removed in previous commits in master but was added back by this PR due to target branch change. --- bindings/java/fdbJNI.cpp | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index e8127060b6..62b9daaeee 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -377,42 +377,6 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyAr } -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getSummary(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { - throwParamNotNull(jenv); - return JNI_NULL; - } - - FDBFuture *f = (FDBFuture *)future; - - const FDBKeyValue *kvs; - int count; - fdb_bool_t more; - fdb_error_t err = fdb_future_get_keyvalue_array( f, &kvs, &count, &more ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); - return JNI_NULL; - } - - jbyteArray lastKey = JNI_NULL; - if(count) { - lastKey = jenv->NewByteArray(kvs[count - 1].key_length); - if( !lastKey ) { - if( !jenv->ExceptionOccurred() ) - throwOutOfMem(jenv); - return JNI_NULL; - } - - jenv->SetByteArrayRegion(lastKey, 0, kvs[count - 1].key_length, (jbyte *)kvs[count - 1].key); - } - - jobject result = jenv->NewObject(range_result_summary_class, range_result_summary_init, lastKey, count, (jboolean)more); - if( jenv->ExceptionOccurred() ) - return JNI_NULL; - - return result; -} - // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) { From 2d0b9fb12b47a1f19e6d430f0fb06604842f6f34 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Fri, 11 Sep 2020 17:44:09 -0400 Subject: [PATCH 093/458] Declared local status variables Added checks to ensure that cluster died when killed via cli Changed error message --- contrib/Joshua/scripts/localClusterStart.sh | 52 ++++++++++++++++----- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 656de162d9..5ee629dcbe 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -103,7 +103,10 @@ function displayMessage } # Create the directories used by the server. -function createDirectories { +function createDirectories +{ + local status=0 + # Display user message if ! displayMessage "Creating directories" then @@ -148,7 +151,10 @@ function createDirectories { } # Create a cluster file for the local cluster. -function createClusterFile { +function createClusterFile +{ + local status=0 + if [ "${status}" -ne 0 ]; then : # Display user message @@ -176,7 +182,10 @@ function createClusterFile { } # Stop the Cluster from running. -function stopCluster { +function stopCluster +{ + local status=0 + # Add an audit entry, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then printf '%-15s (%6s) Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" "${FDBSERVERID}" >> "${AUDITLOG}" @@ -189,8 +198,15 @@ function stopCluster { let status="${status} + 1" elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${CLUSTERSTRING}; sleep 3" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" then - log "Killed cluster (${FDBSERVERID}) via cli" - + # Ensure that process is dead + if ! kill -0 "${FDBSERVERID}" 2> /dev/null; then + log "Killed cluster (${FDBSERVERID}) via cli" + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to kill FDB Server process (${FDBSERVERID}) via cli or kill command" + let status="${status} + 1" + else + log "Forcibly killed FDB Server process (${FDBSERVERID}) since cli failed" + fi elif ! kill -9 "${FDBSERVERID}"; then log "Failed to forcibly kill FDB Server process (${FDBSERVERID})" let status="${status} + 1" @@ -201,7 +217,10 @@ function stopCluster { } # Start the server running. -function startFdbServer { +function startFdbServer +{ + local status=0 + # Add an audit entry, if enabled if [ "${AUDITCLUSTER}" -gt 0 ]; then printf '%-15s (%6s) Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" >> "${AUDITLOG}" @@ -226,14 +245,17 @@ function startFdbServer { log "FDB Server start failed because no process" let status="${status} + 1" elif ! kill -0 "${FDBSERVERID}" ; then - log "FDB Server start failed because no perms" + log "FDB Server start failed because process terminated unexpectedly" let status="${status} + 1" fi return ${status} } -function getStatus { +function getStatus +{ + local status=0 + if [ "${status}" -ne 0 ]; then : elif ! date &>> "${LOGDIR}/fdbclient.log" @@ -254,8 +276,10 @@ function getStatus { } # Verify that the cluster is available. -function verifyAvailable { +function verifyAvailable +{ local status=0 + if [ -z "${FDBSERVERID}" ]; then log "FDB Server process is not defined." let status="${status} + 1" @@ -283,7 +307,10 @@ function verifyAvailable { } # Configure the database on the server. -function createDatabase { +function createDatabase +{ + local status=0 + if [ "${status}" -ne 0 ]; then : # Ensure that the server is running @@ -336,7 +363,10 @@ function createDatabase { } # Begin the local cluster from scratch. -function startCluster { +function startCluster +{ + local status=0 + if [ "${status}" -ne 0 ]; then : elif ! createDirectories From 6dbcd42ebdb18a1fe663b07e3a449bbdc00523b3 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 16:16:56 -0700 Subject: [PATCH 094/458] No need for a new protocol version since the serialization won't change. Added code for deserializing from old binary. --- fdbclient/StorageServerInterface.h | 7 +++---- flow/ProtocolVersion.h | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 1eb733b358..1c2d7aacd6 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -100,10 +100,7 @@ struct StorageServerInterface { getKeyValueStoreType = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(9) ); watchValue = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(10) ); getReadHotRanges = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(11) ); - if(ar.protocolVersion().hasRangeSplit()) { - getRangeSplitPoints = - RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); - } + getRangeSplitPoints = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); } } else { ASSERT(Ar::isDeserializing); @@ -113,6 +110,8 @@ struct StorageServerInterface { serializer(ar, uniqueID, locality, getValue, getKey, getKeyValues, getShardState, waitMetrics, splitMetrics, getStorageMetrics, waitFailure, getQueuingMetrics, getKeyValueStoreType); if (ar.protocolVersion().hasWatches()) serializer(ar, watchValue); + getReadHotRanges.getEndpoint(); + getRangeSplitPoints.getEndpoint(); } } bool operator == (StorageServerInterface const& s) const { return uniqueID == s.uniqueID; } diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index 38e00171d1..39aa05a36b 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -128,7 +128,6 @@ public: // introduced features PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, ReportConflictingKeys); PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, SmallEndpoints); PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, CacheRole); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010000LL, RangeSplit); PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TagThrottleValueReason); }; From 0a7d2d31f106f385bc03116070fe0ba93e2247a2 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 16:17:23 -0700 Subject: [PATCH 095/458] Review some review comments from AJ --- fdbclient/ReadYourWrites.actor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 652aa325bc..090689d622 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1404,10 +1404,14 @@ Future ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyR Future>> ReadYourWritesTransaction::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) { if (checkUsedDuringCommit()) { - throw used_during_commit(); + return used_during_commit(); } if (resetPromise.isSet()) return resetPromise.getFuture().getError(); + KeyRef maxKey = getMaxReadKey(); + if(range.begin > maxKey || range.end > maxKey) + return key_outside_legal_range(); + return waitOrError(tr.getRangeSplitPoints(range, chunkSize), resetPromise.getFuture()); } From 0282a1745a0278c88ff9ed7b7f8ca1a045949d54 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 16:44:09 -0700 Subject: [PATCH 096/458] Address more review comments --- bindings/go/go.mod | 2 +- bindings/python/fdb/impl.py | 8 +++----- bindings/ruby/lib/fdbimpl.rb | 4 ++-- fdbclient/MultiVersionTransaction.actor.cpp | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bindings/go/go.mod b/bindings/go/go.mod index 0700d7cf9f..ec5746bf99 100644 --- a/bindings/go/go.mod +++ b/bindings/go/go.mod @@ -3,4 +3,4 @@ module github.com/apple/foundationdb/bindings/go // The FoundationDB go bindings currently have no external golang dependencies outside of // the go standard library. -require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543// indirect +require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 5b9a414a2c..9d1c825762 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -462,10 +462,8 @@ class TransactionRead(_FDBBase): return self.get(key) def get_estimated_range_size_bytes(self, begin_key, end_key): - if begin_key is None: - begin_key = b'' - if end_key is None: - end_key = b'\xff' + if begin_key is None or end_key is None: + raise Exception('Invalid begin key or end key') return FutureInt64(self.capi.fdb_transaction_get_estimated_range_size_bytes( self.tpointer, begin_key, len(begin_key), @@ -751,7 +749,7 @@ class FutureKeyArray(Future): ks = ctypes.pointer(KeyStruct()) count = ctypes.c_int() self.capi.fdb_future_get_key_array(self.fpointer, ctypes.byref(ks), ctypes.byref(count)) - return ([ctypes.string_at(x.key, x.key_length) for x in ks[0:count.value]], count.value) + return [ctypes.string_at(x.key, x.key_length) for x in ks[0:count.value]] class FutureStringArray(Future): diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index ea0f1bc347..0f2ce4344a 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -488,10 +488,10 @@ module FDB FDBC.check_error FDBC.fdb_future_get_key_array(@fpointer, kvs, count) ks = ks.read_pointer - [(0..count.read_int-1).map{|i| + (0..count.read_int-1).map{|i| x = FDBC::KeyStruct.new(ks + (i * FDBC::KeyStruct.size)) x[:key].read_bytes(x[:key_length]) - }, count.read_int] + } end end diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index ec08c003d6..d01281d2d9 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -340,7 +340,7 @@ void DLApi::init() { loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); loadClientFunction(&api->transactionGetRangeSplitPoints, lib, fdbCPath, "fdb_transaction_get_range_split_points", - headerVersion >= 630); + headerVersion >= 700); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); From 13c9dc6e371c2f73170e9d19e685eb3e608333b0 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 11 Sep 2020 16:57:00 -0700 Subject: [PATCH 097/458] Forgot to update local variables --- bindings/python/fdb/impl.py | 2 +- bindings/ruby/lib/fdbimpl.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 9d1c825762..937f2bfce1 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -477,7 +477,7 @@ class TransactionRead(_FDBBase): self.tpointer, begin_key, len(begin_key), end_key, len(end_key), - chunkSize + chunk_size )) class Transaction(TransactionRead): diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 0f2ce4344a..150f02a550 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -851,7 +851,7 @@ module FDB def get_range_split_points(begin_key, end_key, chunk_size) bkey = FDB.key_to_bytes(begin_key) ekey = FDB.key_to_bytes(end_key) - FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunkSize)) + FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunk_size)) end end From b060f53bab7beff72b491852391766c83a0bfc67 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Sat, 12 Sep 2020 00:39:36 -0400 Subject: [PATCH 098/458] Added support for randomizing the port from 4000 to 4999 --- contrib/Joshua/scripts/bindingTest.sh | 2 +- contrib/Joshua/scripts/bindingTestScript.sh | 18 +++++++++--------- contrib/Joshua/scripts/localClusterStart.sh | 14 ++++++-------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh index 3e926140e0..4a0d7c70da 100755 --- a/contrib/Joshua/scripts/bindingTest.sh +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -7,4 +7,4 @@ WORKDIR="$(pwd)/tmp/$$" if [ ! -d "${WORKDIR}" ] ; then mkdir -p "${WORKDIR}" fi -DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" FDBSERVERPORT="${PORT_FDBSERVER:-4500}" ${SCRIPTDIR}/bindingTestScript.sh 1 +DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" ${SCRIPTDIR}/bindingTestScript.sh 1 diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 898fd39aeb..c9cd5f1a80 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -7,7 +7,7 @@ SCRIPTID="${$}" SAVEONERROR="${SAVEONERROR:-1}" PYTHONDIR="${BINDIR}/tests/python" testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh" -VERSION="1.7" +VERSION="1.8" source ${SCRIPTDIR}/localClusterStart.sh @@ -23,14 +23,14 @@ cycles="${1}" if [ "${DEBUGLEVEL}" -gt 0 ] then - echo "Work dir: ${WORKDIR}" - echo "Bin dir: ${BINDIR}" - echo "Log dir: ${LOGDIR}" - echo "Python path: ${PYTHONDIR}" - echo "Lib dir: ${LIBDIR}" - echo "Server port: ${FDBSERVERPORT}" - echo "Script Id: ${SCRIPTID}" - echo "Version: ${VERSION}" + echo "Work dir: ${WORKDIR}" + echo "Bin dir: ${BINDIR}" + echo "Log dir: ${LOGDIR}" + echo "Python path: ${PYTHONDIR}" + echo "Lib dir: ${LIBDIR}" + echo "Cluster String: ${CLUSTERSTRING}" + echo "Script Id: ${SCRIPTID}" + echo "Version: ${VERSION}" fi # Begin the cluster using the logic in localClusterStart.sh. diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 5ee629dcbe..c79946053f 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -5,7 +5,7 @@ WORKDIR="${WORKDIR:-${SCRIPTDIR}/tmp/fdb.work}" LOGDIR="${WORKDIR}/log" ETCDIR="${WORKDIR}/etc" BINDIR="${BINDIR:-${SCRIPTDIR}}" -FDBSERVERPORT="${FDBSERVERPORT:-4500}" +FDBPORTSTART="${FDBPORTSTART:-4000}" SERVERCHECKS="${SERVERCHECKS:-10}" CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" @@ -20,9 +20,10 @@ messagecount=0 let index2="${RANDOM} % 256" let index3="${RANDOM} % 256" let index4="(${RANDOM} % 255) + 1" -# Define a random ip address on localhost +let FDBPORT="(${RANDOM} % 1000) + ${FDBPORTSTART}" +# Define a random ip address and port on localhost IPADDRESS="127.${index2}.${index3}.${index4}" -CLUSTERSTRING="${IPADDRESS}:${FDBSERVERPORT}" +CLUSTERSTRING="${IPADDRESS}:${FDBPORT}" function log @@ -165,10 +166,7 @@ function createClusterFile else description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) - let index2="${RANDOM} % 256" - let index3="${RANDOM} % 256" - let index4="(${RANDOM} % 255) + 1" - echo "${description}:${random_str}@${IPADDRESS}:${FDBSERVERPORT}" > "${FDBCONF}" + echo "${description}:${random_str}@${CLUSTERSTRING}" > "${FDBCONF}" fi if [ "${status}" -ne 0 ]; then @@ -233,7 +231,7 @@ function startFdbServer log 'Failed to display user message' let status="${status} + 1" - elif ! "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${IPADDRESS}:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & + elif ! "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${CLUSTERSTRING}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" let status="${status} + 1" From 3c3943f64f5c6423f96e111aa92ef69183ce22c8 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Mon, 14 Sep 2020 16:01:45 +0000 Subject: [PATCH 099/458] Disallow calling blockUntilReady from main thread Also fix a data race that apparently hasn't been ported to 6.2 yet --- flow/ThreadHelper.actor.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index e034ccf049..6e4e252a32 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -197,12 +197,12 @@ public: }; void blockUntilReady() { - if(isReadyUnsafe()) { - ThreadSpinLockHolder holder(mutex); - ASSERT(isReadyUnsafe()); + if (g_network->isOnMainThread()) { + TraceEvent(SevWarnAlways, "AttemptToBlockOnMainThread").error(client_invalid_operation()); + throw client_invalid_operation(); } - else { - BlockCallback cb( *this ); + if (!isReady()) { + BlockCallback cb(*this); } } From f9a5b727a68d6813c464bcf608344cbaa8909db2 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Mon, 14 Sep 2020 16:28:01 -0700 Subject: [PATCH 100/458] Add comments questions and TODOs --- fdbbackup/backup.actor.cpp | 2 ++ fdbclient/BackupContainer.actor.cpp | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index efd88087c6..eebcd95420 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2516,6 +2516,8 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine state Reference bc = openBackupContainer(name, destinationContainer); if (restoreVersion == invalidVersion) { BackupDescription desc = wait(bc->describeBackup()); + // TODO: If the keyRangeFilter is restorable but the normalKeys is not, maxRestorableVersion will not + // present, but we should still provide a restorable version for the keyRangeFilter if (!desc.maxRestorableVersion.present()) { reportBackupQueryError(operationId, result, "the specified backup is not restorable to any version"); return Void(); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index f5f5bce2ae..c53a0eb884 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -245,7 +245,7 @@ std::string BackupDescription::toJSON() const { * file written will be after the start version of the snapshot's execution. * * Log files are at file paths like - * /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize + * /plogs/.../log,startVersion,endVersion,UID,tagID-of-N,blocksize * /logs/.../log,startVersion,endVersion,UID,blockSize * where ... is a multi level path which sorts lexically into version order and results in approximately 1 * unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs" @@ -1411,8 +1411,9 @@ public: } } // 'latestVersion' represents using the minimum restorable version in a snapshot. + // MX: Isn't "latestVersion" the earliest restorable version? so below should be minKeyRangeVersion restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; - if (restorable.targetVersion < maxKeyRangeVersion) continue; + if (restorable.targetVersion < maxKeyRangeVersion) continue; // Q: Isn't this always true? restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved From 8224e17a08be94ac20ff16f95d1dc3a7dd4f4f32 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Tue, 15 Sep 2020 09:38:40 -0700 Subject: [PATCH 101/458] Integrate extended getRestoreSet API into fast restore The extended getRestoreSet provides a much smaller set of backup files for small keyrange restore; This commit integrate it into fast restore so that fast restore does not have to filter out unneeded backup files. --- fdbclient/BackupContainer.actor.cpp | 4 ++-- fdbserver/RestoreController.actor.cpp | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index c53a0eb884..ba0dcf3376 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1410,8 +1410,7 @@ public: throw backup_not_overlapped_with_keys_filter(); } } - // 'latestVersion' represents using the minimum restorable version in a snapshot. - // MX: Isn't "latestVersion" the earliest restorable version? so below should be minKeyRangeVersion + // 'latestVersion' represents using the maximum restorable version in a snapshot. restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; if (restorable.targetVersion < maxKeyRangeVersion) continue; // Q: Isn't this always true? @@ -1432,6 +1431,7 @@ public: // No logs needed if there is a complete filtered key space snapshot at the target version. if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; + // TODO: Add a Trace here return Optional(restorable); } diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp index d599711b95..c1bf57bb1c 100644 --- a/fdbserver/RestoreController.actor.cpp +++ b/fdbserver/RestoreController.actor.cpp @@ -747,7 +747,11 @@ ACTOR static Future collectBackupFiles(Reference bc, std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n"; } - Optional restorable = wait(bc->getRestoreSet(request.targetVersion)); + // NOTE: If correctness fails and it is hard to fix, do not add restoreRanges in the current PR. + // We should open a new PR to fix correctness failures. + state Standalone restoreRanges; + restoreRanges.push_back(request.range); + Optional restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges)); if (!restorable.present()) { TraceEvent(SevWarn, "FastRestoreControllerPhaseCollectBackupFiles") From 0beab42b78235b410dc8bdf56be33549a35b739c Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 15 Sep 2020 16:49:12 +0000 Subject: [PATCH 102/458] Add blocked_from_network_thread error --- documentation/sphinx/source/api-error-codes.rst | 4 ++++ flow/ThreadHelper.actor.h | 3 +-- flow/error_definitions.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/documentation/sphinx/source/api-error-codes.rst b/documentation/sphinx/source/api-error-codes.rst index 32518ce86f..6d8795208e 100644 --- a/documentation/sphinx/source/api-error-codes.rst +++ b/documentation/sphinx/source/api-error-codes.rst @@ -110,6 +110,10 @@ FoundationDB may return the following error codes from API functions. If you nee +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | transaction_read_only | 2023| Attempted to commit a transaction specified as read-only | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ +| invalid_cache_eviction_policy | 2024| Invalid cache eviction policy, only random and lru are supported | ++-----------------------------------------------+-----+--------------------------------------------------------------------------------+ +| blocked_from_network_thread | 2025| Attempted to block in a callback called from the network thread. | ++-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | incompatible_protocol_version | 2100| Incompatible protocol version | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | transaction_too_large | 2101| Transaction exceeds byte limit | diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 6e4e252a32..377cd4e488 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -198,8 +198,7 @@ public: void blockUntilReady() { if (g_network->isOnMainThread()) { - TraceEvent(SevWarnAlways, "AttemptToBlockOnMainThread").error(client_invalid_operation()); - throw client_invalid_operation(); + throw blocked_from_network_thread(); } if (!isReady()) { BlockCallback cb(*this); diff --git a/flow/error_definitions.h b/flow/error_definitions.h index fe6ab38ac5..b380978551 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -135,6 +135,7 @@ ERROR( no_commit_version, 2021, "Transaction is read-only and therefore does not ERROR( environment_variable_network_option_failed, 2022, "Environment variable network option could not be set" ) ERROR( transaction_read_only, 2023, "Attempted to commit a transaction specified as read-only" ) ERROR( invalid_cache_eviction_policy, 2024, "Invalid cache eviction policy, only random and lru are supported" ) +ERROR( blocked_from_network_thread, 2025, "Attempted to block in a callback called from the network thread." ) ERROR( incompatible_protocol_version, 2100, "Incompatible protocol version" ) ERROR( transaction_too_large, 2101, "Transaction exceeds byte limit" ) From 734bdb72e141c6ca8312195ce1dd265b91a8bafd Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 15 Sep 2020 16:49:31 +0000 Subject: [PATCH 103/458] Fix simulation trace impl occurence --- flow/Trace.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flow/Trace.cpp b/flow/Trace.cpp index de93602da8..dbfa36ea42 100644 --- a/flow/Trace.cpp +++ b/flow/Trace.cpp @@ -473,7 +473,11 @@ public: barriers->push(f); writer->post( new WriterThread::Barrier ); - f.getBlocking(); + if (g_network->isSimulated()) { + ASSERT(f.isReady()); + } else { + f.getBlocking(); + } opened = false; } From ed2d2612669355863cfef57ed741159e327dffb7 Mon Sep 17 00:00:00 2001 From: Meng Xu Date: Tue, 15 Sep 2020 10:01:36 -0700 Subject: [PATCH 104/458] Add MinV2 and AndV2 into AtomicOps test workload --- fdbserver/workloads/AtomicOps.actor.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fdbserver/workloads/AtomicOps.actor.cpp b/fdbserver/workloads/AtomicOps.actor.cpp index 35bfd04ff0..171c627557 100644 --- a/fdbserver/workloads/AtomicOps.actor.cpp +++ b/fdbserver/workloads/AtomicOps.actor.cpp @@ -55,8 +55,7 @@ struct AtomicOpsWorkload : TestWorkload { ubsum = 0; int64_t randNum = sharedRandomNumber / 10; - if(opType == -1) - opType = randNum % 8; + if (opType == -1) opType = randNum % 10; switch(opType) { case 0: @@ -91,6 +90,15 @@ struct AtomicOpsWorkload : TestWorkload { TEST(true); //Testing atomic ByteMax opType = MutationRef::ByteMax; break; + case 8: + TEST(true); // Testing atomic MinV2 + opType = MutationRef::MinV2; + case 9: + TEST(true); // Testing atomic AndV2 + opType = MutationRef::AndV2; + // case 10: + // TEST(true); // Testing atomic CompareAndClear Not supported yet + // opType = MutationRef::CompareAndClear default: ASSERT(false); } From 37d77ecb64f0d65b969b0f346bbdefac5a55b3b7 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Tue, 15 Sep 2020 13:30:35 -0400 Subject: [PATCH 105/458] WIP of adding tests --- .../workloads/IncrementalBackup.actor.cpp | 8 +++- tests/CMakeLists.txt | 3 ++ .../from_7.0.0/SnapIncrementalRestore-1.toml | 38 +++++++++++++++++++ .../from_7.0.0/SnapIncrementalRestore-2.toml | 28 ++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml create mode 100644 tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.actor.cpp index aea0b8350d..00c7cc4fd4 100644 --- a/fdbserver/workloads/IncrementalBackup.actor.cpp +++ b/fdbserver/workloads/IncrementalBackup.actor.cpp @@ -19,6 +19,7 @@ */ #include "fdbclient/FDBTypes.h" +#include "fdbclient/SystemData.h" #include "fdbrpc/simulator.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" @@ -86,8 +87,13 @@ struct IncrementalBackupWorkload : TestWorkload { try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - Optional versionValue = wait(tr->get(snapshotEndVersionKey)); + state Optional writeFlag = wait(tr->get(writeRecoveryKey)); + state Optional versionValue = wait(tr->get(snapshotEndVersionKey)); + TraceEvent("IBackupCheckSpecialKeys") + .detail("WriteRecoveryValue", writeFlag.present() ? writeFlag.get().toString() : "N/A") + .detail("EndVersionValue", versionValue.present() ? versionValue.get().toString() : "N/A"); beginVersion = BinaryReader::fromStringRef(versionValue.get(), Unversioned()); + TraceEvent("IBackupCheckBeginVersion").detail("Version", beginVersion); break; } catch (Error& e) { TraceEvent("IBackupReadSystemKeysError").error(e); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 54e77bc470..5f0e3b6086 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -167,6 +167,9 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/TransactionTagApiCorrectness.toml) add_fdb_test(TEST_FILES rare/TransactionTagSwizzledApiCorrectness.toml) add_fdb_test(TEST_FILES rare/WriteTagThrottling.toml) + add_fdb_test( + TEST_FILES restarting/from_7.0.0/SnapIncrementalRestore-1.toml + restarting/from_7.0.0/SnapIncrementalRestore-2.toml) add_fdb_test( TEST_FILES restarting/from_7.0.0/ConfigureTestRestart-1.txt restarting/from_7.0.0/ConfigureTestRestart-2.txt) diff --git a/tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml b/tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml new file mode 100644 index 0000000000..ad1e279b71 --- /dev/null +++ b/tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml @@ -0,0 +1,38 @@ +[[test]] +testTitle = 'SubmitBackup' +simBackupAgents = 'BackupToFile' + + [[test.workload]] + testName = 'IncrementalBackup' + tag = 'default' + submitOnly = true + +[[test]] +testTitle = 'SnapRunWorkloads' +clearAfterTest = false + + [[test.workload]] + testName = 'Cycle' + nodeCount = 3000 + transactionsPerSecond = 3000.0 + testDuration = 20.0 + expectedRate = 0 + +[[test]] +testTitle = 'SnapPostWorkloads' +clearAfterTest = false + + [[test.workload]] + testName = 'AtomicOps' + transactionsPerSecond = 2500.0 + testDuration = 20.0 + opType = 0 + nodeCount = 1000 + +[[test]] +testTitle = 'SnapShutdown' + + [[test.workload]] + testName = 'SaveAndKill' + restartInfoLocation = 'simfdb/restartInfo.ini' + testDuration = 10.0 \ No newline at end of file diff --git a/tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml b/tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml new file mode 100644 index 0000000000..6f313c3cda --- /dev/null +++ b/tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml @@ -0,0 +1,28 @@ +[[test]] +testTitle = 'RestoreBackup' +simBackupAgents = 'BackupToFile' +clearAfterTest = false + + [[test.workload]] + testName = 'IncrementalBackup' + tag = 'default' + restoreOnly = true + checkBeginVersion = false + +[[test]] +testTitle = 'VerifyCycleAndAtomics' +checkOnly = true + + [[test.workload]] + testName = 'Cycle' + nodeCount = 3000 + transactionsPerSecond = 3000.0 + testDuration = 10.0 + expectedRate = 0 + + [[test.workload]] + testName = 'AtomicOps' + transactionsPerSecond = 2500.0 + testDuration = 10.0 + opType = 0 + nodeCount = 1000 \ No newline at end of file From acdf04ed187b1b894179387d8843cfb7e46a8f85 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Tue, 15 Sep 2020 14:09:53 -0700 Subject: [PATCH 106/458] Make sure Python and Ruby does null check --- bindings/python/fdb/impl.py | 4 ++-- bindings/ruby/lib/fdbimpl.rb | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 937f2bfce1..3eeffc021e 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -471,8 +471,8 @@ class TransactionRead(_FDBBase): )) def get_range_split_points(self, begin_key, end_key, chunk_size): - if begin_key is None or end_key is None: - raise Exception('Invalid begin key or end key') + if begin_key is None or end_key is None or chunk_size <=0: + raise Exception('Invalid begin key, end key or chunk size') return FutureKeyArray(self.capi.fdb_transaction_get_range_split_points( self.tpointer, begin_key, len(begin_key), diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 150f02a550..8ea5452149 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -843,12 +843,18 @@ module FDB end def get_estimated_range_size_bytes(begin_key, end_key) + if begin_key.nil? || end_key.nil? + raise ArgumentError, "Invalid begin key or end key" + end bkey = FDB.key_to_bytes(begin_key) ekey = FDB.key_to_bytes(end_key) Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize)) end def get_range_split_points(begin_key, end_key, chunk_size) + if begin_key.nil? || end_key.nil? || chunk_size <=0 + raise ArgumentError, "Invalid begin key, end key or chunk size" + end bkey = FDB.key_to_bytes(begin_key) ekey = FDB.key_to_bytes(end_key) FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunk_size)) From 3a68c892812cd24a7de01eaf9c74ca516f64b30b Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard <65554662+sfc-gh-tclinkenbeard@users.noreply.github.com> Date: Tue, 15 Sep 2020 16:32:24 -0700 Subject: [PATCH 107/458] Remove initialisms Co-authored-by: A.J. Beamon --- fdbserver/Ratekeeper.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index af464b4cf5..e25a32ba77 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -925,7 +925,7 @@ void tryAutoThrottleTag(RatekeeperData* self, TransactionTag tag, double rate, d void tryAutoThrottleTag(RatekeeperData* self, StorageQueueInfo& ss, int64_t storageQueue, int64_t storageDurabilityLag) { // NOTE: we just keep it simple and don't differentiate write-saturation and read-saturation at the moment. In most of situation, this works. - // More indicators besides SQ and NDV could be investigated in the future + // More indicators besides queue size and durability lag could be investigated in the future if (storageQueue > SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES || storageDurabilityLag > SERVER_KNOBS->AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS) { if(ss.busiestWriteTag.present()) { tryAutoThrottleTag(self, ss.busiestWriteTag.get(), ss.busiestWriteTagRate, From cc5bc16bd8ac8841a3378849b02b4cdbc8f1b21b Mon Sep 17 00:00:00 2001 From: Young Liu Date: Tue, 15 Sep 2020 22:29:49 -0700 Subject: [PATCH 108/458] Rename more places from proxy to commit proxy --- fdbcli/fdbcli.actor.cpp | 19 ++++---- fdbclient/CommitProxyInterface.h | 4 +- fdbclient/DatabaseConfiguration.cpp | 48 ++++++++++++------- fdbclient/DatabaseContext.h | 2 +- fdbclient/Knobs.cpp | 2 +- fdbclient/ManagementAPI.actor.cpp | 1 - fdbclient/NativeAPI.actor.cpp | 6 +-- fdbclient/vexillographer/fdb.options | 2 +- fdbrpc/Locality.h | 3 -- fdbserver/ApplyMetadataMutation.cpp | 4 +- fdbserver/ClusterController.actor.cpp | 4 +- fdbserver/Knobs.cpp | 2 +- fdbserver/Knobs.h | 2 +- fdbserver/MasterInterface.h | 2 +- fdbserver/Ratekeeper.actor.cpp | 18 +++---- fdbserver/Resolver.actor.cpp | 10 ++-- fdbserver/Status.actor.cpp | 16 +++---- fdbserver/masterserver.actor.cpp | 26 +++++----- fdbserver/storageserver.actor.cpp | 6 +-- .../workloads/ConfigureDatabase.actor.cpp | 2 +- fdbserver/workloads/TargetedKill.actor.cpp | 10 ++-- tests/status/separate_not_enough_servers.txt | 2 +- 22 files changed, 100 insertions(+), 91 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index a304daa2ad..ac219df9e3 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -471,8 +471,7 @@ void initHelp() { helpMap["configure"] = CommandHelp( "configure [new] " "|grv_" - "proxies=|logs=|resolvers=>*", + "COMMIT_PROXIES>|grv_proxies=|logs=|resolvers=>*", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing " "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " @@ -481,15 +480,13 @@ void initHelp() { "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " "datasets.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. Must " - "be at least 1, or set " - "to -1 which restores the number of commit proxies to the default value.\n\ngrv_proxies=: Sets " - "the " - "desired number of GRV proxies in the cluster. Must be at least 1, or set to -1 which restores the number of " - "GRV proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must " - "be " - "at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: " - "Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the " - "number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); + "be at least 1, or set to -1 which restores the number of commit proxies to the default " + "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " + "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " + "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " + "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " + "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the " + "FoundationDB Administration Guide for more information."); helpMap["fileconfigure"] = CommandHelp( "fileconfigure [new] ", "change the database configuration from a file", diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index c6b12dd7f2..ceba2cf0f8 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -100,10 +100,10 @@ struct CommitProxyInterface { struct ClientDBInfo { constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes - vector< GrvProxyInterface > grvProxies; + vector grvProxies; vector commitProxies; Optional - firstCommitProxy; // not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk + firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk double clientTxnInfoSampleRate; int64_t clientTxnInfoSizeLimit; Optional forward; diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index f70fc4275c..464c220555 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -164,24 +164,40 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() { } bool DatabaseConfiguration::isValid() const { - if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 && - tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredCommitProxies() >= 1 && - getDesiredGrvProxies() >= 1 && getDesiredLogs() >= 1 && getDesiredResolvers() >= 1 && - tLogVersion != TLogVersion::UNSET && tLogVersion >= TLogVersion::MIN_RECRUITABLE && - tLogVersion <= TLogVersion::MAX_SUPPORTED && tLogDataStoreType != KeyValueStoreType::END && - tLogSpillType != TLogSpillType::UNSET && - !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && - storageServerStoreType != KeyValueStoreType::END && autoCommitProxyCount >= 1 && autoGrvProxyCount >= 1 && - autoResolverCount >= 1 && autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy && - getDesiredRemoteLogs() >= 1 && remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 && - repopulateRegionAntiQuorum <= 1 && usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 && - (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) && - (regions.size() == 0 || - tLogPolicy->info() != - "dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication + if( !(initialized && + tLogWriteAntiQuorum >= 0 && + tLogWriteAntiQuorum <= tLogReplicationFactor/2 && + tLogReplicationFactor >= 1 && + storageTeamSize >= 1 && + getDesiredCommitProxies() >= 1 && + getDesiredGrvProxies() >= 1 && + getDesiredLogs() >= 1 && + getDesiredResolvers() >= 1 && + tLogVersion != TLogVersion::UNSET && + tLogVersion >= TLogVersion::MIN_RECRUITABLE && + tLogVersion <= TLogVersion::MAX_SUPPORTED && + tLogDataStoreType != KeyValueStoreType::END && + tLogSpillType != TLogSpillType::UNSET && + !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && + storageServerStoreType != KeyValueStoreType::END && + autoCommitProxyCount >= 1 && + autoGrvProxyCount >= 1 && + autoResolverCount >= 1 && + autoDesiredTLogCount >= 1 && + storagePolicy && + tLogPolicy && + getDesiredRemoteLogs() >= 1 && + remoteTLogReplicationFactor >= 0 && + repopulateRegionAntiQuorum >= 0 && + repopulateRegionAntiQuorum <= 1 && + usableRegions >= 1 && + usableRegions <= 2 && + regions.size() <= 2 && + ( usableRegions == 1 || regions.size() == 2 ) && + ( regions.size() == 0 || regions[0].priority >= 0 ) && + ( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication return false; } - std::set dcIds; dcIds.insert(Key()); for(auto& r : regions) { diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index f9367482e5..5652ae7a14 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -221,7 +221,7 @@ public: Future monitorProxiesInfoChange; Reference commitProxies; Reference grvProxies; - bool proxyProvisional; + bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time. UID proxiesLastChange; LocalityData clientLocality; QueueModel queueModel; diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index d1ec7a4f5f..9ec850bc75 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -171,7 +171,7 @@ void ClientKnobs::initialize(bool randomize) { init( MIN_CLEANUP_SECONDS, 3600.0 ); // Configuration - init( DEFAULT_AUTO_COMMIT_PROXIES, 3 ); + init( DEFAULT_AUTO_COMMIT_PROXIES, 3 ); init( DEFAULT_AUTO_GRV_PROXIES, 1 ); init( DEFAULT_AUTO_RESOLVERS, 1 ); init( DEFAULT_AUTO_LOGS, 3 ); diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index e4a5183b95..bd4b3b6e95 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -747,7 +747,6 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { proxyCount = result.old_commit_proxies; } - // Need to configure a good number. result.desired_grv_proxies = std::max(std::min(4, processCount / 20), 1); int grvProxyCount; if (!statusObjConfig.get("grv_proxies", result.old_grv_proxies)) { diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 19683a536b..62a0c1936d 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -1597,9 +1597,9 @@ Reference DatabaseContext::getGrvProxies(bool useProvisionalProxie // Actor which will wait until the MultiInterface returned by the DatabaseContext cx is not NULL ACTOR Future> getCommitProxiesFuture(DatabaseContext* cx, bool useProvisionalProxies) { loop{ - Reference proxies = cx->getCommitProxies(useProvisionalProxies); - if (proxies) - return proxies; + Reference commitProxies = cx->getCommitProxies(useProvisionalProxies); + if (commitProxies) + return commitProxies; wait( cx->onProxiesChanged() ); } } diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index f11956d79c..37e57346ee 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -195,7 +195,7 @@ description is not currently required but encouraged.