diff --git a/CMakeLists.txt b/CMakeLists.txt index e4da141b9d..cd06f1cb71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,7 @@ endif() add_subdirectory(fdbbackup) add_subdirectory(contrib) add_subdirectory(tests) +add_subdirectory(flowbench EXCLUDE_FROM_ALL) if(WITH_PYTHON) add_subdirectory(bindings) endif() @@ -177,11 +178,6 @@ else() include(CPack) endif() -set(BUILD_FLOWBENCH OFF CACHE BOOL "Build microbenchmark program (builds google microbenchmark dependency)") -if(BUILD_FLOWBENCH) - add_subdirectory(flowbench) -endif() - if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") add_link_options(-lexecinfo) endif() diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 5e8d2d66a2..df90adf890 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -157,7 +157,7 @@ class ApiTest(Test): read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] txn_sizes = ['GET_APPROXIMATE_SIZE'] - storage_metrics = ['GET_ESTIMATED_RANGE_SIZE'] + storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] op_choices += reads op_choices += mutations @@ -553,6 +553,23 @@ class ApiTest(Test): instructions.push_args(key1, key2) instructions.append(op) self.add_strings(1) + elif op == 'GET_RANGE_SPLIT_POINTS': + # Protect against inverted range and identical keys + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + while key1 == key2: + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + if key1 > key2: + key1, key2 = key2, key1 + + # TODO: randomize chunkSize but should not exceed 100M(shard limit) + chunkSize = 10000000 # 10M + instructions.push_args(key1, key2, chunkSize) + instructions.append(op) + self.add_strings(1) else: assert False, 'Unknown operation: ' + op diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index 0f4b30544a..f4e5a762ac 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -71,17 +71,27 @@ if(NOT WIN32) test/mako/mako.h test/mako/utils.c test/mako/utils.h) + add_subdirectory(test/unit/third_party) + find_package(Threads REQUIRED) + set(UNIT_TEST_SRCS + test/unit/unit_tests.cpp + test/unit/fdb_api.cpp + test/unit/fdb_api.hpp) if(OPEN_FOR_IDE) add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h) add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h) add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h) add_library(mako OBJECT ${MAKO_SRCS}) + add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp) + add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS}) else() add_executable(fdb_c_performance_test test/performance_test.c test/test.h) add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h) add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h) add_executable(mako ${MAKO_SRCS}) + add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp) + add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS}) strip_debug_symbols(fdb_c_performance_test) strip_debug_symbols(fdb_c_ryw_benchmark) strip_debug_symbols(fdb_c_txn_size_test) @@ -89,9 +99,26 @@ if(NOT WIN32) target_link_libraries(fdb_c_performance_test PRIVATE fdb_c) target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c) target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c) + + add_dependencies(fdb_c_setup_tests doctest) + add_dependencies(fdb_c_unit_tests doctest) + target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) + target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) + target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads) + target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads) + # do not set RPATH for mako set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE) target_link_libraries(mako PRIVATE fdb_c) + + add_fdbclient_test( + NAME fdb_c_setup_tests + COMMAND $) + add_fdbclient_test( + NAME fdb_c_unit_tests + COMMAND $ + @CLUSTER_FILE@ + fdb) endif() set(c_workloads_srcs diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index 750e6fc57b..b92fd7664e 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -153,7 +153,7 @@ void fdb_future_destroy( FDBFuture* f ) { extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) { - CATCH_AND_RETURN( TSAVB(f)->blockUntilReady(); ); + CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread();); } fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) { @@ -171,12 +171,12 @@ public: void* userdata) : callbackf(callbackf), f(f), userdata(userdata) {} - virtual bool canFire(int notMadeActive) { return true; } - virtual void fire(const Void& unused, int& userParam) { + bool canFire(int notMadeActive) const override { return true; } + void fire(const Void& unused, int& userParam) override { (*callbackf)(f, userdata); delete this; } - virtual void error(const Error&, int& userParam) { + void error(const Error&, int& userParam) override { (*callbackf)(f, userdata); delete this; } @@ -281,6 +281,17 @@ fdb_error_t fdb_future_get_string_array( ); } +extern "C" DLLEXPORT +fdb_error_t fdb_future_get_key_array( + FDBFuture* f, FDBKey const** out_key_array, int* out_count) +{ + CATCH_AND_RETURN( + Standalone> na = TSAV(Standalone>, f)->get(); + *out_key_array = (FDBKey*) na.begin(); + *out_count = na.size(); + ); +} + FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) { char *path; if(cluster_file_path) { @@ -601,7 +612,7 @@ fdb_error_t fdb_transaction_set_option_impl( FDBTransaction* tr, void fdb_transaction_set_option_v13( FDBTransaction* tr, FDBTransactionOption option ) { - fdb_transaction_set_option_impl( tr, option, NULL, 0 ); + fdb_transaction_set_option_impl( tr, option, nullptr, 0 ); } extern "C" DLLEXPORT @@ -631,13 +642,20 @@ fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const } -extern "C" DLLEXPORT +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) { KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); } +extern "C" DLLEXPORT +FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) { + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr()); +} + #include "fdb_c_function_pointers.g.h" #define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index f34656db1e..966db53e34 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -91,6 +91,10 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter); #pragma pack(push, 4) + typedef struct key { + const uint8_t* key; + int key_length; + } FDBKey; #if FDB_API_VERSION >= 700 typedef struct keyvalue { const uint8_t* key; @@ -146,6 +150,9 @@ extern "C" { fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_kv, int* out_count, fdb_bool_t* out_more ); #endif + DLLEXPORT WARN_UNUSED_RESULT fdb_error_t + fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, + int* out_count); DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count); @@ -263,6 +270,10 @@ extern "C" { fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* + fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size); + #define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0 #define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0 #define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1 diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index 8d69d1af79..95de0d99e7 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -96,6 +96,7 @@ int commit_transaction(FDBTransaction* transaction) { f = fdb_transaction_commit(transaction); fdb_wait_and_handle_error(commit_transaction, f, transaction); + fdb_future_destroy(f); return FDB_SUCCESS; } diff --git a/bindings/c/test/unit/fdb_api.cpp b/bindings/c/test/unit/fdb_api.cpp new file mode 100644 index 0000000000..4f7c754041 --- /dev/null +++ b/bindings/c/test/unit/fdb_api.cpp @@ -0,0 +1,229 @@ +/* + * fdb_api.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdb_api.hpp" + +#include + +namespace fdb { + +// Future + +Future::~Future() { + fdb_future_destroy(future_); +} + +bool Future::is_ready() { + return fdb_future_is_ready(future_); +} + +[[nodiscard]] fdb_error_t Future::block_until_ready() { + return fdb_future_block_until_ready(future_); +} + +[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, + void* callback_parameter) { + return fdb_future_set_callback(future_, callback, callback_parameter); +} + +[[nodiscard]] fdb_error_t Future::get_error() { + return fdb_future_get_error(future_); +} + +void Future::release_memory() { + fdb_future_release_memory(future_); +} + +void Future::cancel() { + fdb_future_cancel(future_); +} + +// Int64Future + +[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) { + return fdb_future_get_int64(future_, out); +} + +// ValueFuture + +[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, + const uint8_t** out_value, + int* out_value_length) { + return fdb_future_get_value(future_, out_present, out_value, + out_value_length); +} + +// KeyFuture + +[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, + int* out_key_length) { + return fdb_future_get_key(future_, out_key, out_key_length); +} + +// StringArrayFuture + +[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, + int* out_count) { + return fdb_future_get_string_array(future_, out_strings, out_count); +} + +// KeyValueArrayFuture + +[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, + int* out_count, + fdb_bool_t* out_more) { + return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more); +} + +// Transaction + +Transaction::Transaction(FDBDatabase* db) { + if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) { + std::cerr << fdb_get_error(err) << std::endl; + std::abort(); + } +} + +Transaction::~Transaction() { + fdb_transaction_destroy(tr_); +} + +void Transaction::reset() { + fdb_transaction_reset(tr_); +} + +void Transaction::cancel() { + fdb_transaction_cancel(tr_); +} + +[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, + const uint8_t* value, + int value_length) { + return fdb_transaction_set_option(tr_, option, value, value_length); +} + +void Transaction::set_read_version(int64_t version) { + fdb_transaction_set_read_version(tr_, version); +} + +Int64Future Transaction::get_read_version() { + return Int64Future(fdb_transaction_get_read_version(tr_)); +} + +Int64Future Transaction::get_approximate_size() { + return Int64Future(fdb_transaction_get_approximate_size(tr_)); +} + +KeyFuture Transaction::get_versionstamp() { + return KeyFuture(fdb_transaction_get_versionstamp(tr_)); +} + +ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) { + return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), + key.size(), snapshot)); +} + +KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length, + fdb_bool_t or_equal, int offset, + fdb_bool_t snapshot) { + return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, + or_equal, offset, snapshot)); +} + +StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) { + return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, + (const uint8_t*)key.data(), key.size())); +} + +KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, + int begin_offset, + const uint8_t* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, + int end_offset, int limit, + int target_bytes, + FDBStreamingMode mode, + int iteration, fdb_bool_t snapshot, + fdb_bool_t reverse) { + return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name, + begin_key_name_length, + begin_or_equal, + begin_offset, + end_key_name, + end_key_name_length, + end_or_equal, + end_offset, + limit, target_bytes, + mode, iteration, + snapshot, reverse)); +} + +EmptyFuture Transaction::watch(std::string_view key) { + return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size())); +} + +EmptyFuture Transaction::commit() { + return EmptyFuture(fdb_transaction_commit(tr_)); +} + +EmptyFuture Transaction::on_error(fdb_error_t err) { + return EmptyFuture(fdb_transaction_on_error(tr_, err)); +} + +void Transaction::clear(std::string_view key) { + return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size()); +} + +void Transaction::clear_range(std::string_view begin_key, + std::string_view end_key) { + fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(), + begin_key.size(), (const uint8_t*)end_key.data(), + end_key.size()); +} + +void Transaction::set(std::string_view key, std::string_view value) { + fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), + (const uint8_t*)value.data(), value.size()); +} + +void Transaction::atomic_op(std::string_view key, const uint8_t* param, + int param_length, FDBMutationType operationType) { + return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), + param, param_length, operationType); +} + +[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) { + return fdb_transaction_get_committed_version(tr_, out_version); +} + +fdb_error_t Transaction::add_conflict_range(std::string_view begin_key, + std::string_view end_key, + FDBConflictRangeType type) { + return fdb_transaction_add_conflict_range(tr_, + (const uint8_t*)begin_key.data(), + begin_key.size(), + (const uint8_t*)end_key.data(), + end_key.size(), + type); +} + +} // namespace fdb diff --git a/bindings/c/test/unit/fdb_api.hpp b/bindings/c/test/unit/fdb_api.hpp new file mode 100644 index 0000000000..b36e929322 --- /dev/null +++ b/bindings/c/test/unit/fdb_api.hpp @@ -0,0 +1,243 @@ +/* + * fdb_api.hpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A collection of C++ classes to wrap the C API to improve memory management +// and add types to futures. Using the old C API may look something like: +// +// FDBTransaction *tr; +// fdb_database_create_transaction(db, &tr); +// FDBFuture *f = fdb_transaction_get(tr, (const uint8_t*)"foo", 3, true); +// fdb_future_block_until_ready(f); +// fdb_future_get_value(f, ...); +// fdb_future_destroy(f); +// fdb_transaction_destroy(tr); +// +// Using the wrapper classes defined here, it will instead look like: +// +// fdb::Transaction tr(db); +// fdb::ValueFuture f = tr.get((const uint8_t*)"foo", 3, true); +// f.block_until_ready(); +// f.get_value(f, ...); +// + +#pragma once + +#define FDB_API_VERSION 620 +#include + +#include +#include + +namespace fdb { + +// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up +// FDBFuture when this instance goes out of scope. +class Future { + public: + virtual ~Future() = 0; + + // Wrapper around fdb_future_is_ready. + bool is_ready(); + // Wrapper around fdb_future_block_until_ready. + fdb_error_t block_until_ready(); + // Wrapper around fdb_future_set_callback. + fdb_error_t set_callback(FDBCallback callback, void* callback_parameter); + // Wrapper around fdb_future_get_error. + fdb_error_t get_error(); + // Wrapper around fdb_future_release_memory. + void release_memory(); + // Wrapper around fdb_future_cancel. + void cancel(); + + // Conversion operator to allow Future instances to work interchangeably as + // an FDBFuture object. + // operator FDBFuture* () const { + // return future_; + // } + + protected: + Future(FDBFuture *f) : future_(f) {} + FDBFuture* future_; +}; + + +class Int64Future : public Future { + public: + // Call this function instead of fdb_future_get_int64 when using the + // Int64Future type. It's behavior is identical to fdb_future_get_int64. + fdb_error_t get(int64_t* out); + + private: + friend class Transaction; + Int64Future(FDBFuture* f) : Future(f) {} +}; + + +class KeyFuture : public Future { + public: + // Call this function instead of fdb_future_get_key when using the KeyFuture + // type. It's behavior is identical to fdb_future_get_key. + fdb_error_t get(const uint8_t** out_key, int* out_key_length); + + private: + friend class Transaction; + KeyFuture(FDBFuture* f) : Future(f) {} +}; + + +class ValueFuture : public Future { + public: + // Call this function instead of fdb_future_get_value when using the + // ValueFuture type. It's behavior is identical to fdb_future_get_value. + fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, + int* out_value_length); + + private: + friend class Transaction; + ValueFuture(FDBFuture* f) : Future(f) {} +}; + + +class StringArrayFuture : public Future { + public: + // Call this function instead of fdb_future_get_string_array when using the + // StringArrayFuture type. It's behavior is identical to + // fdb_future_get_string_array. + fdb_error_t get(const char*** out_strings, int* out_count); + + private: + friend class Transaction; + StringArrayFuture(FDBFuture* f) : Future(f) {} +}; + + +class KeyValueArrayFuture : public Future { + public: + // Call this function instead of fdb_future_get_keyvalue_array when using + // the KeyValueArrayFuture type. It's behavior is identical to + // fdb_future_get_keyvalue_array. + fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, + fdb_bool_t* out_more); + + private: + friend class Transaction; + KeyValueArrayFuture(FDBFuture* f) : Future(f) {} +}; + + +class EmptyFuture : public Future { + private: + friend class Transaction; + EmptyFuture(FDBFuture* f) : Future(f) {} +}; + + +// Wrapper around FDBTransaction, providing the same set of calls as the C API. +// Handles cleanup of memory, removing the need to call +// fdb_transaction_destroy. +class Transaction final { + public: + // Given an FDBDatabase, initializes a new transaction. + Transaction(FDBDatabase* db); + ~Transaction(); + + // Wrapper around fdb_transaction_reset. + void reset(); + + // Wrapper around fdb_transaction_cancel. + void cancel(); + + // Wrapper around fdb_transaction_set_option. + fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, + int value_length); + + // Wrapper around fdb_transaction_set_read_version. + void set_read_version(int64_t version); + + // Returns a future which will be set to the transaction read version. + Int64Future get_read_version(); + + // Returns a future which will be set to the approximate transaction size so far. + Int64Future get_approximate_size(); + + // Returns a future which will be set to the versionstamp which was used by + // any versionstamp operations in the transaction. + KeyFuture get_versionstamp(); + + // Returns a future which will be set to the value of `key` in the database. + ValueFuture get(std::string_view key, fdb_bool_t snapshot); + + // Returns a future which will be set to the key in the database matching the + // passed key selector. + KeyFuture get_key(const uint8_t* key_name, int key_name_length, + fdb_bool_t or_equal, int offset, fdb_bool_t snapshot); + + // Returns a future which will be set to an array of strings. + StringArrayFuture get_addresses_for_key(std::string_view key); + + // Returns a future which will be set to an FDBKeyValue array. + KeyValueArrayFuture get_range(const uint8_t* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, int begin_offset, + const uint8_t* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, int end_offset, + int limit, int target_bytes, + FDBStreamingMode mode, int iteration, + fdb_bool_t snapshot, fdb_bool_t reverse); + + // Wrapper around fdb_transaction_watch. Returns a future representing an + // empty value. + EmptyFuture watch(std::string_view key); + + // Wrapper around fdb_transaction_commit. Returns a future representing an + // empty value. + EmptyFuture commit(); + + // Wrapper around fdb_transaction_on_error. Returns a future representing an + // empty value. + EmptyFuture on_error(fdb_error_t err); + + // Wrapper around fdb_transaction_clear. + void clear(std::string_view key); + + // Wrapper around fdb_transaction_clear_range. + void clear_range(std::string_view begin_key, std::string_view end_key); + + // Wrapper around fdb_transaction_set. + void set(std::string_view key, std::string_view value); + + // Wrapper around fdb_transaction_atomic_op. + void atomic_op(std::string_view key, const uint8_t* param, int param_length, + FDBMutationType operationType); + + // Wrapper around fdb_transaction_get_committed_version. + fdb_error_t get_committed_version(int64_t* out_version); + + // Wrapper around fdb_transaction_add_conflict_range. + fdb_error_t add_conflict_range(std::string_view begin_key, + std::string_view end_key, + FDBConflictRangeType type); + + private: + FDBTransaction* tr_; +}; + +} // namespace fdb diff --git a/bindings/c/test/unit/setup_tests.cpp b/bindings/c/test/unit/setup_tests.cpp new file mode 100644 index 0000000000..da2a3f9bd7 --- /dev/null +++ b/bindings/c/test/unit/setup_tests.cpp @@ -0,0 +1,75 @@ +/* + * setup_tests.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Unit tests for API setup, network initialization functions from the FDB C API. + +#define FDB_API_VERSION 620 +#include +#include +#include + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest.h" + +void fdb_check(fdb_error_t e) { + if (e) { + std::cerr << fdb_get_error(e) << std::endl; + std::abort(); + } +} + +TEST_CASE("setup") { + fdb_error_t err; + // Version passed here must be <= FDB_API_VERSION + err = fdb_select_api_version(9000); + CHECK(err); + + // Select current API version + fdb_check(fdb_select_api_version(620)); + + // Error to call again after a successful return + err = fdb_select_api_version(620); + CHECK(err); + + CHECK(fdb_get_max_api_version() >= 620); + + fdb_check(fdb_setup_network()); + // Calling a second time should fail + err = fdb_setup_network(); + CHECK(err); + + struct Context { + bool called = false; + }; + Context context; + fdb_check(fdb_add_network_thread_completion_hook( + [](void *param) { + auto *context = static_cast(param); + context->called = true; + }, + &context)); + + std::thread network_thread{&fdb_run_network}; + + CHECK(!context.called); + fdb_check(fdb_stop_network()); + network_thread.join(); + CHECK(context.called); +} diff --git a/bindings/c/test/unit/third_party/CMakeLists.txt b/bindings/c/test/unit/third_party/CMakeLists.txt new file mode 100644 index 0000000000..6229abb0c9 --- /dev/null +++ b/bindings/c/test/unit/third_party/CMakeLists.txt @@ -0,0 +1,18 @@ +# Download doctest repo. +include(ExternalProject) +find_package(Git REQUIRED) + +ExternalProject_Add( + doctest + PREFIX ${CMAKE_BINARY_DIR}/doctest + GIT_REPOSITORY https://github.com/onqtam/doctest.git + GIT_TAG 1c8da00c978c19e00a434b2b1f854fcffc9fba35 # v2.4.0 + TIMEOUT 10 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD ON +) + +ExternalProject_Get_Property(doctest source_dir) +set(DOCTEST_INCLUDE_DIR ${source_dir}/doctest CACHE INTERNAL "Path to include folder for doctest") diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp new file mode 100644 index 0000000000..5876658ae3 --- /dev/null +++ b/bindings/c/test/unit/unit_tests.cpp @@ -0,0 +1,1842 @@ +/* + * unit_tests.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Unit tests for the FoundationDB C API. + +#define FDB_API_VERSION 620 +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCTEST_CONFIG_IMPLEMENT +#include "doctest.h" + +#include "fdb_api.hpp" + +void fdb_check(fdb_error_t e) { + if (e) { + std::cerr << fdb_get_error(e) << std::endl; + std::abort(); + } +} + +FDBDatabase *fdb_open_database(const char *clusterFile) { + FDBDatabase *db; + fdb_check(fdb_create_database(clusterFile, &db)); + return db; +} + +static FDBDatabase *db = nullptr; +static std::string prefix; + +std::string key(const std::string& key) { + return prefix + key; +} + +// Blocks until the given future is ready, returning an error code if there was +// an issue. +fdb_error_t wait_future(fdb::Future &f) { + fdb_check(f.block_until_ready()); + return f.get_error(); +} + +// Given a string s, returns the "lowest" string greater than any string that +// starts with s. Taken from +// https://github.com/apple/foundationdb/blob/e7d72f458c6a985fdfa677ae021f357d6f49945b/flow/flow.cpp#L223. +std::string strinc(const std::string &s) { + int index = -1; + for (index = s.size() - 1; index >= 0; --index) { + if ((uint8_t)s[index] != 255) { + break; + } + } + + assert(index >= 0); + + std::string r = s.substr(0, index + 1); + char *p = r.data(); + p[r.size() - 1]++; + return r; +} + +TEST_CASE("strinc") { + CHECK(strinc("a").compare("b") == 0); + CHECK(strinc("y").compare("z") == 0); + CHECK(strinc("!").compare("\"") == 0); + CHECK(strinc("*").compare("+") == 0); + CHECK(strinc("fdb").compare("fdc") == 0); + CHECK(strinc("foundation database 6").compare("foundation database 7") == 0); + + char terminated[] = {'a', 'b', '\xff'}; + CHECK(strinc(std::string(terminated, 3)).compare("ac") == 0); +} + +// Helper function to add `prefix` to all keys in the given map. Returns a new +// map. +std::map +create_data(std::map &&map) { + std::map out; + for (const auto & [ key, val ] : map) { + out[prefix + key] = val; + } + return out; +} + +// Clears all data in the database, then inserts the given key value pairs. +void insert_data(FDBDatabase *db, + const std::map &data) { + fdb::Transaction tr(db); + auto end_key = strinc(prefix); + while (1) { + tr.clear_range(prefix, end_key); + for (const auto & [ key, val ] : data) { + tr.set(key, val); + } + + fdb::EmptyFuture f1 = tr.commit(); + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } +} + +// Get the value associated with `key_name` from the database. Accepts a list +// of transaction options to apply (values for options not supported). Returns +// an optional which will be populated with the result if one was found. +std::optional +get_value(std::string_view key, fdb_bool_t snapshot, + std::vector options) { + fdb::Transaction tr(db); + while (1) { + for (auto &option : options) { + fdb_check(tr.set_option(option, nullptr, 0)); + } + fdb::ValueFuture f1 = tr.get(key, snapshot); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + return out_present ? std::make_optional(std::string(val, vallen)) : std::nullopt; + } +} + +struct GetRangeResult { + // List of key-value pairs in the range read. + std::vector> kvs; + // True if values remain in the key range requested. + bool more; + // Set to a non-zero value if an error occurred during the transaction. + fdb_error_t err; +}; + +// Helper function to get a range of kv pairs. Returns a GetRangeResult struct +// containing the results of the range read. Caller is responsible for checking +// error on failure and retrying if necessary. +GetRangeResult +get_range(fdb::Transaction& tr, const uint8_t* begin_key_name, + int begin_key_name_length, fdb_bool_t begin_or_equal, + int begin_offset, const uint8_t* end_key_name, + int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, + int limit, int target_bytes, FDBStreamingMode mode, + int iteration, fdb_bool_t snapshot, fdb_bool_t reverse) { + fdb::KeyValueArrayFuture f1 = tr.get_range( + begin_key_name, begin_key_name_length, begin_or_equal, begin_offset, + end_key_name, end_key_name_length, end_or_equal, end_offset, limit, + target_bytes, mode, iteration, snapshot, reverse); + + fdb_error_t err = wait_future(f1); + if (err) { + return GetRangeResult{{}, false, err}; + } + + const FDBKeyValue *out_kv; + int out_count; + fdb_bool_t out_more; + fdb_check(f1.get(&out_kv, &out_count, &out_more)); + + std::vector> results; + for (int i = 0; i < out_count; ++i) { + std::string key((const char *)out_kv[i].key, out_kv[i].key_length); + std::string value((const char *)out_kv[i].value, out_kv[i].value_length); + results.push_back(std::make_pair(key, value)); + } + return GetRangeResult{results, out_more != 0, 0}; +} + +// Clears all data in the database. +void clear_data(FDBDatabase *db) { + insert_data(db, {}); +} + +struct FdbEvent { + void wait() { + std::unique_lock l(mutex); + cv.wait(l, [this]() { return this->complete; }); + } + void set() { + std::unique_lock l(mutex); + complete = true; + cv.notify_all(); + } + + private: + std::mutex mutex; + std::condition_variable cv; + bool complete = false; +}; + +TEST_CASE("fdb_future_set_callback") { + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + + struct Context { + FdbEvent event; + }; + Context context; + fdb_check(f1.set_callback( + +[](FDBFuture *, void *param) { + auto *context = static_cast(param); + context->event.set(); + }, + &context)); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + context.event.wait(); + break; + } +} + +TEST_CASE("fdb_future_cancel after future completion") { + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get("foo", false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + // Should have no effect + f1.cancel(); + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + break; + } +} + +TEST_CASE("fdb_future_is_ready") { + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get("foo", false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + CHECK(f1.is_ready()); + break; + } +} + +TEST_CASE("fdb_future_release_memory") { + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get("foo", false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + // "After [fdb_future_release_memory] has been called the same number of + // times as fdb_future_get_*(), further calls to fdb_future_get_*() will + // return a future_released error". + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + f1.release_memory(); + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + f1.release_memory(); + f1.release_memory(); + err = f1.get(&out_present, (const uint8_t **)&val, &vallen); + CHECK(err == 1102); // future_released + break; + } +} + +TEST_CASE("fdb_future_get_int64") { + fdb::Transaction tr(db); + while (1) { + fdb::Int64Future f1 = tr.get_read_version(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int64_t rv; + fdb_check(f1.get(&rv)); + CHECK(rv > 0); + break; + } +} + +TEST_CASE("fdb_future_get_key") { + insert_data(db, + create_data({ { "a", "1" }, { "baz", "2" }, { "bar", "3" } })); + + fdb::Transaction tr(db); + while (1) { + fdb::KeyFuture f1 = tr.get_key( + FDB_KEYSEL_FIRST_GREATER_THAN( + (const uint8_t *)key("a").c_str(), + key("a").size() + ), /* snapshot */ false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + const uint8_t *key; + int keylen; + fdb_check(f1.get(&key, &keylen)); + + std::string dbKey((const char *)key, keylen); + CHECK(dbKey.compare(prefix + "bar") == 0); + break; + } +} + +TEST_CASE("fdb_future_get_value") { + insert_data(db, create_data({ { "foo", "bar" } })); + + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get(key("foo"), /* snapshot */ false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(out_present); + std::string dbValue(val, vallen); + CHECK(dbValue.compare("bar") == 0); + break; + } +} + +TEST_CASE("fdb_future_get_string_array") { + insert_data(db, create_data({ { "foo", "bar" } })); + + fdb::Transaction tr(db); + while (1) { + fdb::StringArrayFuture f1 = tr.get_addresses_for_key(key("foo")); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + const char **strings; + int count; + fdb_check(f1.get(&strings, &count)); + + CHECK(count > 0); + for (int i = 0; i < count; ++i) { + CHECK(strlen(strings[i]) > 0); + } + break; + } +} + +TEST_CASE("fdb_future_get_keyvalue_array") { + std::map data = + create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } }); + insert_data(db, data); + + fdb::Transaction tr(db); + while (1) { + fdb::KeyValueArrayFuture f1 = tr.get_range( + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( + (const uint8_t *)key("a").c_str(), + key("a").size() + ), + FDB_KEYSEL_LAST_LESS_OR_EQUAL( + (const uint8_t *)key("c").c_str(), + key("c").size() + ) + 1, /* limit */ 0, /* target_bytes */ 0, + /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0, + /* snapshot */ false, /* reverse */ 0); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + FDBKeyValue const *out_kv; + int out_count; + int out_more; + fdb_check(f1.get(&out_kv, &out_count, &out_more)); + + CHECK(out_count > 0); + CHECK(out_count <= 3); + if (out_count < 3) { + CHECK(out_more); + } + + for (int i = 0; i < out_count; ++i) { + FDBKeyValue kv = *out_kv++; + + std::string key((const char *)kv.key, kv.key_length); + std::string value((const char *)kv.value, kv.value_length); + + CHECK(data[key].compare(value) == 0); + } + break; + } +} + +TEST_CASE("cannot read system key") { + fdb::Transaction tr(db); + + fdb::ValueFuture f1 = tr.get("\xff/coordinators", /* snapshot */ false); + + fdb_error_t err = wait_future(f1); + CHECK(err == 2004); // key_outside_legal_range +} + +TEST_CASE("read system key") { + auto value = get_value("\xff/coordinators", /* snapshot */ false, + { FDB_TR_OPTION_READ_SYSTEM_KEYS }); + REQUIRE(value.has_value()); +} + +TEST_CASE("cannot write system key") { + fdb::Transaction tr(db); + + tr.set("\xff\x02", "bar"); + + fdb::EmptyFuture f1 = tr.commit(); + fdb_error_t err = wait_future(f1); + CHECK(err == 2004); // key_outside_legal_range +} + +TEST_CASE("write system key") { + fdb::Transaction tr(db); + + std::string syskey("\xff\x02"); + fdb_check(tr.set_option(FDB_TR_OPTION_ACCESS_SYSTEM_KEYS, nullptr, 0)); + tr.set(syskey, "bar"); + + while (1) { + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(syskey, /* snapshot */ false, + { FDB_TR_OPTION_READ_SYSTEM_KEYS }); + REQUIRE(value.has_value()); + CHECK(value->compare("bar") == 0); +} + +TEST_CASE("fdb_transaction read_your_writes") { + fdb::Transaction tr(db); + clear_data(db); + + while (1) { + tr.set("foo", "bar"); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ false); + + // Read before committing, should read the initial write. + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(out_present); + std::string value(val, vallen); + CHECK(value.compare("bar") == 0); + break; + } +} + +TEST_CASE("fdb_transaction_set_option read_your_writes_disable") { + clear_data(db); + + fdb::Transaction tr(db); + while (1) { + fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0)); + tr.set("foo", "bar"); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ false); + + // Read before committing, shouldn't read the initial write because + // read_your_writes is disabled. + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(!out_present); + break; + } +} + +TEST_CASE("fdb_transaction_set_option snapshot_read_your_writes_enable") { + clear_data(db); + + fdb::Transaction tr(db); + while (1) { + // Enable read your writes for snapshot reads. + fdb_check(tr.set_option(FDB_TR_OPTION_SNAPSHOT_RYW_ENABLE, nullptr, 0)); + tr.set("foo", "bar"); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(out_present); + std::string value(val, vallen); + CHECK(value.compare("bar") == 0); + break; + } +} + +TEST_CASE("fdb_transaction_set_option snapshot_read_your_writes_disable") { + clear_data(db); + + fdb::Transaction tr(db); + while (1) { + // Disable read your writes for snapshot reads. + fdb_check(tr.set_option(FDB_TR_OPTION_SNAPSHOT_RYW_DISABLE, nullptr, 0)); + tr.set("foo", "bar"); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + fdb::ValueFuture f2 = tr.get("foo", /*snapshot*/ false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f3 = tr.on_error(err); + fdb_check(wait_future(f3)); + continue; + } + + int out_present; + char *val; + int vallen; + fdb_check(f1.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(!out_present); + + // Non-snapshot reads should still read writes in the transaction. + err = wait_future(f2); + if (err) { + fdb::EmptyFuture f3 = tr.on_error(err); + fdb_check(wait_future(f3)); + continue; + } + fdb_check(f2.get(&out_present, (const uint8_t **)&val, &vallen)); + + CHECK(out_present); + std::string value(val, vallen); + CHECK(value.compare("bar") == 0); + break; + } +} + +TEST_CASE("fdb_transaction_set_option timeout") { + fdb::Transaction tr(db); + // Set smallest possible timeout, retry until a timeout occurs. + int64_t timeout = 1; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, (const uint8_t *)&timeout, + sizeof(timeout))); + + fdb_error_t err = 0; + while (!err) { + fdb::ValueFuture f1 = tr.get("foo", /* snapshot */ false); + err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + err = wait_future(f2); + } + } + CHECK(err == 1031); // transaction_timed_out +} + +TEST_CASE("FDB_DB_OPTION_TRANSACTION_TIMEOUT") { + // Set smallest possible timeout, retry until a timeout occurs. + int64_t timeout = 1; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, + (const uint8_t *)&timeout, + sizeof(timeout))); + + fdb::Transaction tr(db); + fdb_error_t err = 0; + while (!err) { + fdb::ValueFuture f1 = tr.get("foo", /* snapshot */ false); + err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + err = wait_future(f2); + } + } + CHECK(err == 1031); // transaction_timed_out + + // Reset transaction timeout (disable timeout). + timeout = 0; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, + (const uint8_t *)&timeout, + sizeof(timeout))); +} + +TEST_CASE("fdb_transaction_set_option size_limit too small") { + fdb::Transaction tr(db); + + // Size limit must be at least 32 to be valid, so test a smaller size. + int64_t size_limit = 31; + fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, + (const uint8_t *)&size_limit, sizeof(size_limit))); + tr.set("foo", "bar"); + fdb::EmptyFuture f1 = tr.commit(); + + CHECK(wait_future(f1) == 2006); // invalid_option_value +} + +TEST_CASE("fdb_transaction_set_option size_limit too large") { + fdb::Transaction tr(db); + + // Size limit must be less than or equal to 10,000,000. + int64_t size_limit = 10000001; + fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, + (const uint8_t *)&size_limit, sizeof(size_limit))); + tr.set("foo", "bar"); + fdb::EmptyFuture f1 = tr.commit(); + + CHECK(wait_future(f1) == 2006); // invalid_option_value +} + +TEST_CASE("fdb_transaction_set_option size_limit") { + fdb::Transaction tr(db); + + int64_t size_limit = 32; + fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, + (const uint8_t *)&size_limit, sizeof(size_limit))); + tr.set("foo", "foundation database is amazing"); + fdb::EmptyFuture f1 = tr.commit(); + + CHECK(wait_future(f1) == 2101); // transaction_too_large +} + +// Setting the transaction size limit as a database option causes issues when +// outside the bounds of acceptable values. TODO: Needs investigating... +// TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT too small") { +// // Size limit must be at least 32 to be valid, so test a smaller size. +// int64_t size_limit = 31; +// fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, +// (const uint8_t *)&size_limit, +// sizeof(size_limit))); +// +// fdb::Transaction tr(db); +// tr.set((const uint8_t *)"foo", 3, (const uint8_t *)"bar", 3); +// fdb::EmptyFuture f1 = tr.commit(); +// +// CHECK(wait_future(f1) == 2006); // invalid_option_value +// +// // Set size limit back to default. +// size_limit = 10000000; +// fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, +// (const uint8_t *)&size_limit, +// sizeof(size_limit))); +// } + +// TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT too large") { +// // Size limit must be less than or equal to 10,000,000. +// int64_t size_limit = 10000001; +// fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, +// (const uint8_t *)&size_limit, +// sizeof(size_limit))); +// +// fdb::Transaction tr(db); +// tr.set((const uint8_t *)"foo", 3, (const uint8_t *)"bar", 3); +// fdb::EmptyFuture f1 = tr.commit(); +// +// CHECK(wait_future(f1) == 2006); // invalid_option_value +// +// // Set size limit back to default. +// size_limit = 10000000; +// fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, +// (const uint8_t *)&size_limit, +// sizeof(size_limit))); +// } + +TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT") { + int64_t size_limit = 32; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, + (const uint8_t *)&size_limit, + sizeof(size_limit))); + + fdb::Transaction tr(db); + tr.set("foo", "foundation database is amazing"); + fdb::EmptyFuture f1 = tr.commit(); + + CHECK(wait_future(f1) == 2101); // transaction_too_large + + // Set size limit back to default. + size_limit = 10000000; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, + (const uint8_t *)&size_limit, + sizeof(size_limit))); +} + +TEST_CASE("fdb_transaction_set_read_version old_version") { + fdb::Transaction tr(db); + + tr.set_read_version(1); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + + fdb_error_t err = wait_future(f1); + CHECK(err == 1007); // transaction_too_old +} + +TEST_CASE("fdb_transaction_set_read_version future_version") { + fdb::Transaction tr(db); + + tr.set_read_version(1UL << 62); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + + fdb_error_t err = wait_future(f1); + CHECK(err == 1009); // future_version +} + +TEST_CASE("fdb_transaction_get_range reverse") { + std::map data = + create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } }); + insert_data(db, data); + + fdb::Transaction tr(db); + while (1) { + auto result = get_range( + tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( + (const uint8_t*)key("a").c_str(), + key("a").size() + ), + FDB_KEYSEL_LAST_LESS_OR_EQUAL( + (const uint8_t*)key("d").c_str(), + key("d").size() + ) + 1, /* limit */ 0, /* target_bytes */ 0, + /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0, + /* snapshot */ false, /* reverse */ 1); + + if (result.err) { + fdb::EmptyFuture f1 = tr.on_error(result.err); + fdb_check(wait_future(f1)); + continue; + } + + CHECK(result.kvs.size() > 0); + CHECK(result.kvs.size() <= 4); + if (result.kvs.size() < 4) { + CHECK(result.more); + } + + // Read data in reverse order. + auto it = data.rbegin(); + for (auto results_it = result.kvs.begin(); results_it != result.kvs.end(); ++results_it, ++it) { + std::string data_key = it->first; + std::string data_value = it->second; + + auto [key, value] = *results_it; + + CHECK(data_key.compare(key) == 0); + CHECK(data[data_key].compare(value) == 0); + } + break; + } +} + +TEST_CASE("fdb_transaction_get_range limit") { + std::map data = + create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } }); + insert_data(db, data); + + fdb::Transaction tr(db); + while (1) { + auto result = get_range( + tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( + (const uint8_t*)key("a").c_str(), + key("a").size() + ), + FDB_KEYSEL_LAST_LESS_OR_EQUAL( + (const uint8_t*)key("d").c_str(), + key("d").size() + ) + 1, /* limit */ 2, /* target_bytes */ 0, + /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0, + /* snapshot */ false, /* reverse */ 0); + + if (result.err) { + fdb::EmptyFuture f1 = tr.on_error(result.err); + fdb_check(wait_future(f1)); + continue; + } + + CHECK(result.kvs.size() > 0); + CHECK(result.kvs.size() <= 2); + if (result.kvs.size() < 4) { + CHECK(result.more); + } + + for (const auto& [ key, value ] : result.kvs) { + CHECK(data[key].compare(value) == 0); + } + break; + } +} + +TEST_CASE("fdb_transaction_get_range FDB_STREAMING_MODE_EXACT") { + std::map data = + create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } }); + insert_data(db, data); + + fdb::Transaction tr(db); + while (1) { + auto result = get_range( + tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( + (const uint8_t*)key("a").c_str(), + key("a").size() + ), + FDB_KEYSEL_LAST_LESS_OR_EQUAL( + (const uint8_t*)key("d").c_str(), + key("d").size() + ) + 1, /* limit */ 3, /* target_bytes */ 0, + /* FDBStreamingMode */ FDB_STREAMING_MODE_EXACT, /* iteration */ 0, + /* snapshot */ false, /* reverse */ 0); + + if (result.err) { + fdb::EmptyFuture f1 = tr.on_error(result.err); + fdb_check(wait_future(f1)); + continue; + } + + CHECK(result.kvs.size() == 3); + CHECK(result.more); + + for (const auto& [ key, value ] : result.kvs) { + CHECK(data[key].compare(value) == 0); + } + break; + } +} + +TEST_CASE("fdb_transaction_clear") { + insert_data(db, create_data({ { "foo", "bar" } })); + + fdb::Transaction tr(db); + while (1) { + tr.clear(key("foo")); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(!value.has_value()); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_ADD") { + insert_data(db, create_data({ { "foo", "a" } })); + + fdb::Transaction tr(db); + int8_t param = 1; + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)¶m, sizeof(param), + FDB_MUTATION_TYPE_ADD); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 'b'); // incrementing 'a' results in 'b' +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_AND") { + // Test bitwise and on values of same length: + // db key = foo + // db value = 'a' == 97 + // param = 'b' == 98 + // + // 'a' == 97 == 0b01100001 + // & 'b' == 98 == 0b01100010 + // ----------------------- + // 0b01100000 == 96 == '`' + // + // Test bitwise and on extended database value: + // db key = bar + // db value = 'c' == 99 + // param = "ad" + // + // 'c' == 99 == 0b0110001100000000 (zero extended on right to match length of param) + // & "ad" == 0b0110000101100100 + // ------------------------------- + // 0b0110000100000000 == 'a' followed by null (0) + // + // Test bitwise and on truncated database value: + // db key = baz + // db value = "abc" + // param = 'e' == 101 + // + // "abc" -> 0b01100001 (truncated to "a" to match length of param) + // & 'e' == 101 0b01100101 + // --------------------- + // 0b01100001 == 97 == 'a' + // + insert_data(db, create_data({ { "foo", "a" }, { "bar", "c" }, { "baz", "abc" } })); + + fdb::Transaction tr(db); + char param[] = { 'a', 'd' }; + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BIT_AND); + tr.atomic_op(key("bar"), (const uint8_t *)param, 2, + FDB_MUTATION_TYPE_BIT_AND); + tr.atomic_op(key("baz"), (const uint8_t *)"e", 1, + FDB_MUTATION_TYPE_BIT_AND); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 96); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 2); + CHECK(value->data()[0] == 97); + CHECK(value->data()[1] == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 97); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_OR") { + // Test bitwise or on values of same length: + // db key = foo + // db value = 'a' == 97 + // param = 'b' == 98 + // + // 'a' == 97 == 0b01100001 + // | 'b' == 98 == 0b01100010 + // ----------------------- + // 0b01100011 == 99 == 'c' + // + // Test bitwise or on extended database value: + // db key = bar + // db value = 'b' == 98 + // param = "ad" + // + // 'b' == 98 -> 0b0110001000000000 (zero extended on right to match length of param) + // | "ad" == 0b0110000101100100 + // ------------------------------- + // 0b0110001101100100 == "cd" + // + // Test bitwise or on truncated database value: + // db key = baz + // db value = "abc" + // param = 'd' == 100 + // + // "abc" -> 0b01100001 (truncated to "a" to match length of param) + // | 'd' == 100 0b01100100 + // --------------------- + // 0b01100101 == 101 == 'e' + // + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "abc" } })); + + fdb::Transaction tr(db); + char param[] = { 'a', 'd' }; + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BIT_OR); + tr.atomic_op(key("bar"), (const uint8_t *)param, 2, + FDB_MUTATION_TYPE_BIT_OR); + tr.atomic_op(key("baz"), (const uint8_t *)"d", 1, + FDB_MUTATION_TYPE_BIT_OR); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 99); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("cd") == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 101); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_XOR") { + // Test bitwise xor on values of same length: + // db key = foo + // db value = 'a' == 97 + // param = 'b' == 98 + // + // 'a' == 97 == 0b01100001 + // ^ 'b' == 98 == 0b01100010 + // ----------------------- + // 0b00000011 == 0x3 + // + // Test bitwise xor on extended database value: + // db key = bar + // db value = 'b' == 98 + // param = "ad" + // + // 'b' == 98 -> 0b0110001000000000 (zero extended on right to match length of param) + // ^ "ad" == 0b0110000101100100 + // ------------------------------- + // 0b0000001101100100 == 0x3 followed by 0x64 + // + // Test bitwise xor on truncated database value: + // db key = baz + // db value = "abc" + // param = 'd' == 100 + // + // "abc" -> 0b01100001 (truncated to "a" to match length of param) + // ^ 'd' == 100 0b01100100 + // --------------------- + // 0b00000101 == 0x5 + // + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "abc" } })); + + fdb::Transaction tr(db); + char param[] = { 'a', 'd' }; + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BIT_XOR); + tr.atomic_op(key("bar"), (const uint8_t *)param, 2, + FDB_MUTATION_TYPE_BIT_XOR); + tr.atomic_op(key("baz"), (const uint8_t *)"d", 1, + FDB_MUTATION_TYPE_BIT_XOR); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 0x3); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 2); + CHECK(value->data()[0] == 0x3); + CHECK(value->data()[1] == 0x64); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 1); + CHECK(value->data()[0] == 0x5); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_COMPARE_AND_CLEAR") { + // Atomically remove a key-value pair from the database based on a value + // comparison. + insert_data(db, create_data({ { "foo", "bar" }, { "fdb", "foundation" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"bar", 3, + FDB_MUTATION_TYPE_COMPARE_AND_CLEAR); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + CHECK(!value.has_value()); + + value = get_value(key("fdb"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("foundation") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_APPEND_IF_FITS") { + // Atomically append a value to an existing key-value pair, or insert the + // key-value pair if an existing key-value pair doesn't exist. + insert_data(db, create_data({ { "foo", "f" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"db", 2, + FDB_MUTATION_TYPE_APPEND_IF_FITS); + tr.atomic_op(key("bar"), (const uint8_t *)"foundation", 10, + FDB_MUTATION_TYPE_APPEND_IF_FITS); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("fdb") == 0); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("foundation") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_MAX") { + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "cba" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, FDB_MUTATION_TYPE_MAX); + // Value in database will be extended with zeros to match length of param. + tr.atomic_op(key("bar"), (const uint8_t *)"aa", 2, FDB_MUTATION_TYPE_MAX); + // Value in database will be truncated to match length of param. + tr.atomic_op(key("baz"), (const uint8_t *)"b", 1, FDB_MUTATION_TYPE_MAX); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("b") == 0); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("aa") == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("c") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_MIN") { + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "cba" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, FDB_MUTATION_TYPE_MIN); + // Value in database will be extended with zeros to match length of param. + tr.atomic_op(key("bar"), (const uint8_t *)"aa", 2, FDB_MUTATION_TYPE_MIN); + // Value in database will be truncated to match length of param. + tr.atomic_op(key("baz"), (const uint8_t *)"b", 1, FDB_MUTATION_TYPE_MIN); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("a") == 0); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->size() == 2); + CHECK(value->data()[0] == 'b'); + CHECK(value->data()[1] == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("b") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BYTE_MAX") { + // The difference with FDB_MUTATION_TYPE_MAX is that strings will not be + // extended/truncated so lengths match. + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "cba" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BYTE_MAX); + tr.atomic_op(key("bar"), (const uint8_t *)"cc", 2, + FDB_MUTATION_TYPE_BYTE_MAX); + tr.atomic_op(key("baz"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BYTE_MAX); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("b") == 0); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("cc") == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("cba") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BYTE_MIN") { + // The difference with FDB_MUTATION_TYPE_MIN is that strings will not be + // extended/truncated so lengths match. + insert_data(db, create_data({ { "foo", "a" }, { "bar", "b" }, { "baz", "abc" } })); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BYTE_MIN); + tr.atomic_op(key("bar"), (const uint8_t *)"aa", 2, + FDB_MUTATION_TYPE_BYTE_MIN); + tr.atomic_op(key("baz"), (const uint8_t *)"b", 1, + FDB_MUTATION_TYPE_BYTE_MIN); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("a") == 0); + + value = get_value(key("bar"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("aa") == 0); + + value = get_value(key("baz"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("abc") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY") { + int offset = prefix.size() + 3; + const char *p = reinterpret_cast(&offset); + char keybuf[] = {'f', 'o', 'o', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', p[0], p[1], p[2], p[3]}; + std::string key = prefix + std::string(keybuf, 17); + std::string versionstamp(""); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key, (const uint8_t *)"bar", 3, + FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY); + fdb::KeyFuture f1 = tr.get_versionstamp(); + fdb::EmptyFuture f2 = tr.commit(); + + fdb_error_t err = wait_future(f2); + if (err) { + fdb::EmptyFuture f3 = tr.on_error(err); + fdb_check(wait_future(f3)); + continue; + } + + fdb_check(wait_future(f1)); + + const uint8_t *key; + int keylen; + fdb_check(f1.get(&key, &keylen)); + + versionstamp = std::string((const char *)key, keylen); + break; + } + + REQUIRE(versionstamp.size() > 0); + std::string dbKey(prefix + "foo" + versionstamp); + auto value = get_value(dbKey, /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("bar") == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE") { + // Don't care about prefixing value like we did the key. + char valbuf[] = {'b', 'a', 'r', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 3, 0, 0, 0}; + std::string versionstamp(""); + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(key("foo"), (const uint8_t *)valbuf, 17, + FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); + fdb::KeyFuture f1 = tr.get_versionstamp(); + fdb::EmptyFuture f2 = tr.commit(); + + fdb_error_t err = wait_future(f2); + if (err) { + fdb::EmptyFuture f3 = tr.on_error(err); + fdb_check(wait_future(f3)); + continue; + } + + fdb_check(wait_future(f1)); + + const uint8_t *key; + int keylen; + fdb_check(f1.get(&key, &keylen)); + + versionstamp = std::string((const char *)key, keylen); + break; + } + + REQUIRE(versionstamp.size() > 0); + auto value = get_value(key("foo"), /* snapshot */ false, {}); + REQUIRE(value.has_value()); + CHECK(value->compare("bar" + versionstamp) == 0); +} + +TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY invalid index") { + // Only 9 bytes available starting at index 4 (ten bytes needed), should + // return an error. + char keybuf[] = {'f', 'o', 'o', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 4, 0, 0, 0}; + + fdb::Transaction tr(db); + while (1) { + tr.atomic_op(keybuf, (const uint8_t *)"bar", 3, + FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY); + fdb::EmptyFuture f1 = tr.commit(); + + CHECK(wait_future(f1) != 0); // type of error not specified + break; + } +} + +TEST_CASE("fdb_transaction_get_committed_version read_only") { + // Read-only transaction should have a committed version of -1. + fdb::Transaction tr(db); + while (1) { + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ false); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int64_t out_version; + fdb_check(tr.get_committed_version(&out_version)); + CHECK(out_version == -1); + break; + } +} + +TEST_CASE("fdb_transaction_get_committed_version") { + fdb::Transaction tr(db); + while (1) { + tr.set(key("foo"), "bar"); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int64_t out_version; + fdb_check(tr.get_committed_version(&out_version)); + CHECK(out_version >= 0); + break; + } +} + +TEST_CASE("fdb_transaction_get_approximate_size") { + fdb::Transaction tr(db); + while (1) { + tr.set(key("foo"), "bar"); + fdb::Int64Future f1 = tr.get_approximate_size(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + + int64_t size; + fdb_check(f1.get(&size)); + CHECK(size >= 3); + break; + } +} + +TEST_CASE("fdb_transaction_watch read_your_writes_disable") { + // Watches created on a transaction with the option READ_YOUR_WRITES_DISABLE + // should return a watches_disabled error. + fdb::Transaction tr(db); + fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0)); + fdb::EmptyFuture f1 = tr.watch(key("foo")); + + CHECK(wait_future(f1) == 1034); // watches_disabled +} + +TEST_CASE("fdb_transaction_watch reset") { + // Resetting (or destroying) an uncommitted transaction should cause watches + // created by the transaction to fail with a transaction_cancelled error. + fdb::Transaction tr(db); + fdb::EmptyFuture f1 = tr.watch(key("foo")); + tr.reset(); + CHECK(wait_future(f1) == 1025); // transaction_cancelled +} + +TEST_CASE("fdb_transaction_watch max watches") { + int64_t max_watches = 3; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, + (const uint8_t *)&max_watches, 8)); + + auto event = std::make_shared(); + + fdb::Transaction tr(db); + while (1) { + fdb::EmptyFuture f1 = tr.watch(key("a")); + fdb::EmptyFuture f2 = tr.watch(key("b")); + fdb::EmptyFuture f3 = tr.watch(key("c")); + fdb::EmptyFuture f4 = tr.watch(key("d")); + fdb::EmptyFuture f5 = tr.commit(); + + fdb_error_t err = wait_future(f5); + if (err) { + fdb::EmptyFuture f6 = tr.on_error(err); + fdb_check(wait_future(f6)); + continue; + } + + // Callbacks will be triggered with operation_cancelled errors once the + // too_many_watches error fires, as the other futures will go out of scope + // and be cleaned up. The future which too_many_watches occurs on is + // nondeterministic, so each future is checked. + fdb_check(f1.set_callback( + +[](FDBFuture *f, void *param) { + fdb_error_t err = fdb_future_get_error(f); + if (err != 1101) { // operation_cancelled + CHECK(err == 1032); // too_many_watches + } + auto *event = static_cast *>(param); + (*event)->set(); + delete event; + }, new std::shared_ptr(event))); + fdb_check(f2.set_callback( + +[](FDBFuture *f, void *param) { + fdb_error_t err = fdb_future_get_error(f); + if (err != 1101) { // operation_cancelled + CHECK(err == 1032); // too_many_watches + } + auto *event = static_cast *>(param); + (*event)->set(); + delete event; + }, new std::shared_ptr(event))); + fdb_check(f3.set_callback( + +[](FDBFuture *f, void *param) { + fdb_error_t err = fdb_future_get_error(f); + if (err != 1101) { // operation_cancelled + CHECK(err == 1032); // too_many_watches + } + auto *event = static_cast *>(param); + (*event)->set(); + delete event; + }, new std::shared_ptr(event))); + fdb_check(f4.set_callback( + +[](FDBFuture *f, void *param) { + fdb_error_t err = fdb_future_get_error(f); + if (err != 1101) { // operation_cancelled + CHECK(err == 1032); // too_many_watches + } + auto *event = static_cast *>(param); + (*event)->set(); + delete event; + }, new std::shared_ptr(event))); + + event->wait(); + break; + } + + // Reset available number of watches. + max_watches = 10000; + fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, + (const uint8_t *)&max_watches, 8)); +} + +TEST_CASE("fdb_transaction_watch") { + insert_data(db, create_data({ { "foo", "foo" } })); + + struct Context { + FdbEvent event; + }; + Context context; + + fdb::Transaction tr(db); + while (1) { + fdb::EmptyFuture f1 = tr.watch(key("foo")); + fdb::EmptyFuture f2 = tr.commit(); + + fdb_error_t err = wait_future(f2); + if (err) { + fdb::EmptyFuture f3 = tr.on_error(err); + fdb_check(wait_future(f3)); + continue; + } + + fdb_check(f1.set_callback( + +[](FDBFuture *, void *param) { + auto *context = static_cast(param); + context->event.set(); + }, + &context)); + + // Update value for key "foo" to trigger the watch. + insert_data(db, create_data({ { "foo", "bar" } })); + context.event.wait(); + break; + } +} + +TEST_CASE("fdb_transaction_cancel") { + // Cannot use transaction after cancelling it... + fdb::Transaction tr(db); + tr.cancel(); + fdb::ValueFuture f1 = tr.get("foo", /* snapshot */ false); + CHECK(wait_future(f1) == 1025); // transaction_cancelled + + // ... until the transaction has been reset. + tr.reset(); + fdb::ValueFuture f2 = tr.get("foo", /* snapshot */ false); + fdb_check(wait_future(f2)); +} + +TEST_CASE("fdb_transaction_add_conflict_range") { + bool success = false; + + bool retry = true; + while (retry) { + fdb::Transaction tr(db); + while (1) { + fdb::Int64Future f1 = tr.get_read_version(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + fdb::Transaction tr2(db); + while (1) { + fdb_check(tr2.add_conflict_range(key("a"), strinc(key("a")), + FDB_CONFLICT_RANGE_TYPE_WRITE)); + fdb::EmptyFuture f1 = tr2.commit(); + + fdb_error_t err = wait_future(f1); + if (err) { + fdb::EmptyFuture f2 = tr2.on_error(err); + fdb_check(wait_future(f2)); + continue; + } + break; + } + + while (1) { + fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")), + FDB_CONFLICT_RANGE_TYPE_READ)); + fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")), + FDB_CONFLICT_RANGE_TYPE_WRITE)); + fdb::EmptyFuture f1 = tr.commit(); + + fdb_error_t err = wait_future(f1); + if (err == 1020) { // not_committed + // Test should pass if transactions conflict. + success = true; + retry = false; + } else if (err) { + fdb::EmptyFuture f2 = tr.on_error(err); + fdb_check(wait_future(f2)); + retry = true; + } else { + // If the transaction succeeded, something went wrong. + CHECK(false); + retry = false; + } + break; + } + } + + // Double check that failure was achieved and the loop wasn't just broken out + // of. + CHECK(success); +} + +TEST_CASE("fdb_error_predicate") { + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1007)); // transaction_too_old + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1020)); // not_committed + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1038)); // database_locked + + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1036)); // accessed_unreadable + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2000)); // client_invalid_operation + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2004)); // key_outside_legal_range + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2005)); // inverted_range + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2006)); // invalid_option_value + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2007)); // invalid_option + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2011)); // version_invalid + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2020)); // transaction_invalid_version + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2023)); // transaction_read_only + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2100)); // incompatible_protocol_version + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2101)); // transaction_too_large + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2102)); // key_too_large + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2103)); // value_too_large + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2108)); // unsupported_operation + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 2200)); // api_version_unset + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 4000)); // unknown_error + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 4001)); // internal_error + + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1021)); // commit_unknown_result + + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1000)); // operation_failed + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1004)); // timed_out + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1025)); // transaction_cancelled + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1038)); // database_locked + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 1101)); // operation_cancelled + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_MAYBE_COMMITTED, 2002)); // commit_read_incomplete + + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1007)); // transaction_too_old + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1020)); // not_committed + CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1038)); // database_locked + + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1021)); // commit_unknown_result + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1025)); // transaction_cancelled + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1031)); // transaction_timed_out + CHECK(!fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED, 1040)); // proxy_memory_limit_exceeded +} + +// Feature not live yet, re-enable when checking if a blocking call is made +// from the network thread is live. +TEST_CASE("block_from_callback" + * doctest::skip(true)) { + fdb::Transaction tr(db); + fdb::ValueFuture f1 = tr.get("foo", /*snapshot*/ true); + struct Context { + FdbEvent event; + fdb::Transaction *tr; + }; + Context context; + context.tr = &tr; + fdb_check(f1.set_callback( + +[](FDBFuture *, void *param) { + auto *context = static_cast(param); + fdb::ValueFuture f2 = context->tr->get("bar", /*snapshot*/ true); + fdb_error_t error = f2.block_until_ready(); + if (error) { + CHECK(error == /*blocked_from_network_thread*/ 2025); + } + context->event.set(); + }, + &context)); + context.event.wait(); +} + +int main(int argc, char **argv) { + if (argc != 3) { + std::cout << "Unit tests for the FoundationDB C API.\n" + << "Usage: fdb_c_unit_tests /path/to/cluster_file key_prefix" + << std::endl; + return 1; + } + + doctest::Context context; + + fdb_check(fdb_select_api_version(620)); + fdb_check(fdb_setup_network()); + std::thread network_thread{ &fdb_run_network }; + + db = fdb_open_database(argv[1]); + prefix = argv[2]; + int res = context.run(); + fdb_database_destroy(db); + + if (context.shouldExit()) { + fdb_check(fdb_stop_network()); + network_thread.join(); + return res; + } + fdb_check(fdb_stop_network()); + network_thread.join(); + + return res; +} diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 27355138b1..7ac8dfc1bb 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -134,6 +134,7 @@ namespace FDB { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; + Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override; void addReadConflictRange(KeyRangeRef const& keys) override; void addReadConflictKey(KeyRef const& key) override; @@ -157,14 +158,14 @@ namespace FDB { void cancel() override; void reset() override; - TransactionImpl() : tr(NULL) {} + TransactionImpl() : tr(nullptr) {} TransactionImpl(TransactionImpl&& r) noexcept { tr = r.tr; - r.tr = NULL; + r.tr = nullptr; } TransactionImpl& operator=(TransactionImpl&& r) noexcept { tr = r.tr; - r.tr = NULL; + r.tr = nullptr; return *this; } @@ -207,10 +208,10 @@ namespace FDB { if ( value.present() ) throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) ); else - throw_on_error( fdb_network_set_option( option, NULL, 0 ) ); + throw_on_error( fdb_network_set_option( option, nullptr, 0 ) ); } - API* API::instance = NULL; + API* API::instance = nullptr; API::API(int version) : version(version) {} API* API::selectAPIVersion(int apiVersion) { @@ -234,11 +235,11 @@ namespace FDB { } bool API::isAPIVersionSelected() { - return API::instance != NULL; + return API::instance != nullptr; } API* API::getInstance() { - if(API::instance == NULL) { + if(API::instance == nullptr) { throw api_version_unset(); } else { @@ -280,7 +281,7 @@ namespace FDB { if (value.present()) throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); else - throw_on_error(fdb_database_set_option(db, option, NULL, 0)); + throw_on_error(fdb_database_set_option(db, option, nullptr, 0)); } TransactionImpl::TransactionImpl(FDBDatabase* db) { @@ -356,6 +357,16 @@ namespace FDB { }); } + Future>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) { + return backToFuture>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference f) { + FDBKey const* ks; + int count; + throw_on_error(fdb_future_get_key_array(f->f, &ks, &count)); + + return FDBStandalone>(f, VectorRef((KeyRef*)ks, count)); + }); + } + void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); } @@ -417,7 +428,7 @@ namespace FDB { if ( value.present() ) { throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) ); } else { - throw_on_error( fdb_transaction_set_option( tr, option, NULL, 0 ) ); + throw_on_error( fdb_transaction_set_option( tr, option, nullptr, 0 ) ); } } diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 66049cae0c..cd5be3a974 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -31,7 +31,7 @@ namespace FDB { struct CFuture : NonCopyable, ReferenceCounted, FastAllocated { - CFuture() : f(NULL) {} + CFuture() : f(nullptr) {} explicit CFuture(FDBFuture* f) : f(f) {} ~CFuture() { if (f) { @@ -90,6 +90,7 @@ namespace FDB { } virtual Future getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; + virtual Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0; virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; virtual void addReadConflictKey(KeyRef const& key) = 0; diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index e748a716ed..886a659c4f 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -661,6 +661,33 @@ struct GetEstimatedRangeSize : InstructionFunc { const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE"; REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize); +struct GetRangeSplitPoints : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + state std::vector items = data->stack.pop(3); + if (items.size() != 3) + return Void(); + + Standalone s1 = wait(items[0].value); + state Standalone beginKey = Tuple::unpack(s1).getString(0); + + Standalone s2 = wait(items[1].value); + state Standalone endKey = Tuple::unpack(s2).getString(0); + + Standalone s3 = wait(items[2].value); + state int64_t chunkSize = Tuple::unpack(s3).getInt(0); + + Future>> fsplitPoints = instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize); + FDBStandalone> splitPoints = wait(fsplitPoints); + data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS")); + + return Void(); + } +}; +const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS"; +REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints); + struct GetKeyFunc : InstructionFunc { static const char* name; diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 9737391569..d986ddec53 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -579,6 +579,17 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { if e != nil { panic(e) } + case op == "GET_RANGE_SPLIT_POINTS": + r := sm.popKeyRange() + chunkSize := sm.waitAndPop().item.(int64) + _, e := rt.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) { + _ = rtr.GetRangeSplitPoints(r, chunkSize).MustGet() + sm.store(idx, []byte("GOT_RANGE_SPLIT_POINTS")) + return nil, nil + }) + if e != nil { + panic(e) + } case op == "COMMIT": sm.store(idx, sm.currentTransaction().Commit()) case op == "RESET": diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 43718fe738..e51d5eaa8d 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -306,6 +306,57 @@ func (f *futureKeyValueArray) Get() ([]KeyValue, bool, error) { return ret, (more != 0), nil } +// FutureKeyArray represents the asynchronous result of a function +// that returns an array of keys. FutureKeyArray is a lightweight object +// that may be efficiently copied, and is safe for concurrent use by multiple goroutines. +type FutureKeyArray interface { + + // Get returns an array of keys or an error if the asynchronous operation + // associated with this future did not successfully complete. The current + // goroutine will be blocked until the future is ready. + Get() ([]Key, error) + + // MustGet returns an array of keys, or panics if the asynchronous operations + // associated with this future did not successfully complete. The current goroutine + // will be blocked until the future is ready. + MustGet() []Key +} + +type futureKeyArray struct { + *future +} + +func (f *futureKeyArray) Get() ([]Key, error) { + defer runtime.KeepAlive(f.future) + + f.BlockUntilReady() + + var ks *C.FDBKey + var count C.int + + if err := C.fdb_future_get_key_array(f.ptr, &ks, &count); err != 0 { + return nil, Error{int(err)} + } + + ret := make([]Key, int(count)) + + for i := 0; i < int(count); i++ { + kptr := unsafe.Pointer(uintptr(unsafe.Pointer(ks)) + uintptr(i*12)) + + ret[i] = stringRefToSlice(kptr) + } + + return ret, nil +} + +func (f *futureKeyArray) MustGet() []Key { + val, err := f.Get() + if err != nil { + panic(err) + } + return val +} + // FutureInt64 represents the asynchronous result of a function that returns a // database version. FutureInt64 is a lightweight object that may be efficiently // copied, and is safe for concurrent use by multiple goroutines. diff --git a/bindings/go/src/fdb/snapshot.go b/bindings/go/src/fdb/snapshot.go index ca21818729..d20a67d9ad 100644 --- a/bindings/go/src/fdb/snapshot.go +++ b/bindings/go/src/fdb/snapshot.go @@ -87,6 +87,8 @@ func (s Snapshot) GetDatabase() Database { return s.transaction.db } +// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes +// stored in the given range. func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { beginKey, endKey := r.FDBRangeKeys() return s.getEstimatedRangeSizeBytes( @@ -94,3 +96,15 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { endKey.FDBKey(), ) } + +// GetRangeSplitPoints returns a list of keys that can split the given range +// into (roughly) equally sized chunks based on chunkSize. +// Note: the returned split points contain the start key and end key of the given range. +func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray { + beginKey, endKey := r.FDBRangeKeys() + return s.getRangeSplitPoints( + beginKey.FDBKey(), + endKey.FDBKey(), + chunkSize, + ) +} diff --git a/bindings/go/src/fdb/subspace/subspace.go b/bindings/go/src/fdb/subspace/subspace.go index 65f97048c8..d4600d725c 100644 --- a/bindings/go/src/fdb/subspace/subspace.go +++ b/bindings/go/src/fdb/subspace/subspace.go @@ -78,8 +78,9 @@ type Subspace interface { // FoundationDB keys (corresponding to the prefix of this Subspace). fdb.KeyConvertible - // All Subspaces implement fdb.ExactRange and fdb.Range, and describe all - // keys logically in this Subspace. + // All Subspaces implement fdb.ExactRange and fdb.Range, and describe all + // keys strictly within the subspace that encode tuples. Specifically, + // this will include all keys in [prefix + '\x00', prefix + '\xff'). fdb.ExactRange } diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index a7100df701..9c64b06ac7 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -40,6 +40,7 @@ type ReadTransaction interface { GetDatabase() Database Snapshot() Snapshot GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 + GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray ReadTransactor } @@ -318,7 +319,7 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur } } -// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes +// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes // stored in the given range. // Note: the estimated size is calculated based on the sampling done by FDB server. The sampling // algorithm works roughly in this way: the larger the key-value pair is, the more likely it would @@ -334,6 +335,31 @@ func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { ) } +func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize int64) FutureKeyArray { + return &futureKeyArray{ + future: newFuture(C.fdb_transaction_get_range_split_points( + t.ptr, + byteSliceToPtr(beginKey), + C.int(len(beginKey)), + byteSliceToPtr(endKey), + C.int(len(endKey)), + C.int64_t(chunkSize), + )), + } +} + +// GetRangeSplitPoints returns a list of keys that can split the given range +// into (roughly) equally sized chunks based on chunkSize. +// Note: the returned split points contain the start key and end key of the given range. +func (t Transaction) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray { + beginKey, endKey := r.FDBRangeKeys() + return t.getRangeSplitPoints( + beginKey.FDBKey(), + endKey.FDBKey(), + chunkSize, + ) +} + func (t *transaction) getReadVersion() FutureInt64 { return &futureInt64{ future: newFuture(C.fdb_transaction_get_read_version(t.ptr)), diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index cbcbe07c27..2fc24ecf98 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -1,3 +1,6 @@ +set(RUN_JAVA_TESTS ON CACHE BOOL "Run Java unit tests") +set(RUN_JUNIT_TESTS OFF CACHE BOOL "Compile and run junit tests") + set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/async/AsyncIterable.java src/main/com/apple/foundationdb/async/AsyncIterator.java @@ -29,6 +32,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/FDBTransaction.java src/main/com/apple/foundationdb/FutureInt64.java src/main/com/apple/foundationdb/FutureKey.java + src/main/com/apple/foundationdb/FutureKeyArray.java src/main/com/apple/foundationdb/FutureResult.java src/main/com/apple/foundationdb/FutureResults.java src/main/com/apple/foundationdb/FutureStrings.java @@ -44,6 +48,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/package-info.java src/main/com/apple/foundationdb/Range.java src/main/com/apple/foundationdb/RangeQuery.java + src/main/com/apple/foundationdb/KeyArrayResult.java src/main/com/apple/foundationdb/RangeResult.java src/main/com/apple/foundationdb/RangeResultInfo.java src/main/com/apple/foundationdb/RangeResultSummary.java @@ -102,6 +107,10 @@ set(JAVA_TESTS_SRCS src/test/com/apple/foundationdb/test/WatchTest.java src/test/com/apple/foundationdb/test/WhileTrueTest.java) +set(JAVA_JUNIT_TESTS + src/junit/com/apple/foundationdb/tuple/AllTests.java + src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java) + set(GENERATED_JAVA_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/main/com/apple/foundationdb) file(MAKE_DIRECTORY ${GENERATED_JAVA_DIR}) @@ -173,12 +182,6 @@ add_jar(fdb-java ${JAVA_BINDING_SRCS} ${GENERATED_JAVA_FILES} ${CMAKE_SOURCE_DIR OUTPUT_DIR ${PROJECT_BINARY_DIR}/lib VERSION ${CMAKE_PROJECT_VERSION} MANIFEST ${MANIFEST_FILE}) add_dependencies(fdb-java fdb_java_options fdb_java) -# TODO[mpilman]: The java RPM will require some more effort (mostly on debian). However, -# most people will use the fat-jar, so it is not clear how high this priority is. - -#install_jar(fdb-java DESTINATION ${FDB_SHARE_DIR}/java COMPONENT java) -#install(TARGETS fdb_java DESTINATION ${FDB_LIB_DIR} COMPONENT java) - if(NOT OPEN_FOR_IDE) set(FAT_JAR_BINARIES "NOTFOUND" CACHE STRING "Path of a directory structure with libraries to include in fat jar (a lib directory)") @@ -252,4 +255,30 @@ if(NOT OPEN_FOR_IDE) add_dependencies(fat-jar fdb-java) add_dependencies(fat-jar copy_lib) add_dependencies(packages fat-jar) + + if(RUN_JAVA_TESTS) + set(enabled ENABLED) + else() + set(enabled DISABLED) + endif() + set(TEST_CP ${tests_jar} ${target_jar}) + message(STATUS "TEST_CP ${TEST_CP}") + add_java_test(NAME DirectoryTest CLASS_PATH ${TEST_CP} + CLASS com.apple.foundationdb.test.DirectoryTest ${enabled}) + + if(RUN_JUNIT_TESTS) + file(DOWNLOAD "https://search.maven.org/remotecontent?filepath=junit/junit/4.13/junit-4.13.jar" + ${CMAKE_BINARY_DIR}/packages/junit-4.13.jar + EXPECTED_HASH SHA256=4b8532f63bdc0e0661507f947eb324a954d1dbac631ad19c8aa9a00feed1d863) + file(DOWNLOAD "https://repo1.maven.org/maven2/org/hamcrest/hamcrest-all/1.3/hamcrest-all-1.3.jar" + ${CMAKE_BINARY_DIR}/packages/hamcrest-all-1.3.jar + EXPECTED_HASH SHA256=4877670629ab96f34f5f90ab283125fcd9acb7e683e66319a68be6eb2cca60de) + add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} INCLUDE_JARS fdb-java ${CMAKE_BINARY_DIR}/packages/junit-4.13.jar) + get_property(junit_jar_path TARGET fdb-junit PROPERTY JAR_FILE) + add_test(NAME junit + COMMAND ${Java_JAVA_EXECUTABLE} + -cp "${target_jar}:${junit_jar_path}:${CMAKE_BINARY_DIR}/packages/junit-4.13.jar:${CMAKE_BINARY_DIR}/packages/hamcrest-all-1.3.jar" + -Djava.library.path=${CMAKE_BINARY_DIR}/lib + org.junit.runner.JUnitCore "com.apple.foundationdb.tuple.AllTests") + endif() endif() diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 19a68fca27..1cf889649e 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -39,6 +39,8 @@ static thread_local bool is_external = false; static jclass range_result_summary_class; static jclass range_result_class; static jclass string_class; +static jclass key_array_result_class; +static jmethodID key_array_result_init; static jmethodID range_result_init; static jmethodID range_result_summary_init; @@ -305,6 +307,77 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString return arr; } + +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyArray_1get(JNIEnv *jenv, jobject, jlong future) { + if( !future ) { + throwParamNotNull(jenv); + return JNI_NULL; + } + + FDBFuture *f = (FDBFuture *)future; + + const FDBKey *ks; + int count; + fdb_error_t err = fdb_future_get_key_array( f, &ks, &count ); + if( err ) { + safeThrow( jenv, getThrowable( jenv, err ) ); + return JNI_NULL; + } + + int totalKeySize = 0; + for(int i = 0; i < count; i++) { + totalKeySize += ks[i].key_length; + } + + jbyteArray keyArray = jenv->NewByteArray(totalKeySize); + if( !keyArray ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + return JNI_NULL; + } + uint8_t *keys_barr = (uint8_t *)jenv->GetByteArrayElements(keyArray, JNI_NULL); + if (!keys_barr) { + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return JNI_NULL; + } + + jintArray lengthArray = jenv->NewIntArray(count); + if( !lengthArray ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + return JNI_NULL; + } + + jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); + if( !length_barr ) { + if( !jenv->ExceptionOccurred() ) + throwOutOfMem(jenv); + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + return JNI_NULL; + } + + int offset = 0; + for(int i = 0; i < count; i++) { + memcpy(keys_barr + offset, ks[i].key, ks[i].key_length); + length_barr[i] = ks[i].key_length; + offset += ks[i].key_length; + } + + jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0); + jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0); + + jobject result = jenv->NewObject(key_array_result_class, key_array_result_init, keyArray, lengthArray); + if( jenv->ExceptionOccurred() ) + return JNI_NULL; + + return result; + +} + + // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) { if( !future ) { @@ -695,6 +768,35 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIEnv *jenv, jobject, jlong tPtr, + jbyteArray beginKeyBytes, jbyteArray endKeyBytes, jlong chunkSize) { + if( !tPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction *tr = (FDBTransaction *)tPtr; + + uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL ); + if(!startKey) { + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKey) { + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + FDBFuture *f = fdb_transaction_get_range_split_points( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ), chunkSize ); + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT ); + return (jlong)f; +} + JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { if( !tPtr || !keyBytes || !valueBytes ) { throwParamNotNull(jenv); @@ -1071,6 +1173,10 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { range_result_init = env->GetMethodID(local_range_result_class, "", "([B[IZ)V"); range_result_class = (jclass) (env)->NewGlobalRef(local_range_result_class); + jclass local_key_array_result_class = env->FindClass("com/apple/foundationdb/KeyArrayResult"); + key_array_result_init = env->GetMethodID(local_key_array_result_class, "", "([B[I)V"); + key_array_result_class = (jclass) (env)->NewGlobalRef(local_key_array_result_class); + jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary"); range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "", "([BIZ)V"); range_result_summary_class = (jclass) (env)->NewGlobalRef(local_range_result_summary_class); @@ -1089,13 +1195,13 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { return; } else { // delete global references so the GC can collect them - if (range_result_summary_class != NULL) { + if (range_result_summary_class != JNI_NULL) { env->DeleteGlobalRef(range_result_summary_class); } - if (range_result_class != NULL) { + if (range_result_class != JNI_NULL) { env->DeleteGlobalRef(range_result_class); } - if (string_class != NULL) { + if (string_class != JNI_NULL) { env->DeleteGlobalRef(string_class); } } diff --git a/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java b/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java index 3cd7125a97..0b964b10dc 100644 --- a/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java +++ b/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java @@ -27,9 +27,14 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Random; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.Ignore; /** * @author Ben @@ -251,7 +256,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#bisectLeft(java.math.BigInteger[], java.math.BigInteger)}. */ - @Test + @Test @Ignore public void testBisectLeft() { fail("Not yet implemented"); } @@ -259,7 +264,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#compareUnsigned(byte[], byte[])}. */ - @Test + @Test @Ignore public void testCompare() { fail("Not yet implemented"); } @@ -267,7 +272,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#findNext(byte[], byte, int)}. */ - @Test + @Test @Ignore public void testFindNext() { fail("Not yet implemented"); } @@ -275,7 +280,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#findTerminator(byte[], byte, byte, int)}. */ - @Test + @Test @Ignore public void testFindTerminator() { fail("Not yet implemented"); } @@ -283,7 +288,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#copyOfRange(byte[], int, int)}. */ - @Test + @Test @Ignore public void testCopyOfRange() { fail("Not yet implemented"); } @@ -291,7 +296,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#strinc(byte[])}. */ - @Test + @Test @Ignore public void testStrinc() { fail("Not yet implemented"); } @@ -299,7 +304,7 @@ public class ArrayUtilTests { /** * Test method for {@link ByteArrayUtil#printable(byte[])}. */ - @Test + @Test @Ignore public void testPrintable() { fail("Not yet implemented"); } diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index 7bcf39d43c..4455c3ef06 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -80,6 +80,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC return FDBTransaction.this.getEstimatedRangeSizeBytes(range); } + @Override + public CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) { + return FDBTransaction.this.getRangeSplitPoints(begin, end, chunkSize); + } + + @Override + public CompletableFuture getRangeSplitPoints(Range range, long chunkSize) { + return FDBTransaction.this.getRangeSplitPoints(range, chunkSize); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -282,6 +292,21 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC return this.getEstimatedRangeSizeBytes(range.begin, range.end); } + @Override + public CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) { + pointerReadLock.lock(); + try { + return new FutureKeyArray(Transaction_getRangeSplitPoints(getPtr(), begin, end, chunkSize), executor); + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public CompletableFuture getRangeSplitPoints(Range range, long chunkSize) { + return this.getRangeSplitPoints(range.begin, range.end, chunkSize); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -686,4 +711,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC private native void Transaction_cancel(long cPtr); private native long Transaction_getKeyLocations(long cPtr, byte[] key); private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd); + private native long Transaction_getRangeSplitPoints(long cPtr, byte[] keyBegin, byte[] keyEnd, long chunkSize); } diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java b/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java new file mode 100644 index 0000000000..527d7076b0 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java @@ -0,0 +1,37 @@ +/* + * FutureKeyArray.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.concurrent.Executor; + +class FutureKeyArray extends NativeFuture { + FutureKeyArray(long cPtr, Executor executor) { + super(cPtr); + registerMarshalCallback(executor); + } + + @Override + protected KeyArrayResult getIfDone_internal(long cPtr) throws FDBException { + return FutureKeyArray_get(cPtr); + } + + private native KeyArrayResult FutureKeyArray_get(long cPtr) throws FDBException; +} diff --git a/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java new file mode 100644 index 0000000000..174bc89b19 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java @@ -0,0 +1,44 @@ +/* + * KeyArrayResult.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb; + +import java.util.ArrayList; +import java.util.List; + +public class KeyArrayResult { + final List keys; + + KeyArrayResult(byte[] keyBytes, int[] keyLengths) { + int count = keyLengths.length; + keys = new ArrayList(count); + + int offset = 0; + for(int i = 0; i < count; i++) { + int keyLength = keyLengths[i]; + + byte[] key = new byte[keyLength]; + System.arraycopy(keyBytes, offset, key, 0, keyLength); + + offset += keyLength; + keys.add(key); + } + } +} diff --git a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java index 43f7dd2650..1dabc08c93 100644 --- a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java @@ -455,6 +455,28 @@ public interface ReadTransaction extends ReadTransactionContext { */ CompletableFuture getEstimatedRangeSizeBytes(Range range); + /** + * Gets a list of keys that can split the given range into (roughly) equally sized chunks based on chunkSize. + * Note: the returned split points contain the start key and end key of the given range. + * + * @param begin the beginning of the range (inclusive) + * @param end the end of the range (exclusive) + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize); + + /** + * Gets a list of keys that can split the given range into (roughly) equally sized chunks based on chunkSize + * Note: the returned split points contain the start key and end key of the given range. + * + * @param range the range of the keys + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getRangeSplitPoints(Range range, long chunkSize); + + /** * Returns a set of options that can be set on a {@code Transaction} * diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index 97defab88f..f584f452a9 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -38,6 +38,7 @@ import com.apple.foundationdb.FDB; import com.apple.foundationdb.FDBException; import com.apple.foundationdb.KeySelector; import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.KeyArrayResult; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; import com.apple.foundationdb.StreamingMode; @@ -229,6 +230,12 @@ public class AsyncStackTester { inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); }, FDB.DEFAULT_EXECUTOR); } + else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) { + List params = inst.popParams(3).join(); + return inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).thenAcceptAsync(splitPoints -> { + inst.push("GOT_RANGE_SPLIT_POINTS".getBytes()); + }, FDB.DEFAULT_EXECUTOR); + } else if(op == StackOperation.GET_RANGE) { return inst.popParams(5).thenComposeAsync(params -> { int limit = StackUtils.getInt(params.get(2)); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java index 9f838d8eeb..ae701363e5 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java @@ -34,7 +34,7 @@ public class DirectoryTest { public static void main(String[] args) throws Exception { try { FDB fdb = FDB.selectAPIVersion(700); - try(Database db = fdb.open()) { + try(Database db = args.length > 0 ? fdb.open(args[0]) : fdb.open()) { runTests(db); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index 634a217c7f..bece744605 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -57,6 +57,7 @@ enum StackOperation { GET_APPROXIMATE_SIZE, GET_VERSIONSTAMP, GET_ESTIMATED_RANGE_SIZE, + GET_RANGE_SPLIT_POINTS, SET_READ_VERSION, ON_ERROR, SUB, diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index f196301865..0490e2a5fb 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.FDB; import com.apple.foundationdb.FDBException; import com.apple.foundationdb.KeySelector; import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.KeyArrayResult; import com.apple.foundationdb.LocalityUtil; import com.apple.foundationdb.MutationType; import com.apple.foundationdb.Range; @@ -211,6 +212,11 @@ public class StackTester { Long size = inst.readTr.getEstimatedRangeSizeBytes((byte[])params.get(0), (byte[])params.get(1)).join(); inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); } + else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) { + List params = inst.popParams(3).join(); + KeyArrayResult splitPoints = inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).join(); + inst.push("GOT_RANGE_SPLIT_POINTS".getBytes()); + } else if(op == StackOperation.GET_RANGE) { List params = inst.popParams(5).join(); diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 91bdc2f3a0..d385d12aeb 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -462,16 +462,29 @@ class TransactionRead(_FDBBase): return self.get(key) def get_estimated_range_size_bytes(self, begin_key, end_key): - if begin_key is None: - begin_key = b'' - if end_key is None: - end_key = b'\xff' + if begin_key is None or end_key is None: + if fdb.get_api_version() >= 700: + raise Exception('Invalid begin key or end key') + else: + if begin_key is None: + begin_key = b'' + if end_key is None: + end_key = b'\xff' return FutureInt64(self.capi.fdb_transaction_get_estimated_range_size_bytes( self.tpointer, begin_key, len(begin_key), end_key, len(end_key) )) - + + def get_range_split_points(self, begin_key, end_key, chunk_size): + if begin_key is None or end_key is None or chunk_size <=0: + raise Exception('Invalid begin key, end key or chunk size') + return FutureKeyArray(self.capi.fdb_transaction_get_range_split_points( + self.tpointer, + begin_key, len(begin_key), + end_key, len(end_key), + chunk_size + )) class Transaction(TransactionRead): """A modifiable snapshot of a Database. @@ -736,6 +749,14 @@ class FutureKeyValueArray(Future): # the KVs on the python side and in most cases we are about to # destroy the future anyway +class FutureKeyArray(Future): + def wait(self): + self.block_until_ready() + ks = ctypes.pointer(KeyStruct()) + count = ctypes.c_int() + self.capi.fdb_future_get_key_array(self.fpointer, ctypes.byref(ks), ctypes.byref(count)) + return [ctypes.string_at(x.key, x.key_length) for x in ks[0:count.value]] + class FutureStringArray(Future): def wait(self): @@ -1217,6 +1238,11 @@ class KeyValueStruct(ctypes.Structure): ('value_length', ctypes.c_int)] _pack_ = 4 +class KeyStruct(ctypes.Structure): + _fields_ = [('key', ctypes.POINTER(ctypes.c_byte)), + ('key_length', ctypes.c_int)] + _pack_ = 4 + class KeyValue(object): def __init__(self, key, value): @@ -1406,6 +1432,11 @@ def init_c_api(): _capi.fdb_future_get_keyvalue_array.restype = int _capi.fdb_future_get_keyvalue_array.errcheck = check_error_code + _capi.fdb_future_get_key_array.argtypes = [ctypes.c_void_p, ctypes.POINTER( + ctypes.POINTER(KeyStruct)), ctypes.POINTER(ctypes.c_int)] + _capi.fdb_future_get_key_array.restype = int + _capi.fdb_future_get_key_array.errcheck = check_error_code + _capi.fdb_future_get_string_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)), ctypes.POINTER(ctypes.c_int)] _capi.fdb_future_get_string_array.restype = int _capi.fdb_future_get_string_array.errcheck = check_error_code @@ -1451,6 +1482,9 @@ def init_c_api(): _capi.fdb_transaction_get_estimated_range_size_bytes.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int] _capi.fdb_transaction_get_estimated_range_size_bytes.restype = ctypes.c_void_p + _capi.fdb_transaction_get_range_split_points.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] + _capi.fdb_transaction_get_range_split_points.restype = ctypes.c_void_p + _capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] _capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int _capi.fdb_transaction_add_conflict_range.errcheck = check_error_code diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index f6eab9c207..6aa41dea4a 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -393,6 +393,10 @@ class Tester: begin, end = inst.pop(2) estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() inst.push(b"GOT_ESTIMATED_RANGE_SIZE") + elif inst.op == six.u("GET_RANGE_SPLIT_POINTS"): + begin, end, chunkSize = inst.pop(3) + estimatedSize = obj.get_range_split_points(begin, end, chunkSize).wait() + inst.push(b"GOT_RANGE_SPLIT_POINTS") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 043cce23b6..c679f004bc 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -109,6 +109,7 @@ module FDB attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer attach_function :fdb_transaction_get_estimated_range_size_bytes, [ :pointer, :pointer, :int, :pointer, :int ], :pointer + attach_function :fdb_transaction_get_range_split_points, [ :pointer, :pointer, :int, :pointer, :int, :int64 ], :pointer attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void @@ -129,6 +130,12 @@ module FDB :value_length, :int end + class KeyStruct < FFI::Struct + pack 4 + layout :key, :pointer, + :key_length, :int + end + def self.check_error(code) raise Error.new(code) if code.nonzero? nil @@ -472,6 +479,22 @@ module FDB end end + class FutureKeyArray < Future + def wait + block_until_ready + + ks = FFI::MemoryPointer.new :pointer + count = FFI::MemoryPointer.new :int + FDBC.check_error FDBC.fdb_future_get_key_array(@fpointer, kvs, count) + ks = ks.read_pointer + + (0..count.read_int-1).map{|i| + x = FDBC::KeyStruct.new(ks + (i * FDBC::KeyStruct.size)) + x[:key].read_bytes(x[:key_length]) + } + end + end + class FutureStringArray < LazyFuture def getter strings = FFI::MemoryPointer.new :pointer @@ -825,6 +848,15 @@ module FDB Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize)) end + def get_range_split_points(begin_key, end_key, chunk_size) + if chunk_size <=0 + raise ArgumentError, "Invalid chunk size" + end + bkey = FDB.key_to_bytes(begin_key) + ekey = FDB.key_to_bytes(end_key) + FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunk_size)) + end + end TransactionRead.class_variable_set("@@StreamingMode", @@StreamingMode) diff --git a/bindings/ruby/tests/tester.rb b/bindings/ruby/tests/tester.rb index 3860e7d190..e653bdaf93 100755 --- a/bindings/ruby/tests/tester.rb +++ b/bindings/ruby/tests/tester.rb @@ -320,6 +320,9 @@ class Tester when "GET_ESTIMATED_RANGE_SIZE" inst.tr.get_estimated_range_size_bytes(inst.wait_and_pop, inst.wait_and_pop).to_i inst.push("GOT_ESTIMATED_RANGE_SIZE") + when "GET_RANGE_SPLIT_POINTS" + inst.tr.get_range_split_points(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop).length() + inst.push("GOT_RANGE_SPLIT_POINTS") when "GET_KEY" selector = FDB::KeySelector.new(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop) prefix = inst.wait_and_pop diff --git a/build/Dockerfile b/build/Dockerfile index e99c45357d..ad9669fba9 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -45,13 +45,13 @@ RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip - cd .. && rm -rf ninja-1.9.0 ninja.zip # install openssl -RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1d.tar.gz -o openssl.tar.gz &&\ - echo "1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2 openssl.tar.gz" > openssl-sha.txt &&\ +RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz &&\ + echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt &&\ sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\ - cd openssl-1.1.1d && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ + cd openssl-1.1.1h && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\ ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\ - cd /tmp/ && rm -rf /tmp/openssl-1.1.1d /tmp/openssl.tar.gz + cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\ echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt &&\ @@ -61,8 +61,8 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar. ARG TIMEZONEINFO=America/Los_Angeles RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -LABEL version=0.1.15 -ENV DOCKER_IMAGEVER=0.1.15 +LABEL version=0.1.17 +ENV DOCKER_IMAGEVER=0.1.17 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ diff --git a/build/Dockerfile.c7.layered b/build/Dockerfile.c7.layered new file mode 100644 index 0000000000..9505ec531c --- /dev/null +++ b/build/Dockerfile.c7.layered @@ -0,0 +1,80 @@ +FROM centos:7 + +RUN yum install -y centos-release-scl scl-utils +RUN rpmkeys --import "http://pool.sks-keyservers.net/pks/lookup?op=get&search=0x3fa7e0328081bff6a14da29aa6a19b38d3d831ef" +RUN curl https://download.mono-project.com/repo/centos7-stable.repo | tee /etc/yum.repos.d/mono-centos7-stable.repo +RUN yum install -y curl rpm-build wget git unzip devtoolset-8 devtoolset-8-libubsan-devel devtoolset-8-valgrind-devel \ + rh-ruby26 go-toolset-7 rh-git218 rh-python36-devel java-11-openjdk-devel.x86_64 mono-devel dos2unix dpkg rh-python36 + +# install Ninja +RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\ + unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\ + cd .. && rm -rf ninja-1.9.0 ninja.zip + +# install cmake +RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o /tmp/cmake.tar.gz &&\ + echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\ + sha256sum -c /tmp/cmake-sha.txt &&\ + cd /tmp && tar xf cmake.tar.gz &&\ + cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ &&\ + rm -rf cmake.tar.gz cmake-3.13.4-Linux-x86_64 cmake-sha.txt + +# install LLVM +RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz > /tmp/llvm.tar.xz +RUN cd tmp &&\ + echo "b7b639fc675fa1c86dd6d0bc32267be9eb34451748d2efd03f674b773000e92b llvm.tar.xz" > llvm-sha.txt &&\ + sha256sum -c llvm-sha.txt +RUN cd /tmp && tar xf llvm.tar.xz --no-same-owner +RUN mkdir /tmp/build && cd /tmp/build &&\ + scl enable devtoolset-8 -- cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb"\ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON ../llvm-project-11.0.0/llvm +RUN cd /tmp/build && cmake --build . +RUN cd /tmp/build && cmake --build . --target install +RUN rm -rf /tmp/* + +# install openssl +RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz &&\ + echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt &&\ + sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\ + cd openssl-1.1.1h && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ + scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\ + ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\ + cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz + +# install RocksDB +RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\ + echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt &&\ + sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt + +# install Boost +# wget of bintray without forcing UTF-8 encoding results in 403 Forbidden +RUN cd /opt/ &&\ + curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 &&\ + echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt &&\ + sha256sum -c boost-sha-67.txt &&\ + tar -xjf boost_1_67_0.tar.bz2 &&\ + rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs &&\ + curl -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 &&\ + echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt &&\ + sha256sum -c boost-sha-72.txt &&\ + tar -xjf boost_1_72_0.tar.bz2 &&\ + rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0/libs + + +# Install CCACHE +RUN cd /tmp && curl -L https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz > ccache.tar.gz &&\ + echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt &&\ + sha256sum -c ccache-sha256.txt &&\ + tar xf ccache.tar.gz && rm -rf build && mkdir build && cd build && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache-4.0 &&\ + cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/ccache-4.0 + +# Install toml11 +RUN cd /tmp && curl -L https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz > toml.tar.gz &&\ + echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt &&\ + sha256sum -c toml-sha256.txt &&\ + tar xf toml.tar.gz && rm -rf build && mkdir build && cd build && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml11-3.4.0 &&\ + cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/toml11-3.4.0 + +# do some cleanup +RUN rm -rf /tmp/* && yum clean all && rm -rf /var/cache/yum diff --git a/build/Dockerfile.devel b/build/Dockerfile.devel index 9552a064dc..ba0ca44c94 100644 --- a/build/Dockerfile.devel +++ b/build/Dockerfile.devel @@ -1,4 +1,4 @@ -FROM foundationdb/foundationdb-build:0.1.15 +FROM foundationdb/foundationdb-build:0.1.17 USER root @@ -7,7 +7,7 @@ ADD artifacts /mnt/artifacts # Install build tools for building via make RUN \ - yum install -y distcc-server gperf rubygems python34 libmpc-devel npm cgdb + yum install -y distcc-server gperf rubygems python34 libmpc-devel npm cgdb jq # Download and install llvm-10.0.0 RUN cd / &&\ @@ -50,8 +50,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\ ldconfig &&\ rm -rf /mnt/artifacts -LABEL version=0.11.8 -ENV DOCKER_IMAGEVER=0.11.8 +LABEL version=0.11.9 +ENV DOCKER_IMAGEVER=0.11.9 ENV CLANGCC=/usr/local/bin/clang.de8a65ef ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml index 8241c48cb3..3ba3c6fb2b 100644 --- a/build/docker-compose.yaml +++ b/build/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: common: &common - image: foundationdb/foundationdb-build:0.1.15 + image: foundationdb/foundationdb-build:0.1.17 build-setup: &build-setup <<: *common @@ -84,7 +84,7 @@ services: snapshot-ctest: &snapshot-ctest <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' prb-ctest: <<: *snapshot-ctest diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 46134571a3..94c79e97d8 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -159,7 +159,7 @@ endif() # - OUT_DIR the directory where files will be staged # - CONTEXT the type of correctness package being built (e.g. 'valgrind correctness') function(stage_correctness_package) - set(oneValueArgs OUT_DIR CONTEXT) + set(oneValueArgs OUT_DIR CONTEXT OUT_FILES) cmake_parse_arguments(STAGE "" "${oneValueArgs}" "" "${ARGN}") file(MAKE_DIRECTORY ${STAGE_OUT_DIR}/bin) string(LENGTH "${CMAKE_SOURCE_DIR}/tests/" base_length) @@ -202,6 +202,10 @@ function(stage_correctness_package) endforeach() endforeach() list(APPEND package_files ${STAGE_OUT_DIR}/bin/fdbserver + ${STAGE_OUT_DIR}/bin/coverage.fdbserver.xml + ${STAGE_OUT_DIR}/bin/coverage.fdbclient.xml + ${STAGE_OUT_DIR}/bin/coverage.fdbrpc.xml + ${STAGE_OUT_DIR}/bin/coverage.flow.xml ${STAGE_OUT_DIR}/bin/TestHarness.exe ${STAGE_OUT_DIR}/bin/TraceLogHelper.dll ${STAGE_OUT_DIR}/CMakeCache.txt @@ -210,17 +214,27 @@ function(stage_correctness_package) OUTPUT ${package_files} DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml + ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/CMakeCache.txt ${STAGE_OUT_DIR} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml + ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll ${STAGE_OUT_DIR}/bin COMMENT "Copying files for ${STAGE_CONTEXT} package" ) list(APPEND package_files ${test_files} ${external_files}) - set(package_files ${package_files} PARENT_SCOPE) + if(STAGE_OUT_FILES) + set(${STAGE_OUT_FILES} ${package_files} PARENT_SCOPE) + endif() endfunction() function(create_correctness_package) @@ -228,7 +242,7 @@ function(create_correctness_package) return() endif() set(out_dir "${CMAKE_BINARY_DIR}/correctness") - stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness") + stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness" OUT_FILES package_files) set(tar_file ${CMAKE_BINARY_DIR}/packages/correctness-${CMAKE_PROJECT_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} @@ -255,7 +269,7 @@ function(create_valgrind_correctness_package) endif() if(USE_VALGRIND) set(out_dir "${CMAKE_BINARY_DIR}/valgrind_correctness") - stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness") + stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness" OUT_FILES package_files) set(tar_file ${CMAKE_BINARY_DIR}/packages/valgrind-${CMAKE_PROJECT_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} @@ -363,3 +377,60 @@ function(package_bindingtester) add_custom_target(bindingtester ALL DEPENDS ${tar_file}) add_dependencies(bindingtester copy_bindingtester_binaries) endfunction() + +function(add_fdbclient_test) + set(options DISABLED ENABLED) + set(oneValueArgs NAME) + set(multiValueArgs COMMAND) + cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + if(NOT T_ENABLED AND T_DISABLED) + return() + endif() + if(NOT T_NAME) + message(FATAL_ERROR "NAME is a required argument for add_fdbclient_test") + endif() + if(NOT T_COMMAND) + message(FATAL_ERROR "COMMAND is a required argument for add_fdbclient_test") + endif() + message(STATUS "Adding Client test ${T_NAME}") + add_test(NAME "${T_NAME}" + COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py + --build-dir ${CMAKE_BINARY_DIR} + -- + ${T_COMMAND}) +endfunction() + +function(add_java_test) + set(options DISABLED ENABLED) + set(oneValueArgs NAME CLASS) + set(multiValueArgs CLASS_PATH) + cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + if(NOT T_ENABLED AND T_DISABLED) + return() + endif() + if(NOT T_NAME) + message(FATAL_ERROR "NAME is a required argument for add_fdbclient_test") + endif() + if(NOT T_CLASS) + message(FATAL_ERROR "CLASS is a required argument for add_fdbclient_test") + endif() + set(cp "") + set(separator ":") + if (WIN32) + set(separator ";") + endif() + message(STATUS "CLASSPATH ${T_CLASS_PATH}") + foreach(path ${T_CLASS_PATH}) + if(cp) + set(cp "${cp}${separator}${path}") + else() + set(cp "${path}") + endif() + endforeach() + add_fdbclient_test( + NAME ${T_NAME} + COMMAND ${Java_JAVA_EXECUTABLE} + -cp "${cp}" + -Djava.library.path=${CMAKE_BINARY_DIR}/lib + ${T_CLASS} "@CLUSTER_FILE@") +endfunction() diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 3815ad2cf5..2a3ef1da23 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -4,12 +4,13 @@ env_set(USE_GPERFTOOLS OFF BOOL "Use gperfools for profiling") env_set(USE_DTRACE ON BOOL "Enable dtrace probes on supported platforms") env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage") env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest") -env_set(VALGRIND_ARENA OFF BOOL "Inform valgrind about arena-allocated memory. Makes valgrind slower but more precise.") env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc") env_set(WITH_UNDODB OFF BOOL "Use rr or undodb") env_set(USE_ASAN OFF BOOL "Compile with address sanitizer") +env_set(USE_GCOV OFF BOOL "Compile with gcov instrumentation") +env_set(USE_MSAN OFF BOOL "Compile with memory sanitizer. To avoid false positives you need to dynamically link to a msan-instrumented libc++ and libc++abi, which you must compile separately. See https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo#instrumented-libc.") +env_set(USE_TSAN OFF BOOL "Compile with thread sanitizer. It is recommended to dynamically link to a tsan-instrumented libc++ and libc++abi, which you can compile separately.") env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer") -env_set(USE_TSAN OFF BOOL "Compile with thread sanitizer") env_set(FDB_RELEASE OFF BOOL "This is a building of a final release") env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available") env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info") @@ -28,6 +29,9 @@ endif() if(STATIC_LINK_LIBCXX AND USE_TSAN) message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX doesn't work with tsan") endif() +if(STATIC_LINK_LIBCXX AND USE_MSAN) + message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX doesn't work with msan") +endif() set(rel_debug_paths OFF) if(RELATIVE_DEBUG_PATHS) @@ -41,6 +45,7 @@ endif() add_compile_options(-DCMAKE_BUILD) add_compile_definitions(BOOST_ERROR_CODE_HEADER_ONLY BOOST_SYSTEM_NO_DEPRECATED) +set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) if(ALLOC_INSTRUMENTATION) add_compile_options(-DALLOC_INSTRUMENTATION) @@ -163,10 +168,28 @@ else() if(USE_ASAN) add_compile_options( -fsanitize=address - -DUSE_SANITIZER) - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address ${CMAKE_THREAD_LIBS_INIT}") + -DUSE_SANITIZER + -DADDRESS_SANITIZER + ) + add_link_options(-fsanitize=address) + endif() + + if(USE_MSAN) + if(NOT CLANG) + message(FATAL_ERROR "Unsupported configuration: USE_MSAN only works with Clang") + endif() + add_compile_options( + -fsanitize=memory + -fsanitize-memory-track-origins=2 + -DUSE_SANITIZER + -DMEMORY_SANITIZER + ) + add_link_options(-fsanitize=memory) + endif() + + if(USE_GCOV) + add_compile_options(--coverage -DUSE_GCOV) + add_link_options(--coverage) endif() if(USE_UBSAN) @@ -174,19 +197,20 @@ else() -fsanitize=undefined # TODO(atn34) Re-enable -fsanitize=alignment once https://github.com/apple/foundationdb/issues/1434 is resolved -fno-sanitize=alignment - -DUSE_SANITIZER) - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=undefined") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=undefined") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined ${CMAKE_THREAD_LIBS_INIT}") + -DUSE_SANITIZER + -DUNDEFINED_BEHAVIOR_SANITIZER + ) + add_link_options(-fsanitize=undefined) endif() if(USE_TSAN) add_compile_options( -fsanitize=thread - -DUSE_SANITIZER) - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=thread") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=thread") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread ${CMAKE_THREAD_LIBS_INIT}") + -DUSE_SANITIZER + -DTHREAD_SANITIZER + -DDYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1 + ) + add_link_options(-fsanitize=thread) endif() if(PORTABLE_BINARY) @@ -248,9 +272,6 @@ else() if (USE_VALGRIND) add_compile_options(-DVALGRIND=1 -DUSE_VALGRIND=1) endif() - if (VALGRIND_ARENA) - add_compile_options(-DVALGRIND_ARENA=1) - endif() if (CLANG) add_compile_options() # Clang has link errors unless `atomic` is specifically requested. @@ -363,4 +384,3 @@ else() endif() endif() endif() - diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index be6b044a73..1d1266edaf 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -121,7 +121,24 @@ endif() # TOML can download and install itself into the binary directory, so it should # always be available. -find_package(TOML11) +find_package(toml11 QUIET) +if(toml11_FOUND) + add_library(toml11_target INTERFACE) + add_dependencies(toml11_target INTERFACE toml11::toml11) +else() + include(ExternalProject) + + ExternalProject_add(toml11Project + URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz" + URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d + CMAKE_CACHE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11 + -Dtoml11_BUILD_TEST:BOOL=OFF + BUILD_ALWAYS ON) + add_library(toml11_target INTERFACE) + add_dependencies(toml11_target toml11Project) + target_include_directories(toml11_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/toml11/include) +endif() ################################################################################ diff --git a/cmake/FindGperftools.cmake b/cmake/FindGperftools.cmake index 1374f2c2f7..e35cad4e87 100644 --- a/cmake/FindGperftools.cmake +++ b/cmake/FindGperftools.cmake @@ -52,6 +52,7 @@ mark_as_advanced( if (GPERFTOOLS_FOUND) add_library(gperftools UNKNOWN IMPORTED) + target_compile_definitions(gperftools PUBLIC USE_GPERFTOOLS) set_target_properties(gperftools PROPERTIES IMPORTED_LOCATION ${GPERFTOOLS_TCMALLOC_AND_PROFILER} INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}") diff --git a/cmake/FindTOML11.cmake b/cmake/FindTOML11.cmake deleted file mode 100644 index 5cd9640c99..0000000000 --- a/cmake/FindTOML11.cmake +++ /dev/null @@ -1,28 +0,0 @@ -find_path(TOML11_INCLUDE_DIR - NAMES - toml.hpp - PATH_SUFFIXES - include - toml11 - include/toml11 - HINTS - "${_TOML11_HINTS}" -) - -if (NOT TOML11_INCLUDE_DIR) - include(ExternalProject) - - ExternalProject_add(toml11 - URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz" - URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d - CMAKE_CACHE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11 - -Dtoml11_BUILD_TEST:BOOL=OFF) - - set(TOML11_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/toml11/include") -endif() - -find_package_handle_standard_args(TOML11 - REQUIRED_VARS - TOML11_INCLUDE_DIR -) diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh index 8e2fde1f7d..4a0d7c70da 100755 --- a/contrib/Joshua/scripts/bindingTest.sh +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -1,6 +1,5 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pkill fdbserver ulimit -S -c unlimited unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY @@ -8,4 +7,4 @@ WORKDIR="$(pwd)/tmp/$$" if [ ! -d "${WORKDIR}" ] ; then mkdir -p "${WORKDIR}" fi -DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" FDBSERVERPORT="${PORT_FDBSERVER:-4500}" ${SCRIPTDIR}/bindingTestScript.sh 1 +DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" ${SCRIPTDIR}/bindingTestScript.sh 1 diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 9ef19ab1a6..f4e0e8eb8b 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -7,7 +7,7 @@ SCRIPTID="${$}" SAVEONERROR="${SAVEONERROR:-1}" PYTHONDIR="${BINDIR}/tests/python" testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh" -VERSION="1.6" +VERSION="1.9" source ${SCRIPTDIR}/localClusterStart.sh @@ -23,19 +23,22 @@ cycles="${1}" if [ "${DEBUGLEVEL}" -gt 0 ] then - echo "Work dir: ${WORKDIR}" - echo "Bin dir: ${BINDIR}" - echo "Log dir: ${LOGDIR}" - echo "Python path: ${PYTHONDIR}" - echo "Lib dir: ${LIBDIR}" - echo "Server port: ${FDBSERVERPORT}" - echo "Script Id: ${SCRIPTID}" - echo "Version: ${VERSION}" + echo "Work dir: ${WORKDIR}" + echo "Bin dir: ${BINDIR}" + echo "Log dir: ${LOGDIR}" + echo "Python path: ${PYTHONDIR}" + echo "Lib dir: ${LIBDIR}" + echo "Cluster String: ${FDBCLUSTERTEXT}" + echo "Script Id: ${SCRIPTID}" + echo "Version: ${VERSION}" fi # Begin the cluster using the logic in localClusterStart.sh. startCluster +# Stop the cluster on exit +trap "stopCluster" EXIT + # Display user message if [ "${status}" -ne 0 ]; then : @@ -58,8 +61,8 @@ fi # Display directory and log information, if an error occurred if [ "${status}" -ne 0 ] then - ls "${WORKDIR}" > "${LOGDIR}/dir.log" - ps -eafw > "${LOGDIR}/process-preclean.log" + ls "${WORKDIR}" &> "${LOGDIR}/dir.log" + ps -eafwH &> "${LOGDIR}/process-preclean.log" if [ -f "${FDBCONF}" ]; then cp -f "${FDBCONF}" "${LOGDIR}/" fi @@ -71,10 +74,15 @@ fi # Save debug information files, environment, and log information, if an error occurred if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then - ps -eafw > "${LOGDIR}/process-exit.log" - netstat -na > "${LOGDIR}/netstat.log" - df -h > "${LOGDIR}/disk.log" - env > "${LOGDIR}/env.log" + ps -eafwH &> "${LOGDIR}/process-exit.log" + netstat -na &> "${LOGDIR}/netstat.log" + df -h &> "${LOGDIR}/disk.log" + env &> "${LOGDIR}/env.log" +fi + +# Stop the cluster +if stopCluster; then + unset FDBSERVERID fi exit "${status}" diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 3ba4cb9dcb..d1280267b3 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -5,311 +5,398 @@ WORKDIR="${WORKDIR:-${SCRIPTDIR}/tmp/fdb.work}" LOGDIR="${WORKDIR}/log" ETCDIR="${WORKDIR}/etc" BINDIR="${BINDIR:-${SCRIPTDIR}}" -FDBSERVERPORT="${FDBSERVERPORT:-4500}" +FDBPORTSTART="${FDBPORTSTART:-4000}" +FDBPORTTOTAL="${FDBPORTTOTAL:-1000}" +SERVERCHECKS="${SERVERCHECKS:-10}" +CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" +AUDITCLUSTER="${AUDITCLUSTER:-0}" +AUDITLOG="${AUDITLOG:-/tmp/audit-cluster.log}" # Initialize the variables status=0 messagetime=0 messagecount=0 +# Do nothing, if cluster string is already defined +if [ -n "${FDBCLUSTERTEXT}" ] +then + : +# Otherwise, define the cluster text +else + # Define a random ip address and port on localhost + if [ -z "${IPADDRESS}" ]; then + let index2="${RANDOM} % 256" + let index3="${RANDOM} % 256" + let index4="(${RANDOM} % 255) + 1" + IPADDRESS="127.${index2}.${index3}.${index4}" + fi + if [ -z "${FDBPORT}" ]; then + let FDBPORT="(${RANDOM} % ${FDBPORTTOTAL}) + ${FDBPORTSTART}" + fi + FDBCLUSTERTEXT="${IPADDRESS}:${FDBPORT}" +fi + function log { - local status=0 - if [ "$#" -lt 1 ] - then - echo "Usage: log [echo]" - echo - echo "Logs the message and timestamp to LOGFILE (${LOGFILE}) and, if the" - echo "second argument is either not present or is set to 1, stdout." - let status="${status} + 1" - else - # Log to stdout. - if [ "$#" -lt 2 ] || [ "${2}" -ge 1 ] - then - echo "${1}" - fi + local status=0 + if [ "$#" -lt 1 ] + then + echo "Usage: log [echo]" + echo + echo "Logs the message and timestamp to LOGFILE (${LOGFILE}) and, if the" + echo "second argument is either not present or is set to 1, stdout." + let status="${status} + 1" + else + # Log to stdout. + if [ "$#" -lt 2 ] || [ "${2}" -ge 1 ] + then + echo "${1}" + fi - # Log to file. - datestr=$(date +"%Y-%m-%d %H:%M:%S (%s)") - dir=$(dirname "${LOGFILE}") - if ! [ -d "${dir}" ] && ! mkdir -p "${dir}" - then - echo "Could not create directory to log output." - let status="${status} + 1" - elif ! [ -f "${LOGFILE}" ] && ! touch "${LOGFILE}" - then - echo "Could not create file ${LOGFILE} to log output." - let status="${status} + 1" - elif ! echo "[ ${datestr} ] ${1}" >> "${LOGFILE}" - then - echo "Could not log output to ${LOGFILE}." - let status="${status} + 1" - fi - fi + # Log to file. + datestr=$(date +"%Y-%m-%d %H:%M:%S (%s)") + dir=$(dirname "${LOGFILE}") + if ! [ -d "${dir}" ] && ! mkdir -p "${dir}" + then + echo "Could not create directory to log output." + let status="${status} + 1" + elif ! [ -f "${LOGFILE}" ] && ! touch "${LOGFILE}" + then + echo "Could not create file ${LOGFILE} to log output." + let status="${status} + 1" + elif ! echo "[ ${datestr} ] ${1}" >> "${LOGFILE}" + then + echo "Could not log output to ${LOGFILE}." + let status="${status} + 1" + fi + fi - return "${status}" + return "${status}" } # Display a message for the user. function displayMessage { - local status=0 + local status=0 - if [ "$#" -lt 1 ] - then - echo "displayMessage " - let status="${status} + 1" - elif ! log "${1}" 0 - then - log "Could not write message to file." - else - # Increment the message counter - let messagecount="${messagecount} + 1" + if [ "$#" -lt 1 ] + then + echo "displayMessage " + let status="${status} + 1" + elif ! log "${1}" 0 + then + log "Could not write message to file." + else + # Increment the message counter + let messagecount="${messagecount} + 1" - # Display successful message, if previous message - if [ "${messagecount}" -gt 1 ] - then - # Determine the amount of transpired time - let timespent="${SECONDS}-${messagetime}" + # Display successful message, if previous message + if [ "${messagecount}" -gt 1 ] + then + # Determine the amount of transpired time + let timespent="${SECONDS}-${messagetime}" - if [ "${DEBUGLEVEL}" -gt 0 ]; then - printf "... done in %3d seconds\n" "${timespent}" - fi - fi + if [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "... done in %3d seconds\n" "${timespent}" + fi + fi - # Display message - if [ "${DEBUGLEVEL}" -gt 0 ]; then - printf "%-16s %-35s " "$(date "+%F %H-%M-%S")" "$1" - fi + # Display message + if [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "%-16s %-35s " "$(date "+%F %H-%M-%S")" "$1" + fi - # Update the variables - messagetime="${SECONDS}" - fi + # Update the variables + messagetime="${SECONDS}" + fi - return "${status}" + return "${status}" } # Create the directories used by the server. -function createDirectories { - # Display user message - if ! displayMessage "Creating directories" - then - echo 'Failed to display user message' - let status="${status} + 1" - - elif ! mkdir -p "${LOGDIR}" "${ETCDIR}" - then - log "Failed to create directories" - let status="${status} + 1" - - # Display user message - elif ! displayMessage "Setting file permissions" - then - log 'Failed to display user message' - let status="${status} + 1" - - elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli" - then - log "Failed to set file permissions" - let status="${status} + 1" - - else - while read filepath - do - if [ -f "${filepath}" ] && [ ! -x "${filepath}" ] - then - # if [ "${DEBUGLEVEL}" -gt 1 ]; then - # log " Enable executable: ${filepath}" - # fi - log " Enable executable: ${filepath}" "${DEBUGLEVEL}" - if ! chmod 755 "${filepath}" - then - log "Failed to set executable for file: ${filepath}" - let status="${status} + 1" - fi - fi - done < <(find "${BINDIR}" -iname '*.py' -o -iname '*.rb' -o -iname 'fdb_flow_tester' -o -iname '_stacktester' -o -iname '*.js' -o -iname '*.sh' -o -iname '*.ksh') - fi +function createDirectories +{ + local status=0 - return ${status} + # Display user message + if ! displayMessage "Creating directories" + then + echo 'Failed to display user message' + let status="${status} + 1" + + elif ! mkdir -p "${LOGDIR}" "${ETCDIR}" + then + log "Failed to create directories" + let status="${status} + 1" + + # Display user message + elif ! displayMessage "Setting file permissions" + then + log 'Failed to display user message' + let status="${status} + 1" + + elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli" + then + log "Failed to set file permissions" + let status="${status} + 1" + + else + while read filepath + do + if [ -f "${filepath}" ] && [ ! -x "${filepath}" ] + then + # if [ "${DEBUGLEVEL}" -gt 1 ]; then + # log " Enable executable: ${filepath}" + # fi + log " Enable executable: ${filepath}" "${DEBUGLEVEL}" + if ! chmod 755 "${filepath}" + then + log "Failed to set executable for file: ${filepath}" + let status="${status} + 1" + fi + fi + done < <(find "${BINDIR}" -iname '*.py' -o -iname '*.rb' -o -iname 'fdb_flow_tester' -o -iname '_stacktester' -o -iname '*.js' -o -iname '*.sh' -o -iname '*.ksh') + fi + + return ${status} } # Create a cluster file for the local cluster. -function createClusterFile { - if [ "${status}" -ne 0 ]; then - : - # Display user message - elif ! displayMessage "Creating Fdb Cluster file" - then - log 'Failed to display user message' - let status="${status} + 1" - else - description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) - random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) - echo "$description:$random_str@127.0.0.1:${FDBSERVERPORT}" > "${FDBCONF}" - fi +function createClusterFile +{ + local status=0 - if [ "${status}" -ne 0 ]; then - : - elif ! chmod 0664 "${FDBCONF}"; then - log "Failed to set permissions on fdbconf: ${FDBCONF}" - let status="${status} + 1" - fi + if [ "${status}" -ne 0 ]; then + : + # Display user message + elif ! displayMessage "Creating Fdb Cluster file" + then + log 'Failed to display user message' + let status="${status} + 1" + else + description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) + random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) + echo "${description}:${random_str}@${FDBCLUSTERTEXT}" > "${FDBCONF}" + fi - return ${status} + if [ "${status}" -ne 0 ]; then + : + elif ! chmod 0664 "${FDBCONF}"; then + log "Failed to set permissions on fdbconf: ${FDBCONF}" + let status="${status} + 1" + fi + + return ${status} +} + +# Stop the Cluster from running. +function stopCluster +{ + local status=0 + + # Add an audit entry, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBCLUSTERTEXT}" "${FDBSERVERID}" >> "${AUDITLOG}" + fi + if [ -z "${FDBSERVERID}" ]; then + log 'FDB Server process is not defined' + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}"; then + log "Failed to locate FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${FDBCLUSTERTEXT}; sleep 3" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + then + # Ensure that process is dead + if ! kill -0 "${FDBSERVERID}" 2> /dev/null; then + log "Killed cluster (${FDBSERVERID}) via cli" + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to kill FDB Server process (${FDBSERVERID}) via cli or kill command" + let status="${status} + 1" + else + log "Forcibly killed FDB Server process (${FDBSERVERID}) since cli failed" + fi + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to forcibly kill FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + else + log "Forcibly killed FDB Server process (${FDBSERVERID})" + fi + return "${status}" } # Start the server running. -function startFdbServer { - if [ "${status}" -ne 0 ]; then - : - elif ! displayMessage "Starting Fdb Server" - then - log 'Failed to display user message' - let status="${status} + 1" +function startFdbServer +{ + local status=0 - elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "auto:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/$$" &> "${LOGDIR}/fdbserver.log" & - then - log "Failed to start FDB Server" - # Maybe the server is already running - FDBSERVERID="$(pidof fdbserver)" - let status="${status} + 1" - else - FDBSERVERID="${!}" - fi + # Add an audit entry, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBCLUSTERTEXT}" >> "${AUDITLOG}" + fi - if ! kill -0 ${FDBSERVERID} ; then - log "FDB Server start failed." - let status="${status} + 1" - fi + if ! displayMessage "Starting Fdb Server" + then + log 'Failed to display user message' + let status="${status} + 1" - return ${status} + else + "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${FDBCLUSTERTEXT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & + if [ "${?}" -ne 0 ] + then + log "Failed to start FDB Server" + let status="${status} + 1" + else + FDBSERVERID="${!}" + fi + fi + + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server start failed because no process" + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}" ; then + log "FDB Server start failed because process terminated unexpectedly" + let status="${status} + 1" + fi + + return ${status} } -function getStatus { - if [ "${status}" -ne 0 ]; then - : - elif ! date &>> "${LOGDIR}/fdbclient.log" - then - log 'Failed to get date' - let status="${status} + 1" - elif ! "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 120 &>> "${LOGDIR}/fdbclient.log" - then - log 'Failed to get status from fdbcli' - let status="${status} + 1" - elif ! date &>> "${LOGDIR}/fdbclient.log" - then - log 'Failed to get date' - let status="${status} + 1" - fi +function getStatus +{ + local status=0 - return ${status} + if [ "${status}" -ne 0 ]; then + : + elif ! date &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get date' + let status="${status} + 1" + elif ! "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 120 &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get status from fdbcli' + let status="${status} + 1" + elif ! date &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get date' + let status="${status} + 1" + fi + + return ${status} } # Verify that the cluster is available. -function verifyAvailable { - # Verify that the server is running. - if ! kill -0 "${FDBSERVERID}" - then - log "FDB server process (${FDBSERVERID}) is not running" - let status="${status} + 1" - return 1 +function verifyAvailable +{ + local status=0 - # Display user message. - elif ! displayMessage "Checking cluster availability" - then - log 'Failed to display user message' - let status="${status} + 1" - return 1 - - # Determine if status json says the database is available. - else - avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 10 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` - log "Avail value: ${avail}" "${DEBUGLEVEL}" - if [[ -n "${avail}" ]] ; then - return 0 - else - return 1 - fi - fi + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server process is not defined." + let status="${status} + 1" + # Verify that the server is running. + elif ! kill -0 "${FDBSERVERID}" + then + log "FDB server process (${FDBSERVERID}) is not running" + let status="${status} + 1" + # Display user message. + elif ! displayMessage "Checking cluster availability" + then + log 'Failed to display user message' + let status="${status} + 1" + # Determine if status json says the database is available. + else + avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout "${SERVERCHECKS}" 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` + log "Avail value: ${avail}" "${DEBUGLEVEL}" + if [[ -n "${avail}" ]] ; then + : + else + let status="${status} + 1" + fi + fi + return "${status}" } # Configure the database on the server. -function createDatabase { - if [ "${status}" -ne 0 ]; then - : - # Ensure that the server is running - elif ! kill -0 "${FDBSERVERID}" - then - log "FDB server process: (${FDBSERVERID}) is not running" - let status="${status} + 1" +function createDatabase +{ + local status=0 - # Display user message - elif ! displayMessage "Creating database" - then - log 'Failed to display user message' - let status="${status} + 1" - elif ! echo "Client log:" &> "${LOGDIR}/fdbclient.log" - then - log 'Failed to create fdbclient.log' - let status="${status} + 1" - elif ! getStatus - then - log 'Failed to get status' - let status="${status} + 1" + if [ "${status}" -ne 0 ]; then + : + # Ensure that the server is running + elif ! kill -0 "${FDBSERVERID}" + then + log "FDB server process: (${FDBSERVERID}) is not running" + let status="${status} + 1" - # Configure the database. - else - "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout 240 --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" + # Display user message + elif ! displayMessage "Creating database" + then + log 'Failed to display user message' + let status="${status} + 1" + elif ! echo "Client log:" &> "${LOGDIR}/fdbclient.log" + then + log 'Failed to create fdbclient.log' + let status="${status} + 1" + elif ! getStatus + then + log 'Failed to get status' + let status="${status} + 1" - if ! displayMessage "Checking if config succeeded" - then - log 'Failed to display user message.' - fi + # Configure the database. + else + "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout "${CONFIGUREWAIT}" --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" - iteration=0 - while [[ "${iteration}" -lt 10 ]] && ! verifyAvailable - do - log "Database not created (iteration ${iteration})." - let iteration="${iteration} + 1" - done + if ! displayMessage "Checking if config succeeded" + then + log 'Failed to display user message.' + fi - if ! verifyAvailable - then - log "Failed to create database via cli" - getStatus - cat "${LOGDIR}/fdbclient.log" - log "Ignoring -- moving on" - #let status="${status} + 1" - fi - fi + iteration=0 + while [[ "${iteration}" -lt "${SERVERCHECKS}" ]] && ! verifyAvailable + do + log "Database not created (iteration ${iteration})." + let iteration="${iteration} + 1" + done - return ${status} + if ! verifyAvailable + then + log "Failed to create database via cli" + getStatus + cat "${LOGDIR}/fdbclient.log" + log "Ignoring -- moving on" + #let status="${status} + 1" + fi + fi + + return ${status} } # Begin the local cluster from scratch. -function startCluster { - if [ "${status}" -ne 0 ]; then - : - elif ! createDirectories - then - log "Could not create directories." - let status="${status} + 1" - elif ! createClusterFile - then - log "Could not create cluster file." - let status="${status} + 1" - elif ! startFdbServer - then - log "Could not start FDB server." - let status="${status} + 1" - elif ! createDatabase - then - log "Could not create database." - let status="${status} + 1" - fi +function startCluster +{ + local status=0 - return ${status} + if [ "${status}" -ne 0 ]; then + : + elif ! createDirectories + then + log "Could not create directories." + let status="${status} + 1" + elif ! createClusterFile + then + log "Could not create cluster file." + let status="${status} + 1" + elif ! startFdbServer + then + log "Could not start FDB server." + let status="${status} + 1" + elif ! createDatabase + then + log "Could not create database." + let status="${status} + 1" + fi + + return ${status} } diff --git a/contrib/TestHarness/Program.cs.cmake b/contrib/TestHarness/Program.cs.cmake index 4693784c23..81324d5d61 100644 --- a/contrib/TestHarness/Program.cs.cmake +++ b/contrib/TestHarness/Program.cs.cmake @@ -307,10 +307,13 @@ namespace SummarizeTest int unseed; string uid = Guid.NewGuid().ToString(); bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0"); - result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout); + bool useToml = File.Exists(testFile + "-1.toml"); + string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt"; + result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout); if (result == 0) { - result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout); + string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt"; + result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout); } } else diff --git a/contrib/commit_debug.py b/contrib/commit_debug.py index 7f6de3ff91..b37b5260d0 100755 --- a/contrib/commit_debug.py +++ b/contrib/commit_debug.py @@ -24,22 +24,22 @@ def parse_args(): # (e)nd of a span with a better given name locationToPhase = { "NativeAPI.commit.Before": [], - "MasterProxyServer.batcher": [("b", "Commit")], - "MasterProxyServer.commitBatch.Before": [], - "MasterProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")], - "MasterProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")], + "CommitProxyServer.batcher": [("b", "Commit")], + "CommitProxyServer.commitBatch.Before": [], + "CommitProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")], + "CommitProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")], "Resolver.resolveBatch.Before": [("b", "Resolver.PipelineWait")], "Resolver.resolveBatch.AfterQueueSizeCheck": [], "Resolver.resolveBatch.AfterOrderer": [("e", "Resolver.PipelineWait"), ("b", "Resolver.Conflicts")], "Resolver.resolveBatch.After": [("e", "Resolver.Conflicts")], - "MasterProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")], - "MasterProxyServer.commitBatch.ProcessingMutations": [], - "MasterProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")], + "CommitProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")], + "CommitProxyServer.commitBatch.ProcessingMutations": [], + "CommitProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")], "TLog.tLogCommit.BeforeWaitForVersion": [("b", "TLog.PipelineWait")], "TLog.tLogCommit.Before": [("e", "TLog.PipelineWait")], "TLog.tLogCommit.AfterTLogCommit": [("b", "TLog.FSync")], "TLog.tLogCommit.After": [("e", "TLog.FSync")], - "MasterProxyServer.commitBatch.AfterLogPush": [("e", "Commit")], + "CommitProxyServer.commitBatch.AfterLogPush": [("e", "Commit")], "NativeAPI.commit.After": [], } diff --git a/design/backup_v2_partitioned_logs.md b/design/backup_v2_partitioned_logs.md index 18369cdd6f..3768643891 100644 --- a/design/backup_v2_partitioned_logs.md +++ b/design/backup_v2_partitioned_logs.md @@ -16,7 +16,7 @@ As an essential component of a database system, backup and restore is commonly u ## Background -FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at Proxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of Proxy CPU time. +FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at CommitProxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of CommitProxy CPU time. The design of old backup system is [here](https://github.com/apple/foundationdb/blob/master/design/backup.md), and the data format of range files and mutations files is [here](https://github.com/apple/foundationdb/blob/master/design/backup-dataFormat.md). The technical overview of FDB is [here](https://github.com/apple/foundationdb/wiki/Technical-Overview-of-the-Database). The FDB recovery is described in this [doc](https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md). @@ -37,7 +37,7 @@ The design of old backup system is [here](https://github.com/apple/foundationdb/ Feature priorities: Feature 1, 2, 3, 4, 5 are must-have; Feature 6 is better to have. -1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the Proxy, thus reduce TLog write bandwidth usage by half and significantly improve Proxy CPU usage; +1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the CommitProxy, thus reduce TLog write bandwidth usage by half and significantly improve CommitProxy CPU usage; 2. **Correctness**: The restored database must be consistent: each *restored* state (i.e., key-value pair) at a version `v` must match the original state at version `v`. 3. **Performance**: The backup system should be performant, mostly measured as a small CPU overhead on transaction logs and backup workers. The version lag on backup workers is an indicator of performance. 4. **Fault-tolerant**: The backup system should be fault-tolerant to node failures in the FDB cluster. @@ -153,9 +153,9 @@ The requirement of the new backup system raises several design challenges: **Master**: The master is responsible for coordinating the transition of the FDB transaction sub-system from one generation to the next. In particular, the master recruits backup workers during the recovery. -**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs. +**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the commit proxy in version order, and only respond to the commit proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs. -**Proxy**: The proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, Proxies are responsible to group mutations into backup mutations and write them to the database. +**CommitProxy**: The commit proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, commit proxies are responsible to group mutations into backup mutations and write them to the database. **GrvProxy**: The GRV proxies are responsible for providing read versions. ## System overview @@ -229,7 +229,7 @@ The operator’s backup request can indicate if an old backup or a new backup is 2. All backup workers monitor the key `\xff\x02/backupStarted`, see the change, and start logging mutations. 3. After all backup workers have started, the `fdbbackup` tool initiates the backup of all or specified key ranges by issuing a transaction `Ts`. -Compared to the old backup system, the above step 1 and 2 are new and is only triggered if client requests for a new type of backup. The purpose is to allow backup workers to function as no-op if there are no ongoing backups. However, the backup workers should still continuously pop their corresponding tags, otherwise mutations will be kept in the TLog. In order to know the version to pop, backup workers can obtain the read version from any proxy. Because the read version must be a committed version, so popping to this version is safe. +Compared to the old backup system, the above step 1 and 2 are new and is only triggered if client requests for a new type of backup. The purpose is to allow backup workers to function as no-op if there are no ongoing backups. However, the backup workers should still continuously pop their corresponding tags, otherwise mutations will be kept in the TLog. In order to know the version to pop, backup workers can obtain the read version from any GRV proxy. Because the read version must be a committed version, so popping to this version is safe. **Backup Submission Protocol** Protocol for `submitBackup()` to ensure that all backup workers of the current epoch have started logging mutations: diff --git a/design/data-distributor-internals.md b/design/data-distributor-internals.md index 661e35874b..ce432bfe67 100644 --- a/design/data-distributor-internals.md +++ b/design/data-distributor-internals.md @@ -22,7 +22,7 @@ Data distribution manages the lifetime of storage servers, decides which storage **Data distribution queue (`struct DDQueueData`)**: It receives shards to be relocated (i.e., RelocateShards), decides which shard should be moved to which server team, prioritizes the data movement based on relocate shard’s priority, and controls the progress of data movement based on servers’ workload. -**Special keys in the system keyspace**: DD saves its state in the system keyspace to recover from failure and to ensure every process (e.g., proxies, tLogs and storage servers) has a consistent view of which storage server is responsible for which key range. +**Special keys in the system keyspace**: DD saves its state in the system keyspace to recover from failure and to ensure every process (e.g., commit proxies, tLogs and storage servers) has a consistent view of which storage server is responsible for which key range. *serverKeys* sub-space (`\xff/serverKeys/`): It records the start key of each shard a server is responsible for. The format is *\xff/serverKeys/[serverID]/[start_key]*. To get start keys of all shards for a server, DD can read the key range with prefix *\xff/serverKeys/[serverID]/* and decode the value of [start_key]. @@ -32,9 +32,9 @@ Data distribution manages the lifetime of storage servers, decides which storage When a new DD is initialized, it will set itself as the owner by setting its random UID to the `moveKeysLockOwnerKey`. Since the owner key has only one value, at most one DD can own the DD-related system subspace. This avoids the potential race condition between multiple DDs which may co-exit during DD recruitment. -**Transaction State Store (txnStateStore)**: It is a replica of the special keyspace that stores the cluster’s states, such as which SS is responsible for which shard. Because proxies use txnStateStore to decide which tLog and SS should receive a mutation, proxies must have a consistent view of txnStateStore. Therefore, changes to txnStateStore must be populated to all proxies in total order. To achieve that, we use the special transaction (`applyMetaMutations`) to update txnStateStore and use resolvers to ensure the total ordering (serializable snapshot isolation). +**Transaction State Store (txnStateStore)**: It is a replica of the special keyspace that stores the cluster’s states, such as which SS is responsible for which shard. Because commit proxies use txnStateStore to decide which tLog and SS should receive a mutation, commit proxies must have a consistent view of txnStateStore. Therefore, changes to txnStateStore must be populated to all commit proxies in total order. To achieve that, we use the special transaction (`applyMetaMutations`) to update txnStateStore and use resolvers to ensure the total ordering (serializable snapshot isolation). -**Private mutation**: A private mutation is a mutation updating a special system key, such as keyServersKey (`\xff/keyServers/`) and serverKeysKey (`\xff/serverKeys/`). Like a normal mutation, a private mutation will be processed by the transaction systems (i.e., proxy, resolver and tLog) and be routed to a set of storage servers, based on the mutation’s tag, to update the key-value in the storage engine. Private mutations also keep the serializable snapshot isolation and consensus: The results of committed concurrent private mutations can be reproduced by sequentially executing the mutations, and all components in FDB have the same view of the mutations. +**Private mutation**: A private mutation is a mutation updating a special system key, such as keyServersKey (`\xff/keyServers/`) and serverKeysKey (`\xff/serverKeys/`). Like a normal mutation, a private mutation will be processed by the transaction systems (i.e., commit proxy, resolver and tLog) and be routed to a set of storage servers, based on the mutation’s tag, to update the key-value in the storage engine. Private mutations also keep the serializable snapshot isolation and consensus: The results of committed concurrent private mutations can be reproduced by sequentially executing the mutations, and all components in FDB have the same view of the mutations. ## Operations @@ -51,7 +51,7 @@ Whenever the team builder is invoked, it aims to build the desired number of ser **Data distribution queue server (`dataDistributionQueue` actor)**: It is created when DD is initialized. It behaves as a server to handle RelocateShard related requests. For example, it waits on the stream of RelocateShard. When a new RelocateShard is sent by teamTracker, it enqueues the new shard, and cancels the inflight shards that overlap with the new relocate shard. -**`applyMetaMutations`**: This is special logic to handle *private transactions* that modify txnStateStore and special system keys. Transaction systems (i.e., proxy, resolver and tLogs) and storage servers perform extra operations for the special transactions. For any update, it will be executed on all proxies in order so that all proxies have a consistent view of the txnStateStore. It will also send special keys to storage servers so that storage servers know the new keyspace they are now responsible for. +**`applyMetaMutations`**: This is special logic to handle *private transactions* that modify txnStateStore and special system keys. Transaction systems (i.e., commit proxy, resolver and tLogs) and storage servers perform extra operations for the special transactions. For any update, it will be executed on all commit proxies in order so that all commit proxies have a consistent view of the txnStateStore. It will also send special keys to storage servers so that storage servers know the new keyspace they are now responsible for. A storage server (SS) processes all requests sent to the server in its `storageServerCore` actor. When a (private) mutation request is sent to a SS, the server will call the `update()` function. Eventually, the `StorageUpdater` class will be invoked to apply the mutation in `applyMutation()` function, which handles private mutations `applyPrivateData()` function. @@ -84,9 +84,9 @@ Actors are created to monitor the reasons of key movement: A key range is a shard. A shard is the minimum unit of moving data. The storage server’s ownership of a shard -- which SS owns which shard -- is stored in the system keyspace *serverKeys* (`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). To simplify the explanation, we refer to the storage server’s ownership of a shard as a shard’s ownership. -A shard’s ownership is used in transaction systems (proxy and tLogs) to route mutations to tLogs and storage servers. When a proxy receives a mutation,dd it uses the shard’s ownership to decide which *k* tLogs receive the mutation, assuming *k* is the replias factor. When a storage server pulls mutations from tLogs, it uses the shard’s ownership to decide which shards the SS is responsible for and which tLog the SS should pull the data from. +A shard’s ownership is used in transaction systems (commit proxy and tLogs) to route mutations to tLogs and storage servers. When a commit proxy receives a mutation, it uses the shard’s ownership to decide which *k* tLogs receive the mutation, assuming *k* is the replias factor. When a storage server pulls mutations from tLogs, it uses the shard’s ownership to decide which shards the SS is responsible for and which tLog the SS should pull the data from. -A shard’s ownership must be consistent across transaction systems and SSes, so that mutations can be correctly routed to SSes. Moving keys from a SS to another requires changing the shard’s ownership under ACID property. The ACID property is achieved by using FDB transactions to change the *serverKeys *(`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). The mutation on the *serverKeys *and* keyServers *will be categorized as private mutations in transaction system. Compared to normal mutation, the private mutations will change the transaction state store (txnStateStore) that maintains the *serverKeys* and *keyServers* for transaction systems (proxy and tLog) when it arrives on each transaction component (e.g., tLog). Because mutations are processed in total order with the ACID guarantees, the change to the txnStateStore will be executed in total order on each node and the change on the shard’s ownership will also be consistent. +A shard’s ownership must be consistent across transaction systems and SSes, so that mutations can be correctly routed to SSes. Moving keys from a SS to another requires changing the shard’s ownership under ACID property. The ACID property is achieved by using FDB transactions to change the *serverKeys *(`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). The mutation on the *serverKeys *and* keyServers *will be categorized as private mutations in transaction system. Compared to normal mutation, the private mutations will change the transaction state store (txnStateStore) that maintains the *serverKeys* and *keyServers* for transaction systems (commit proxy and tLog) when it arrives on each transaction component (e.g., tLog). Because mutations are processed in total order with the ACID guarantees, the change to the txnStateStore will be executed in total order on each node and the change on the shard’s ownership will also be consistent. The data movement from one server (called source server) to another (called destination server) has four steps: (1) DD adds the destination server as the shard’s new owner; diff --git a/design/recovery-internals.md b/design/recovery-internals.md index 338304f988..cf9cc0b413 100644 --- a/design/recovery-internals.md +++ b/design/recovery-internals.md @@ -8,12 +8,12 @@ This document explains at the high level how the recovery works in a single clus ## `ServerDBInfo` data structure -This data structure contains transient information which is broadcast to all workers for a database, permitting them to communicate with each other. It contains, for example, the interfaces for cluster controller (CC), master, ratekeeper, and resolver, and holds the log system's configuration. Only part of the data structure, such as `ClientDBInfo` that contains the list of proxies, is available to the client. +This data structure contains transient information which is broadcast to all workers for a database, permitting them to communicate with each other. It contains, for example, the interfaces for cluster controller (CC), master, ratekeeper, and resolver, and holds the log system's configuration. Only part of the data structure, such as `ClientDBInfo` that contains the list of GRV proxies and commit proxies, is available to the client. Whenever a field of the `ServerDBInfo`is changed, the new value of the field, say new master's interface, will be sent to the CC and CC will propagate the new `ServerDBInfo` to all workers in the cluster. ## When will recovery happen? -Failure of certain roles in FDB can cause recovery. Those roles are cluster controller, master, proxy, transaction logs (tLog), resolvers, and log router. +Failure of certain roles in FDB can cause recovery. Those roles are cluster controller, master, GRV proxy, commit proxy, transaction logs (tLog), resolvers, log router, and backup workers. Network partition or failures can make CC unable to reach some roles, treating those roles as dead and causing recovery. If CC cannot connect to a majority of coordinators, it will be treated as dead by coordinators and recovery will happen. @@ -97,7 +97,7 @@ Master interface is stored in `serverDBInfo`. Once the CC recruits the master, i Once the master locks the cstate, it will recruit the still-alive tLogs from the previous generation for the benefit of faster recovery. The master gets the old tLogs’ interfaces from the READING_CSTATE phase and uses those interfaces to track which old tLog are still alive, the implementation of which is in `trackRejoins()`. -Once the master gets enough tLogs, it calculates the known committed version (i.e., `knownCommittedVersion` in code). `knownCommittedVersion` is the highest version that a proxy tells a given tLog that it had durably committed on *all* tLogs. The master's is the maximum of all of that. `knownCommittedVersion` is important, because it defines the lower bound of what version range of mutations need to be copied to the new generation. That is, any versions larger than the master's `knownCommittedVersion` is not guaranteed to persist on all replicas. The master chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy. +Once the master gets enough tLogs, it calculates the known committed version (i.e., `knownCommittedVersion` in code). `knownCommittedVersion` is the highest version that a commit proxy tells a given tLog that it had durably committed on *all* tLogs. The master's is the maximum of all of that. `knownCommittedVersion` is important, because it defines the lower bound of what version range of mutations need to be copied to the new generation. That is, any versions larger than the master's `knownCommittedVersion` is not guaranteed to persist on all replicas. The master chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy. Later, the master will use the recruited tLogs to create a new `TagPartitionedLogSystem` for the new generation. @@ -121,9 +121,9 @@ Consider an old generation with three TLogs: `A, B, C`. Their durable versions a Once we have a `knownCommittedVersion`, the master will reconstruct the transaction state store (txnStateStore) by peeking the txnStateTag in oldLogSystem. Recall that the txnStateStore includes the transaction system’s configuration, such as the assignment of shards to SS and to tLogs and that the txnStateStore was durable on disk in the oldLogSystem. -Once we get the txnStateStore, we know the configuration of the transaction system, such as the number of proxies. The master then can ask the CC to recruit roles for the new generation in the `recruitEverything()` function. Those recruited roles includes proxies, tLogs and seed SSes, which are the storage servers created for an empty database in the first generation to host the first shard and serve as the starting point of the bootstrap process to recruit more SSes. Once all roles are recruited, the master starts a new epoch in `newEpoch()`. +Once we get the txnStateStore, we know the configuration of the transaction system, such as the number of GRV proxies and commit proxies. The master then can ask the CC to recruit roles for the new generation in the `recruitEverything()` function. Those recruited roles includes GRV proxies, commit proxies, tLogs and seed SSes, which are the storage servers created for an empty database in the first generation to host the first shard and serve as the starting point of the bootstrap process to recruit more SSes. Once all roles are recruited, the master starts a new epoch in `newEpoch()`. -At this point, we have recovered the txnStateStore, recruited new proxies and tLogs, and copied data from old tLogs to new tLogs. We have a working transaction system in the new generation now. +At this point, we have recovered the txnStateStore, recruited new GRV proxies, commit proxies and tLogs, and copied data from old tLogs to new tLogs. We have a working transaction system in the new generation now. ### Where can the recovery get stuck in this phase? @@ -151,7 +151,7 @@ Not every FDB role participates in the recovery phases 1-3. This phase tells the Storage servers (SSes) are not involved in the recovery phase 1 - 3. To notify SSes about the recovery, the master commits a recovery transaction, the first transaction in the new generation, which contains the txnStateStore information. Once storage servers receive the recovery transaction, it will compare its latest data version and the recovery version, and rollback to the recovery version if its data version is newer. Note that storage servers may have newer data than the recovery version because they pre-fetch mutations from tLogs before the mutations are durable to reduce the latency to read newly written data. -Proxies haven’t recovered the transaction system state and cannot accept transactions yet. The master recovers proxies’ states by sending the txnStateStore to proxies through proxies’ (`txnState`) interfaces in `sendIntialCommitToResolvers()` function. Once proxies have recovered their states, they can start processing transactions. The recovery transaction that was waiting on proxies will be processed. +Commit proxies haven’t recovered the transaction system state and cannot accept transactions yet. The master recovers proxies’ states by sending the txnStateStore to commit proxies through commit proxies’ (`txnState`) interfaces in `sendIntialCommitToResolvers()` function. Once commit proxies have recovered their states, they can start processing transactions. The recovery transaction that was waiting on commit proxies will be processed. The resolvers haven’t known the recovery version either. The master needs to send the lastEpochEnd version (i.e., last commit of the previous generation) to resolvers via resolvers’ (`resolve`) interface. @@ -162,7 +162,7 @@ At the end of this phase, every role should be aware of the recovery and start r ## Phase 5: WRITING_CSTATE -Coordinators store the transaction systems’ information. The master needs to write the new tLogs into coordinators’ states to achieve consensus and fault tolerance. Only when the coordinators’ states are updated with the new transaction system’s configuration will the cluster controller tell clients about the new transaction system (such as the new proxies). +Coordinators store the transaction systems’ information. The master needs to write the new tLogs into coordinators’ states to achieve consensus and fault tolerance. Only when the coordinators’ states are updated with the new transaction system’s configuration will the cluster controller tell clients about the new transaction system (such as the new GRV proxies and commit proxies). The master only needs to write the new tLogs to a quorum of coordinators for a running cluster. The only time the master has to write all coordinators is when creating a brand new database. diff --git a/design/tlog-spilling.md.html b/design/tlog-spilling.md.html index aee572b597..4fce3e8e90 100644 --- a/design/tlog-spilling.md.html +++ b/design/tlog-spilling.md.html @@ -7,17 +7,17 @@ (This assumes a basic familiarity with [FoundationDB's architecture](https://www.youtu.be/EMwhsGsxfPU).) Transaction logs are a distributed Write-Ahead-Log for FoundationDB. They -receive commits from proxies, and are responsible for durably storing those -commits, and making them available to storage servers for reading. +receive commits from commit proxies, and are responsible for durably storing +those commits, and making them available to storage servers for reading. Clients send *mutations*, the list of their set, clears, atomic operations, -etc., to proxies. Proxies collect mutations into a *batch*, which is the list -of all changes that need to be applied to the database to bring it from version -`N-1` to `N`. Proxies then walk through their in-memory mapping of shard -boundaries to associate one or more *tags*, a small integer uniquely -identifying a destination storage server, with each mutation. They then send a -*commit*, the full list of `(tags, mutation)` for each mutation in a batch, to -the transaction logs. +etc., to commit proxies. Commit proxies collect mutations into a *batch*, which +is the list of all changes that need to be applied to the database to bring it +from version `N-1` to `N`. Commit proxies then walk through their in-memory +mapping of shard boundaries to associate one or more *tags*, a small integer +uniquely identifying a destination storage server, with each mutation. They +then send a *commit*, the full list of `(tags, mutation)` for each mutation in +a batch, to the transaction logs. The transaction log has two responsibilities: it must persist the commits to disk and notify the proxy when a commit is durably stored, and it must make the diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst index 173f05f312..5f6369d889 100644 --- a/documentation/sphinx/source/administration.rst +++ b/documentation/sphinx/source/administration.rst @@ -28,8 +28,6 @@ Starting and stopping After installation, FoundationDB is set to start automatically. You can manually start and stop the database with the commands shown below. -These commands start and stop the master ``fdbmonitor`` process, which in turn starts ``fdbserver`` and ``backup-agent`` processes. See :ref:`administration_fdbmonitor` for details. - Linux ----- @@ -58,6 +56,15 @@ It can be stopped and prevented from starting at boot as follows:: host:~ user$ sudo launchctl unload -w /Library/LaunchDaemons/com.foundationdb.fdbmonitor.plist +Start, stop and restart behavior +================================= + +These commands above start and stop the master ``fdbmonitor`` process, which in turn starts ``fdbserver`` and ``backup-agent`` processes. See :ref:`administration_fdbmonitor` for details. + +After any child process has terminated by any reason, ``fdbmonitor`` tries to restart it. See :ref:`restarting parameters `. + +When ``fdbmonitor`` itself is killed unexpectedly (for example, by the ``out-of-memory killer``), all the child processes are also terminated. Then the operating system is responsible for restarting it. See :ref:`Configuring autorestart of fdbmonitor `. + .. _foundationdb-cluster-file: Cluster files @@ -259,7 +266,8 @@ Use the ``status`` command of ``fdbcli`` to determine if the cluster is up and r Redundancy mode - triple Storage engine - ssd-2 Coordinators - 5 - Desired Proxies - 5 + Desired GRV Proxies - 1 + Desired Commit Proxies - 4 Desired Logs - 8 Cluster: @@ -299,7 +307,8 @@ The summary fields are interpreted as follows: Redundancy mode The currently configured redundancy mode (see the section :ref:`configuration-choosing-redundancy-mode`) Storage engine The currently configured storage engine (see the section :ref:`configuration-configuring-storage-subsystem`) Coordinators The number of FoundationDB coordination servers -Desired Proxies Number of proxies desired. If replication mode is 3 then default number of proxies is 3 +Desired GRV Proxies Number of GRV proxies desired. (default 1) +Desired Commit Proxies Number of commit proxies desired. If replication mode is 3 then default number of commit proxies is 3 Desired Logs Number of logs desired. If replication mode is 3 then default number of logs is 3 FoundationDB processes Number of FoundationDB processes participating in the cluster Machines Number of physical machines running at least one FoundationDB process that is participating in the cluster @@ -565,7 +574,7 @@ When configured, the ``status json`` output will include additional fields to re filtered: 1 } -The ``grv_latency_bands`` and ``commit_latency_bands`` objects will only be logged for ``proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. +The ``grv_latency_bands`` objects will only be logged for ``grv_proxy`` roles, ``commit_latency_bands`` objects will only be logged for ``commit_proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. For example, ``0.1: 1`` in ``read_latency_bands`` indicates that there has been 1 read request with a latency in the range ``[0.01, 0.1)``. For the smallest specified threshold, the lower bound is 0 (e.g. ``[0, 0.01)`` in the example above). Requests that took longer than any defined latency band will be reported in the ``inf`` (infinity) band. Requests that were filtered by the configuration (e.g. using ``max_read_bytes``) are reported in the ``filtered`` category. diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index 02cfaf1682..8b85a12dcf 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -263,9 +263,9 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe .. function:: fdb_error_t fdb_future_block_until_ready(FDBFuture* future) - Blocks the calling thread until the given Future is ready. It will return success even if the Future is set to an error -- you must call :func:`fdb_future_get_error()` to determine that. :func:`fdb_future_block_until_ready()` will return an error only in exceptional conditions (e.g. out of memory or other operating system resources). + Blocks the calling thread until the given Future is ready. It will return success even if the Future is set to an error -- you must call :func:`fdb_future_get_error()` to determine that. :func:`fdb_future_block_until_ready()` will return an error only in exceptional conditions (e.g. deadlock detected, out of memory or other operating system resources). - .. warning:: Never call this function from a callback passed to :func:`fdb_future_set_callback()`. This may block the thread on which :func:`fdb_run_network()` was invoked, resulting in a deadlock. + .. warning:: Never call this function from a callback passed to :func:`fdb_future_set_callback()`. This may block the thread on which :func:`fdb_run_network()` was invoked, resulting in a deadlock. In some cases the client can detect the deadlock and throw a ``blocked_from_network_thread`` error. .. function:: fdb_bool_t fdb_future_is_ready(FDBFuture* future) @@ -301,6 +301,12 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe |future-get-return1| |future-get-return2|. +.. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count) + + Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning| + + |future-get-return1| |future-get-return2|. + .. function:: fdb_error_t fdb_future_get_key(FDBFuture* future, uint8_t const** out_key, int* out_key_length) Extracts a key from an :type:`FDBFuture` into caller-provided variables of type ``uint8_t*`` (a pointer to the beginning of the key) and ``int`` (the length of the key). |future-warning| @@ -480,6 +486,12 @@ Applications must provide error handling and an appropriate retry loop around th |future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2| +.. function:: FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) + Returns a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. + .. note:: The returned split points contain the start key and end key of the given range + + |future-return0| the list of split points. |future-return1| call :func:`fdb_future_get_key_array()` to extract the array, |future-return2| + .. function:: FDBFuture* fdb_transaction_get_key(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t or_equal, int offset, fdb_bool_t snapshot) Resolves a :ref:`key selector ` against the keys in the database snapshot represented by ``transaction``. diff --git a/documentation/sphinx/source/api-error-codes.rst b/documentation/sphinx/source/api-error-codes.rst index f013f4aabd..e8f564d80d 100644 --- a/documentation/sphinx/source/api-error-codes.rst +++ b/documentation/sphinx/source/api-error-codes.rst @@ -40,7 +40,7 @@ FoundationDB may return the following error codes from API functions. If you nee +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | external_client_already_loaded | 1040| External client has already been loaded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ -| proxy_memory_limit_exceeded | 1042| Proxy commit memory limit exceeded | +| proxy_memory_limit_exceeded | 1042| CommitProxy commit memory limit exceeded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | batch_transaction_throttled | 1051| Batch GRV request rate limit exceeded | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ @@ -114,8 +114,12 @@ FoundationDB may return the following error codes from API functions. If you nee +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | transaction_read_only | 2023| Attempted to commit a transaction specified as read-only | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ +| invalid_cache_eviction_policy | 2024| Invalid cache eviction policy, only random and lru are supported | ++-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | network_cannot_be_restarted | 2025| Network can only be started once | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ +| blocked_from_network_thread | 2026| Detected a deadlock in a callback called from the network thread | ++-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | incompatible_protocol_version | 2100| Incompatible protocol version | +-----------------------------------------------+-----+--------------------------------------------------------------------------------+ | transaction_too_large | 2101| Transaction exceeds byte limit | diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 4fcddfbfb9..59b82406e0 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -799,9 +799,15 @@ Transaction misc functions .. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) - Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`. + Gets the estimated byte size of the given key range. Returns a :class:`FutureInt64`. .. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate. +.. method:: Transaction.get_range_split_points(self, begin_key, end_key, chunk_size) + + Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`. + .. note:: The returned split points contain the start key and end key of the given range + + .. _api-python-transaction-options: Transaction misc functions diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index 170f800e30..7c707f445b 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -741,11 +741,16 @@ Most applications should use the read version that FoundationDB determines autom Transaction misc functions -------------------------- -.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) +.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) -> Int64Future - Get the estimated byte size of the given key range. Returns a :class:`Int64Future`. + Gets the estimated byte size of the given key range. Returns a :class:`Int64Future`. .. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate. +.. method:: Transaction.get_range_split_points(begin_key, end_key, chunk_size) -> FutureKeyArray + + Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`. + .. note:: The returned split points contain the start key and end key of the given range + .. method:: Transaction.get_approximate_size() -> Int64Future |transaction-get-approximate-size-blurb|. Returns a :class:`Int64Future`. diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst index 370cdb6526..83486986a6 100644 --- a/documentation/sphinx/source/api-version-upgrade-guide.rst +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -9,6 +9,19 @@ This document provides an overview of changes that an application developer may For more details about API versions, see :ref:`api-versions`. +.. _api-version-upgrade-guide-700: + +API version 700 +=============== + +General +------- + +Python bindings +--------------- + +* The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. + .. _api-version-upgrade-guide-630: API version 630 diff --git a/documentation/sphinx/source/architecture.rst b/documentation/sphinx/source/architecture.rst index b04e0dc963..f0a902dfe2 100644 --- a/documentation/sphinx/source/architecture.rst +++ b/documentation/sphinx/source/architecture.rst @@ -26,7 +26,7 @@ and servers use the coordinators to connect with the cluster controller. The servers will attempt to become the cluster controller if one does not exist, and register with the cluster controller once one has been elected. Clients use the cluster controller to keep an up-to-date list -of proxies. +of GRV proxies and commit proxies. Cluster Controller ~~~~~~~~~~~~~~~~~~ @@ -42,10 +42,11 @@ Master The master is responsible for coordinating the transition of the write sub-system from one generation to the next. The write sub-system -includes the master, proxies, resolvers, and transaction logs. The three -roles are treated as a unit, and if any of them fail, we will recruit a -replacement for all three roles. The master provides the commit versions -for batches of the mutations to the proxies. +includes the master, GRV proxies, commit proxies, resolvers, and +transaction logs. The three roles are treated as a unit, and if any of +them fail, we will recruit a replacement for all three roles. The master +provides the commit versions for batches of the mutations to the commit +proxies. Historically, Ratekeeper and Data Distributor are coupled with Master on the same process. Since 6.2, both have become a singleton in the @@ -53,16 +54,22 @@ cluster. The life time is no longer tied with Master. |image1| -Proxies -~~~~~~~ +GRV Proxies +~~~~~~~~~~~ -The proxies are responsible for providing read versions, committing -transactions, and tracking the storage servers responsible for each -range of keys. To provide a read version, a proxy will ask all other -proxies to see the largest committed version at this point in time, -while simultaneously checking that the transaction logs have not been -stopped. Ratekeeper will artificially slow down the rate at which the -proxy provides read versions. +The GRV proxies are responsible for providing read versions, communicating +with ratekeeper to control the rate providing read versions. To provide a +read version, a GRV proxy will ask all master to see the largest committed +version at this point in time, while simultaneously checking that the +transaction logs have not been stopped. Ratekeeper will artificially slow +down the rate at which the GRV proxy provides read versions. + +Commit Proxies +~~~~~~~~~~~~~~ + +The proxies are responsible for committing transactions, report committed +versions to master and tracking the storage servers responsible for each +range of keys. Commits are accomplished by: @@ -73,20 +80,20 @@ Commits are accomplished by: The key space starting with the ``\xff`` byte is reserved for system metadata. All mutations committed into this key space are distributed to -all of the proxies through the resolvers. This metadata includes a +all of the commit proxies through the resolvers. This metadata includes a mapping between key ranges and the storage servers which have the data -for that range of keys. The proxies provides this information to clients -on-demand. The clients cache this mapping; if they ask a storage server -for a key it does not have, they will clear their cache and get a more -up-to-date list of servers from the proxies. +for that range of keys. The commit proxies provides this information to +clients on-demand. The clients cache this mapping; if they ask a storage +server for a key it does not have, they will clear their cache and get a +more up-to-date list of servers from the commit proxies. Transaction Logs ~~~~~~~~~~~~~~~~ The transaction logs make mutations durable to disk for fast commit -latencies. The logs receive commits from the proxy in version order, and -only respond to the proxy once the data has been written and fsync’ed to -an append only mutation log on disk. Before the data is even written to +latencies. The logs receive commits from the commit proxy in version order, +and only respond to the commit proxy once the data has been written and fsync’ed +to an append only mutation log on disk. Before the data is even written to disk we forward it to the storage servers responsible for that mutation. Once the storage servers have made the mutation durable, they pop it from the log. This generally happens roughly 6 seconds after the @@ -153,7 +160,7 @@ Transaction Processing ---------------------- A database transaction in FoundationDB starts by a client contacting one -of the Proxies to obtain a read version, which is guaranteed to be +of the GRV proxies to obtain a read version, which is guaranteed to be larger than any of commit version that client may know about (even through side channels outside the FoundationDB cluster). This is needed so that a client will see the result of previous commits that have @@ -165,64 +172,51 @@ memory without contacting the cluster. By default, reading a key that was written in the same transaction will return the newly written value. At commit time, the client sends the transaction data (all reads and -writes) to one of the Proxies and waits for commit or abort response -from the proxy. If the transaction conflicts with another one and cannot -commit, the client may choose to retry the transaction from the -beginning again. If the transaction commits, the proxy also returns the -commit version back to the client. Note this commit version is larger -than the read version and is chosen by the master. +writes) to one of the commit proxies and waits for commit or abort response +from the commit proxy. If the transaction conflicts with another one and +cannot commit, the client may choose to retry the transaction from the +beginning again. If the transaction commits, the commit proxy also returns +the commit version back to the client and to master so that GRV proxies can +get access to the latest committed version. Note this commit version is +larger than the read version and is chosen by the master. The FoundationDB architecture separates the scaling of client reads and writes (i.e., transaction commits). Because clients directly issue reads to sharded storage servers, reads scale linearly to the number of storage servers. Similarly, writes are scaled by adding more processes -to Proxies, Resolvers, and Log Servers in the transaction system. +to Commit Proxies, Resolvers, and Log Servers in the transaction system. Determine Read Version ~~~~~~~~~~~~~~~~~~~~~~ -When a client requests a read version from a proxy, the proxy asks all -other proxies for their last commit versions, and checks a set of -transaction logs satisfying replication policy are live. Then the proxy -returns the maximum commit version as the read version to the client. +When a client requests a read version from a GRV proxy, the GRV proxy asks +master for the latest committed version, and checks a set of transaction +logs satisfying replication policy are live. Then the GRV proxy returns +the maximum committed version as the read version to the client. |image2| -The reason for the proxy to contact all other proxies for commit -versions is to ensure the read version is larger than any previously -committed version. Consider that if proxy ``A`` commits a transaction, -and then the client asks proxy ``B`` for a read version. The read -version from proxy ``B`` must be larger than the version committed by -proxy ``A``. The only way to get this information is by asking proxy -``A`` for its largest committed version. +The reason for the GRV proxy to contact master for the latest committed +versions is to because master is a central place to keep the largest of +all commit proxies' committed version. The reason for checking a set of transaction logs satisfying replication -policy are live is to ensure the proxy is not replaced with newer -generation of proxies. This is because proxy is a stateless role -recruited in each generation. If a recovery has happened and the old -proxy is still live, this old proxy could still give out read versions. +policy are live is to ensure the GRV proxy is not replaced with newer +generation of GRV proxies. This is because GRV proxy is a stateless role +recruited in each generation. If a recovery has happened and the old GRV +proxy is still live, this old GRV proxy could still give out read versions. As a result, a *read-only* transaction may see stale results (a read-write transaction will be aborted). By checking a set of -transaction logs satisfying replication policy are live, the proxy makes +transaction logs satisfying replication policy are live, the GRV proxy makes sure no recovery has happened, thus the *read-only* transaction sees the latest data. -Note that the client cannot simply ask the master for read versions. The -master gives out versions to proxies to be committed, but the master -does not know when the versions it gives out are durable on the -transaction logs. Therefore it is not safe to do reads at the largest -version the master has provided because that version might be rolled -back in the event of a failure, so the client could end up reading data -that was never committed. In order for the client to use versions from -the master, the client needs to wait until all in-flight -transaction-batches (a write version is used for a batch of -transactions) to commit. This can take a long time and thus is -inefficient. Another drawback of this approach is putting more work -towards the master, because the master role can’t be scaled. Even though -giving out read-versions isn’t very expensive, it still requires the -master to get a transaction budget from the Ratekeeper, batches -requests, and potentially maintains thousands of network connections -from clients. +Note that the client cannot simply ask the master for read versions because +this approach is putting more work towards the master, because the master +role can’t be scaled. Even though giving out read-versions isn’t very +expensive, it still requires the master to get a transaction budget from the +Ratekeeper, batches requests, and potentially maintains thousands of network +connections from clients. |image3| @@ -231,27 +225,27 @@ Transaction Commit A client transaction commits in the following steps: -1. A client sends a transaction to a proxy. -2. The proxy asks the master for a commit version. +1. A client sends a transaction to a commit proxy. +2. The commit proxy asks the master for a commit version. 3. The master sends back a commit version that is higher than any commit version seen before. -4. The proxy sends the read and write conflict ranges to the resolver(s) +4. The commit proxy sends the read and write conflict ranges to the resolver(s) with the commit version included. 5. The resolver responds back with whether the transaction has any conflicts with previous transactions by sorting transactions according to their commit versions and computing if such a serial execution order is conflict-free. - - If there are conflicts, the proxy responds back to the client with + - If there are conflicts, the commit proxy responds back to the client with a not_committed error. - - If there are no conflicts, the proxy sends the mutations and + - If there are no conflicts, the commit proxy sends the mutations and commit version of this transaction to the transaction logs. -6. Once the mutations are durable on the logs, the proxy responds back +6. Once the mutations are durable on the logs, the commit proxy responds back success to the user. -Note the proxy sends each resolver their respective key ranges, if any -one of the resolvers detects a conflict then the transaction is not +Note the commit proxy sends each resolver their respective key ranges, if +any one of the resolvers detects a conflict then the transaction is not committed. This has the flaw that if only one of the resolvers detects a conflict, the other resolver will still think the transaction has succeeded and may fail future transactions with overlapping write @@ -273,8 +267,8 @@ Background Work There are a number of background work happening besides the transaction processing: -- **Ratekeeper** collects statistic information from proxies, - transaction logs, and storage servers and compute the target +- **Ratekeeper** collects statistic information from GRV proxies, Commit + proxies, transaction logs, and storage servers and compute the target transaction rate for the cluster. - **Data distribution** monitors all storage servers and perform load @@ -284,7 +278,7 @@ processing: - **Storage servers** pull mutations from transaction logs, write them into storage engine to persist on disks. -- **Proxies** periodically send empty commits to transaction logs to +- **Commit proxies** periodically send empty commits to transaction logs to keep commit versions increasing, in case there is no client generated transactions. @@ -299,9 +293,9 @@ latency. A typical recovery takes about a few hundred milliseconds, but longer recovery time (usually a few seconds) can happen. Whenever there is a failure in the transaction system, a recovery process is performed to restore the transaction system to a new configuration, i.e., a clean -state. Specifically, the Master process monitors the health of Proxies, -Resolvers, and Transaction Logs. If any one of the monitored process -failed, the Master process terminates. The Cluster Controller will +state. Specifically, the Master process monitors the health of GRV Proxies, +Commit Proxies, Resolvers, and Transaction Logs. If any one of the monitored +process failed, the Master process terminates. The Cluster Controller will detect this event, and then recruits a new Master, which coordinates the recovery and recruits a new transaction system instance. In this way, the transaction processing is divided into a number of epochs, where @@ -314,20 +308,20 @@ Coordinators and lock the coordinated states to prevent another Master process from recovering at the same time. Then the Master recovers previous transaction system states, including all Log Servers’ Information, stops these Log Servers from accepting transactions, and -recruits a new set of Proxies, Resolvers, and Transaction Logs. After -previous Log Servers are stopped and new transaction system is -recruited, the Master writes the coordinated states with current +recruits a new set of GRV Proxies, Commit Proxies, Resolvers, and +Transaction Logs. After previous Log Servers are stopped and new transaction +system is recruited, the Master writes the coordinated states with current transaction system information. Finally, the Master accepts new transaction commits. See details in this `documentation `__. -Because Proxies and Resolvers are stateless, their recoveries have no -extra work. In contrast, Transaction Logs save the logs of committed -transactions, and we need to ensure all previously committed -transactions are durable and retrievable by storage servers. That is, -for any transactions that the Proxies may have sent back commit -response, their logs are persisted in multiple Log Servers (e.g., three -servers if replication degree is 3). +Because GRV Proxies, Commit Proxies and Resolvers are stateless, their +recoveries have no extra work. In contrast, Transaction Logs save the +logs of committed transactions, and we need to ensure all previously +committed transactions are durable and retrievable by storage servers. +That is, for any transactions that the Commit Proxies may have sent back +commit response, their logs are persisted in multiple Log Servers (e.g., +three servers if replication degree is 3). Finally, a recovery will *fast forward* time by 90 seconds, which would abort any in-progress client transactions with ``transaction_too_old`` @@ -335,7 +329,7 @@ error. During retry, these client transactions will find the new generation of transaction system and commit. **``commit_result_unknown`` error:** If a recovery happened while a -transaction is committing (i.e., a proxy has sent mutations to +transaction is committing (i.e., a commit proxy has sent mutations to transaction logs). A client would have received ``commit_result_unknown``, and then retried the transaction. It’s completely permissible for FDB to commit both the first attempt, and the diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index a051f5cbad..cf695b825a 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu configure --------- -The ``configure`` command changes the database configuration. Its syntax is ``configure [new] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [proxies=] [resolvers=] [logs=]``. +The ``configure`` command changes the database configuration. Its syntax is ``configure [new] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=]``. The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified. @@ -98,11 +98,12 @@ A FoundationDB cluster employs server processes of different types. It automatic For large clusters, you can manually set the allocated number of processes of a given type. Valid process types are: -* ``proxies`` +* ``grv_proxies`` +* ``commit_proxies`` * ``resolvers`` * ``logs`` -Set the process using ``configure [proxies|resolvers|logs]=``, where ```` is an integer greater than 0, or -1 to reset the value to its default. +Set the process using ``configure [grv_proxies|commit_proxies|resolvers|logs]=``, where ```` is an integer greater than 0, or -1 to reset the value to its default. For recommendations on appropriate values for process types in large clusters, see :ref:`guidelines-process-class-config`. @@ -357,7 +358,7 @@ setclass The ``setclass`` command can be used to change the :ref:`process class ` for a given process. Its syntax is ``setclass [
]``. If no arguments are specified, then the process classes of all processes are listed. Setting the class to ``default`` to revert to the process class specified on the command line. -The available process classes are ``unset``, ``storage``, ``transaction``, ``resolution``, ``proxy``, ``master``, ``test``, ``unset``, ``stateless``, ``log``, ``router``, ``cluster_controller``, ``fast_restore``, ``data_distributor``, ``coordinator``, ``ratekeeper``, ``storage_cache``, ``backup``, and ``default``. +The available process classes are ``unset``, ``storage``, ``transaction``, ``resolution``, ``grv_proxy``, ``commit_proxy``, ``master``, ``test``, ``unset``, ``stateless``, ``log``, ``router``, ``cluster_controller``, ``fast_restore``, ``data_distributor``, ``coordinator``, ``ratekeeper``, ``storage_cache``, ``backup``, and ``default``. sleep ----- diff --git a/documentation/sphinx/source/configuration.rst b/documentation/sphinx/source/configuration.rst index 0c0cb54332..dee34986c5 100644 --- a/documentation/sphinx/source/configuration.rst +++ b/documentation/sphinx/source/configuration.rst @@ -229,6 +229,8 @@ Contains settings applicable to all processes (e.g. fdbserver, backup_agent). * ``kill_on_configuration_change``: If ``true``, affected processes will be restarted whenever the configuration file changes. Defaults to ``true``. * ``disable_lifecycle_logging``: If ``true``, ``fdbmonitor`` will not write log events when processes start or terminate. Defaults to ``false``. +.. _configuration-restarting: + The ``[general]`` section also contains some parameters to control how processes are restarted when they die. ``fdbmonitor`` uses backoff logic to prevent a process that dies repeatedly from cycling too quickly, and it also introduces up to +/-10% random jitter into the delay to avoid multiple processes all restarting simultaneously. ``fdbmonitor`` tracks separate backoff state for each process, so the restarting of one process will have no effect on the backoff behavior of another. * ``restart_delay``: The maximum number of seconds (subject to jitter) that fdbmonitor will delay before restarting a failed process. @@ -236,6 +238,8 @@ The ``[general]`` section also contains some parameters to control how processes * ``restart_backoff``: Controls how quickly ``fdbmonitor`` backs off when a process dies repeatedly. The previous delay (or 1, if the previous delay is 0) is multiplied by ``restart_backoff`` to get the next delay, maxing out at the value of ``restart_delay``. Defaults to the value of ``restart_delay``, meaning that the second and subsequent failures will all delay ``restart_delay`` between restarts. * ``restart_delay_reset_interval``: The number of seconds a process must be running before resetting the backoff back to the value of ``initial_restart_delay``. Defaults to the value of ``restart_delay``. + These ``restart_`` parameters are not applicable to the ``fdbmonitor`` process itself. See :ref:`Configuring autorestart of fdbmonitor ` for details. + As an example, let's say the following parameters have been set: .. code-block:: ini @@ -322,6 +326,24 @@ Backup agent sections These sections run and configure the backup agent process used for :doc:`point-in-time backups ` of FoundationDB. These don't usually need to be modified. The structure and functionality is similar to the ``[fdbserver]`` and ``[fdbserver.]`` sections. +.. _configuration-restart-fdbmonitor: + +Configuring autorestart of fdbmonitor +===================================== + +Configuring the restart parameters for ``fdbmonitor`` is operating system-specific. + +Linux (RHEL/CentOS) +------------------- + + ``systemd`` controls the ``foundationdb`` service. When ``fdbmonitor`` is killed unexpectedly, by default, systemd restarts it in 60 seconds. To adjust this value you have to create a file ``/etc/systemd/system/foundationdb.service.d/override.conf`` with the overriding values. For example: + +.. code-block:: ini + + [Service] + RestartSec=20s + +To disable auto-restart of ``fdbmonitor``, put ``Restart=no`` in the same section. .. _configuration-choosing-redundancy-mode: @@ -777,16 +799,17 @@ The 6.2 release still has a number of rough edges related to region configuratio Guidelines for setting process class ==================================== -In a FoundationDB cluster, each of the ``fdbserver`` processes perform different tasks. Each process is recruited to do a particular task based on its process ``class``. For example, processes with ``class=storage`` are given preference to be recruited for doing storage server tasks, ``class=transaction`` are for log server processes and ``class=stateless`` are for stateless processes like proxies, resolvers, etc., +In a FoundationDB cluster, each of the ``fdbserver`` processes perform different tasks. Each process is recruited to do a particular task based on its process ``class``. For example, processes with ``class=storage`` are given preference to be recruited for doing storage server tasks, ``class=transaction`` are for log server processes and ``class=stateless`` are for stateless processes like commit proxies, resolvers, etc., -The recommended minimum number of ``class=transaction`` (log server) processes is 8 (active) + 2 (standby) and the recommended minimum number for ``class=stateless`` processes is 4 (proxy) + 1 (resolver) + 1 (cluster controller) + 1 (master) + 2 (standby). It is better to spread the transaction and stateless processes across as many machines as possible. +The recommended minimum number of ``class=transaction`` (log server) processes is 8 (active) + 2 (standby) and the recommended minimum number for ``class=stateless`` processes is 1 (GRV proxy) + 3 (commit proxy) + 1 (resolver) + 1 (cluster controller) + 1 (master) + 2 (standby). It is better to spread the transaction and stateless processes across as many machines as possible. ``fdbcli`` is used to set the desired number of processes of a particular process type. To do so, you would issue the ``fdbcli`` commands:: - fdb> configure proxies=5 + fdb> configure grv_proxies=1 + fdb> configure grv_proxies=4 fdb> configure logs=8 -.. note:: In the present release, the default value for proxies and log servers is 3 and for resolvers is 1. You should not set the value of a process type to less than its default. +.. note:: In the present release, the default value for commit proxies and log servers is 3 and for GRV proxies and resolvers is 1. You should not set the value of a process type to less than its default. .. warning:: The conflict-resolution algorithm used by FoundationDB is conservative: it guarantees that no conflicting transactions will be committed, but it may fail to commit some transactions that theoretically could have been. The effects of this conservatism may increase as you increase the number of resolvers. It is therefore important to employ the recommended techniques for :ref:`minimizing conflicts ` when increasing the number of resolvers. diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst index 95e7d3986a..e82b73392b 100644 --- a/documentation/sphinx/source/developer-guide.rst +++ b/documentation/sphinx/source/developer-guide.rst @@ -838,7 +838,7 @@ Caveats #. ``\xff\xff/transaction/read_conflict_range/`` The conflict range for a read is sometimes not known until that read completes (e.g. range reads with limits, key selectors). When you read from these special keys, the returned future first blocks until all pending reads are complete so it can give an accurate response. #. ``\xff\xff/transaction/write_conflict_range/`` The conflict range range for a ``set_versionstamped_key`` atomic op is not known until commit time. You'll get an approximate range (the actual range will be a subset of the approximate range) until the precise range is known. -#. ``\xff\xff/transaction/conflicting_keys/`` Since using this feature costs server (i.e., proxy and resolver) resources, it's disabled by default. You must opt in by setting the ``report_conflicting_keys`` transaction option. +#. ``\xff\xff/transaction/conflicting_keys/`` Since using this feature costs server (i.e., commit proxy and resolver) resources, it's disabled by default. You must opt in by setting the ``report_conflicting_keys`` transaction option. Metrics module -------------- @@ -942,7 +942,7 @@ that process, and wait for necessary data to be moved away. #. ``\xff\xff/management/excluded/`` Read/write. Indicates that the cluster should move data away from processes matching ````, so that they can be safely removed. See :ref:`removing machines from a cluster ` for documentation for the corresponding fdbcli command. #. ``\xff\xff/management/failed/`` Read/write. Indicates that the cluster should consider matching processes as permanently failed. This allows the cluster to avoid maintaining extra state and doing extra work in the hope that these processes come back. See :ref:`removing machines from a cluster ` for documentation for the corresponding fdbcli command. -#. ``\xff\xff/management/inProgressExclusion/
`` Read-only. Indicates that the process matching ``
`` matches an exclusion, but still has necessary data and can't yet be safely removed. +#. ``\xff\xff/management/in_progress_exclusion/
`` Read-only. Indicates that the process matching ``
`` matches an exclusion, but still has necessary data and can't yet be safely removed. #. ``\xff\xff/management/options/excluded/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/excluded/``. Setting this key only has an effect in the current transaction and is not persisted on commit. #. ``\xff\xff/management/options/failed/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/failed/``. Setting this key only has an effect in the current transaction and is not persisted on commit. @@ -1059,22 +1059,21 @@ How Versions are Generated and Assigned Versions are generated by the process that runs the *master* role. FoundationDB guarantees that no version will be generated twice and that the versions are monotonically increasing. -In order to assign read and commit versions to transactions, a client will never talk to the master. Instead it will get both from a proxy. Getting a read version is more complex than a commit version. Let's first look at commit versions: +In order to assign read and commit versions to transactions, a client will never talk to the master. Instead it will get them from a GRV proxy and a commit proxy. Getting a read version is more complex than a commit version. Let's first look at commit versions: -#. The client will send a commit message to a proxy. -#. The proxy will put this commit message in a queue in order to build a batch. -#. In parallel, the proxy will ask for a new version from the master (note that this means that only proxies will ever ask for new versions - which scales much better as it puts less stress on the network). -#. The proxy will then resolve all transactions within that batch (discussed later) and assign the version it got from the master to *all* transactions within that batch. It will then write the transactions to the transaction log system to make it durable. +#. The client will send a commit message to a commit proxy. +#. The commit proxy will put this commit message in a queue in order to build a batch. +#. In parallel, the commit proxy will ask for a new version from the master (note that this means that only commit proxies will ever ask for new versions - which scales much better as it puts less stress on the network). +#. The commit proxy will then resolve all transactions within that batch (discussed later) and assign the version it got from the master to *all* transactions within that batch. It will then write the transactions to the transaction log system to make it durable. #. If the transaction succeeded, it will send back the version as commit version to the client. Otherwise it will send back an error. -As mentioned before, the algorithm to assign read versions is a bit more complex. At the start of a transaction, a client will ask a proxy server for a read version. The proxy will reply with the last committed version as of the time it received the request - this is important to guarantee external consistency. This is how this is achieved: +As mentioned before, the algorithm to assign read versions is a bit more complex. At the start of a transaction, a client will ask a GRV proxy server for a read version. The GRV proxy will reply with the last committed version as of the time it received the request - this is important to guarantee external consistency. This is how this is achieved: -#. The client will send a GRV (get read version) request to a proxy. -#. The proxy will batch GRV requests for a short amount of time (it depends on load and configuartion how big these batches will be). +#. The client will send a GRV (get read version) request to a GRV proxy. +#. The GRV proxy will batch GRV requests for a short amount of time (it depends on load and configuartion how big these batches will be). #. The proxy will do the following steps in parallel: - * Ask all other proxies for their most recent committed version (the largest version they received from the master for which it successfully wrote the transactions to the transaction log system). - * Send a message to the transaction log system to verify that it is still writable. This is to prevent that we fetch read versions from a proxy that has been declared to be dead. -#. It will then take the largest committed version from all proxies (including its own) and send it back to the clients. + * Ask master for their most recent committed version (the largest version of proxies' committed version for which the transactions are successfully written to the transaction log system). + * Send a message to the transaction log system to verify that it is still writable. This is to prevent that we fetch read versions from a GRV proxy that has been declared to be dead. Checking whether the log-system is still writeable can be especially expensive if a clusters runs in a multi-region configuration. If a user is fine to sacrifice strict serializability they can use :ref:`option-causal-read-risky `. @@ -1148,8 +1147,8 @@ The ``commit_unknown_result`` Error ``commit_unknown_result`` can be thrown during a commit. This error is difficult to handle as you won't know whether your transaction was committed or not. There are mostly two reasons why you might see this error: -#. The client lost the connection to the proxy to which it did send the commit. So it never got a reply and therefore can't know whether the commit was successful or not. -#. There was a FoundationDB failure - for example a proxy failed during the commit. In that case there is no way for the client know whether the transaction succeeded or not. +#. The client lost the connection to the commit proxy to which it did send the commit. So it never got a reply and therefore can't know whether the commit was successful or not. +#. There was a FoundationDB failure - for example a commit proxy failed during the commit. In that case there is no way for the client know whether the transaction succeeded or not. However, there is one guarantee FoundationDB gives to the caller: at the point of time where you receive this error, the transaction either committed or not and if it didn't commit, it will never commit in the future. Or: it is guaranteed that the transaction is not in-flight anymore. This is an important guarantee as it means that if your transaction is idempotent you can simply retry. For more explanations see developer-guide-unknown-results_. diff --git a/documentation/sphinx/source/disk-snapshot-backup.rst b/documentation/sphinx/source/disk-snapshot-backup.rst index e5eccd8051..33b97b8c09 100644 --- a/documentation/sphinx/source/disk-snapshot-backup.rst +++ b/documentation/sphinx/source/disk-snapshot-backup.rst @@ -104,7 +104,7 @@ Field Name Description ``Name for the snapshot file`` recommended name for the disk snapshot cluster-name:ip-addr:port:UID ================================ ======================================================== ======================================================== -``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or MasterProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally. +``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or CommitProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally. Management of disk snapshots ---------------------------- diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index 1aae48d2eb..8d766f0e7a 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.3.5.pkg `_ +* `FoundationDB-6.3.9.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.3.5-1_amd64.deb `_ -* `foundationdb-server-6.3.5-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.3.9-1_amd64.deb `_ +* `foundationdb-server-6.3.9-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.3.5-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.3.5-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.9-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.3.9-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.3.5-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.3.5-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.9-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.3.9-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.3.5-x64.msi `_ +* `foundationdb-6.3.9-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package: -* `foundationdb-6.3.5.tar.gz `_ +* `foundationdb-6.3.9.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.3.5.gem `_ +* `fdb-6.3.9.gem `_ Java 8+ ------- -* `fdb-java-6.3.5.jar `_ -* `fdb-java-6.3.5-javadoc.jar `_ +* `fdb-java-6.3.9.jar `_ +* `fdb-java-6.3.9-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/kv-architecture.rst b/documentation/sphinx/source/kv-architecture.rst index 6375072d5f..a30b4b1941 100644 --- a/documentation/sphinx/source/kv-architecture.rst +++ b/documentation/sphinx/source/kv-architecture.rst @@ -5,7 +5,7 @@ FoundationDB Architecture Coordinators ============ -All clients and servers connect to a FoundationDB cluster with a cluster file, which contains the IP:PORT of the coordinators. Both the clients and servers use the coordinators to connect with the cluster controller. The servers will attempt to become the cluster controller if one does not exist, and register with the cluster controller once one has been elected. Clients use the cluster controller to keep an up-to-date list of proxies. +All clients and servers connect to a FoundationDB cluster with a cluster file, which contains the IP:PORT of the coordinators. Both the clients and servers use the coordinators to connect with the cluster controller. The servers will attempt to become the cluster controller if one does not exist, and register with the cluster controller once one has been elected. Clients use the cluster controller to keep an up-to-date list of GRV proxies and commit proxies. Cluster Controller ================== @@ -15,12 +15,12 @@ The cluster controller is a singleton elected by a majority of coordinators. It Master ====== -The master is responsible for coordinating the transition of the write sub-system from one generation to the next. The write sub-system includes the master, proxies, resolvers, and transaction logs. The three roles are treated as a unit, and if any of them fail, we will recruit a replacement for all three roles. The master provides the commit versions for batches of the mutations to the proxies, runs data distribution algorithm, and runs ratekeeper. +The master is responsible for coordinating the transition of the write sub-system from one generation to the next. The write sub-system includes the master, GRV proxies, commit proxies, resolvers, and transaction logs. The three roles are treated as a unit, and if any of them fail, we will recruit a replacement for all three roles. The master keeps commit proxies' committed version, provides read version for GRV proxies, provides the commit versions for batches of the mutations to the commit proxies, runs data distribution algorithm, and runs ratekeeper. -Proxies -======= +GRV Proxies and Commit Proxies +============================== -The proxies are responsible for providing read versions, committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a proxy will ask all other proxies to see the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the proxy provides read versions. +The GRV proxies are responsible for providing read versions. The commit proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a GRV proxy will ask master the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the GRV proxy provides read versions. Commits are accomplished by: @@ -33,7 +33,7 @@ The key space starting with the '\xff' byte is reserved for system metadata. All Transaction Logs ================ -The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Before the data is even written to disk we forward it to the storage servers responsible for that mutation. Once the storage servers have made the mutation durable, they pop it from the log. This generally happens roughly 6 seconds after the mutation was originally committed to the log. We only read from the log's disk when the process has been rebooted. If a storage server has failed, mutations bound for that storage server will build up on the logs. Once data distribution makes a different storage server responsible for all of the missing storage server's data we will discard the log data bound for the failed server. +The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the commit proxy in version order, and only respond to the commit proxy once the data has been written and fsync'ed to an append only mutation log on disk. Before the data is even written to disk we forward it to the storage servers responsible for that mutation. Once the storage servers have made the mutation durable, they pop it from the log. This generally happens roughly 6 seconds after the mutation was originally committed to the log. We only read from the log's disk when the process has been rebooted. If a storage server has failed, mutations bound for that storage server will build up on the logs. Once data distribution makes a different storage server responsible for all of the missing storage server's data we will discard the log data bound for the failed server. Resolvers ========= @@ -48,4 +48,4 @@ The vast majority of processes in a cluster are storage servers. Storage servers Clients ======= -Clients must get a read version at the start of every transaction. During the transaction all of the reads are done at that read version, and write are kept in memory until transaction is committed. When the transaction is committed, all of the reads and writes are sent to the proxy. If the transaction conflicts with another transaction the client is responsible for retrying the transaction. By default, reading a key that was written in the same transaction will return the newly written value. +Clients must get a read version at the start of every transaction. During the transaction all of the reads are done at that read version, and write are kept in memory until transaction is committed. When the transaction is committed, all of the reads and writes are sent to the commit proxy. If the transaction conflicts with another transaction the client is responsible for retrying the transaction. By default, reading a key that was written in the same transaction will return the newly written value. diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 0f4b6a9aa9..106c4c19c2 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -27,7 +27,7 @@ "storage", "transaction", "resolution", - "proxy", + "commit_proxy", "grv_proxy", "master", "test", @@ -61,7 +61,7 @@ "role":{ "$enum":[ "master", - "proxy", + "commit_proxy", "grv_proxy", "log", "storage", @@ -446,8 +446,9 @@ } ], "recovery_state":{ + "seconds_since_last_recovered":1, "required_resolvers":1, - "required_proxies":1, + "required_commit_proxies":1, "required_grv_proxies":1, "name":{ // "fully_recovered" is the healthy state; other states are normal to transition through but not to persist in "$enum":[ @@ -633,11 +634,11 @@ "address":"10.0.4.1" } ], - "auto_proxies":3, + "auto_commit_proxies":3, "auto_resolvers":1, "auto_logs":3, "backup_worker_enabled":1, - "proxies":5 // this field will be absent if a value has not been explicitly set + "commit_proxies":5 // this field will be absent if a value has not been explicitly set }, "data":{ "least_operating_space_bytes_log_server":0, diff --git a/documentation/sphinx/source/release-notes/release-notes-620.rst b/documentation/sphinx/source/release-notes/release-notes-620.rst index 32b9d11e6f..596c2826fe 100644 --- a/documentation/sphinx/source/release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/release-notes/release-notes-620.rst @@ -2,6 +2,18 @@ Release Notes ############# +6.2.27 +====== +* For clusters with a large number of shards, avoid slow tasks in the data distributor by adding yields to the shard map destruction. `(PR #3834) `_ +* Reset the network connection between a proxy and master or resolvers if the proxy is too far behind in processing transactions. `(PR #3891) `_ + +6.2.26 +====== + +* Fixed undefined behavior in configuring supported FoundationDB versions while starting up a client. `(PR #3849) `_ +* Updated OpenSSL to version 1.1.1h. `(PR #3809) `_ +* Attempt to detect when calling :func:`fdb_future_block_until_ready` would cause a deadlock, and throw ``blocked_from_network_thread`` if it would definitely cause a deadlock. `(PR #3786) `_ + 6.2.25 ====== @@ -11,50 +23,31 @@ Release Notes 6.2.24 ====== -Features --------- - * Added the ``suspend`` command to ``fdbcli`` which kills a process and prevents it from rejoining the cluster for a specified duration. `(PR #3550) `_ 6.2.23 ====== -Fixes ------ - * When configured with ``usable_regions=2`` data distribution could temporarily lower the replication of a shard when moving it. `(PR #3487) `_ * Prevent data distribution from running out of memory by fetching the source servers for too many shards in parallel. `(PR #3487) `_ * Reset network connections between log routers and satellite tlogs if the latencies are larger than 500ms. `(PR #3487) `_ - -Status ------- - * Added per-process server request latency statistics reported in the role section of relevant processes. These are named ``grv_latency_statistics`` and ``commit_latency_statistics`` on proxy roles and ``read_latency_statistics`` on storage roles. `(PR #3480) `_ * Added ``cluster.active_primary_dc`` that indicates which datacenter is serving as the primary datacenter in multi-region setups. `(PR #3320) `_ 6.2.22 ====== -Fixes ------ - * Coordinator class processes could be recruited as the cluster controller. `(PR #3282) `_ * HTTPS requests made by backup would fail (introduced in 6.2.21). `(PR #3284) `_ 6.2.21 ====== -Fixes ------ - * HTTPS requests made by backup could hang indefinitely. `(PR #3027) `_ * ``fdbrestore`` prefix options required exactly a single hyphen instead of the standard two. `(PR #3056) `_ * Commits could stall on a newly elected proxy because of inaccurate compute estimates. `(PR #3123) `_ * A transaction class process with a bad disk could be repeatedly recruited as a transaction log. `(PR #3268) `_ * Fix a potential race condition that could lead to undefined behavior when connecting to a database using the multi-version client API. `(PR #3265) `_ - -Features --------- * Added the ``getversion`` command to ``fdbcli`` which returns the current read version of the cluster. `(PR #2882) `_ * Added the ``advanceversion`` command to ``fdbcli`` which increases the current version of a cluster. `(PR #2965) `_ * Added the ``lock`` and ``unlock`` commands to ``fdbcli`` which lock or unlock a cluster. `(PR #2890) `_ @@ -62,9 +55,6 @@ Features 6.2.20 ====== -Fixes ------ - * In rare scenarios, clients could send corrupted data to the server. `(PR #2976) `_ * Internal tools like ``fdbbackup`` are no longer tracked as clients in status (introduced in 6.2.18) `(PR #2849) `_ * Changed TLS error handling to match the behavior of 6.2.15. `(PR #2993) `_ `(PR #2977) `_ @@ -72,9 +62,6 @@ Fixes 6.2.19 ====== -Fixes ------ - * Protect the proxies from running out of memory when bombarded with requests from clients. `(PR #2812) `_. * One process with a ``proxy`` class would not become the first proxy when put with other ``stateless`` class processes. `(PR #2819) `_. * If a transaction log stalled on a disk operation during recruitment the cluster would become unavailable until the process died. `(PR #2815) `_. @@ -82,70 +69,37 @@ Fixes * Prevent the cluster from having too many active generations as a safety measure against repeated failures. `(PR #2814) `_. * ``fdbcli`` status JSON could become truncated because of unprintable characters. `(PR #2807) `_. * The data distributor used too much CPU in large clusters (broken in 6.2.16). `(PR #2806) `_. - -Status ------- - * Added ``cluster.workload.operations.memory_errors`` to measure the number of requests rejected by the proxies because the memory limit has been exceeded. `(PR #2812) `_. * Added ``cluster.workload.operations.location_requests`` to measure the number of outgoing key server location responses from the proxies. `(PR #2812) `_. * Added ``cluster.recovery_state.active_generations`` to track the number of generations for which the cluster still requires transaction logs. `(PR #2814) `_. * Added ``network.tls_policy_failures`` to the ``processes`` section to record the number of TLS policy failures each process has observed. `(PR #2811) `_. - -Features --------- - * Added ``--debug-tls`` as a command line argument to ``fdbcli`` to help diagnose TLS issues. `(PR #2810) `_. 6.2.18 ====== -Fixes ------ - * When configuring a cluster to usable_regions=2, data distribution would not react to machine failures while copying data to the remote region. `(PR #2774) `_. * When a cluster is configured with usable_regions=2, data distribution could push a cluster into saturation by relocating too many shards simulatenously. `(PR #2776) `_. * Do not allow the cluster controller to mark any process as failed within 30 seconds of startup. `(PR #2780) `_. * Backup could not establish TLS connections (broken in 6.2.16). `(PR #2775) `_. * Certificates were not refreshed automatically (broken in 6.2.16). `(PR #2781) `_. - -Performance ------------ - * Improved the efficiency of establishing large numbers of network connections. `(PR #2777) `_. - -Features --------- - * Add support for setting knobs to modify the behavior of ``fdbcli``. `(PR #2773) `_. - -Other Changes -------------- - * Setting invalid knobs in backup and DR binaries is now a warning instead of an error and will not result in the application being terminated. `(PR #2773) `_. 6.2.17 ====== -Fixes ------ - * Restored the ability to set TLS configuration using environment variables (broken in 6.2.16). `(PR #2755) `_. 6.2.16 ====== -Performance ------------ - * Reduced tail commit latencies by improving commit pipelining on the proxies. `(PR #2589) `_. * Data distribution does a better job balancing data when disks are more than 70% full. `(PR #2722) `_. * Reverse range reads could read too much data from disk, resulting in poor performance relative to forward range reads. `(PR #2650) `_. * Switched from LibreSSL to OpenSSL to improve the speed of establishing connections. `(PR #2646) `_. * The cluster controller does a better job avoiding multiple recoveries when first recruited. `(PR #2698) `_. - -Fixes ------ - * Storage servers could fail to advance their version correctly in response to empty commits. `(PR #2617) `_. * Status could not label more than 5 processes as proxies. `(PR #2653) `_. * The ``TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER``, ``TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS``, ``TR_FLAG_DISABLE_SERVER_TEAM_REMOVER``, and ``BUGGIFY_ALL_COORDINATION`` knobs could not be set at runtime. `(PR #2661) `_. @@ -157,17 +111,11 @@ Fixes 6.2.15 ====== -Fixes ------ - * TLS throttling could block legitimate connections. `(PR #2575) `_. 6.2.14 ====== -Fixes ------ - * Data distribution was prioritizing shard merges too highly. `(PR #2562) `_. * Status would incorrectly mark clusters as having no fault tolerance. `(PR #2562) `_. * A proxy could run out of memory if disconnected from the cluster for too long. `(PR #2562) `_. @@ -175,26 +123,16 @@ Fixes 6.2.13 ====== -Performance ------------ - * Optimized the commit path the proxies to significantly reduce commit latencies in large clusters. `(PR #2536) `_. * Data distribution could create temporarily untrackable shards which could not be split if they became hot. `(PR #2546) `_. 6.2.12 ====== -Performance ------------ - * Throttle TLS connect attempts from misconfigured clients. `(PR #2529) `_. * Reduced master recovery times in large clusters. `(PR #2430) `_. * Improved performance while a remote region is catching up. `(PR #2527) `_. * The data distribution algorithm does a better job preventing hot shards while recovering from machine failures. `(PR #2526) `_. - -Fixes ------ - * Improve the reliability of a ``kill`` command from ``fdbcli``. `(PR #2512) `_. * The ``--traceclock`` parameter to fdbserver incorrectly had no effect. `(PR #2420) `_. * Clients could throw an internal error during ``commit`` if client buggification was enabled. `(PR #2427) `_. @@ -204,9 +142,6 @@ Fixes 6.2.11 ====== -Fixes ------ - * Clients could hang indefinitely on reads if all storage servers holding a keyrange were removed from a cluster since the last time the client read a key in the range. `(PR #2377) `_. * In rare scenarios, status could falsely report no replicas remain of some data. `(PR #2380) `_. * Latency band tracking could fail to configure correctly after a recovery or upon process startup. `(PR #2371) `_. @@ -214,17 +149,11 @@ Fixes 6.2.10 ====== -Fixes ------ - * ``backup_agent`` crashed on startup. `(PR #2356) `_. 6.2.9 ===== -Fixes ------ - * Small clusters using specific sets of process classes could cause the data distributor to be continuously killed and re-recruited. `(PR #2344) `_. * The data distributor and ratekeeper could be recruited on non-optimal processes. `(PR #2344) `_. * A ``kill`` command from ``fdbcli`` could take a long time before being executed by a busy process. `(PR #2339) `_. @@ -234,9 +163,6 @@ Fixes 6.2.8 ===== -Fixes ------ - * Significantly improved the rate at which the transaction logs in a remote region can pull data from the primary region. `(PR #2307) `_ `(PR #2323) `_. * The ``system_kv_size_bytes`` status field could report a size much larger than the actual size of the system keyspace. `(PR #2305) `_. diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index bcbddc91d4..f5983e3bb0 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -2,7 +2,7 @@ Release Notes ############# -6.3.5 +6.3.9 ===== Features @@ -61,6 +61,8 @@ Fixes * In very rare scenarios, the data distributor process would crash when being shutdown. `(PR #3530) `_ * The master would die immediately if it did not have the correct cluster controller interface when recruited. [6.3.4] `(PR #3537) `_ * Fix an issue where ``fdbcli --exec 'exclude no_wait ...'`` would incorrectly report that processes can safely be removed from the cluster. [6.3.5] `(PR #3566) `_ +* Commit latencies could become large because of inaccurate compute estimates. [6.3.9] `(PR #3845) `_ +* Added a timeout on TLS handshakes to prevent them from hanging indefinitely. [6.3.9] `(PR #3850) `_ Status ------ @@ -108,6 +110,10 @@ Other Changes * Updated boost to 1.72. `(PR #2684) `_ * Calling ``fdb_run_network`` multiple times in a single run of a client program now returns an error instead of causing undefined behavior. [6.3.1] `(PR #3229) `_ * Blob backup URL parameter ``request_timeout`` changed to ``request_timeout_min``, with prior name still supported. `(PR #3533) `_ +* Support query command in backup CLI that allows users to query restorable files by key ranges. [6.3.6] `(PR #3703) `_ +* Report missing old tlogs information when in recovery before storage servers are fully recovered. [6.3.6] `(PR #3706) `_ +* Updated OpenSSL to version 1.1.1h. [6.3.7] `(PR #3809) `_ +* Lowered the amount of time a watch will remain registered on a storage server from 900 seconds to 30 seconds. [6.3.8] `(PR #3833) `_ Fixes from previous versions ---------------------------- @@ -124,6 +130,9 @@ Fixes only impacting 6.3.0+ * Refreshing TLS certificates could cause crashes. [6.3.2] `(PR #3352) `_ * All storage class processes attempted to connect to the same coordinator. [6.3.2] `(PR #3361) `_ * Adjusted the proxy load balancing algorithm to be based on the CPU usage of the process instead of the number of requests processed. [6.3.5] `(PR #3653) `_ +* Only return the error code ``batch_transaction_throttled`` for API versions greater than or equal to 630. [6.3.6] `(PR #3799) `_ +* The fault tolerance calculation in status did not take into account region configurations. [6.3.8] `(PR #3836) `_ +* Get read version tail latencies were high because some proxies were serving more read versions than other proxies. [6.3.9] `(PR #3845) `_ Earlier release notes --------------------- diff --git a/documentation/sphinx/source/release-notes/release-notes-700.rst b/documentation/sphinx/source/release-notes/release-notes-700.rst index 664d96c692..cac8e12be6 100644 --- a/documentation/sphinx/source/release-notes/release-notes-700.rst +++ b/documentation/sphinx/source/release-notes/release-notes-700.rst @@ -9,7 +9,7 @@ Release Notes Features -------- - +* Added a new API in all bindings that can be used to get a list of split points that will split the given range into (roughly) equally sized chunks. `(PR #3394) `_ Performance @@ -34,7 +34,7 @@ Status Bindings -------- - +* Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) `_ Other Changes diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 53260c923c..eb64bcbe3a 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -18,6 +18,10 @@ * limitations under the License. */ +#include "fdbclient/JsonBuilder.h" +#include "flow/Arena.h" +#include "flow/Error.h" +#include "flow/Trace.h" #define BOOST_DATE_TIME_NO_LIB #include @@ -81,7 +85,22 @@ enum enumProgramExe { }; enum enumBackupType { - BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP + BACKUP_UNDEFINED = 0, + BACKUP_START, + BACKUP_MODIFY, + BACKUP_STATUS, + BACKUP_ABORT, + BACKUP_WAIT, + BACKUP_DISCONTINUE, + BACKUP_PAUSE, + BACKUP_RESUME, + BACKUP_EXPIRE, + BACKUP_DELETE, + BACKUP_DESCRIBE, + BACKUP_LIST, + BACKUP_QUERY, + BACKUP_DUMP, + BACKUP_CLEANUP }; enum enumDBType { @@ -121,6 +140,7 @@ enum { OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, + OPT_BACKUPKEYS_FILTER, OPT_INCREMENTALONLY, // Backup Modify @@ -624,6 +644,40 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = { SO_END_OF_OPTIONS }; +CSimpleOpt::SOption g_rgBackupQueryOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_RESTORE_TIMESTAMP, "--query_restore_timestamp", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, +#ifndef TLS_DISABLED + TLS_OPTION_FLAGS +#endif + SO_END_OF_OPTIONS +}; + // g_rgRestoreOptions is used by fdbrestore and fastrestore_tool CSimpleOpt::SOption g_rgRestoreOptions[] = { #ifdef _WIN32 @@ -959,13 +1013,16 @@ void printBackupContainerInfo() { static void printBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str()); + printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | " + "list | query | cleanup) [OPTIONS]\n\n", + exeBackup.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" " FoundationDB cluster. The default is first the value of the\n" " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); printf(" -d, --destcontainer URL\n" - " The Backup container URL for start, modify, describe, expire, and delete operations.\n"); + " The Backup container URL for start, modify, describe, query, expire, and delete " + "operations.\n"); printBackupContainerInfo(); printf(" -b, --base_url BASEURL\n" " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n"); @@ -979,6 +1036,12 @@ static void printBackupUsage(bool devhelp) { printf(" --delete_before_days NUM_DAYS\n" " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); + printf(" -qrv --query_restore_version VERSION\n" + " For query operations, set target version for restoring a backup. Set -1 for maximum\n" + " restorable version (default) and -2 for minimum restorable version.\n"); + printf(" --query_restore_timestamp DATETIME\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); + printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" @@ -997,8 +1060,8 @@ static void printBackupUsage(bool devhelp) { " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" + " If not specified, the entire database will be backed up or no filter will be applied.\n"); printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" @@ -1320,6 +1383,7 @@ enumBackupType getBackupType(std::string backupType) values["delete"] = BACKUP_DELETE; values["describe"] = BACKUP_DESCRIBE; values["list"] = BACKUP_LIST; + values["query"] = BACKUP_QUERY; values["dump"] = BACKUP_DUMP; values["modify"] = BACKUP_MODIFY; } @@ -1449,7 +1513,7 @@ ACTOR Future getLayerStatus(Reference tr for (KeyBackedTag eachTag : backupTags) { Version last_restorable_version = tagLastRestorableVersions[j].get(); double last_restorable_seconds_behind = ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - BackupAgentBase::enumState status = (BackupAgentBase::enumState)tagStates[j].get(); + EBackupState status = tagStates[j].get(); const char *statusText = fba.getStateText(status); // The object for this backup tag inside this instance's subdocument @@ -1458,8 +1522,9 @@ ACTOR Future getLayerStatus(Reference tr tagRoot.create("current_status") = statusText; tagRoot.create("last_restorable_version") = tagLastRestorableVersions[j].get(); tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind; - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); - tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); + tagRoot.create("running_backup") = + (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); + tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); tagRoot.create("mutation_stream_id") = backupTagUids[j].toString(); @@ -1474,7 +1539,7 @@ ACTOR Future getLayerStatus(Reference tr tr2->setOption(FDBTransactionOptions::LOCK_AWARE); state Standalone tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000, snapshot)); state std::vector>> backupVersion; - state std::vector> backupStatus; + state std::vector> backupStatus; state std::vector> tagRangeBytesDR; state std::vector> tagLogBytesDR; state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); @@ -1499,11 +1564,12 @@ ACTOR Future getLayerStatus(Reference tr for (int i = 0; i < tagNames.size(); i++) { std::string tagName = dba.sourceTagNames.unpack(tagNames[i].key).getString(0).toString(); - BackupAgentBase::enumState status = (BackupAgentBase::enumState)backupStatus[i].get(); + auto status = backupStatus[i].get(); JSONDoc tagRoot = tagsRoot.create(tagName); - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); - tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); + tagRoot.create("running_backup") = + (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); + tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get(); tagRoot.create("mutation_stream_id") = drTagUids[i].toString(); @@ -1787,7 +1853,7 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv EBackupState backupStatus = wait(config.stateEnum().getOrThrow(db)); // Throw error if a backup is currently running until we support parallel backups - if (BackupAgentBase::isRunnable((BackupAgentBase::enumState)backupStatus)) { + if (BackupAgentBase::isRunnable(backupStatus)) { throw backup_duplicate(); } } @@ -2012,10 +2078,10 @@ ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDon { state FileBackupAgent backupAgent; - int status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone)); + EBackupState status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone)); printf("The backup on tag `%s' %s.\n", printable(StringRef(tagName)).c_str(), - BackupAgentBase::getStateText((BackupAgentBase::enumState) status)); + BackupAgentBase::getStateText(status)); } catch (Error& e) { if(e.code() == error_code_actor_cancelled) @@ -2456,6 +2522,135 @@ ACTOR Future describeBackup(const char *name, std::string destinationConta return Void(); } +static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) { + result["error"] = errorMessage; + printf("%s\n", result.getJson().c_str()); + TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage); +} + +// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for +// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version +// resolved to that timestamp. +ACTOR Future queryBackup(const char* name, std::string destinationContainer, + Standalone> keyRangesFilter, Version restoreVersion, + std::string originalClusterFile, std::string restoreTimestamp, bool verbose) { + state UID operationId = deterministicRandom()->randomUniqueID(); + state JsonBuilderObject result; + state std::string errorMessage; + result["key_ranges_filter"] = printable(keyRangesFilter); + result["destination_container"] = destinationContainer; + + TraceEvent("BackupQueryStart") + .detail("OperationId", operationId) + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("SpecifiedRestoreVersion", restoreVersion) + .detail("RestoreTimestamp", restoreTimestamp) + .detail("BackupClusterFile", originalClusterFile); + + // Resolve restoreTimestamp if given + if (!restoreTimestamp.empty()) { + if (originalClusterFile.empty()) { + reportBackupQueryError( + operationId, result, + format("an original cluster file must be given in order to resolve restore target timestamp '%s'", + restoreTimestamp.c_str())); + return Void(); + } + + if (!fileExists(originalClusterFile)) { + reportBackupQueryError(operationId, result, + format("The specified original source database cluster file '%s' does not exist\n", + originalClusterFile.c_str())); + return Void(); + } + + Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST); + Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); + result["restore_timestamp"] = restoreTimestamp; + result["restore_timestamp_resolved_version"] = v; + restoreVersion = v; + } + + try { + state Reference bc = openBackupContainer(name, destinationContainer); + if (restoreVersion == invalidVersion) { + BackupDescription desc = wait(bc->describeBackup()); + if (desc.maxRestorableVersion.present()) { + restoreVersion = desc.maxRestorableVersion.get(); + // Use continuous log end version for the maximum restorable version for the key ranges. + } else if (keyRangesFilter.size() && desc.contiguousLogEnd.present()) { + restoreVersion = desc.contiguousLogEnd.get(); + } else { + reportBackupQueryError( + operationId, result, + errorMessage = format("the backup for the specified key ranges is not restorable to any version")); + } + } + + if (restoreVersion < 0 && restoreVersion != latestVersion) { + reportBackupQueryError(operationId, result, + errorMessage = + format("the specified restorable version %ld is not valid", restoreVersion)); + return Void(); + } + Optional fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter)); + if (fileSet.present()) { + int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0; + result["restore_version"] = fileSet.get().targetVersion; + JsonBuilderArray rangeFilesJson; + JsonBuilderArray logFilesJson; + for (const auto& rangeFile : fileSet.get().ranges) { + JsonBuilderObject object; + object["file_name"] = rangeFile.fileName; + object["file_size"] = rangeFile.fileSize; + object["version"] = rangeFile.version; + object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0 + ? "none" + : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); + rangeFilesJson.push_back(object); + totalRangeFilesSize += rangeFile.fileSize; + } + for (const auto& log : fileSet.get().logs) { + JsonBuilderObject object; + object["file_name"] = log.fileName; + object["file_size"] = log.fileSize; + object["begin_version"] = log.beginVersion; + object["end_version"] = log.endVersion; + logFilesJson.push_back(object); + totalLogFilesSize += log.fileSize; + } + + result["total_range_files_size"] = totalRangeFilesSize; + result["total_log_files_size"] = totalLogFilesSize; + + if (verbose) { + result["ranges"] = rangeFilesJson; + result["logs"] = logFilesJson; + } + + TraceEvent("BackupQueryReceivedRestorableFilesSet") + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("ActualRestoreVersion", fileSet.get().targetVersion) + .detail("NumRangeFiles", fileSet.get().ranges.size()) + .detail("NumLogFiles", fileSet.get().logs.size()) + .detail("RangeFilesBytes", totalRangeFilesSize) + .detail("LogFilesBytes", totalLogFilesSize); + } else { + reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges"); + return Void(); + } + + } catch (Error& e) { + reportBackupQueryError(operationId, result, e.what()); + return Void(); + } + + printf("%s\n", result.getJson().c_str()); + return Void(); +} + ACTOR Future listBackup(std::string baseUrl) { try { std::vector containers = wait(IBackupContainer::listContainers(baseUrl)); @@ -2825,6 +3020,9 @@ int main(int argc, char* argv[]) { case BACKUP_LIST: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT); break; + case BACKUP_QUERY: + args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT); + break; case BACKUP_MODIFY: args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT); break; @@ -2964,6 +3162,7 @@ int main(int argc, char* argv[]) { std::string addPrefix; std::string removePrefix; Standalone> backupKeys; + Standalone> backupKeysFilter; int maxErrors = 20; Version beginVersion = invalidVersion; Version restoreVersion = invalidVersion; @@ -3186,6 +3385,15 @@ int main(int argc, char* argv[]) { return FDB_EXIT_ERROR; } break; + case OPT_BACKUPKEYS_FILTER: + try { + addKeyRange(args->OptionArg(), backupKeysFilter); + } + catch (Error &) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + break; case OPT_DESTCONTAINER: destinationContainer = args->OptionArg(); // If the url starts with '/' then prepend "file://" for backwards compatibility @@ -3725,6 +3933,12 @@ int main(int argc, char* argv[]) { f = stopAfter( listBackup(baseUrl) ); break; + case BACKUP_QUERY: + initTraceFile(); + f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion, + restoreClusterFileOrig, restoreTimestamp, !quietDisplay)); + break; + case BACKUP_DUMP: initTraceFile(); f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) ); diff --git a/fdbcli/FlowLineNoise.actor.cpp b/fdbcli/FlowLineNoise.actor.cpp index 6c101ca666..8725a7255b 100644 --- a/fdbcli/FlowLineNoise.actor.cpp +++ b/fdbcli/FlowLineNoise.actor.cpp @@ -36,18 +36,18 @@ #endif #include "flow/actorcompiler.h" // This must be the last #include. -struct LineNoiseReader : IThreadPoolReceiver { - virtual void init() {} +struct LineNoiseReader final : IThreadPoolReceiver { + void init() override {} - struct Read : TypedAction { - std::string prompt; + struct Read final : TypedAction { + std::string prompt; ThreadReturnPromise> result; - virtual double getTimeEstimate() { return 0.0; } - explicit Read(std::string const& prompt) : prompt(prompt) {} - }; + double getTimeEstimate() const override { return 0.0; } + explicit Read(std::string const& prompt) : prompt(prompt) {} + }; - void action(Read& r) { + void action(Read& r) { try { r.result.send( read(r.prompt) ); } catch (Error& e) { @@ -117,7 +117,7 @@ LineNoise::LineNoise( Hint h = onMainThread( [line]() -> Future { return hint_callback(line); }).getBlocking(); - if (!h.valid) return NULL; + if (!h.valid) return nullptr; *color = h.color; *bold = h.bold; return strdup( h.text.c_str() ); diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 6351219341..9d6d4d0824 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -20,6 +20,7 @@ #include "boost/lexical_cast.hpp" #include "fdbclient/NativeAPI.actor.h" +#include "fdbclient/FDBTypes.h" #include "fdbclient/Status.h" #include "fdbclient/StatusClient.h" #include "fdbclient/DatabaseContext.h" @@ -102,7 +103,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, void printAtCol(const char* text, int col) { const char* iter = text; const char* start = text; - const char* space = NULL; + const char* space = nullptr; do { iter++; @@ -112,7 +113,7 @@ void printAtCol(const char* text, int col) { printf("%.*s\n", (int)(space - start), start); start = space; if (*start == ' ' || *start == '\n') start++; - space = NULL; + space = nullptr; } } while (*iter); } @@ -120,7 +121,7 @@ void printAtCol(const char* text, int col) { std::string lineWrap(const char* text, int col) { const char* iter = text; const char* start = text; - const char* space = NULL; + const char* space = nullptr; std::string out = ""; do { iter++; @@ -130,7 +131,7 @@ std::string lineWrap(const char* text, int col) { out += format("%.*s\n", (int)(space - start), start); start = space; if (*start == ' '/* || *start == '\n'*/) start++; - space = NULL; + space = nullptr; } } while (*iter); return out; @@ -470,8 +471,8 @@ void initHelp() { "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); helpMap["configure"] = CommandHelp( "configure [new] " - "|grv_" - "proxies=|logs=|resolvers=>*", + "|" + "commit_proxies=|grv_proxies=|logs=|resolvers=>*", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing " "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " @@ -479,13 +480,19 @@ void initHelp() { "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " - "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set " - "to -1 which restores the number of proxies to the default value.\n\ngrv_proxies=: Sets the " - "desired number of GRV proxies in the cluster. Must be at least 1, or set to -1 which restores the number of " - "proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must be " - "at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: " - "Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the " - "number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); + "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. The proxy role is being " + "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' " + "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and 'grv_proxies' " + "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and 'commit_proxies'. " + "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the " + "default value.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. " + "Must be at least 1, or set to -1 which restores the number of commit proxies to the default " + "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " + "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " + "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " + "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " + "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the " + "FoundationDB Administration Guide for more information."); helpMap["fileconfigure"] = CommandHelp( "fileconfigure [new] ", "change the database configuration from a file", @@ -871,12 +878,13 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, fatalRecoveryState = true; if (name == "recruiting_transaction_servers") { - description += format("\nNeed at least %d log servers across unique zones, %d proxies, " - "%d GRV proxies and %d resolvers.", - recoveryState["required_logs"].get_int(), - recoveryState["required_proxies"].get_int(), - recoveryState["required_grv_proxies"].get_int(), - recoveryState["required_resolvers"].get_int()); + description += + format("\nNeed at least %d log servers across unique zones, %d commit proxies, " + "%d GRV proxies and %d resolvers.", + recoveryState["required_logs"].get_int(), + recoveryState["required_commit_proxies"].get_int(), + recoveryState["required_grv_proxies"].get_int(), + recoveryState["required_resolvers"].get_int()); if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) { auto numOfNonExcludedProcessesAndZones = getNumOfNonExcludedProcessAndZones(statusObjCluster); description += format("\nHave %d non-excluded processes on %d machines across %d zones.", numOfNonExcludedProcessesAndZones.first, getNumofNonExcludedMachines(statusObjCluster), numOfNonExcludedProcessesAndZones.second); @@ -1026,8 +1034,8 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, outputString += format("\n Exclusions - %d (type `exclude' for details)", excludedServersArr.size()); } - if (statusObjConfig.get("proxies", intVal)) - outputString += format("\n Desired Proxies - %d", intVal); + if (statusObjConfig.get("commit_proxies", intVal)) + outputString += format("\n Desired Commit Proxies - %d", intVal); if (statusObjConfig.get("grv_proxies", intVal)) outputString += format("\n Desired GRV Proxies - %d", intVal); @@ -1055,10 +1063,10 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, if (statusObjConfig.has("regions")) { outputString += "\n Regions: "; regions = statusObjConfig["regions"].get_array(); - bool isPrimary = false; - std::vector regionSatelliteDCs; - std::string regionDC; for (StatusObjectReader region : regions) { + bool isPrimary = false; + std::vector regionSatelliteDCs; + std::string regionDC; for (StatusObjectReader dc : region["datacenters"].get_array()) { if (!dc.has("satellite")) { regionDC = dc["id"].get_str(); @@ -1233,14 +1241,54 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, int minLoss = std::min(availLoss, dataLoss); const char *faultDomain = machinesAreZones ? "machine" : "zone"; - if (minLoss == 1) - outputString += format("1 %s", faultDomain); - else - outputString += format("%d %ss", minLoss, faultDomain); + outputString += format("%d %ss", minLoss, faultDomain); if (dataLoss > availLoss){ outputString += format(" (%d without data loss)", dataLoss); } + + if (dataLoss == -1) { + ASSERT_WE_THINK(availLoss == -1); + outputString += format( + "\n\n Warning: the database may have data loss and availability loss. Please restart " + "following tlog interfaces, otherwise storage servers may never be able to catch " + "up.\n"); + StatusObjectReader logs; + if (statusObjCluster.has("logs")) { + for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) { + bool possiblyLosingData; + if (logEpoch.get("possibly_losing_data", possiblyLosingData) && + !possiblyLosingData) { + continue; + } + // Current epoch doesn't have an end version. + int64_t epoch, beginVersion, endVersion = invalidVersion; + bool current; + logEpoch.get("epoch", epoch); + logEpoch.get("begin_version", beginVersion); + logEpoch.get("end_version", endVersion); + logEpoch.get("current", current); + std::string missing_log_interfaces; + if (logEpoch.has("log_interfaces")) { + for (StatusObjectReader logInterface : logEpoch.last().get_array()) { + bool healthy; + std::string address, id; + if (logInterface.get("healthy", healthy) && !healthy) { + logInterface.get("id", id); + logInterface.get("address", address); + missing_log_interfaces += format("%s,%s ", id.c_str(), address.c_str()); + } + } + } + outputString += format( + " %s log epoch: %ld begin: %ld end: %s, missing " + "log interfaces(id,address): %s\n", + current ? "Current" : "Old", epoch, beginVersion, + endVersion == invalidVersion ? "(unknown)" : format("%ld", endVersion).c_str(), + missing_log_interfaces.c_str()); + } + } + } } } @@ -1764,7 +1812,7 @@ ACTOR Future commitTransaction( Reference tr ) } ACTOR Future configure( Database db, std::vector tokens, Reference ccf, LineNoise* linenoise, Future warn ) { - state ConfigurationResult::Type result; + state ConfigurationResult result; state int startToken = 1; state bool force = false; if (tokens.size() < 2) @@ -1790,14 +1838,14 @@ ACTOR Future configure( Database db, std::vector tokens, Refere bool noChanges = conf.get().old_replication == conf.get().auto_replication && conf.get().old_logs == conf.get().auto_logs && - conf.get().old_proxies == conf.get().auto_proxies && + conf.get().old_commit_proxies == conf.get().auto_commit_proxies && conf.get().old_grv_proxies == conf.get().auto_grv_proxies && conf.get().old_resolvers == conf.get().auto_resolvers && conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && - conf.get().old_proxies == conf.get().desired_proxies && + conf.get().old_commit_proxies == conf.get().desired_commit_proxies && conf.get().old_grv_proxies == conf.get().desired_grv_proxies && conf.get().old_resolvers == conf.get().desired_resolvers; @@ -1816,8 +1864,11 @@ ACTOR Future configure( Database db, std::vector tokens, Refere outputString += format("| replication | %16s | %16s |\n", conf.get().old_replication.c_str(), conf.get().auto_replication.c_str()); outputString += format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); outputString += conf.get().auto_logs != conf.get().desired_logs ? format(" (manually set; would be %d)\n", conf.get().desired_logs) : "\n"; - outputString += format("| proxies | %16d | %16d |", conf.get().old_proxies, conf.get().auto_proxies); - outputString += conf.get().auto_proxies != conf.get().desired_proxies ? format(" (manually set; would be %d)\n", conf.get().desired_proxies) : "\n"; + outputString += format("| commit_proxies | %16d | %16d |", conf.get().old_commit_proxies, + conf.get().auto_commit_proxies); + outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) + : "\n"; outputString += format("| grv_proxies | %16d | %16d |", conf.get().old_grv_proxies, conf.get().auto_grv_proxies); outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies @@ -1842,7 +1893,8 @@ ACTOR Future configure( Database db, std::vector tokens, Refere } } - ConfigurationResult::Type r = wait( makeInterruptable( changeConfig( db, std::vector(tokens.begin()+startToken,tokens.end()), conf, force) ) ); + ConfigurationResult r = wait(makeInterruptable( + changeConfig(db, std::vector(tokens.begin() + startToken, tokens.end()), conf, force))); result = r; } @@ -1968,7 +2020,7 @@ ACTOR Future fileConfigure(Database db, std::string filePath, bool isNewDa return true; } } - ConfigurationResult::Type result = wait( makeInterruptable( changeConfig(db, configString, force) ) ); + ConfigurationResult result = wait(makeInterruptable(changeConfig(db, configString, force))); // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but // there are various results specific to changeConfig() that we need to report: bool ret; @@ -2099,7 +2151,7 @@ ACTOR Future coordinators( Database db, std::vector tokens, boo } if(setName.size()) change = nameQuorumChange( setName.toString(), change ); - CoordinatorsResult::Type r = wait( makeInterruptable( changeQuorum( db, change ) ) ); + CoordinatorsResult r = wait(makeInterruptable(changeQuorum(db, change))); // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but // there are various results specific to changeConfig() that we need to report: @@ -2472,7 +2524,7 @@ void compGenerator(const char* text, bool help, std::vector& lc) { std::map::const_iterator iter; int len = strlen(text); - const char* helpExtra[] = {"escaping", "options", NULL}; + const char* helpExtra[] = {"escaping", "options", nullptr}; const char** he = helpExtra; @@ -2531,11 +2583,24 @@ void onOffGenerator(const char* text, const char *line, std::vector } void configureGenerator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = { - "new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", - "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "memory-radixtree-beta", "proxies=", - "grv_proxies=", "logs=", "resolvers=", nullptr - }; + const char* opts[] = { "new", + "single", + "double", + "triple", + "three_data_hall", + "three_datacenter", + "ssd", + "ssd-1", + "ssd-2", + "memory", + "memory-1", + "memory-2", + "memory-radixtree-beta", + "commit_proxies=", + "grv_proxies=", + "logs=", + "resolvers=", + nullptr }; arrayGenerator(text, line, opts, lc); } @@ -2973,7 +3038,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { .detail("SourceVersion", getSourceVersion()) .detail("Version", FDB_VT_VERSION) .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) .detail("ClusterFile", ccf->getFilename().c_str()) .detail("ConnectionString", ccf->getConnectionString().toString()) .setMaxFieldLength(10000) @@ -4548,7 +4613,7 @@ int main(int argc, char **argv) { sigemptyset( &act.sa_mask ); act.sa_flags = 0; act.sa_handler = SIG_IGN; - sigaction(SIGINT, &act, NULL); + sigaction(SIGINT, &act, nullptr); #endif CLIOptions opt(argc, argv); diff --git a/fdbclient/AsyncFileBlobStore.actor.h b/fdbclient/AsyncFileBlobStore.actor.h index 681c28ac6a..b070cd65d9 100644 --- a/fdbclient/AsyncFileBlobStore.actor.h +++ b/fdbclient/AsyncFileBlobStore.actor.h @@ -59,7 +59,7 @@ public: virtual void delref() { ReferenceCounted::delref(); } struct Part : ReferenceCounted { - Part(int n, int minSize) : number(n), writer(content.getWriteBuffer(minSize), NULL, Unversioned()), length(0) { + Part(int n, int minSize) : number(n), writer(content.getWriteBuffer(minSize), nullptr, Unversioned()), length(0) { etag = std::string(); ::MD5_Init(&content_md5_buf); } diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index 927295952e..4b27c9e8d9 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -46,13 +46,15 @@ public: return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; } - // Type of program being executed - enum enumActionResult { - RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3 - }; - - enum enumState { - STATE_ERRORED = 0, STATE_SUBMITTED = 1, STATE_RUNNING = 2, STATE_RUNNING_DIFFERENTIAL = 3, STATE_COMPLETED = 4, STATE_NEVERRAN = 5, STATE_ABORTED = 6, STATE_PARTIALLY_ABORTED = 7 + enum class EnumState { + STATE_ERRORED = 0, + STATE_SUBMITTED = 1, + STATE_RUNNING = 2, + STATE_RUNNING_DIFFERENTIAL = 3, + STATE_COMPLETED = 4, + STATE_NEVERRAN = 5, + STATE_ABORTED = 6, + STATE_PARTIALLY_ABORTED = 7 }; static const Key keyFolderId; @@ -85,70 +87,68 @@ public: static const int logHeaderSize; // Convert the status text to an enumerated value - static enumState getState(std::string stateText) - { - enumState enState = STATE_ERRORED; + static EnumState getState(std::string stateText) { + auto enState = EnumState::STATE_ERRORED; if (stateText.empty()) { - enState = STATE_NEVERRAN; + enState = EnumState::STATE_NEVERRAN; } else if (!stateText.compare("has been submitted")) { - enState = STATE_SUBMITTED; + enState = EnumState::STATE_SUBMITTED; } else if (!stateText.compare("has been started")) { - enState = STATE_RUNNING; + enState = EnumState::STATE_RUNNING; } else if (!stateText.compare("is differential")) { - enState = STATE_RUNNING_DIFFERENTIAL; + enState = EnumState::STATE_RUNNING_DIFFERENTIAL; } else if (!stateText.compare("has been completed")) { - enState = STATE_COMPLETED; + enState = EnumState::STATE_COMPLETED; } else if (!stateText.compare("has been aborted")) { - enState = STATE_ABORTED; + enState = EnumState::STATE_ABORTED; } else if (!stateText.compare("has been partially aborted")) { - enState = STATE_PARTIALLY_ABORTED; + enState = EnumState::STATE_PARTIALLY_ABORTED; } return enState; } // Convert the status enum to a text description - static const char* getStateText(enumState enState) - { + static const char* getStateText(EnumState enState) { const char* stateText; switch (enState) { - case STATE_ERRORED: + case EnumState::STATE_ERRORED: stateText = "has errored"; break; - case STATE_NEVERRAN: + case EnumState::STATE_NEVERRAN: stateText = "has never been started"; break; - case STATE_SUBMITTED: + case EnumState::STATE_SUBMITTED: stateText = "has been submitted"; break; - case STATE_RUNNING: + case EnumState::STATE_RUNNING: stateText = "has been started"; break; - case STATE_RUNNING_DIFFERENTIAL: + case EnumState::STATE_RUNNING_DIFFERENTIAL: stateText = "is differential"; break; - case STATE_COMPLETED: + case EnumState::STATE_COMPLETED: stateText = "has been completed"; break; - case STATE_ABORTED: + case EnumState::STATE_ABORTED: stateText = "has been aborted"; break; - case STATE_PARTIALLY_ABORTED: + case EnumState::STATE_PARTIALLY_ABORTED: stateText = "has been partially aborted"; break; default: @@ -160,34 +160,33 @@ public: } // Convert the status enum to a name - static const char* getStateName(enumState enState) - { + static const char* getStateName(EnumState enState) { const char* s; switch (enState) { - case STATE_ERRORED: + case EnumState::STATE_ERRORED: s = "Errored"; break; - case STATE_NEVERRAN: + case EnumState::STATE_NEVERRAN: s = "NeverRan"; break; - case STATE_SUBMITTED: + case EnumState::STATE_SUBMITTED: s = "Submitted"; break; - case STATE_RUNNING: + case EnumState::STATE_RUNNING: s = "Running"; break; - case STATE_RUNNING_DIFFERENTIAL: + case EnumState::STATE_RUNNING_DIFFERENTIAL: s = "RunningDifferentially"; break; - case STATE_COMPLETED: + case EnumState::STATE_COMPLETED: s = "Completed"; break; - case STATE_ABORTED: + case EnumState::STATE_ABORTED: s = "Aborted"; break; - case STATE_PARTIALLY_ABORTED: + case EnumState::STATE_PARTIALLY_ABORTED: s = "Aborting"; break; default: @@ -199,16 +198,15 @@ public: } // Determine if the specified state is runnable - static bool isRunnable(enumState enState) - { + static bool isRunnable(EnumState enState) { bool isRunnable = false; switch (enState) { - case STATE_SUBMITTED: - case STATE_RUNNING: - case STATE_RUNNING_DIFFERENTIAL: - case STATE_PARTIALLY_ABORTED: + case EnumState::STATE_SUBMITTED: + case EnumState::STATE_RUNNING: + case EnumState::STATE_RUNNING_DIFFERENTIAL: + case EnumState::STATE_PARTIALLY_ABORTED: isRunnable = true; break; default: @@ -359,7 +357,8 @@ public: // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. - Future waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Reference *pContainer = nullptr, UID *pUID = nullptr); + Future waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, + Reference* pContainer = nullptr, UID* pUID = nullptr); static const Key keyLastRestorable; @@ -432,8 +431,8 @@ public: Future getStatus(Database cx, int errorLimit, Key tagName); - Future getStateValue(Reference tr, UID logUid, bool snapshot = false); - Future getStateValue(Database cx, UID logUid) { + Future getStateValue(Reference tr, UID logUid, bool snapshot = false); + Future getStateValue(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr){ return getStateValue(tr, logUid); }); } @@ -452,8 +451,8 @@ public: // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. - Future waitBackup(Database cx, Key tagName, bool stopWhenDone = true); - Future waitSubmitted(Database cx, Key tagName); + Future waitBackup(Database cx, Key tagName, bool stopWhenDone = true); + Future waitSubmitted(Database cx, Key tagName); Future waitUpgradeToLatestDrVersion(Database cx, Key tagName); static const Key keyAddPrefix; @@ -522,9 +521,15 @@ ACTOR Future applyMutations(Database cx, Key uid, Key addPrefix, Key remov NotifiedVersion* committedVersion, Reference> keyVersion); ACTOR Future cleanupBackup(Database cx, bool deleteData); -typedef BackupAgentBase::enumState EBackupState; -template<> inline Tuple Codec::pack(EBackupState const &val) { return Tuple().append(val); } -template<> inline EBackupState Codec::unpack(Tuple const &val) { return (EBackupState)val.getInt(0); } +using EBackupState = BackupAgentBase::EnumState; +template <> +inline Tuple Codec::pack(EBackupState const& val) { + return Tuple().append(static_cast(val)); +} +template <> +inline EBackupState Codec::unpack(Tuple const& val) { + return static_cast(val.getInt(0)); +} // Key backed tags are a single-key slice of the TagUidMap, defined below. // The Value type of the key is a UidAndAbortedFlagT which is a pair of {UID, aborted_flag} diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 912510553b..223ef1949b 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -23,6 +23,7 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/JsonBuilder.h" +#include "flow/Arena.h" #include "flow/Trace.h" #include "flow/UnitTest.h" #include "flow/Hash3.h" @@ -245,7 +246,7 @@ std::string BackupDescription::toJSON() const { * file written will be after the start version of the snapshot's execution. * * Log files are at file paths like - * /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize + * /plogs/.../log,startVersion,endVersion,UID,tagID-of-N,blocksize * /logs/.../log,startVersion,endVersion,UID,blockSize * where ... is a multi level path which sorts lexically into version order and results in approximately 1 * unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs" @@ -1403,8 +1404,15 @@ public: } ACTOR static Future> getRestoreSet_impl(Reference bc, - Version targetVersion, bool logsOnly, - Version beginVersion) { + Version targetVersion, + VectorRef keyRangesFilter, bool logsOnly = false, + Version beginVersion = invalidVersion) { + // Does not support use keyRangesFilter for logsOnly yet + if (logsOnly && !keyRangesFilter.empty()) { + TraceEvent(SevError, "BackupContainerRestoreSetUnsupportedAPI").detail("KeyRangesFilter", keyRangesFilter.size()); + return Optional(); + } + if (logsOnly) { state RestorableFileSet restorableSet; state std::vector logFiles; @@ -1416,23 +1424,55 @@ public: return getRestoreSetFromLogs(logFiles, targetVersion, restorableSet); } } - // Find the most recent keyrange snapshot to end at or before targetVersion - state Optional snapshot; - std::vector snapshots = wait(bc->listKeyspaceSnapshots()); - for(auto const &s : snapshots) { - if(s.endVersion <= targetVersion) - snapshot = s; - } - if(snapshot.present()) { + // Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion. + state std::vector snapshots = wait(bc->listKeyspaceSnapshots()); + state int i = snapshots.size() - 1; + for (; i >= 0; i--) { + // The smallest version of filtered range files >= snapshot beginVersion > targetVersion + if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) { + continue; + } + state RestorableFileSet restorable; - restorable.snapshot = snapshot.get(); - restorable.targetVersion = targetVersion; + state Version minKeyRangeVersion = MAX_VERSION; + state Version maxKeyRangeVersion = -1; std::pair, std::map> results = - wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = std::move(results.first); - restorable.keyRanges = std::move(results.second); + wait(bc->readKeyspaceSnapshot(snapshots[i])); + + // Old backup does not have metadata about key ranges and can not be filtered with key ranges. + if (keyRangesFilter.size() && results.second.empty() && !results.first.empty()) { + throw backup_not_filterable_with_key_ranges(); + } + + // Filter by keyRangesFilter. + if (keyRangesFilter.empty()) { + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + minKeyRangeVersion = snapshots[i].beginVersion; + maxKeyRangeVersion = snapshots[i].endVersion; + } else { + for (const auto& rangeFile : results.first) { + const auto& keyRange = results.second.at(rangeFile.fileName); + if (keyRange.intersects(keyRangesFilter)) { + restorable.ranges.push_back(rangeFile); + restorable.keyRanges[rangeFile.fileName] = keyRange; + minKeyRangeVersion = std::min(minKeyRangeVersion, rangeFile.version); + maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version); + } + } + // No range file matches 'keyRangesFilter'. + if (restorable.ranges.empty()) { + throw backup_not_overlapped_with_keys_filter(); + } + } + // 'latestVersion' represents using the minimum restorable version in a snapshot. + restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; + // Any version < maxKeyRangeVersion is not restorable. + if (restorable.targetVersion < maxKeyRangeVersion) continue; + + restorable.snapshot = snapshots[i]; // TODO: Reenable the sanity check after TooManyFiles error is resolved if (false && g_network->isSimulated()) { // Sanity check key ranges @@ -1446,18 +1486,21 @@ public: } } - // No logs needed if there is a complete key space snapshot at the target version. - if (snapshot.get().beginVersion == snapshot.get().endVersion && - snapshot.get().endVersion == targetVersion) { + // No logs needed if there is a complete filtered key space snapshot at the target version. + if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) { restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; + TraceEvent("BackupContainerGetRestorableFilesWithoutLogs") + .detail("KeyRangeVersion", restorable.targetVersion) + .detail("NumberOfRangeFiles", restorable.ranges.size()) + .detail("KeyRangesFilter", printable(keyRangesFilter)); return Optional(restorable); } // FIXME: check if there are tagged logs. for each tag, there is no version gap. state std::vector logs; state std::vector plogs; - wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && - store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) && + store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true))); if (plogs.size() > 0) { logs.swap(plogs); @@ -1469,13 +1512,12 @@ public: // Remove duplicated log files that can happen for old epochs. std::vector filtered = filterDuplicates(logs); - restorable.logs.swap(filtered); // sort by version order again for continuous analysis std::sort(restorable.logs.begin(), restorable.logs.end()); - if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { - restorable.continuousBeginVersion = snapshot.get().beginVersion; - restorable.continuousEndVersion = targetVersion + 1; // not inclusive + if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) { + restorable.continuousBeginVersion = minKeyRangeVersion; + restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive return Optional(restorable); } return Optional(); @@ -1483,20 +1525,19 @@ public: // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); - - // If there are logs and the first one starts at or before the snapshot begin version then proceed - if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { + // If there are logs and the first one starts at or before the keyrange's snapshot begin version, then + // it is valid restore set and proceed + if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) { return getRestoreSetFromLogs(logs, targetVersion, restorable); } } - return Optional(); } - Future> getRestoreSet(Version targetVersion, bool logsOnly, - Version beginVersion) final { - return getRestoreSet_impl(Reference::addRef(this), targetVersion, logsOnly, - beginVersion); + Future> getRestoreSet(Version targetVersion, VectorRef keyRangesFilter, + bool logsOnly, Version beginVersion) final { + return getRestoreSet_impl(Reference::addRef(this), targetVersion, keyRangesFilter, + logsOnly, beginVersion); } private: @@ -1796,7 +1837,7 @@ private: std::string m_path; }; -class BackupContainerBlobStore : public BackupContainerFileSystem, ReferenceCounted { +class BackupContainerBlobStore final : public BackupContainerFileSystem, ReferenceCounted { private: // Backup files to under a single folder prefix with subfolders for each named backup static const std::string DATAFOLDER; @@ -1836,15 +1877,13 @@ public: } } - void addref() final { return ReferenceCounted::addref(); } - void delref() final { return ReferenceCounted::delref(); } + void addref() override { return ReferenceCounted::addref(); } + void delref() override { return ReferenceCounted::delref(); } static std::string getURLFormat() { return BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)"; } - virtual ~BackupContainerBlobStore() {} - Future> readFile(std::string path) final { return Reference( new AsyncFileReadAheadCache( diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index c3f0fd08fa..1b7fa7a7eb 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -280,10 +280,13 @@ public: virtual Future dumpFileList(Version begin = 0, Version end = std::numeric_limits::max()) = 0; - // Get exactly the files necessary to restore to targetVersion. Returns non-present if - // restore to given version is not possible. - virtual Future> getRestoreSet(Version targetVersion, bool logsOnly = false, - Version beginVersion = -1) = 0; + // Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion. + // If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. + // If logsOnly is set, only use log files in [beginVersion, targetVervions) in restore set. + // Returns non-present if restoring to the given version is not possible. + virtual Future> getRestoreSet(Version targetVersion, + VectorRef keyRangesFilter = {}, + bool logsOnly = false, Version beginVersion = -1) = 0; // Get an IBackupContainer based on a container spec string static Reference openContainer(std::string url); diff --git a/fdbclient/BlobStore.actor.cpp b/fdbclient/BlobStore.actor.cpp index 664d82bd8d..b29af56172 100644 --- a/fdbclient/BlobStore.actor.cpp +++ b/fdbclient/BlobStore.actor.cpp @@ -277,7 +277,7 @@ ACTOR Future bucketExists_impl(Reference b, std::string std::string resource = std::string("/") + bucket; HTTP::Headers headers; - Reference r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404})); + Reference r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404})); return r->code == 200; } @@ -291,7 +291,7 @@ ACTOR Future objectExists_impl(Reference b, std::string std::string resource = std::string("/") + bucket + "/" + object; HTTP::Headers headers; - Reference r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404})); + Reference r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404})); return r->code == 200; } @@ -305,7 +305,7 @@ ACTOR Future deleteObject_impl(Reference b, std::string std::string resource = std::string("/") + bucket + "/" + object; HTTP::Headers headers; // 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered successful - Reference r = wait(b->doRequest("DELETE", resource, headers, NULL, 0, {200, 204, 404})); + Reference r = wait(b->doRequest("DELETE", resource, headers, nullptr, 0, {200, 204, 404})); // But if the object already did not exist then the 'delete' is assumed to be successful but a warning is logged. if(r->code == 404) { @@ -386,7 +386,7 @@ ACTOR Future createBucket_impl(Reference b, std::string if(!exists) { std::string resource = std::string("/") + bucket; HTTP::Headers headers; - Reference r = wait(b->doRequest("PUT", resource, headers, NULL, 0, {200, 409})); + Reference r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, {200, 409})); } return Void(); } @@ -401,7 +401,7 @@ ACTOR Future objectSize_impl(Reference b, std::strin std::string resource = std::string("/") + bucket + "/" + object; HTTP::Headers headers; - Reference r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404})); + Reference r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404})); if(r->code == 404) throw file_not_found(); return r->contentLen; @@ -737,7 +737,7 @@ ACTOR Future listObjectsStream_impl(Reference bstore, s HTTP::Headers headers; state std::string fullResource = resource + HTTP::urlEncode(lastFile); lastFile.clear(); - Reference r = wait(bstore->doRequest("GET", fullResource, headers, NULL, 0, {200})); + Reference r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, {200})); listReleaser.release(); try { @@ -782,7 +782,7 @@ ACTOR Future listObjectsStream_impl(Reference bstore, s if(size == nullptr) { throw http_bad_response(); } - object.size = strtoull(size->value(), NULL, 10); + object.size = strtoull(size->value(), nullptr, 10); listResult.objects.push_back(object); } @@ -893,7 +893,7 @@ ACTOR Future> listBuckets_impl(Reference r = wait(bstore->doRequest("GET", fullResource, headers, NULL, 0, {200})); + Reference r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, {200})); listReleaser.release(); try { @@ -1024,7 +1024,7 @@ ACTOR Future readEntireFile_impl(Reference bstor std::string resource = std::string("/") + bucket + "/" + object; HTTP::Headers headers; - Reference r = wait(bstore->doRequest("GET", resource, headers, NULL, 0, {200, 404})); + Reference r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, {200, 404})); if(r->code == 404) throw file_not_found(); return r->content; @@ -1057,7 +1057,7 @@ ACTOR Future writeEntireFileFromBuffer_impl(Reference b ACTOR Future writeEntireFile_impl(Reference bstore, std::string bucket, std::string object, std::string content) { state UnsentPacketQueue packets; - PacketWriter pw(packets.getWriteBuffer(content.size()), NULL, Unversioned()); + PacketWriter pw(packets.getWriteBuffer(content.size()), nullptr, Unversioned()); pw.serializeBytes(content); if(content.size() > bstore->knobs.multipart_max_part_size) throw file_too_large(); @@ -1095,7 +1095,7 @@ ACTOR Future readObject_impl(Reference bstore, std::stri std::string resource = std::string("/") + bucket + "/" + object; HTTP::Headers headers; headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1); - Reference r = wait(bstore->doRequest("GET", resource, headers, NULL, 0, {200, 206, 404})); + Reference r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, {200, 206, 404})); if(r->code == 404) throw file_not_found(); if(r->contentLen != r->content.size()) // Double check that this wasn't a header-only response, probably unnecessary @@ -1114,7 +1114,7 @@ ACTOR static Future beginMultiPartUpload_impl(Reference r = wait(bstore->doRequest("POST", resource, headers, NULL, 0, {200})); + Reference r = wait(bstore->doRequest("POST", resource, headers, nullptr, 0, {200})); try { xml_document<> doc; @@ -1180,7 +1180,7 @@ ACTOR Future finishMultiPartUpload_impl(Reference bstor std::string resource = format("/%s/%s?uploadId=%s", bucket.c_str(), object.c_str(), uploadID.c_str()); HTTP::Headers headers; - PacketWriter pw(part_list.getWriteBuffer(manifest.size()), NULL, Unversioned()); + PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned()); pw.serializeBytes(manifest); Reference r = wait(bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), {200})); // TODO: In the event that the client times out just before the request completes (so the client is unaware) then the next retry diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 43f9343b28..3f7333b632 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -33,7 +33,7 @@ set(FDBCLIENT_SRCS Knobs.h ManagementAPI.actor.cpp ManagementAPI.actor.h - MasterProxyInterface.h + CommitProxyInterface.h MetricLogger.actor.cpp MetricLogger.h MonitorLeader.actor.cpp diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index 4b4f822fc9..c4bdb2bc1b 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/FailureMonitor.h" #include "fdbclient/Status.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" // Streams from WorkerInterface that are safe and useful to call from a client. // A ClientWorkerInterface is embedded as the first element of a WorkerInterface. diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h index c957ae8633..2570666b12 100644 --- a/fdbclient/ClusterInterface.h +++ b/fdbclient/ClusterInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/FailureMonitor.h" #include "fdbclient/Status.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ClientWorkerInterface.h" struct ClusterInterface { diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/CommitProxyInterface.h similarity index 95% rename from fdbclient/MasterProxyInterface.h rename to fdbclient/CommitProxyInterface.h index 9e2b49037c..1cf63fcd2b 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -1,6 +1,6 @@ /* - * MasterProxyInterface.h + * CommitProxyInterface.h * * This source file is part of the FoundationDB open source project * @@ -19,8 +19,8 @@ * limitations under the License. */ -#ifndef FDBCLIENT_MASTERPROXYINTERFACE_H -#define FDBCLIENT_MASTERPROXYINTERFACE_H +#ifndef FDBCLIENT_COMMITPROXYINTERFACE_H +#define FDBCLIENT_COMMITPROXYINTERFACE_H #pragma once #include @@ -36,7 +36,7 @@ #include "fdbrpc/TimedRequest.h" #include "GrvProxyInterface.h" -struct MasterProxyInterface { +struct CommitProxyInterface { constexpr static FileIdentifier file_identifier = 8954922; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 1 }; @@ -59,8 +59,8 @@ struct MasterProxyInterface { UID id() const { return commit.getEndpoint().token; } std::string toString() const { return id().shortString(); } - bool operator == (MasterProxyInterface const& r) const { return id() == r.id(); } - bool operator != (MasterProxyInterface const& r) const { return id() != r.id(); } + bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); } + bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); } template @@ -100,9 +100,10 @@ struct MasterProxyInterface { struct ClientDBInfo { constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes - vector< GrvProxyInterface > grvProxies; - vector< MasterProxyInterface > masterProxies; - Optional firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk + vector grvProxies; + vector commitProxies; + Optional + firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk double clientTxnInfoSampleRate; int64_t clientTxnInfoSizeLimit; Optional forward; @@ -122,7 +123,7 @@ struct ClientDBInfo { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, grvProxies, masterProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, + serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, transactionTagSampleRate, transactionTagSampleCost); } }; @@ -165,7 +166,8 @@ struct CommitTransactionRequest : TimedRequest { Optional commitCostEstimation; Optional tagSet; - CommitTransactionRequest() : flags(0) {} + CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {} + CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {} template void serialize(Ar& ar) { diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index bc74941704..b2776a4dcd 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -49,6 +49,7 @@ static const char* typeString[] = { "SetValue", "MinV2", "AndV2", "CompareAndClear", + "Reserved_For_SpanContextMessage", "MAX_ATOMIC_OP" }; struct MutationRef { @@ -75,6 +76,7 @@ struct MutationRef { MinV2, AndV2, CompareAndClear, + Reserved_For_SpanContextMessage /* See fdbserver/SpanContextMessage.h */, MAX_ATOMIC_OP }; // This is stored this way for serialization purposes. diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h index 0dc2970ca1..95423bf6ca 100644 --- a/fdbclient/CoordinationInterface.h +++ b/fdbclient/CoordinationInterface.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/fdbrpc.h" #include "fdbrpc/Locality.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ClusterInterface.h" const int MAX_CLUSTER_FILE_BYTES = 60000; diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp index 1162986e10..da2f38f24b 100644 --- a/fdbclient/DatabaseBackupAgent.actor.cpp +++ b/fdbclient/DatabaseBackupAgent.actor.cpp @@ -129,9 +129,9 @@ namespace dbBackup { struct BackupRangeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam bytesWritten() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -421,16 +421,15 @@ namespace dbBackup { }; StringRef BackupRangeTaskFunc::name = LiteralStringRef("dr_backup_range"); - const uint32_t BackupRangeTaskFunc::version = 1; const Key BackupRangeTaskFunc::keyAddBackupRangeTasks = LiteralStringRef("addBackupRangeTasks"); const Key BackupRangeTaskFunc::keyBackupRangeBeginKey = LiteralStringRef("backupRangeBeginKey"); REGISTER_TASKFUNC(BackupRangeTaskFunc); struct FinishFullBackupTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]); wait(checkTaskVersion(tr, task, FinishFullBackupTaskFunc::name, FinishFullBackupTaskFunc::version)); @@ -467,14 +466,13 @@ namespace dbBackup { }; StringRef FinishFullBackupTaskFunc::name = LiteralStringRef("dr_finish_full_backup"); - const uint32_t FinishFullBackupTaskFunc::version = 1; REGISTER_TASKFUNC(FinishFullBackupTaskFunc); struct EraseLogRangeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - StringRef getName() const { return name; }; + StringRef getName() const { return name; }; Future execute(Database cx, Reference tb, Reference fb, Reference task) { return _execute(cx, tb, fb, task); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; @@ -523,14 +521,13 @@ namespace dbBackup { } }; StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("dr_erase_log_range"); - const uint32_t EraseLogRangeTaskFunc::version = 1; REGISTER_TASKFUNC(EraseLogRangeTaskFunc); struct CopyLogRangeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam bytesWritten() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -773,15 +770,14 @@ namespace dbBackup { } }; StringRef CopyLogRangeTaskFunc::name = LiteralStringRef("dr_copy_log_range"); - const uint32_t CopyLogRangeTaskFunc::version = 1; const Key CopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion"); REGISTER_TASKFUNC(CopyLogRangeTaskFunc); struct CopyLogsTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { state Subspace conf = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(task->params[BackupAgentBase::keyConfigLogUid]); state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]); wait(checkTaskVersion(tr, task, CopyLogsTaskFunc::name, CopyLogsTaskFunc::version)); @@ -876,13 +872,12 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef CopyLogsTaskFunc::name = LiteralStringRef("dr_copy_logs"); - const uint32_t CopyLogsTaskFunc::version = 1; REGISTER_TASKFUNC(CopyLogsTaskFunc); struct FinishedFullBackupTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; - static const Key keyInsertTask; + static constexpr uint32_t version = 1; + static const Key keyInsertTask; StringRef getName() const { return name; }; @@ -965,9 +960,10 @@ namespace dbBackup { tr->clear(KeyRangeRef(logsPath, strinc(logsPath))); tr->clear(conf.range()); - tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_COMPLETED))); + tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_COMPLETED))); - wait(taskBucket->finish(tr, task)); + wait(taskBucket->finish(tr, task)); return Void(); } @@ -975,15 +971,14 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("dr_finished_full_backup"); - const uint32_t FinishedFullBackupTaskFunc::version = 1; const Key FinishedFullBackupTaskFunc::keyInsertTask = LiteralStringRef("insertTask"); REGISTER_TASKFUNC(FinishedFullBackupTaskFunc); struct CopyDiffLogsTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { state Subspace conf = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(task->params[BackupAgentBase::keyConfigLogUid]); state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]); wait(checkTaskVersion(tr, task, CopyDiffLogsTaskFunc::name, CopyDiffLogsTaskFunc::version)); @@ -1058,15 +1053,14 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef CopyDiffLogsTaskFunc::name = LiteralStringRef("dr_copy_diff_logs"); - const uint32_t CopyDiffLogsTaskFunc::version = 1; REGISTER_TASKFUNC(CopyDiffLogsTaskFunc); // Skip unneeded EraseLogRangeTaskFunc in 5.1 struct SkipOldEraseLogRangeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { state Reference taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]); wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task)); return Void(); @@ -1078,16 +1072,15 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef SkipOldEraseLogRangeTaskFunc::name = LiteralStringRef("dr_skip_legacy_task"); - const uint32_t SkipOldEraseLogRangeTaskFunc::version = 1; REGISTER_TASKFUNC(SkipOldEraseLogRangeTaskFunc); REGISTER_TASKFUNC_ALIAS(SkipOldEraseLogRangeTaskFunc, db_erase_log_range); // This is almost the same as CopyLogRangeTaskFunc in 5.1. The only purpose is to support DR upgrade struct OldCopyLogRangeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam bytesWritten() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -1254,15 +1247,14 @@ namespace dbBackup { } }; StringRef OldCopyLogRangeTaskFunc::name = LiteralStringRef("db_copy_log_range"); - const uint32_t OldCopyLogRangeTaskFunc::version = 1; const Key OldCopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion"); REGISTER_TASKFUNC(OldCopyLogRangeTaskFunc); struct AbortOldBackupTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { state DatabaseBackupAgent srcDrAgent(taskBucket->src); state Reference tr(new ReadYourWritesTransaction(cx)); state Key tagNameKey; @@ -1315,7 +1307,6 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef AbortOldBackupTaskFunc::name = LiteralStringRef("dr_abort_legacy_backup"); - const uint32_t AbortOldBackupTaskFunc::version = 1; REGISTER_TASKFUNC(AbortOldBackupTaskFunc); REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_backup_range); REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_finish_full_backup); @@ -1327,9 +1318,9 @@ namespace dbBackup { //Upgrade DR from 5.1 struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid]; state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue); state Subspace config = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue); @@ -1434,14 +1425,13 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef CopyDiffLogsUpgradeTaskFunc::name = LiteralStringRef("db_copy_diff_logs"); - const uint32_t CopyDiffLogsUpgradeTaskFunc::version = 1; REGISTER_TASKFUNC(CopyDiffLogsUpgradeTaskFunc); struct BackupRestorableTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(task->params[BackupAgentBase::keyConfigLogUid]); wait(checkTaskVersion(cx, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version)); state Transaction tr(taskBucket->src); @@ -1449,9 +1439,10 @@ namespace dbBackup { try { tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.addReadConflictRange(singleKeyRange(sourceStates.pack(DatabaseBackupAgent::keyStateStatus))); - tr.set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING_DIFFERENTIAL))); + tr.set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING_DIFFERENTIAL))); - Key versionKey = task->params[DatabaseBackupAgent::keyConfigLogUid].withPrefix(task->params[BackupAgentBase::destUid]).withPrefix(backupLatestVersionsPrefix); + Key versionKey = task->params[DatabaseBackupAgent::keyConfigLogUid].withPrefix(task->params[BackupAgentBase::destUid]).withPrefix(backupLatestVersionsPrefix); Optional prevBeginVersion = wait(tr.get(versionKey)); if (!prevBeginVersion.present()) { return Void(); @@ -1489,9 +1480,10 @@ namespace dbBackup { wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal()))); } else { // Start the writing of logs, if differential - tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING_DIFFERENTIAL))); + tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING_DIFFERENTIAL))); - allPartsDone = futureBucket->future(tr); + allPartsDone = futureBucket->future(tr); Version prevBeginVersion = BinaryReader::fromStringRef(task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned()); wait(success(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone)))); @@ -1524,14 +1516,13 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef BackupRestorableTaskFunc::name = LiteralStringRef("dr_backup_restorable"); - const uint32_t BackupRestorableTaskFunc::version = 1; REGISTER_TASKFUNC(BackupRestorableTaskFunc); struct StartFullBackupTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid]; state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue); wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version)); @@ -1623,9 +1614,10 @@ namespace dbBackup { srcTr2->set( Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceTagName).pack(task->params[BackupAgentBase::keyTagName]), logUidValue ); srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyFolderId), task->params[DatabaseBackupAgent::keyFolderId] ); - srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING))); + srcTr2->set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING))); - state Key destPath = destUidValue.withPrefix(backupLogKeys.begin); + state Key destPath = destUidValue.withPrefix(backupLogKeys.begin); // Start logging the mutations for the specified ranges of the tag for (auto &backupRange : backupRanges) { srcTr2->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, destPath)); @@ -1666,9 +1658,10 @@ namespace dbBackup { tr->set(logUidValue.withPrefix(applyMutationsBeginRange.begin), BinaryWriter::toValue(beginVersion, Unversioned())); tr->set(logUidValue.withPrefix(applyMutationsEndRange.begin), BinaryWriter::toValue(beginVersion, Unversioned())); - tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING))); + tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING))); - state Reference kvBackupRangeComplete = futureBucket->future(tr); + state Reference kvBackupRangeComplete = futureBucket->future(tr); state Reference kvBackupComplete = futureBucket->future(tr); state int rangeCount = 0; @@ -1721,7 +1714,6 @@ namespace dbBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef StartFullBackupTaskFunc::name = LiteralStringRef("dr_start_full_backup"); - const uint32_t StartFullBackupTaskFunc::version = 1; REGISTER_TASKFUNC(StartFullBackupTaskFunc); } @@ -1817,7 +1809,7 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade class DatabaseBackupAgentImpl { public: - static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; + static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; ACTOR static Future waitUpgradeToLatestDrVersion(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) { state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); @@ -1851,7 +1843,8 @@ public: } // This method will return the final status of the backup - ACTOR static Future waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, bool stopWhenDone) { + ACTOR static Future waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, + bool stopWhenDone) { state std::string backTrace; state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus); @@ -1862,15 +1855,15 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); try { - state int status = wait(backupAgent->getStateValue(tr, logUid)); + state EBackupState status = wait(backupAgent->getStateValue(tr, logUid)); // Break, if no longer runnable - if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status) || BackupAgentBase::STATE_PARTIALLY_ABORTED == status) { + if (!DatabaseBackupAgent::isRunnable(status) || EBackupState::STATE_PARTIALLY_ABORTED == status) { return status; } // Break, if in differential mode (restorable) and stopWhenDone is not enabled - if ((!stopWhenDone) && (BackupAgentBase::STATE_RUNNING_DIFFERENTIAL == status)) { + if ((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status)) { return status; } @@ -1885,7 +1878,7 @@ public: } // This method will return the final status of the backup - ACTOR static Future waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) { + ACTOR static Future waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) { state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus); @@ -1895,10 +1888,10 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); try { - state int status = wait(backupAgent->getStateValue(tr, logUid)); + state EBackupState status = wait(backupAgent->getStateValue(tr, logUid)); // Break, if no longer runnable - if( BackupAgentBase::STATE_SUBMITTED != status) { + if (EBackupState::STATE_SUBMITTED != status) { return status; } @@ -1924,9 +1917,9 @@ public: tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); // We will use the global status for now to ensure that multiple backups do not start place with different tags - state int status = wait(backupAgent->getStateValue(tr, logUidCurrent)); + state EBackupState status = wait(backupAgent->getStateValue(tr, logUidCurrent)); - if (DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) { + if (DatabaseBackupAgent::isRunnable(status)) { throw backup_duplicate(); } @@ -1987,7 +1980,8 @@ public: tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid); tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid); // written to config and states because it's also used by abort tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyConfigBackupRanges), BinaryWriter::toValue(backupRanges, IncludeVersion(ProtocolVersion::withDRBackupRanges()))); - tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_SUBMITTED))); + tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_SUBMITTED))); if (stopWhenDone) { tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey), StringRef()); } @@ -2033,10 +2027,10 @@ public: ACTOR static Future atomicSwitchover(DatabaseBackupAgent* backupAgent, Database dest, Key tagName, Standalone> backupRanges, Key addPrefix, Key removePrefix, bool forceAction) { state DatabaseBackupAgent drAgent(dest); state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName)); - state int status = wait(backupAgent->getStateValue(dest, destlogUid)); + state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid)); TraceEvent("DBA_SwitchoverStart").detail("Status", status); - if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL && status != BackupAgentBase::STATE_COMPLETED) { + if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL && status != EBackupState::STATE_COMPLETED) { throw backup_duplicate(); } @@ -2153,10 +2147,10 @@ public: ACTOR static Future discontinueBackup(DatabaseBackupAgent* backupAgent, Reference tr, Key tagName) { tr->setOption(FDBTransactionOptions::LOCK_AWARE); state UID logUid = wait(backupAgent->getLogUid(tr, tagName)); - state int status = wait(backupAgent->getStateValue(tr, logUid)); + state EBackupState status = wait(backupAgent->getStateValue(tr, logUid)); TraceEvent("DBA_Discontinue").detail("Status", status); - if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) { + if (!DatabaseBackupAgent::isRunnable(status)) { throw backup_unneeded(); } @@ -2189,7 +2183,7 @@ public: logUid = _logUid; logUidValue = BinaryWriter::toValue(logUid, Unversioned()); - state Future statusFuture= backupAgent->getStateValue(tr, logUid); + state Future statusFuture = backupAgent->getStateValue(tr, logUid); state Future destUidFuture = backupAgent->getDestUid(tr, logUid); wait(success(statusFuture) && success(destUidFuture)); @@ -2197,8 +2191,8 @@ public: if (destUid.isValid()) { destUidValue = BinaryWriter::toValue(destUid, Unversioned()); } - int status = statusFuture.get(); - if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) { + EBackupState status = statusFuture.get(); + if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); } @@ -2213,7 +2207,8 @@ public: tr->clear(prefixRange(logUidValue.withPrefix(applyLogKeys.begin))); - tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED))); + tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), + StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_PARTIALLY_ABORTED))); wait(tr->commit()); TraceEvent("DBA_Abort").detail("CommitVersion", tr->getCommittedVersion()); @@ -2286,7 +2281,8 @@ public: } if (abortOldBackup) { - srcTr->set( backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_ABORTED) )); + srcTr->set(backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(BackupAgentBase::getStateText(EBackupState::STATE_ABORTED))); srcTr->set( backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid ); srcTr->clear(prefixRange(logUidValue.withPrefix(backupLogKeys.begin))); srcTr->clear(prefixRange(logUidValue.withPrefix(logRangesRange.begin))); @@ -2307,7 +2303,8 @@ public: break; } - srcTr->set( backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED) )); + srcTr->set(backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), + StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_PARTIALLY_ABORTED))); srcTr->set( backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid ); wait( eraseLogData(srcTr, logUidValue, destUidValue) || partialTimeout ); @@ -2341,7 +2338,8 @@ public: return Void(); } - tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_ABORTED))); + tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), + StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_ABORTED))); wait(tr->commit()); @@ -2382,13 +2380,11 @@ public: state Future> fStopVersionKey = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop)); state Future> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges)); - int backupStateInt = wait(backupAgent->getStateValue(tr, logUid)); - state BackupAgentBase::enumState backupState = (BackupAgentBase::enumState)backupStateInt; - - if (backupState == DatabaseBackupAgent::STATE_NEVERRAN) { + state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid)); + + if (backupState == EBackupState::STATE_NEVERRAN) { statusText += "No previous backups found.\n"; - } - else { + } else { state std::string tagNameDisplay; Optional tagName = wait(fTagName); @@ -2408,23 +2404,20 @@ public: } switch (backupState) { - case BackupAgentBase::STATE_SUBMITTED: + case EBackupState::STATE_SUBMITTED: statusText += "The DR on tag `" + tagNameDisplay + "' is NOT a complete copy of the primary database (just started).\n"; break; - case BackupAgentBase::STATE_RUNNING: + case EBackupState::STATE_RUNNING: statusText += "The DR on tag `" + tagNameDisplay + "' is NOT a complete copy of the primary database.\n"; break; - case BackupAgentBase::STATE_RUNNING_DIFFERENTIAL: + case EBackupState::STATE_RUNNING_DIFFERENTIAL: statusText += "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database.\n"; break; - case BackupAgentBase::STATE_COMPLETED: - { + case EBackupState::STATE_COMPLETED: { Version stopVersion = stopVersionKey.present() ? BinaryReader::fromStringRef(stopVersionKey.get(), Unversioned()) : -1; statusText += "The previous DR on tag `" + tagNameDisplay + "' completed at version " + format("%lld", stopVersion) + ".\n"; - } - break; - case BackupAgentBase::STATE_PARTIALLY_ABORTED: - { + } break; + case EBackupState::STATE_PARTIALLY_ABORTED: { statusText += "The previous DR on tag `" + tagNameDisplay + "' " + BackupAgentBase::getStateText(backupState) + ".\n"; statusText += "Abort the DR with --cleanup before starting a new DR.\n"; break; @@ -2485,13 +2478,15 @@ public: return statusText; } - ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, bool snapshot) { + ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, + Reference tr, UID logUid, + bool snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus); Optional status = wait(tr->get(statusKey, snapshot)); - return (!status.present()) ? DatabaseBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString()); + return (!status.present()) ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString()); } ACTOR static Future getDestUid(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, bool snapshot) { @@ -2536,7 +2531,8 @@ Future DatabaseBackupAgent::getStatus(Database cx, int errorLimit, return DatabaseBackupAgentImpl::getStatus(this, cx, errorLimit, tagName); } -Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid, bool snapshot) { +Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid, + bool snapshot) { return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot); } @@ -2552,11 +2548,11 @@ Future DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key return DatabaseBackupAgentImpl::waitUpgradeToLatestDrVersion(this, cx, tagName); } -Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) { +Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) { return DatabaseBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone); } -Future DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName) { +Future DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName) { return DatabaseBackupAgentImpl::waitSubmitted(this, cx, tagName); } diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index b1580205a0..53769e8140 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -29,12 +29,12 @@ DatabaseConfiguration::DatabaseConfiguration() void DatabaseConfiguration::resetInternal() { // does NOT reset rawConfiguration initialized = false; - proxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = + commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1; tLogVersion = TLogVersion::DEFAULT; tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END; tLogSpillType = TLogSpillType::DEFAULT; - autoProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES; + autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES; autoGrvProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES; autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS; autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS; @@ -165,40 +165,39 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() { bool DatabaseConfiguration::isValid() const { if( !(initialized && - tLogWriteAntiQuorum >= 0 && - tLogWriteAntiQuorum <= tLogReplicationFactor/2 && - tLogReplicationFactor >= 1 && - storageTeamSize >= 1 && - getDesiredProxies() >= 1 && - getDesiredGrvProxies() >= 1 && - getDesiredLogs() >= 1 && - getDesiredResolvers() >= 1 && - tLogVersion != TLogVersion::UNSET && - tLogVersion >= TLogVersion::MIN_RECRUITABLE && - tLogVersion <= TLogVersion::MAX_SUPPORTED && - tLogDataStoreType != KeyValueStoreType::END && - tLogSpillType != TLogSpillType::UNSET && - !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && - storageServerStoreType != KeyValueStoreType::END && - autoProxyCount >= 1 && - autoGrvProxyCount >= 1 && - autoResolverCount >= 1 && - autoDesiredTLogCount >= 1 && - storagePolicy && - tLogPolicy && - getDesiredRemoteLogs() >= 1 && - remoteTLogReplicationFactor >= 0 && - repopulateRegionAntiQuorum >= 0 && - repopulateRegionAntiQuorum <= 1 && - usableRegions >= 1 && - usableRegions <= 2 && - regions.size() <= 2 && - ( usableRegions == 1 || regions.size() == 2 ) && - ( regions.size() == 0 || regions[0].priority >= 0 ) && - ( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication + tLogWriteAntiQuorum >= 0 && + tLogWriteAntiQuorum <= tLogReplicationFactor/2 && + tLogReplicationFactor >= 1 && + storageTeamSize >= 1 && + getDesiredCommitProxies() >= 1 && + getDesiredGrvProxies() >= 1 && + getDesiredLogs() >= 1 && + getDesiredResolvers() >= 1 && + tLogVersion != TLogVersion::UNSET && + tLogVersion >= TLogVersion::MIN_RECRUITABLE && + tLogVersion <= TLogVersion::MAX_SUPPORTED && + tLogDataStoreType != KeyValueStoreType::END && + tLogSpillType != TLogSpillType::UNSET && + !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && + storageServerStoreType != KeyValueStoreType::END && + autoCommitProxyCount >= 1 && + autoGrvProxyCount >= 1 && + autoResolverCount >= 1 && + autoDesiredTLogCount >= 1 && + storagePolicy && + tLogPolicy && + getDesiredRemoteLogs() >= 1 && + remoteTLogReplicationFactor >= 0 && + repopulateRegionAntiQuorum >= 0 && + repopulateRegionAntiQuorum <= 1 && + usableRegions >= 1 && + usableRegions <= 2 && + regions.size() <= 2 && + ( usableRegions == 1 || regions.size() == 2 ) && + ( regions.size() == 0 || regions[0].priority >= 0 ) && + ( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication return false; } - std::set dcIds; dcIds.insert(Key()); for(auto& r : regions) { @@ -318,11 +317,11 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { if (desiredTLogCount != -1 || isOverridden("logs")) { result["logs"] = desiredTLogCount; } - if (proxyCount != -1 || isOverridden("proxies")) { - result["proxies"] = proxyCount; + if (commitProxyCount != -1 || isOverridden("commit_proxies")) { + result["commit_proxies"] = commitProxyCount; } if (grvProxyCount != -1 || isOverridden("grv_proxies")) { - result["grv_proxies"] = proxyCount; + result["grv_proxies"] = grvProxyCount; } if (resolverCount != -1 || isOverridden("resolvers")) { result["resolvers"] = resolverCount; @@ -336,8 +335,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { if (repopulateRegionAntiQuorum != 0 || isOverridden("repopulate_anti_quorum")) { result["repopulate_anti_quorum"] = repopulateRegionAntiQuorum; } - if (autoProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_PROXIES || isOverridden("auto_proxies")) { - result["auto_proxies"] = autoProxyCount; + if (autoCommitProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES || isOverridden("auto_commit_proxies")) { + result["auto_commit_proxies"] = autoCommitProxyCount; } if (autoGrvProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES || isOverridden("auto_grv_proxies")) { result["auto_grv_proxies"] = autoGrvProxyCount; @@ -419,8 +418,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { if (ck == LiteralStringRef("initialized")) { initialized = true; - } else if (ck == LiteralStringRef("proxies")) { - parse(&proxyCount, value); + } else if (ck == LiteralStringRef("commit_proxies")) { + parse(&commitProxyCount, value); } else if (ck == LiteralStringRef("grv_proxies")) { parse(&grvProxyCount, value); } else if (ck == LiteralStringRef("resolvers")) { @@ -459,8 +458,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { } else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; - } else if (ck == LiteralStringRef("auto_proxies")) { - parse(&autoProxyCount, value); + } else if (ck == LiteralStringRef("auto_commit_proxies")) { + parse(&autoCommitProxyCount, value); } else if (ck == LiteralStringRef("auto_grv_proxies")) { parse(&autoGrvProxyCount, value); } else if (ck == LiteralStringRef("auto_resolvers")) { diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 4a045200e8..0e374457da 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -133,15 +133,19 @@ struct DatabaseConfiguration { } //Killing an entire datacenter counts as killing one zone in modes that support it - int32_t maxZoneFailuresTolerated() const { + int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const { int worstSatellite = regions.size() ? std::numeric_limits::max() : 0; + int regionsWithNonNegativePriority = 0; for(auto& r : regions) { + if(r.priority >= 0) { + regionsWithNonNegativePriority++; + } worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum); if(r.satelliteTLogUsableDcsFallback > 0) { worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback); } } - if(usableRegions > 1 && worstSatellite > 0) { + if(usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && (!forAvailability || regionsWithNonNegativePriority > 1)) { return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1), storageTeamSize - 1); } else if(worstSatellite > 0) { return std::min(tLogReplicationFactor + worstSatellite - 2 - tLogWriteAntiQuorum, storageTeamSize - 1); @@ -149,9 +153,9 @@ struct DatabaseConfiguration { return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1); } - // Proxy Servers - int32_t proxyCount; - int32_t autoProxyCount; + // CommitProxy Servers + int32_t commitProxyCount; + int32_t autoCommitProxyCount; int32_t grvProxyCount; int32_t autoGrvProxyCount; @@ -192,7 +196,10 @@ struct DatabaseConfiguration { bool isExcludedServer( NetworkAddressList ) const; std::set getExcludedServers() const; - int32_t getDesiredProxies() const { if(proxyCount == -1) return autoProxyCount; return proxyCount; } + int32_t getDesiredCommitProxies() const { + if (commitProxyCount == -1) return autoCommitProxyCount; + return commitProxyCount; + } int32_t getDesiredGrvProxies() const { if (grvProxyCount == -1) return autoGrvProxyCount; return grvProxyCount; diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 29206ca039..e2de3ea77d 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -29,7 +29,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/KeyRangeMap.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/SpecialKeySpace.actor.h" #include "fdbrpc/QueueModel.h" #include "fdbrpc/MultiInterface.h" @@ -68,7 +68,7 @@ struct LocationInfo : MultiInterface } }; -using ProxyInfo = ModelInterface; +using CommitProxyInfo = ModelInterface; using GrvProxyInfo = ModelInterface; class ClientTagThrottleData : NonCopyable { @@ -165,8 +165,8 @@ public: bool sampleOnCost(uint64_t cost) const; void updateProxies(); - Reference getMasterProxies(bool useProvisionalProxies); - Future> getMasterProxiesFuture(bool useProvisionalProxies); + Reference getCommitProxies(bool useProvisionalProxies); + Future> getCommitProxiesFuture(bool useProvisionalProxies); Reference getGrvProxies(bool useProvisionalProxies); Future onProxiesChanged(); Future getHealthMetrics(bool detailed); @@ -223,9 +223,9 @@ public: Reference>> connectionFile; AsyncTrigger proxiesChangeTrigger; Future monitorProxiesInfoChange; - Reference masterProxies; + Reference commitProxies; Reference grvProxies; - bool proxyProvisional; + bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time. UID proxiesLastChange; LocalityData clientLocality; QueueModel queueModel; diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 7e16dcd75f..7bae9ec78d 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -257,6 +257,7 @@ struct Traceable> : std::true_type { std::string printable( const StringRef& val ); std::string printable( const std::string& val ); std::string printable( const KeyRangeRef& range ); +std::string printable( const VectorRef& val); std::string printable( const VectorRef& val ); std::string printable( const VectorRef& val ); std::string printable( const KeyValueRef& val ); @@ -289,6 +290,14 @@ struct KeyRangeRef { bool contains( const KeyRef& key ) const { return begin <= key && key < end; } bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; } bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; } + bool intersects(const VectorRef& keysVec) const { + for (const auto& keys : keysVec) { + if (intersects(keys)) { + return true; + } + } + return false; + } bool empty() const { return begin == end; } bool singleKeyRange() const { return equalsKeyAfter(begin, end); } @@ -745,14 +754,16 @@ struct TLogVersion { // V3 was the introduction of spill by reference; // V4 changed how data gets written to satellite TLogs so that we can peek from them; // V5 merged reference and value spilling + // V6 added span context to list of serialized mutations sent from proxy to tlogs // V1 = 1, // 4.6 is dispatched to via 6.0 V2 = 2, // 6.0 V3 = 3, // 6.1 V4 = 4, // 6.2 V5 = 5, // 6.3 + V6 = 6, // 7.0 MIN_SUPPORTED = V2, - MAX_SUPPORTED = V5, - MIN_RECRUITABLE = V4, + MAX_SUPPORTED = V6, + MIN_RECRUITABLE = V5, DEFAULT = V5, } version; @@ -775,6 +786,7 @@ struct TLogVersion { if (s == LiteralStringRef("3")) return V3; if (s == LiteralStringRef("4")) return V4; if (s == LiteralStringRef("5")) return V5; + if (s == LiteralStringRef("6")) return V6; return default_error_or(); } }; diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 7b8833431e..3e96a5f84d 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -623,8 +623,6 @@ namespace fileBackup { // Very simple format compared to KeyRange files. // Header, [Key, Value]... Key len struct LogFileWriter { - static const std::string &FFs; - LogFileWriter(Reference file = Reference(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0) {} @@ -748,9 +746,10 @@ namespace fileBackup { state Subspace newConfigSpace = uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(fileBackupPrefixRange.begin), uid); Optional statusStr = wait(tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus))); - state EBackupState status = !statusStr.present() ? FileBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString()); + state EBackupState status = + !statusStr.present() ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString()); - TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup") + TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup") .detail("TagName", tagName.c_str()) .detail("Status", BackupAgentBase::getStateText(status)); @@ -770,9 +769,9 @@ namespace fileBackup { // Set old style state key to Aborted if it was Runnable if(backupAgent->isRunnable(status)) - tr->set(statusKey, StringRef(FileBackupAgent::getStateText(BackupAgentBase::STATE_ABORTED))); + tr->set(statusKey, StringRef(FileBackupAgent::getStateText(EBackupState::STATE_ABORTED))); - return Void(); + return Void(); } struct AbortFiveZeroBackupTask : TaskFuncBase { @@ -792,13 +791,13 @@ namespace fileBackup { return Void(); } - virtual StringRef getName() const { - TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called"); + StringRef getName() const override { + TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called"); ASSERT(false); return StringRef(); - } + } - Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Future(Void()); }; + Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Future(Void()); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef AbortFiveZeroBackupTask::name = LiteralStringRef("abort_legacy_backup"); @@ -822,11 +821,11 @@ namespace fileBackup { state BackupConfig config(current.first); EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); - if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) { - throw backup_unneeded(); - } + if (!backupAgent->isRunnable(status)) { + throw backup_unneeded(); + } - TraceEvent(SevInfo, "FBA_AbortFileOneBackup") + TraceEvent(SevInfo, "FBA_AbortFileOneBackup") .detail("TagName", tagName.c_str()) .detail("Status", BackupAgentBase::getStateText(status)); @@ -862,13 +861,13 @@ namespace fileBackup { return Void(); } - virtual StringRef getName() const { - TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called"); + StringRef getName() const override { + TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called"); ASSERT(false); return StringRef(); - } + } - Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Future(Void()); }; + Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Future(Void()); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef AbortFiveOneBackupTask::name = LiteralStringRef("abort_legacy_backup_5.2"); @@ -939,24 +938,18 @@ namespace fileBackup { // Backup and Restore taskFunc definitions will inherit from one of the following classes which // servers to catch and log to the appropriate config any error that execute/finish didn't catch and log. struct RestoreTaskFuncBase : TaskFuncBase { - virtual Future handleError(Database cx, Reference task, Error const &error) { - return RestoreConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str())); - } - virtual std::string toString(Reference task) - { - return ""; - } - }; + Future handleError(Database cx, Reference task, Error const& error) final { + return RestoreConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str())); + } + virtual std::string toString(Reference task) const { return ""; } + }; struct BackupTaskFuncBase : TaskFuncBase { - virtual Future handleError(Database cx, Reference task, Error const &error) { - return BackupConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str())); - } - virtual std::string toString(Reference task) - { - return ""; - } - }; + Future handleError(Database cx, Reference task, Error const& error) final { + return BackupConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str())); + } + virtual std::string toString(Reference task) const { return ""; } + }; ACTOR static Future>> getBlockOfShards(Reference tr, Key beginKey, Key endKey, int limit) { @@ -975,9 +968,9 @@ namespace fileBackup { struct BackupRangeTaskFunc : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam beginKey() { return LiteralStringRef(__FUNCTION__); } @@ -989,15 +982,15 @@ namespace fileBackup { } } Params; - std::string toString(Reference task) { - return format("beginKey '%s' endKey '%s' addTasks %d", + std::string toString(Reference task) const override { + return format("beginKey '%s' endKey '%s' addTasks %d", Params.beginKey().get(task).printable().c_str(), Params.endKey().get(task).printable().c_str(), Params.addBackupRangeTasks().get(task) ); - } + } - StringRef getName() const { return name; }; + StringRef getName() const { return name; }; Future execute(Database cx, Reference tb, Reference fb, Reference task) { return _execute(cx, tb, fb, task); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; @@ -1270,14 +1263,13 @@ namespace fileBackup { }; StringRef BackupRangeTaskFunc::name = LiteralStringRef("file_backup_write_range_5.2"); - const uint32_t BackupRangeTaskFunc::version = 1; REGISTER_TASKFUNC(BackupRangeTaskFunc); struct BackupSnapshotDispatchTask : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { // Set by Execute, used by Finish static TaskParam shardsBehind() { return LiteralStringRef(__FUNCTION__); @@ -1791,14 +1783,13 @@ namespace fileBackup { }; StringRef BackupSnapshotDispatchTask::name = LiteralStringRef("file_backup_dispatch_ranges_5.2"); - const uint32_t BackupSnapshotDispatchTask::version = 1; REGISTER_TASKFUNC(BackupSnapshotDispatchTask); struct BackupLogRangeTaskFunc : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam addBackupLogRangeTasks() { return LiteralStringRef(__FUNCTION__); } @@ -1993,14 +1984,13 @@ namespace fileBackup { }; StringRef BackupLogRangeTaskFunc::name = LiteralStringRef("file_backup_write_logs_5.2"); - const uint32_t BackupLogRangeTaskFunc::version = 1; REGISTER_TASKFUNC(BackupLogRangeTaskFunc); //This task stopped being used in 6.2, however the code remains here to handle upgrades. struct EraseLogRangeTaskFunc : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; - StringRef getName() const { return name; }; + static constexpr uint32_t version = 1; + StringRef getName() const { return name; }; static struct { static TaskParam beginVersion() { @@ -2050,16 +2040,15 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("file_backup_erase_logs_5.2"); - const uint32_t EraseLogRangeTaskFunc::version = 1; REGISTER_TASKFUNC(EraseLogRangeTaskFunc); struct BackupLogsDispatchTask : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam prevBeginVersion() { return LiteralStringRef(__FUNCTION__); } @@ -2100,10 +2089,10 @@ namespace fileBackup { } // If the backup is restorable but the state is not differential then set state to differential - if(restorableVersion.present() && backupState != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL) - config.stateEnum().set(tr, BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); + if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL) + config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL); - // If stopWhenDone is set and there is a restorable version, set the done future and do not create further tasks. + // If stopWhenDone is set and there is a restorable version, set the done future and do not create further tasks. if(stopWhenDone && restorableVersion.present()) { wait(onDone->set(tr, taskBucket) && taskBucket->finish(tr, task)); @@ -2178,14 +2167,13 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs_5.2"); - const uint32_t BackupLogsDispatchTask::version = 1; REGISTER_TASKFUNC(BackupLogsDispatchTask); struct FileBackupFinishedTask : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - StringRef getName() const { return name; }; + StringRef getName() const { return name; }; ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version)); @@ -2219,13 +2207,12 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished_5.2"); - const uint32_t FileBackupFinishedTask::version = 1; REGISTER_TASKFUNC(FileBackupFinishedTask); struct BackupSnapshotManifest : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; - static struct { + static constexpr uint32_t version = 1; + static struct { static TaskParam endVersion() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -2350,10 +2337,10 @@ namespace fileBackup { } // If the backup is restorable and the state isn't differential the set state to differential - if(restorableVersion.present() && backupState != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL) - config.stateEnum().set(tr, BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); + if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL) + config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL); - // Unless we are to stop, start the next snapshot using the default interval + // Unless we are to stop, start the next snapshot using the default interval Reference snapshotDoneFuture = task->getDoneFuture(futureBucket); if(!stopWhenDone) { wait(config.initNewSnapshot(tr) && success(BackupSnapshotDispatchTask::addTask(tr, taskBucket, task, 1, TaskCompletionKey::signal(snapshotDoneFuture)))); @@ -2380,7 +2367,6 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest_5.2"); - const uint32_t BackupSnapshotManifest::version = 1; REGISTER_TASKFUNC(BackupSnapshotManifest); Future BackupSnapshotDispatchTask::addSnapshotManifestTask(Reference tr, Reference taskBucket, Reference parentTask, TaskCompletionKey completionKey, Reference waitFor) { @@ -2389,9 +2375,9 @@ namespace fileBackup { struct StartFullBackupTaskFunc : BackupTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam beginVersion() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -2533,7 +2519,6 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start_5.2"); - const uint32_t StartFullBackupTaskFunc::version = 1; REGISTER_TASKFUNC(StartFullBackupTaskFunc); struct RestoreCompleteTaskFunc : RestoreTaskFuncBase { @@ -2576,15 +2561,14 @@ namespace fileBackup { } static StringRef name; - static const uint32_t version; - StringRef getName() const { return name; }; + static constexpr uint32_t version = 1; + StringRef getName() const { return name; }; Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Void(); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef RestoreCompleteTaskFunc::name = LiteralStringRef("restore_complete"); - const uint32_t RestoreCompleteTaskFunc::version = 1; REGISTER_TASKFUNC(RestoreCompleteTaskFunc); struct RestoreFileTaskFuncBase : RestoreTaskFuncBase { @@ -2594,13 +2578,13 @@ namespace fileBackup { static TaskParam readLen() { return LiteralStringRef(__FUNCTION__); } } Params; - std::string toString(Reference task) { - return format("fileName '%s' readLen %lld readOffset %lld", + std::string toString(Reference task) const override { + return format("fileName '%s' readLen %lld readOffset %lld", Params.inputFile().get(task).fileName.c_str(), Params.readLen().get(task), Params.readOffset().get(task)); - } - }; + } + }; struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase { static struct : InputParams { @@ -2621,14 +2605,14 @@ namespace fileBackup { } } Params; - std::string toString(Reference task) { - std::string returnStr = RestoreFileTaskFuncBase::toString(task); + std::string toString(Reference task) const override { + std::string returnStr = RestoreFileTaskFuncBase::toString(task); for(auto &range : Params.getOriginalFileRanges(task)) returnStr += format(" originalFileRange '%s'", printable(range).c_str()); return returnStr; - } + } - ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { + ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { state RestoreConfig restore(task); state RestoreFile rangeFile = Params.inputFile().get(task); @@ -2840,20 +2824,19 @@ namespace fileBackup { } static StringRef name; - static const uint32_t version; - StringRef getName() const { return name; }; + static constexpr uint32_t version = 1; + StringRef getName() const { return name; }; Future execute(Database cx, Reference tb, Reference fb, Reference task) { return _execute(cx, tb, fb, task); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef RestoreRangeTaskFunc::name = LiteralStringRef("restore_range_data"); - const uint32_t RestoreRangeTaskFunc::version = 1; REGISTER_TASKFUNC(RestoreRangeTaskFunc); struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase { static StringRef name; - static const uint32_t version; - StringRef getName() const { return name; }; + static constexpr uint32_t version = 1; + StringRef getName() const { return name; }; static struct : InputParams { } Params; @@ -2995,13 +2978,12 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef RestoreLogDataTaskFunc::name = LiteralStringRef("restore_log_data"); - const uint32_t RestoreLogDataTaskFunc::version = 1; REGISTER_TASKFUNC(RestoreLogDataTaskFunc); struct RestoreDispatchTaskFunc : RestoreTaskFuncBase { static StringRef name; - static const uint32_t version; - StringRef getName() const { return name; }; + static constexpr uint32_t version = 1; + StringRef getName() const { return name; }; static struct { static TaskParam beginVersion() { return LiteralStringRef(__FUNCTION__); } @@ -3308,7 +3290,6 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef RestoreDispatchTaskFunc::name = LiteralStringRef("restore_dispatch"); - const uint32_t RestoreDispatchTaskFunc::version = 1; REGISTER_TASKFUNC(RestoreDispatchTaskFunc); ACTOR Future restoreStatus(Reference tr, Key tagName) { @@ -3402,9 +3383,9 @@ namespace fileBackup { struct StartFullRestoreTaskFunc : RestoreTaskFuncBase { static StringRef name; - static const uint32_t version; + static constexpr uint32_t version = 1; - static struct { + static struct { static TaskParam firstVersion() { return LiteralStringRef(__FUNCTION__); } } Params; @@ -3469,12 +3450,13 @@ namespace fileBackup { if (beginVersion == invalidVersion) { beginVersion = 0; } - Optional restorable = wait(bc->getRestoreSet(restoreVersion, incremental, beginVersion)); - if (!incremental) { - beginVersion = restorable.get().snapshot.beginVersion; - } + Optional restorable = + wait(bc->getRestoreSet(restoreVersion, VectorRef(), incremental, beginVersion)); + if (!incremental) { + beginVersion = restorable.get().snapshot.beginVersion; + } - if(!restorable.present()) + if(!restorable.present()) throw restore_missing_data(); // First version for which log data should be applied @@ -3596,7 +3578,6 @@ namespace fileBackup { Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; }; StringRef StartFullRestoreTaskFunc::name = LiteralStringRef("restore_start"); - const uint32_t StartFullRestoreTaskFunc::version = 1; REGISTER_TASKFUNC(StartFullRestoreTaskFunc); } @@ -3612,7 +3593,7 @@ struct LogInfo : public ReferenceCounted { class FileBackupAgentImpl { public: - static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; + static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; // Parallel restore ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true) { @@ -3746,7 +3727,9 @@ public: // This method will return the final status of the backup at tag, and return the URL that was used on the tag // when that status value was read. - ACTOR static Future waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, bool stopWhenDone, Reference *pContainer = nullptr, UID *pUID = nullptr) { + ACTOR static Future waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, + bool stopWhenDone, Reference* pContainer = nullptr, + UID* pUID = nullptr) { state std::string backTrace; state KeyBackedTag tag = makeBackupTag(tagName); @@ -3767,7 +3750,8 @@ public: // Break, if one of the following is true // - no longer runnable // - in differential mode (restorable) and stopWhenDone is not enabled - if( !FileBackupAgent::isRunnable(status) || ((!stopWhenDone) && (BackupAgentBase::STATE_RUNNING_DIFFERENTIAL == status) )) { + if (!FileBackupAgent::isRunnable(status) || + ((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status))) { if(pContainer != nullptr) { Reference c = wait(config.backupContainer().getOrThrow(tr, false, backup_invalid_info())); @@ -4103,7 +4087,7 @@ public: state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); - if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) { + if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); } @@ -4206,13 +4190,13 @@ public: JsonBuilderObject statusDoc; statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState)); statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState)); - statusDoc.setKey("Completed", backupState == BackupAgentBase::STATE_COMPLETED); + statusDoc.setKey("Completed", backupState == EBackupState::STATE_COMPLETED); statusDoc.setKey("Running", BackupAgentBase::isRunnable(backupState)); doc.setKey("Status", statusDoc); state Future done = Void(); - if(backupState != BackupAgentBase::STATE_NEVERRAN) { + if (backupState != EBackupState::STATE_NEVERRAN) { state Reference bc; state TimestampedVersion latestRestorable; @@ -4224,7 +4208,7 @@ public: if(latestRestorable.present()) { JsonBuilderObject o = latestRestorable.toJSON(); - if(backupState != BackupAgentBase::STATE_COMPLETED) { + if (backupState != EBackupState::STATE_COMPLETED) { o.setKey("LagSeconds", (recentReadVersion - latestRestorable.version.get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND); } doc.setKey("LatestRestorablePoint", o); @@ -4232,7 +4216,8 @@ public: doc.setKey("DestinationURL", bc->getURL()); } - if(backupState == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || backupState == BackupAgentBase::STATE_RUNNING) { + if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL || + backupState == EBackupState::STATE_RUNNING) { state int64_t snapshotInterval; state int64_t logBytesWritten; state int64_t rangeBytesWritten; @@ -4355,23 +4340,28 @@ public: bool snapshotProgress = false; switch (backupState) { - case BackupAgentBase::STATE_SUBMITTED: - statusText += "The backup on tag `" + tagName + "' is in progress (just started) to " + bc->getURL() + ".\n"; - break; - case BackupAgentBase::STATE_RUNNING: - statusText += "The backup on tag `" + tagName + "' is in progress to " + bc->getURL() + ".\n"; - snapshotProgress = true; - break; - case BackupAgentBase::STATE_RUNNING_DIFFERENTIAL: - statusText += "The backup on tag `" + tagName + "' is restorable but continuing to " + bc->getURL() + ".\n"; - snapshotProgress = true; - break; - case BackupAgentBase::STATE_COMPLETED: - statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " completed at version " + format("%lld", latestRestorableVersion.orDefault(-1)) + ".\n"; - break; - default: - statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " " + backupStatus + ".\n"; - break; + case EBackupState::STATE_SUBMITTED: + statusText += "The backup on tag `" + tagName + "' is in progress (just started) to " + + bc->getURL() + ".\n"; + break; + case EBackupState::STATE_RUNNING: + statusText += "The backup on tag `" + tagName + "' is in progress to " + bc->getURL() + ".\n"; + snapshotProgress = true; + break; + case EBackupState::STATE_RUNNING_DIFFERENTIAL: + statusText += "The backup on tag `" + tagName + "' is restorable but continuing to " + + bc->getURL() + ".\n"; + snapshotProgress = true; + break; + case EBackupState::STATE_COMPLETED: + statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + + " completed at version " + format("%lld", latestRestorableVersion.orDefault(-1)) + + ".\n"; + break; + default: + statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " " + + backupStatus + ".\n"; + break; } statusText += format("BackupUID: %s\n", uidAndAbortedFlag.get().first.toString().c_str()); statusText += format("BackupURL: %s\n", bc->getURL().c_str()); @@ -4407,7 +4397,7 @@ public: ); statusText += format("Snapshot interval is %lld seconds. ", snapshotInterval); - if(backupState == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL) + if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL) statusText += format("Current snapshot progress target is %3.2f%% (>100%% means the snapshot is supposed to be done)\n", 100.0 * (recentReadVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion)) ; else statusText += "The initial snapshot is still running.\n"; @@ -4495,7 +4485,7 @@ public: Version beginVersion, UID randomUid) { state Reference bc = IBackupContainer::openContainer(url.toString()); - state BackupDescription desc = wait(bc->describeBackup()); + state BackupDescription desc = wait(bc->describeBackup(true)); if(cxOrig.present()) { wait(desc.resolveVersionTimes(cxOrig.get())); } @@ -4509,11 +4499,12 @@ public: } Optional restoreSet = - wait(bc->getRestoreSet(targetVersion, incrementalBackupOnly, beginVersion)); + wait(bc->getRestoreSet(targetVersion, VectorRef(), incrementalBackupOnly, beginVersion)); if(!restoreSet.present()) { TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible") .detail("BackupContainer", bc->getURL()) + .detail("BeginVersion", beginVersion) .detail("TargetVersion", targetVersion); fprintf(stderr, "ERROR: Restore version %" PRId64 " is not possible from %s\n", targetVersion, bc->getURL().c_str()); throw restore_invalid_version(); @@ -4565,7 +4556,7 @@ public: backupConfig = BackupConfig(uidFlag.first); state EBackupState status = wait(backupConfig.stateEnum().getOrThrow(ryw_tr)); - if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL ) { + if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL) { throw backup_duplicate(); } @@ -4766,7 +4757,8 @@ void FileBackupAgent::setLastRestorable(Reference tr, tr->set(lastRestorable.pack(tagName), BinaryWriter::toValue(version, Unversioned())); } -Future FileBackupAgent::waitBackup(Database cx, std::string tagName, bool stopWhenDone, Reference *pContainer, UID *pUID) { +Future FileBackupAgent::waitBackup(Database cx, std::string tagName, bool stopWhenDone, + Reference* pContainer, UID* pUID) { return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID); } @@ -5029,4 +5021,4 @@ void simulateBlobFailure() { throw lookup_failed(); } } -} \ No newline at end of file +} diff --git a/fdbclient/GrvProxyInterface.h b/fdbclient/GrvProxyInterface.h index 06d4b7e946..94820a175f 100644 --- a/fdbclient/GrvProxyInterface.h +++ b/fdbclient/GrvProxyInterface.h @@ -27,6 +27,8 @@ // with RateKeeper to gather health information of the cluster. struct GrvProxyInterface { constexpr static FileIdentifier file_identifier = 8743216; + enum { LocationAwareLoadBalance = 1 }; + enum { AlwaysFresh = 1 }; Optional processId; bool provisional; diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp index 0b02740b17..e61d203444 100644 --- a/fdbclient/HTTP.actor.cpp +++ b/fdbclient/HTTP.actor.cpp @@ -72,7 +72,7 @@ namespace HTTP { } PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) { - PacketWriter writer(dest, NULL, Unversioned()); + PacketWriter writer(dest, nullptr, Unversioned()); writer.serializeBytes(verb); writer.serializeBytes(" ", 1); writer.serializeBytes(resource); @@ -238,7 +238,7 @@ namespace HTTP { { // Read the line that contains the chunk length as text in hex size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos)); - state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), NULL, 16); + state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16); // Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer r->content.erase(pos, lineLen + 2); @@ -301,7 +301,7 @@ namespace HTTP { state TraceEvent event(SevDebug, "HTTPRequest"); state UnsentPacketQueue empty; - if(pContent == NULL) + if(pContent == nullptr) pContent = ∅ // There is no standard http request id header field, so either a global default can be set via a knob diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index 397dd81f68..7c6a5b32f1 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -49,6 +49,8 @@ public: virtual void addReadConflictRange(const KeyRangeRef& keys) = 0; virtual ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) = 0; + virtual ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) = 0; virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0; virtual void set(const KeyRef& key, const ValueRef& value) = 0; diff --git a/fdbclient/JSONDoc.h b/fdbclient/JSONDoc.h index aafd1bb87f..0c8fe14bba 100644 --- a/fdbclient/JSONDoc.h +++ b/fdbclient/JSONDoc.h @@ -67,11 +67,11 @@ // // The following would throw if a.b.c did not exist, or if it was not an int. // int x = r["a.b.c"].get_int(); struct JSONDoc { - JSONDoc() : pObj(NULL) {} + JSONDoc() : pObj(nullptr) {} // Construction from const json_spirit::mObject, trivial and will never throw. // Resulting JSONDoc will not allow modifications. - JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(NULL) {} + JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(nullptr) {} // Construction from json_spirit::mObject. Allows modifications. JSONDoc(json_spirit::mObject &o) : pObj(&o), wpObj(&o) {} @@ -79,7 +79,7 @@ struct JSONDoc { // Construction from const json_spirit::mValue (which is a Variant type) which will try to // convert it to an mObject. This will throw if that fails, just as it would // if the caller called get_obj() itself and used the previous constructor instead. - JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(NULL) {} + JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(nullptr) {} // Construction from non-const json_spirit::mValue - will convert the mValue to // an object if it isn't already and then attach to it. @@ -98,13 +98,13 @@ struct JSONDoc { // path into on the "dot" character. // When a path is found, pLast is updated. bool has(std::string path, bool split=true) { - if (pObj == NULL) + if (pObj == nullptr) return false; if (path.empty()) return false; size_t start = 0; - const json_spirit::mValue *curVal = NULL; + const json_spirit::mValue *curVal = nullptr; while (start < path.size()) { // If a path segment is found then curVal must be an object @@ -140,7 +140,7 @@ struct JSONDoc { // Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there) // and returns a reference to the Value at that location. json_spirit::mValue & create(std::string path, bool split=true) { - if (wpObj == NULL || path.empty()) + if (wpObj == nullptr || path.empty()) throw std::runtime_error("JSON Object not writable or bad JSON path"); size_t start = 0; @@ -280,7 +280,7 @@ struct JSONDoc { } const json_spirit::mValue & last() const { return *pLast; } - bool valid() const { return pObj != NULL; } + bool valid() const { return pObj != nullptr; } const json_spirit::mObject & obj() { // This dummy object is necessary to make working with obj() easier when this does not currently @@ -304,7 +304,7 @@ struct JSONDoc { static uint64_t expires_reference_version; private: const json_spirit::mObject *pObj; - // Writeable pointer to the same object. Will be NULL if initialized from a const object. + // Writeable pointer to the same object. Will be nullptr if initialized from a const object. json_spirit::mObject *wpObj; const json_spirit::mValue *pLast; }; diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index c2e99f63fb..e194f8827e 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -52,7 +52,7 @@ void ClientKnobs::initialize(bool randomize) { init( COORDINATOR_RECONNECTION_DELAY, 1.0 ); init( CLIENT_EXAMPLE_AMOUNT, 20 ); init( MAX_CLIENT_STATUS_AGE, 1.0 ); - init( MAX_MASTER_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_MASTER_PROXY_CONNECTIONS = 1; + init( MAX_COMMIT_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_COMMIT_PROXY_CONNECTIONS = 1; init( MAX_GRV_PROXY_CONNECTIONS, 3 ); if( randomize && BUGGIFY ) MAX_GRV_PROXY_CONNECTIONS = 1; init( STATUS_IDLE_TIMEOUT, 120.0 ); @@ -104,7 +104,7 @@ void ClientKnobs::initialize(bool randomize) { init( WATCH_POLLING_TIME, 1.0 ); if( randomize && BUGGIFY ) WATCH_POLLING_TIME = 5.0; init( NO_RECENT_UPDATES_DURATION, 20.0 ); if( randomize && BUGGIFY ) NO_RECENT_UPDATES_DURATION = 0.1; init( FAST_WATCH_TIMEOUT, 20.0 ); if( randomize && BUGGIFY ) FAST_WATCH_TIMEOUT = 1.0; - init( WATCH_TIMEOUT, 900.0 ); if( randomize && BUGGIFY ) WATCH_TIMEOUT = 20.0; + init( WATCH_TIMEOUT, 30.0 ); if( randomize && BUGGIFY ) WATCH_TIMEOUT = 20.0; // Core init( CORE_VERSIONSPERSECOND, 1e6 ); @@ -171,10 +171,12 @@ void ClientKnobs::initialize(bool randomize) { init( MIN_CLEANUP_SECONDS, 3600.0 ); // Configuration - init( DEFAULT_AUTO_PROXIES, 3 ); + init( DEFAULT_AUTO_COMMIT_PROXIES, 3 ); init( DEFAULT_AUTO_GRV_PROXIES, 1 ); init( DEFAULT_AUTO_RESOLVERS, 1 ); init( DEFAULT_AUTO_LOGS, 3 ); + init( DEFAULT_COMMIT_GRV_PROXIES_RATIO, 3 ); + init( DEFAULT_MAX_GRV_PROXIES, 4 ); init( IS_ACCEPTABLE_DELAY, 1.5 ); diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index 30e7e7f687..11a06d52cc 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -46,7 +46,7 @@ public: double COORDINATOR_RECONNECTION_DELAY; int CLIENT_EXAMPLE_AMOUNT; double MAX_CLIENT_STATUS_AGE; - int MAX_MASTER_PROXY_CONNECTIONS; + int MAX_COMMIT_PROXY_CONNECTIONS; int MAX_GRV_PROXY_CONNECTIONS; double STATUS_IDLE_TIMEOUT; @@ -167,8 +167,10 @@ public: double MIN_CLEANUP_SECONDS; // Configuration - int32_t DEFAULT_AUTO_PROXIES; + int32_t DEFAULT_AUTO_COMMIT_PROXIES; int32_t DEFAULT_AUTO_GRV_PROXIES; + int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO; + int32_t DEFAULT_MAX_GRV_PROXIES; int32_t DEFAULT_AUTO_RESOLVERS; int32_t DEFAULT_AUTO_LOGS; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index a05fce601e..cca55fafcc 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -19,8 +19,10 @@ */ #include +#include #include +#include "fdbclient/Knobs.h" #include "flow/Arena.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/FDBTypes.h" @@ -33,6 +35,7 @@ #include "fdbclient/DatabaseContext.h" #include "fdbrpc/simulator.h" #include "fdbclient/StatusClient.h" +#include "flow/Trace.h" #include "flow/UnitTest.h" #include "fdbrpc/ReplicationPolicy.h" #include "fdbrpc/Replication.h" @@ -78,8 +81,42 @@ std::map configForToken( std::string const& mode ) { std::string key = mode.substr(0, pos); std::string value = mode.substr(pos+1); - if ((key == "logs" || key == "proxies" || key == "grv_proxies" || key == "resolvers" || key == "remote_logs" || - key == "log_routers" || key == "usable_regions" || key == "repopulate_anti_quorum") && + if (key == "proxies" && isInteger(value)) { + printf("Warning: Proxy role is being split into GRV Proxy and Commit Proxy, now prefer configuring " + "'grv_proxies' and 'commit_proxies' separately. Generally we should follow that 'commit_proxies'" + " is three times of 'grv_proxies' count and 'grv_proxies' should be not more than 4.\n"); + int proxiesCount = atoi(value.c_str()); + if (proxiesCount == -1) { + proxiesCount = CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES + CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES; + ASSERT_WE_THINK(proxiesCount >= 2); + } + + if (proxiesCount < 2) { + printf("Error: At least 2 proxies (1 GRV proxy and 1 Commit proxy) are required.\n"); + return out; + } + + int grvProxyCount = + std::max(1, std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES, + proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1))); + int commitProxyCount = proxiesCount - grvProxyCount; + ASSERT_WE_THINK(grvProxyCount >= 1 && commitProxyCount >= 1); + + out[p + "grv_proxies"] = std::to_string(grvProxyCount); + out[p + "commit_proxies"] = std::to_string(commitProxyCount); + printf("%d proxies are automatically converted into %d GRV proxies and %d Commit proxies.\n", proxiesCount, + grvProxyCount, commitProxyCount); + + TraceEvent("DatabaseConfigurationProxiesSpecified") + .detail("SpecifiedProxies", atoi(value.c_str())) + .detail("EffectiveSpecifiedProxies", proxiesCount) + .detail("ConvertedGrvProxies", grvProxyCount) + .detail("ConvertedCommitProxies", commitProxyCount); + } + + if ((key == "logs" || key == "commit_proxies" || key == "grv_proxies" || key == "resolvers" || + key == "remote_logs" || key == "log_routers" || key == "usable_regions" || + key == "repopulate_anti_quorum") && isInteger(value)) { out[p+key] = value; } @@ -229,7 +266,8 @@ std::map configForToken( std::string const& mode ) { return out; } -ConfigurationResult::Type buildConfiguration( std::vector const& modeTokens, std::map& outConf ) { +ConfigurationResult buildConfiguration(std::vector const& modeTokens, + std::map& outConf) { for(auto it : modeTokens) { std::string mode = it.toString(); auto m = configForToken( mode ); @@ -265,7 +303,7 @@ ConfigurationResult::Type buildConfiguration( std::vector const& mode return ConfigurationResult::SUCCESS; } -ConfigurationResult::Type buildConfiguration( std::string const& configMode, std::map& outConf ) { +ConfigurationResult buildConfiguration(std::string const& configMode, std::map& outConf) { std::vector modes; int p = 0; @@ -305,7 +343,7 @@ ACTOR Future getDatabaseConfiguration( Database cx ) { } } -ACTOR Future changeConfig( Database cx, std::map m, bool force ) { +ACTOR Future changeConfig(Database cx, std::map m, bool force) { state StringRef initIdKey = LiteralStringRef( "\xff/init_id" ); state Transaction tr(cx); @@ -656,7 +694,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { } if (processClass.classType() == ProcessClass::TransactionClass || - processClass.classType() == ProcessClass::ProxyClass || + processClass.classType() == ProcessClass::CommitProxyClass || processClass.classType() == ProcessClass::GrvProxyClass || processClass.classType() == ProcessClass::ResolutionClass || processClass.classType() == ProcessClass::StatelessClass || @@ -701,7 +739,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { if (proc.second == ProcessClass::StatelessClass) { existingStatelessCount++; } - if(proc.second == ProcessClass::ProxyClass) { + if (proc.second == ProcessClass::CommitProxyClass) { existingProxyCount++; } if (proc.second == ProcessClass::GrvProxyClass) { @@ -734,19 +772,18 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { resolverCount = result.old_resolvers; } - result.desired_proxies = std::max(std::min(12, processCount / 15), 1); + result.desired_commit_proxies = std::max(std::min(12, processCount / 15), 1); int proxyCount; - if (!statusObjConfig.get("proxies", result.old_proxies)) { - result.old_proxies = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES; - statusObjConfig.get("auto_proxies", result.old_proxies); - result.auto_proxies = result.desired_proxies; - proxyCount = result.auto_proxies; + if (!statusObjConfig.get("commit_proxies", result.old_commit_proxies)) { + result.old_commit_proxies = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES; + statusObjConfig.get("auto_commit_proxies", result.old_commit_proxies); + result.auto_commit_proxies = result.desired_commit_proxies; + proxyCount = result.auto_commit_proxies; } else { - result.auto_proxies = result.old_proxies; - proxyCount = result.old_proxies; + result.auto_commit_proxies = result.old_commit_proxies; + proxyCount = result.old_commit_proxies; } - // Need to configure a good number. result.desired_grv_proxies = std::max(std::min(4, processCount / 20), 1); int grvProxyCount; if (!statusObjConfig.get("grv_proxies", result.old_grv_proxies)) { @@ -823,7 +860,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) { return result; } -ACTOR Future autoConfig( Database cx, ConfigureAutoResult conf ) { +ACTOR Future autoConfig(Database cx, ConfigureAutoResult conf) { state Transaction tr(cx); state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(),Unversioned()); @@ -857,8 +894,8 @@ ACTOR Future autoConfig( Database cx, ConfigureAutoRe if (conf.auto_logs != conf.old_logs) tr.set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs)); - if(conf.auto_proxies != conf.old_proxies) - tr.set(configKeysPrefix.toString() + "auto_proxies", format("%d", conf.auto_proxies)); + if (conf.auto_commit_proxies != conf.old_commit_proxies) + tr.set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies)); if (conf.auto_grv_proxies != conf.old_grv_proxies) tr.set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies)); @@ -890,7 +927,8 @@ ACTOR Future autoConfig( Database cx, ConfigureAutoRe } } -Future changeConfig( Database const& cx, std::vector const& modes, Optional const& conf, bool force ) { +Future changeConfig(Database const& cx, std::vector const& modes, + Optional const& conf, bool force) { if( modes.size() && modes[0] == LiteralStringRef("auto") && conf.present() ) { return autoConfig(cx, conf.get()); } @@ -902,7 +940,7 @@ Future changeConfig( Database const& cx, std::vector< return changeConfig(cx, m, force); } -Future changeConfig( Database const& cx, std::string const& modes, bool force ) { +Future changeConfig(Database const& cx, std::string const& modes, bool force) { TraceEvent("ChangeConfig").detail("Mode", modes); std::map m; auto r = buildConfiguration( modes, m ); @@ -971,7 +1009,7 @@ ACTOR Future> getCoordinators( Database cx ) { } } -ACTOR Future changeQuorum( Database cx, Reference change ) { +ACTOR Future changeQuorum(Database cx, Reference change) { state Transaction tr(cx); state int retries = 0; state std::vector desiredCoordinators; @@ -991,7 +1029,7 @@ ACTOR Future changeQuorum( Database cx, ReferencegetConnectionFile() && old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName() ) return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database?? - state CoordinatorsResult::Type result = CoordinatorsResult::SUCCESS; + state CoordinatorsResult result = CoordinatorsResult::SUCCESS; if(!desiredCoordinators.size()) { std::vector _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference(new ClusterConnectionFile(old)), result ) ); desiredCoordinators = _desiredCoordinators; @@ -1058,42 +1096,48 @@ ACTOR Future changeQuorum( Database cx, Reference desired; explicit SpecifiedQuorumChange( vector const& desired ) : desired(desired) {} - virtual Future> getDesiredCoordinators( Transaction* tr, vector oldCoordinators, Reference, CoordinatorsResult::Type& ) { + Future> getDesiredCoordinators(Transaction* tr, vector oldCoordinators, + Reference, + CoordinatorsResult&) override { return desired; } }; Reference specifiedQuorumChange(vector const& addresses) { return Reference(new SpecifiedQuorumChange(addresses)); } -struct NoQuorumChange : IQuorumChange { - virtual Future> getDesiredCoordinators( Transaction* tr, vector oldCoordinators, Reference, CoordinatorsResult::Type& ) { +struct NoQuorumChange final : IQuorumChange { + Future> getDesiredCoordinators(Transaction* tr, vector oldCoordinators, + Reference, + CoordinatorsResult&) override { return oldCoordinators; } }; Reference noQuorumChange() { return Reference(new NoQuorumChange); } -struct NameQuorumChange : IQuorumChange { +struct NameQuorumChange final : IQuorumChange { std::string newName; Reference otherChange; explicit NameQuorumChange( std::string const& newName, Reference const& otherChange ) : newName(newName), otherChange(otherChange) {} - virtual Future> getDesiredCoordinators( Transaction* tr, vector oldCoordinators, Reference cf, CoordinatorsResult::Type& t ) { + Future> getDesiredCoordinators(Transaction* tr, vector oldCoordinators, + Reference cf, + CoordinatorsResult& t) override { return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t); } - virtual std::string getDesiredClusterKeyName() { - return newName; - } + std::string getDesiredClusterKeyName() const override { return newName; } }; Reference nameQuorumChange(std::string const& name, Reference const& other) { return Reference(new NameQuorumChange( name, other )); } -struct AutoQuorumChange : IQuorumChange { +struct AutoQuorumChange final : IQuorumChange { int desired; explicit AutoQuorumChange( int desired ) : desired(desired) {} - virtual Future> getDesiredCoordinators( Transaction* tr, vector oldCoordinators, Reference ccf, CoordinatorsResult::Type& err ) { + Future> getDesiredCoordinators(Transaction* tr, vector oldCoordinators, + Reference ccf, + CoordinatorsResult& err) override { return getDesired( this, tr, oldCoordinators, ccf, &err ); } @@ -1145,7 +1189,10 @@ struct AutoQuorumChange : IQuorumChange { return true; // The status quo seems fine } - ACTOR static Future> getDesired( AutoQuorumChange* self, Transaction* tr, vector oldCoordinators, Reference ccf, CoordinatorsResult::Type* err ) { + ACTOR static Future> getDesired(AutoQuorumChange* self, Transaction* tr, + vector oldCoordinators, + Reference ccf, + CoordinatorsResult* err) { state int desiredCount = self->desired; if(desiredCount == -1) { diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h index 20b2a447d9..bbd69b589f 100644 --- a/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/ManagementAPI.actor.h @@ -43,41 +43,35 @@ standard API and some knowledge of the contents of the system key space. // ConfigurationResult enumerates normal outcomes of changeConfig() and various error // conditions specific to it. changeConfig may also throw an Error to report other problems. -class ConfigurationResult { -public: - enum Type { - NO_OPTIONS_PROVIDED, - CONFLICTING_OPTIONS, - UNKNOWN_OPTION, - INCOMPLETE_CONFIGURATION, - INVALID_CONFIGURATION, - DATABASE_ALREADY_CREATED, - DATABASE_CREATED, - DATABASE_UNAVAILABLE, - STORAGE_IN_UNKNOWN_DCID, - REGION_NOT_FULLY_REPLICATED, - MULTIPLE_ACTIVE_REGIONS, - REGIONS_CHANGED, - NOT_ENOUGH_WORKERS, - REGION_REPLICATION_MISMATCH, - DCID_MISSING, - LOCKED_NOT_NEW, - SUCCESS, - }; +enum class ConfigurationResult { + NO_OPTIONS_PROVIDED, + CONFLICTING_OPTIONS, + UNKNOWN_OPTION, + INCOMPLETE_CONFIGURATION, + INVALID_CONFIGURATION, + DATABASE_ALREADY_CREATED, + DATABASE_CREATED, + DATABASE_UNAVAILABLE, + STORAGE_IN_UNKNOWN_DCID, + REGION_NOT_FULLY_REPLICATED, + MULTIPLE_ACTIVE_REGIONS, + REGIONS_CHANGED, + NOT_ENOUGH_WORKERS, + REGION_REPLICATION_MISMATCH, + DCID_MISSING, + LOCKED_NOT_NEW, + SUCCESS, }; -class CoordinatorsResult { -public: - enum Type { - INVALID_NETWORK_ADDRESSES, - SAME_NETWORK_ADDRESSES, - NOT_COORDINATORS, //FIXME: not detected - DATABASE_UNREACHABLE, //FIXME: not detected - BAD_DATABASE_STATE, - COORDINATOR_UNREACHABLE, - NOT_ENOUGH_MACHINES, - SUCCESS - }; +enum class CoordinatorsResult { + INVALID_NETWORK_ADDRESSES, + SAME_NETWORK_ADDRESSES, + NOT_COORDINATORS, // FIXME: not detected + DATABASE_UNREACHABLE, // FIXME: not detected + BAD_DATABASE_STATE, + COORDINATOR_UNREACHABLE, + NOT_ENOUGH_MACHINES, + SUCCESS }; struct ConfigureAutoResult { @@ -86,7 +80,7 @@ struct ConfigureAutoResult { int32_t machines; std::string old_replication; - int32_t old_proxies; + int32_t old_commit_proxies; int32_t old_grv_proxies; int32_t old_resolvers; int32_t old_logs; @@ -94,38 +88,46 @@ struct ConfigureAutoResult { int32_t old_machines_with_transaction; std::string auto_replication; - int32_t auto_proxies; + int32_t auto_commit_proxies; int32_t auto_grv_proxies; int32_t auto_resolvers; int32_t auto_logs; int32_t auto_processes_with_transaction; int32_t auto_machines_with_transaction; - int32_t desired_proxies; + int32_t desired_commit_proxies; int32_t desired_grv_proxies; int32_t desired_resolvers; int32_t desired_logs; ConfigureAutoResult() - : processes(-1), machines(-1), old_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1), - old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_proxies(-1), auto_grv_proxies(-1), - auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1), - desired_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1), desired_logs(-1) {} + : processes(-1), machines(-1), old_commit_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1), + old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_commit_proxies(-1), + auto_grv_proxies(-1), auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), + auto_machines_with_transaction(-1), desired_commit_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1), + desired_logs(-1) {} bool isValid() const { return processes != -1; } }; -ConfigurationResult::Type buildConfiguration( std::vector const& modeTokens, std::map& outConf ); // Accepts a vector of configuration tokens -ConfigurationResult::Type buildConfiguration( std::string const& modeString, std::map& outConf ); // Accepts tokens separated by spaces in a single string +ConfigurationResult buildConfiguration( + std::vector const& modeTokens, + std::map& outConf); // Accepts a vector of configuration tokens +ConfigurationResult buildConfiguration( + std::string const& modeString, + std::map& outConf); // Accepts tokens separated by spaces in a single string bool isCompleteConfiguration( std::map const& options ); // All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error). -Future changeConfig( Database const& cx, std::string const& configMode, bool force ); // Accepts tokens separated by spaces in a single string +Future changeConfig(Database const& cx, std::string const& configMode, + bool force); // Accepts tokens separated by spaces in a single string ConfigureAutoResult parseConfig( StatusObject const& status ); -Future changeConfig( Database const& cx, std::vector const& modes, Optional const& conf, bool force ); // Accepts a vector of configuration tokens -ACTOR Future changeConfig( +Future changeConfig(Database const& cx, std::vector const& modes, + Optional const& conf, + bool force); // Accepts a vector of configuration tokens +ACTOR Future changeConfig( Database cx, std::map m, bool force); // Accepts a full configuration in key/value format (from buildConfiguration) @@ -134,12 +136,15 @@ ACTOR Future waitForFullReplication(Database cx); struct IQuorumChange : ReferenceCounted { virtual ~IQuorumChange() {} - virtual Future> getDesiredCoordinators( Transaction* tr, vector oldCoordinators, Reference, CoordinatorsResult::Type& ) = 0; - virtual std::string getDesiredClusterKeyName() { return std::string(); } + virtual Future> getDesiredCoordinators(Transaction* tr, + vector oldCoordinators, + Reference, + CoordinatorsResult&) = 0; + virtual std::string getDesiredClusterKeyName() const { return std::string(); } }; // Change to use the given set of coordination servers -ACTOR Future changeQuorum(Database cx, Reference change); +ACTOR Future changeQuorum(Database cx, Reference change); Reference autoQuorumChange(int desired = -1); Reference noQuorumChange(); Reference specifiedQuorumChange(vector const&); diff --git a/fdbclient/MetricLogger.actor.cpp b/fdbclient/MetricLogger.actor.cpp index 7b5a16cb97..8d6778545a 100644 --- a/fdbclient/MetricLogger.actor.cpp +++ b/fdbclient/MetricLogger.actor.cpp @@ -171,7 +171,7 @@ ACTOR Future metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri // Implementation of IMetricDB class MetricDB : public IMetricDB { public: - MetricDB(ReadYourWritesTransaction *tr = NULL) : tr(tr) {} + MetricDB(ReadYourWritesTransaction *tr = nullptr) : tr(tr) {} ~MetricDB() {} // levelKey is the prefix for the entire level, no timestamp at the end diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index 1e13b18560..e3ac757840 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -624,8 +624,8 @@ ACTOR Future getClientInfoFromLeader( Referenceget().get().clientInterface.openDatabase.getReply( req ) ) ) ) { TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()) - .detail("MasterProxy0", ni.masterProxies.size() ? ni.masterProxies[0].id() : UID()) - .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) + .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID()) + .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) .detail("ClientID", ni.id); clientData->clientInfo->set(CachedSerialization(ni)); } @@ -681,24 +681,25 @@ ACTOR Future monitorLeaderForProxies( Key clusterKey, vector& lastMasterProxyUIDs, std::vector& lastMasterProxies, - std::vector& lastGrvProxyUIDs, std::vector& lastGrvProxies) { - if(ni.masterProxies.size() > CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS) { - std::vector masterProxyUIDs; - for(auto& masterProxy : ni.masterProxies) { - masterProxyUIDs.push_back(masterProxy.id()); +void shrinkProxyList(ClientDBInfo& ni, std::vector& lastCommitProxyUIDs, + std::vector& lastCommitProxies, std::vector& lastGrvProxyUIDs, + std::vector& lastGrvProxies) { + if (ni.commitProxies.size() > CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS) { + std::vector commitProxyUIDs; + for (auto& commitProxy : ni.commitProxies) { + commitProxyUIDs.push_back(commitProxy.id()); } - if(masterProxyUIDs != lastMasterProxyUIDs) { - lastMasterProxyUIDs.swap(masterProxyUIDs); - lastMasterProxies = ni.masterProxies; - deterministicRandom()->randomShuffle(lastMasterProxies); - lastMasterProxies.resize(CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS); - for(int i = 0; i < lastMasterProxies.size(); i++) { - TraceEvent("ConnectedMasterProxy").detail("MasterProxy", lastMasterProxies[i].id()); + if (commitProxyUIDs != lastCommitProxyUIDs) { + lastCommitProxyUIDs.swap(commitProxyUIDs); + lastCommitProxies = ni.commitProxies; + deterministicRandom()->randomShuffle(lastCommitProxies); + lastCommitProxies.resize(CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS); + for (int i = 0; i < lastCommitProxies.size(); i++) { + TraceEvent("ConnectedCommitProxy").detail("CommitProxy", lastCommitProxies[i].id()); } } - ni.firstProxy = ni.masterProxies[0]; - ni.masterProxies = lastMasterProxies; + ni.firstCommitProxy = ni.commitProxies[0]; + ni.commitProxies = lastCommitProxies; } if(ni.grvProxies.size() > CLIENT_KNOBS->MAX_GRV_PROXY_CONNECTIONS) { std::vector grvProxyUIDs; @@ -719,14 +720,16 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector& lastMasterProxyUIDs, s } // Leader is the process that will be elected by coordinators as the cluster controller -ACTOR Future monitorProxiesOneGeneration( Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, Reference>>> supportedVersions, Key traceLogGroup) { +ACTOR Future monitorProxiesOneGeneration( + Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, + Reference>>> supportedVersions, Key traceLogGroup) { state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString(); state vector addrs = cs.coordinators(); state int idx = 0; state int successIdx = 0; state Optional incorrectTime; - state std::vector lastProxyUIDs; - state std::vector lastProxies; + state std::vector lastCommitProxyUIDs; + state std::vector lastCommitProxies; state std::vector lastGrvProxyUIDs; state std::vector lastGrvProxies; @@ -780,7 +783,7 @@ ACTOR Future monitorProxiesOneGeneration( ReferencenotifyConnected(); auto& ni = rep.get().mutate(); - shrinkProxyList(ni, lastProxyUIDs, lastProxies, lastGrvProxyUIDs, lastGrvProxies); + shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies); clientInfo->set( ni ); successIdx = idx; } else { diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index 58f1fd3bbd..643cf361c7 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -25,7 +25,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #define CLUSTER_FILE_ENV_VAR_NAME "FDB_CLUSTER_FILE" @@ -67,8 +67,9 @@ Future monitorLeaderForProxies( Value const& key, vector c Future monitorProxies( Reference>> const& connFile, Reference> const& clientInfo, Reference>>> const& supportedVersions, Key const& traceLogGroup ); -void shrinkProxyList( ClientDBInfo& ni, std::vector& lastMasterProxyUIDs, std::vector& lastMasterProxies, - std::vector& lastGrvProxyUIDs, std::vector& lastGrvProxies); +void shrinkProxyList(ClientDBInfo& ni, std::vector& lastCommitProxyUIDs, + std::vector& lastCommitProxies, std::vector& lastGrvProxyUIDs, + std::vector& lastGrvProxies); #ifndef __INTEL_COMPILER #pragma region Implementation diff --git a/fdbclient/MultiVersionAssignmentVars.h b/fdbclient/MultiVersionAssignmentVars.h index b4c84f11b9..b270198ecf 100644 --- a/fdbclient/MultiVersionAssignmentVars.h +++ b/fdbclient/MultiVersionAssignmentVars.h @@ -24,8 +24,8 @@ #include "flow/ThreadHelper.actor.h" -template -class AbortableSingleAssignmentVar : public ThreadSingleAssignmentVar, public ThreadCallback { +template +class AbortableSingleAssignmentVar final : public ThreadSingleAssignmentVar, public ThreadCallback { public: AbortableSingleAssignmentVar(ThreadFuture future, ThreadFuture abortSignal) : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) { int userParam; @@ -36,21 +36,21 @@ public: // abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove this callback from abortSignal prematurely abortSignal.callOrSetAsCallback(this, userParam, 0); future.callOrSetAsCallback(this, userParam, 0); - } + } - virtual void cancel() { + void cancel() override { cancelCallbacks(); ThreadSingleAssignmentVar::cancel(); } - virtual void cleanupUnsafe() { + void cleanupUnsafe() override { future.getPtr()->releaseMemory(); ThreadSingleAssignmentVar::cleanupUnsafe(); } - bool canFire(int notMadeActive) { return true; } + bool canFire(int notMadeActive) const override { return true; } - void fire(const Void &unused, int& userParam) { + void fire(const Void& unused, int& userParam) override { lock.enter(); if(!hasBeenSet) { hasBeenSet = true; @@ -74,7 +74,7 @@ public: ThreadSingleAssignmentVar::delref(); } - void error(const Error& e, int& userParam) { + void error(const Error& e, int& userParam) override { ASSERT(future.isError()); lock.enter(); if(!hasBeenSet) { @@ -124,8 +124,8 @@ ThreadFuture abortableFuture(ThreadFuture f, ThreadFuture abortSigna return ThreadFuture(new AbortableSingleAssignmentVar(f, abortSignal)); } -template -class DLThreadSingleAssignmentVar : public ThreadSingleAssignmentVar { +template +class DLThreadSingleAssignmentVar final : public ThreadSingleAssignmentVar { public: DLThreadSingleAssignmentVar(Reference api, FdbCApi::FDBFuture *f, std::function extractValue) : api(api), f(f), extractValue(extractValue), futureRefCount(1) { ThreadSingleAssignmentVar::addref(); @@ -163,13 +163,13 @@ public: if(destroyNow) { api->futureDestroy(f); - f = NULL; + f = nullptr; } return destroyNow; } - virtual void cancel() { + void cancel() override { if(addFutureRef()) { api->futureCancel(f); delFutureRef(); @@ -178,7 +178,7 @@ public: ThreadSingleAssignmentVar::cancel(); } - virtual void cleanupUnsafe() { + void cleanupUnsafe() override { delFutureRef(); ThreadSingleAssignmentVar::cleanupUnsafe(); } @@ -202,7 +202,7 @@ public: auto sav = (DLThreadSingleAssignmentVar*)param; if(MultiVersionApi::api->callbackOnMainThread) { - onMainThreadVoid([sav](){ sav->apply(); }, NULL); + onMainThreadVoid([sav](){ sav->apply(); }, nullptr); } else { sav->apply(); @@ -223,8 +223,8 @@ ThreadFuture toThreadFuture(Reference api, FdbCApi::FDBFuture *f, st return ThreadFuture(new DLThreadSingleAssignmentVar(api, f, extractValue)); } -template -class MapSingleAssignmentVar : public ThreadSingleAssignmentVar, ThreadCallback { +template +class MapSingleAssignmentVar final : public ThreadSingleAssignmentVar, ThreadCallback { public: MapSingleAssignmentVar(ThreadFuture source, std::function(ErrorOr)> mapValue) : source(source), mapValue(mapValue) { ThreadSingleAssignmentVar::addref(); @@ -233,25 +233,25 @@ public: source.callOrSetAsCallback(this, userParam, 0); } - virtual void cancel() { + void cancel() override { source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed source.getPtr()->cancel(); ThreadSingleAssignmentVar::cancel(); } - - virtual void cleanupUnsafe() { + + void cleanupUnsafe() override { source.getPtr()->releaseMemory(); ThreadSingleAssignmentVar::cleanupUnsafe(); } - bool canFire(int notMadeActive) { return true; } + bool canFire(int notMadeActive) const override { return true; } - void fire(const Void &unused, int& userParam) { + void fire(const Void& unused, int& userParam) override { sendResult(mapValue(source.get())); ThreadSingleAssignmentVar::delref(); } - void error(const Error& e, int& userParam) { + void error(const Error& e, int& userParam) override { sendResult(mapValue(source.getError())); ThreadSingleAssignmentVar::delref(); } @@ -275,8 +275,8 @@ ThreadFuture mapThreadFuture(ThreadFuture source, std::function return ThreadFuture(new MapSingleAssignmentVar(source, mapValue)); } -template -class FlatMapSingleAssignmentVar : public ThreadSingleAssignmentVar, ThreadCallback { +template +class FlatMapSingleAssignmentVar final : public ThreadSingleAssignmentVar, ThreadCallback { public: FlatMapSingleAssignmentVar(ThreadFuture source, std::function>(ErrorOr)> mapValue) : source(source), mapValue(mapValue), cancelled(false), released(false) { ThreadSingleAssignmentVar::addref(); @@ -285,7 +285,7 @@ public: source.callOrSetAsCallback(this, userParam, 0); } - virtual void cancel() { + void cancel() override { source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed source.getPtr()->cancel(); @@ -302,8 +302,8 @@ public: ThreadSingleAssignmentVar::cancel(); } - - virtual void cleanupUnsafe() { + + void cleanupUnsafe() override { source.getPtr()->releaseMemory(); lock.enter(); @@ -319,9 +319,9 @@ public: ThreadSingleAssignmentVar::cleanupUnsafe(); } - bool canFire(int notMadeActive) { return true; } + bool canFire(int notMadeActive) const override { return true; } - void fire(const Void &unused, int& userParam) { + void fire(const Void& unused, int& userParam) override { if(mappedFuture.isValid()) { sendResult(mappedFuture.get()); } @@ -332,7 +332,7 @@ public: ThreadSingleAssignmentVar::delref(); } - void error(const Error& e, int& userParam) { + void error(const Error& e, int& userParam) override { if(mappedFuture.isValid()) { sendResult(mappedFuture.getError()); } diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index aebfaeb0eb..2e023bee0d 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -159,6 +159,23 @@ ThreadFuture DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRe }); } +ThreadFuture>> DLTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + if (!api->transactionGetRangeSplitPoints) { + return unsupported_operation(); + } + FdbCApi::FDBFuture* f = api->transactionGetRangeSplitPoints(tr, range.begin.begin(), range.begin.size(), + range.end.begin(), range.end.size(), chunkSize); + + return toThreadFuture>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + const FdbCApi::FDBKey* splitKeys; + int keysArrayLength; + FdbCApi::fdb_error_t error = api->futureGetKeyArray(f, &splitKeys, &keysArrayLength); + ASSERT(!error); + return Standalone>(VectorRef((KeyRef*)splitKeys, keysArrayLength), Arena()); + }); +} + void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ)); } @@ -224,7 +241,7 @@ ThreadFuture DLTransaction::getApproximateSize() { } void DLTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { - throwIfError(api->transactionSetOption(tr, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0)); + throwIfError(api->transactionSetOption(tr, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); } ThreadFuture DLTransaction::onError(Error const& e) { @@ -262,7 +279,7 @@ Reference DLDatabase::createTransaction() { } void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional value) { - throwIfError(api->databaseSetOption(db, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0)); + throwIfError(api->databaseSetOption(db, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); } ThreadFuture DLDatabase::rebootWorker(const ValueRef& value, bool check, uint32_t duration) { @@ -273,7 +290,7 @@ ThreadFuture DLDatabase::rebootWorker(const ValueRef& value, bool check, u template void loadClientFunction(T *fp, void *lib, std::string libPath, const char *functionName, bool requireFunction = true) { *(void**)(fp) = loadFunction(lib, functionName); - if(*fp == NULL && requireFunction) { + if(*fp == nullptr && requireFunction) { TraceEvent(SevError, "ErrorLoadingFunction").detail("LibraryPath", libPath).detail("Function", functionName); throw platform_error(); } @@ -287,7 +304,7 @@ void DLApi::init() { } void* lib = loadLibrary(fdbCPath.c_str()); - if(lib == NULL) { + if(lib == nullptr) { TraceEvent(SevError, "ErrorLoadingExternalClientLibrary").detail("LibraryPath", fdbCPath); throw platform_error(); } @@ -326,12 +343,15 @@ void DLApi::init() { loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel"); loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); + loadClientFunction(&api->transactionGetRangeSplitPoints, lib, fdbCPath, "fdb_transaction_get_range_split_points", + headerVersion >= 700); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); loadClientFunction(&api->futureGetStringArray, lib, fdbCPath, "fdb_future_get_string_array"); + loadClientFunction(&api->futureGetKeyArray, lib, fdbCPath, "fdb_future_get_key_array", headerVersion >= 700); loadClientFunction(&api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array"); loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback"); loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel"); @@ -351,7 +371,7 @@ void DLApi::selectApiVersion(int apiVersion) { init(); throwIfError(api->selectApiVersion(apiVersion, headerVersion)); - throwIfError(api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT, NULL, 0)); + throwIfError(api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT, nullptr, 0)); } const char* DLApi::getClientVersion() { @@ -363,7 +383,7 @@ const char* DLApi::getClientVersion() { } void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional value) { - throwIfError(api->setNetworkOption(option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0)); + throwIfError(api->setNetworkOption(option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); } void DLApi::setupNetwork() { @@ -572,6 +592,14 @@ ThreadFuture MultiVersionTransaction::getEstimatedRangeSizeBytes(const return abortableFuture(f, tr.onChange); } +ThreadFuture>> MultiVersionTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + auto tr = getTransaction(); + auto f = tr.transaction ? tr.transaction->getRangeSplitPoints(range, chunkSize) + : ThreadFuture>>(Never()); + return abortableFuture(f, tr.onChange); +} + void MultiVersionTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) { auto tr = getTransaction(); if(tr.transaction) { @@ -797,7 +825,7 @@ void MultiVersionDatabase::Connector::connect() { else { delref(); } - }, NULL); + }, nullptr); } // Only called from main thread @@ -816,7 +844,7 @@ void MultiVersionDatabase::Connector::fire(const Void &unused, int& userParam) { dbState->stateChanged(); } delref(); - }, NULL); + }, nullptr); } void MultiVersionDatabase::Connector::error(const Error& e, int& userParam) { @@ -831,7 +859,7 @@ void MultiVersionDatabase::Connector::error(const Error& e, int& userParam) { } MultiVersionDatabase::DatabaseState::DatabaseState() - : dbVar(new ThreadSafeAsyncVar>(Reference(NULL))), currentClientIndex(-1) {} + : dbVar(new ThreadSafeAsyncVar>(Reference(nullptr))), currentClientIndex(-1) {} // Only called from main thread void MultiVersionDatabase::DatabaseState::stateChanged() { @@ -909,7 +937,7 @@ void MultiVersionDatabase::DatabaseState::cancelConnections() { connectionAttempts.clear(); clients.clear(); delref(); - }, NULL); + }, nullptr); } // MultiVersionApi @@ -1054,7 +1082,7 @@ void MultiVersionApi::setSupportedClientVersions(Standalone versions) // This option must be set on the main thread because it modifes structures that can be used concurrently by the main thread onMainThreadVoid([this, versions](){ localClient->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions); - }, NULL); + }, nullptr); if(!bypassMultiClientApi) { runOnExternalClients([versions](Reference client) { @@ -1447,18 +1475,18 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing" ) { return Void(); } -class ValidateFuture : public ThreadCallback { +class ValidateFuture final : public ThreadCallback { public: ValidateFuture(ThreadFuture f, ErrorOr expectedValue, std::set legalErrors) : f(f), expectedValue(expectedValue), legalErrors(legalErrors) { } - virtual bool canFire(int notMadeActive) { return true; } + bool canFire(int notMadeActive) const override { return true; } - virtual void fire(const Void &unused, int& userParam) { + void fire(const Void& unused, int& userParam) override { ASSERT(!f.isError() && !expectedValue.isError() && f.get() == expectedValue.get()); delete this; } - virtual void error(const Error& e, int& userParam) { + void error(const Error& e, int& userParam) override { ASSERT(legalErrors.count(e.code()) > 0 || (f.isError() && expectedValue.isError() && f.getError().code() == expectedValue.getError().code())); delete this; } @@ -1665,7 +1693,7 @@ THREAD_FUNC runSingleAssignmentVarTest(void *arg) { onMainThreadVoid([done](){ *done = true; - }, NULL); + }, nullptr); } catch(Error &e) { printf("Caught error in test: %s\n", e.name()); @@ -1704,17 +1732,17 @@ TEST_CASE("/fdbclient/multiversionclient/AbortableSingleAssignmentVar" ) { return Void(); } -class CAPICallback : public ThreadCallback { +class CAPICallback final : public ThreadCallback { public: CAPICallback(void (*callbackf)(FdbCApi::FDBFuture*, void*), FdbCApi::FDBFuture* f, void* userdata) : callbackf(callbackf), f(f), userdata(userdata) {} - virtual bool canFire(int notMadeActive) { return true; } - virtual void fire(const Void& unused, int& userParam) { + bool canFire(int notMadeActive) const override { return true; } + void fire(const Void& unused, int& userParam) override { (*callbackf)(f, userdata); delete this; } - virtual void error(const Error& e, int& userParam) { + void error(const Error& e, int& userParam) override { (*callbackf)(f, userdata); delete this; } diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index 7e23129f29..b231202310 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -35,6 +35,10 @@ struct FdbCApi : public ThreadSafeReferenceCounted { typedef struct transaction FDBTransaction; #pragma pack(push, 4) + typedef struct key { + const uint8_t* key; + int keyLength; + } FDBKey; typedef struct keyvalue { const void *key; int keyLength; @@ -85,7 +89,11 @@ struct FdbCApi : public ThreadSafeReferenceCounted { FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); - + + FDBFuture* (*transactionGetRangeSplitPoints)(FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, + int end_key_name_length, int64_t chunkSize); + FDBFuture* (*transactionCommit)(FDBTransaction *tr); fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion); FDBFuture* (*transactionGetApproximateSize)(FDBTransaction *tr); @@ -104,6 +112,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { fdb_error_t (*futureGetKey)(FDBFuture *f, uint8_t const **outKey, int *outKeyLength); fdb_error_t (*futureGetValue)(FDBFuture *f, fdb_bool_t *outPresent, uint8_t const **outValue, int *outValueLength); fdb_error_t (*futureGetStringArray)(FDBFuture *f, const char ***outStrings, int *outCount); + fdb_error_t (*futureGetKeyArray)(FDBFuture* f, FDBKey const** outKeys, int* outCount); fdb_error_t (*futureGetKeyValueArray)(FDBFuture *f, FDBKeyValue const ** outKV, int *outCount, fdb_bool_t *outMore); fdb_error_t (*futureSetCallback)(FDBFuture *f, FDBCallback callback, void *callback_parameter); void (*futureCancel)(FDBFuture *f); @@ -134,7 +143,9 @@ public: ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; - + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; + void addReadConflictRange(const KeyRangeRef& keys) override; void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; @@ -240,6 +251,8 @@ public: void addReadConflictRange(const KeyRangeRef& keys) override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; void set(const KeyRef& key, const ValueRef& value) override; @@ -289,7 +302,7 @@ struct ClientInfo : ThreadSafeReferenceCounted { bool failed; std::vector> threadCompletionHooks; - ClientInfo() : protocolVersion(0), api(NULL), external(false), failed(true) {} + ClientInfo() : protocolVersion(0), api(nullptr), external(false), failed(true) {} ClientInfo(IClientApi *api) : protocolVersion(0), api(api), libPath("internal"), external(false), failed(false) {} ClientInfo(IClientApi *api, std::string libPath) : protocolVersion(0), api(api), libPath(libPath), external(true), failed(false) {} @@ -299,7 +312,7 @@ struct ClientInfo : ThreadSafeReferenceCounted { class MultiVersionApi; -class MultiVersionDatabase : public IDatabase, ThreadSafeReferenceCounted { +class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted { public: MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference db, bool openConnectors=true); ~MultiVersionDatabase(); @@ -323,9 +336,9 @@ private: void connect(); void cancel(); - bool canFire(int notMadeActive) { return true; } - void fire(const Void &unused, int& userParam); - void error(const Error& e, int& userParam); + bool canFire(int notMadeActive) const override { return true; } + void fire(const Void& unused, int& userParam) override; + void error(const Error& e, int& userParam) override; const Reference client; const std::string clusterFilePath; diff --git a/fdbclient/MutationList.h b/fdbclient/MutationList.h index bcc9b0db76..145c50b0f1 100644 --- a/fdbclient/MutationList.h +++ b/fdbclient/MutationList.h @@ -62,7 +62,7 @@ public: auto e = ptr->end(); // e points to the end of the current blob if (e == blob->data.end()) { // the condition sanity checks e is at the end of current blob blob = blob->next; - e = blob ? blob->data.begin() : NULL; + e = blob ? blob->data.begin() : nullptr; } ptr = (Header*)e; decode(); @@ -70,7 +70,7 @@ public: bool operator == ( Iterator const& i ) const { return ptr == i.ptr; } bool operator != ( Iterator const& i) const { return ptr != i.ptr; } - explicit operator bool() const { return blob!=NULL; } + explicit operator bool() const { return blob!=nullptr; } typedef std::forward_iterator_tag iterator_category; typedef const MutationRef value_type; @@ -79,7 +79,7 @@ public: typedef const MutationRef& reference; Iterator( Blob* blob, const Header* ptr ) : blob(blob), ptr(ptr) { decode(); } - Iterator() : blob(NULL), ptr(NULL) { } + Iterator() : blob(nullptr), ptr(nullptr) { } private: friend struct MutationListRef; const Blob* blob; // The blob containing the indicated mutation @@ -95,16 +95,16 @@ public: } }; - MutationListRef() : blob_begin(NULL), blob_end(NULL), totalBytes(0) { + MutationListRef() : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) { } - MutationListRef( Arena& ar, MutationListRef const& r ) : blob_begin(NULL), blob_end(NULL), totalBytes(0) { + MutationListRef( Arena& ar, MutationListRef const& r ) : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) { append_deep(ar, r.begin(), r.end()); } Iterator begin() const { if (blob_begin) return Iterator(blob_begin, (Header*)blob_begin->data.begin()); - return Iterator(NULL, NULL); + return Iterator(nullptr, nullptr); } - Iterator end() const { return Iterator(NULL, NULL); } + Iterator end() const { return Iterator(nullptr, nullptr); } size_t expectedSize() const { return sizeof(Blob) + totalBytes; } int totalSize() const { return totalBytes; } @@ -146,12 +146,13 @@ public: if(totalBytes > 0) { blob_begin = blob_end = new (ar.arena()) Blob; - blob_begin->next = NULL; + blob_begin->next = nullptr; blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes), totalBytes); // Zero-copy read when deserializing from an ArenaReader } } - //FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also done there + // FIXME: this is re-implemented on the commit proxy to include a yield, any changes to this function should also + // done there template void serialize_save( Ar& ar ) const { serializer(ar, totalBytes); @@ -180,7 +181,7 @@ private: } blob_end->data = StringRef(b, bytes); - blob_end->next = NULL; + blob_end->next = nullptr; return b; } diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index d2be931fa2..3db6f20b8a 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -40,7 +40,7 @@ #include "fdbclient/KeyRangeMap.h" #include "fdbclient/Knobs.h" #include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/MutationList.h" #include "fdbclient/ReadYourWrites.h" @@ -95,7 +95,7 @@ Future loadBalance( DatabaseContext* ctx, const Reference alternatives, RequestStream Interface::*channel, const Request& request = Request(), TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint, bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically - QueueModel* model = NULL) { + QueueModel* model = nullptr) { if (alternatives->hasCaches) { return loadBalance(alternatives->locations(), channel, request, taskID, atMostOnce, model); } @@ -147,7 +147,7 @@ Reference StorageServerInfo::getInterface( DatabaseContext *c } void StorageServerInfo::notifyContextDestroyed() { - cx = NULL; + cx = nullptr; } StorageServerInfo::~StorageServerInfo() { @@ -155,7 +155,7 @@ StorageServerInfo::~StorageServerInfo() { auto it = cx->server_interf.find( interf.id() ); if( it != cx->server_interf.end() ) cx->server_interf.erase( it ); - cx = NULL; + cx = nullptr; } } @@ -189,6 +189,12 @@ std::string printable( const KeyRangeRef& range ) { return printable(range.begin) + " - " + printable(range.end); } +std::string printable(const VectorRef& val) { + std::string s; + for (int i = 0; i < val.size(); i++) s = s + printable(val[i]) + " "; + return s; +} + int unhex( char c ) { if (c >= '0' && c <= '9') return c-'0'; @@ -484,15 +490,15 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext *cx) { } ACTOR static Future monitorProxiesChange(Reference> clientDBInfo, AsyncTrigger *triggerVar) { - state vector< MasterProxyInterface > curProxies; + state vector curCommitProxies; state vector< GrvProxyInterface > curGrvProxies; - curProxies = clientDBInfo->get().masterProxies; + curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; loop{ wait(clientDBInfo->onChange()); - if (clientDBInfo->get().masterProxies != curProxies || clientDBInfo->get().grvProxies != curGrvProxies) { - curProxies = clientDBInfo->get().masterProxies; + if (clientDBInfo->get().commitProxies != curCommitProxies || clientDBInfo->get().grvProxies != curGrvProxies) { + curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; triggerVar->trigger(); } @@ -571,13 +577,6 @@ ACTOR Future updateCachedRanges(DatabaseContext* self, std::map iter->range().begin) { - // self->locationCache.insert( - // KeyRangeRef{ containedRangesEnd, iter->range().begin }, - // Reference{ new LocationInfo{ cacheInterfaces, true } }); - //} containedRangesEnd = iter->range().end; if (iter->value() && !iter->value()->hasCaches) { iter->value() = addCaches(iter->value(), cacheInterfaces); @@ -586,7 +585,8 @@ ACTOR Future updateCachedRanges(DatabaseContext* self, std::maplocationCache.rangeContaining(begin); if (iter->value() && !iter->value()->hasCaches) { if (end>=iter->range().end) { - self->locationCache.insert(KeyRangeRef{ begin, iter->range().end }, + Key endCopy = iter->range().end; // Copy because insertion invalidates iterator + self->locationCache.insert(KeyRangeRef{ begin, endCopy }, addCaches(iter->value(), cacheInterfaces)); } else { self->locationCache.insert(KeyRangeRef{ begin, end }, @@ -595,7 +595,8 @@ ACTOR Future updateCachedRanges(DatabaseContext* self, std::maplocationCache.rangeContainingKeyBefore(end); if (iter->value() && !iter->value()->hasCaches) { - self->locationCache.insert(KeyRangeRef{iter->range().begin, end}, addCaches(iter->value(), cacheInterfaces)); + Key beginCopy = iter->range().begin; // Copy because insertion invalidates iterator + self->locationCache.insert(KeyRangeRef{beginCopy, end}, addCaches(iter->value(), cacheInterfaces)); } } } @@ -881,7 +882,7 @@ DatabaseContext::DatabaseContext(Reference(specialKeys.begin, specialKeys.end, /* test */ false)) { dbId = deterministicRandom()->randomUniqueID(); - connected = (clientInfo->get().masterProxies.size() && clientInfo->get().grvProxies.size()) + connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) ? Void() : clientInfo->onChange(); @@ -928,8 +929,27 @@ DatabaseContext::DatabaseContext(Reference( - KeyRangeRef(LiteralStringRef("inProgressExclusion/"), LiteralStringRef("inProgressExclusion0")) + KeyRangeRef(LiteralStringRef("in_progress_exclusion/"), LiteralStringRef("in_progress_exclusion0")) .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + KeyRangeRef(LiteralStringRef("process/class_type/"), LiteralStringRef("process/class_type0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READONLY, + std::make_unique( + KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique(singleKeyRange(LiteralStringRef("db_locked")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + singleKeyRange(LiteralStringRef("consistency_check_suspended")) + .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin))); } if (apiVersionAtLeast(630)) { registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY, @@ -1164,10 +1184,10 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional(value.get()) : Optional>(), clientLocality.machineId(), clientLocality.dcId() ); - if( clientInfo->get().masterProxies.size() ) - masterProxies = Reference( new ProxyInfo( clientInfo->get().masterProxies) ); - if( clientInfo->get().grvProxies.size() ) - grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies ) ); + if (clientInfo->get().commitProxies.size()) + commitProxies = Reference(new CommitProxyInfo(clientInfo->get().commitProxies, false)); + if( clientInfo->get().grvProxies.size() ) + grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies, true) ); server_interf.clear(); locationCache.insert( allKeys, Reference() ); break; @@ -1176,10 +1196,10 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional(value.get()) : Optional>()); - if( clientInfo->get().masterProxies.size() ) - masterProxies = Reference( new ProxyInfo( clientInfo->get().masterProxies)); - if( clientInfo->get().grvProxies.size() ) - grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies )); + if (clientInfo->get().commitProxies.size()) + commitProxies = Reference( new CommitProxyInfo(clientInfo->get().commitProxies, false)); + if( clientInfo->get().grvProxies.size() ) + grvProxies = Reference( new GrvProxyInfo( clientInfo->get().grvProxies, true)); server_interf.clear(); locationCache.insert( allKeys, Reference() ); break; @@ -1220,13 +1240,13 @@ ACTOR static Future switchConnectionFileImpl(ReferencegetConnectionString().toString()); // Reset state from former cluster. - self->masterProxies.clear(); + self->commitProxies.clear(); self->grvProxies.clear(); self->minAcceptableReadVersion = std::numeric_limits::max(); self->invalidateCache(allKeys); auto clearedClientInfo = self->clientInfo->get(); - clearedClientInfo.masterProxies.clear(); + clearedClientInfo.commitProxies.clear(); clearedClientInfo.grvProxies.clear(); clearedClientInfo.id = deterministicRandom()->randomUniqueID(); self->clientInfo->set(clearedClientInfo); @@ -1307,7 +1327,7 @@ Database Database::createDatabase( Reference connFile, in .detail("PackageName", FDB_VT_PACKAGE_NAME) .detail("ClusterFile", connFile->getFilename().c_str()) .detail("ConnectionString", connFile->getConnectionString().toString()) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) .detail("ApiVersion", apiVersion) .detailf("ImageOffset", "%p", platform::getImageOffset()) .trackLatest("ClientStart"); @@ -1561,29 +1581,29 @@ void stopNetwork() { void DatabaseContext::updateProxies() { if (proxiesLastChange == clientInfo->get().id) return; proxiesLastChange = clientInfo->get().id; - masterProxies.clear(); + commitProxies.clear(); grvProxies.clear(); - bool masterProxyProvisional = false, grvProxyProvisional = false; - if (clientInfo->get().masterProxies.size()) { - masterProxies = Reference(new ProxyInfo(clientInfo->get().masterProxies)); - masterProxyProvisional = clientInfo->get().masterProxies[0].provisional; + bool commitProxyProvisional = false, grvProxyProvisional = false; + if (clientInfo->get().commitProxies.size()) { + commitProxies = Reference(new CommitProxyInfo(clientInfo->get().commitProxies, false)); + commitProxyProvisional = clientInfo->get().commitProxies[0].provisional; } if (clientInfo->get().grvProxies.size()) { - grvProxies = Reference(new GrvProxyInfo(clientInfo->get().grvProxies)); + grvProxies = Reference(new GrvProxyInfo(clientInfo->get().grvProxies, true)); grvProxyProvisional = clientInfo->get().grvProxies[0].provisional; } - if (clientInfo->get().masterProxies.size() && clientInfo->get().grvProxies.size()) { - ASSERT(masterProxyProvisional == grvProxyProvisional); - proxyProvisional = masterProxyProvisional; + if (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) { + ASSERT(commitProxyProvisional == grvProxyProvisional); + proxyProvisional = commitProxyProvisional; } } -Reference DatabaseContext::getMasterProxies(bool useProvisionalProxies) { +Reference DatabaseContext::getCommitProxies(bool useProvisionalProxies) { updateProxies(); if (proxyProvisional && !useProvisionalProxies) { - return Reference(); + return Reference(); } - return masterProxies; + return commitProxies; } Reference DatabaseContext::getGrvProxies(bool useProvisionalProxies) { @@ -1594,19 +1614,19 @@ Reference DatabaseContext::getGrvProxies(bool useProvisionalProxie return grvProxies; } -//Actor which will wait until the MultiInterface returned by the DatabaseContext cx is not NULL -ACTOR Future> getMasterProxiesFuture(DatabaseContext *cx, bool useProvisionalProxies) { +// Actor which will wait until the MultiInterface returned by the DatabaseContext cx is not nullptr +ACTOR Future> getCommitProxiesFuture(DatabaseContext* cx, bool useProvisionalProxies) { loop{ - Reference proxies = cx->getMasterProxies(useProvisionalProxies); - if (proxies) - return proxies; + Reference commitProxies = cx->getCommitProxies(useProvisionalProxies); + if (commitProxies) + return commitProxies; wait( cx->onProxiesChanged() ); } } -//Returns a future which will not be set until the ProxyInfo of this DatabaseContext is not NULL -Future> DatabaseContext::getMasterProxiesFuture(bool useProvisionalProxies) { - return ::getMasterProxiesFuture(this, useProvisionalProxies); +// Returns a future which will not be set until the CommitProxyInfo of this DatabaseContext is not nullptr +Future> DatabaseContext::getCommitProxiesFuture(bool useProvisionalProxies) { + return ::getCommitProxiesFuture(this, useProvisionalProxies); } void GetRangeLimits::decrement( VectorRef const& data ) { @@ -1733,8 +1753,8 @@ ACTOR Future>> getKeyLocation_internal(Da ++cx->transactionKeyServerLocationRequests; choose { when (wait(cx->onProxiesChanged())) {} - when (GetKeyServerLocationsReply rep = wait(basicLoadBalance( - cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, + when(GetKeyServerLocationsReply rep = wait(basicLoadBalance( + cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(span.context, key, Optional(), 100, isBackward, key.arena()), TaskPriority::DefaultPromiseEndpoint))) { ++cx->transactionKeyServerLocationRequestsCompleted; @@ -1782,8 +1802,8 @@ ACTOR Future>>> getKeyRangeLocatio ++cx->transactionKeyServerLocationRequests; choose { when ( wait( cx->onProxiesChanged() ) ) {} - when ( GetKeyServerLocationsReply _rep = wait(basicLoadBalance( - cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, + when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance( + cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(span.context, keys.begin, keys.end, limit, reverse, keys.arena()), TaskPriority::DefaultPromiseEndpoint))) { ++cx->transactionKeyServerLocationRequestsCompleted; @@ -1806,6 +1826,7 @@ ACTOR Future>>> getKeyRangeLocatio } } +// Returns a vector of pairs. template Future< vector< pair> > > getKeyRangeLocations( Database const& cx, KeyRange const& keys, int limit, bool reverse, F StorageServerInterface::*member, TransactionInfo const& info ) { ASSERT (!keys.empty()); @@ -2512,7 +2533,7 @@ ACTOR Future> getRange( Database cx, ReferenceenableLocalityLoadBalance ? &cx->queueModel : NULL)); + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr)); rep = _rep; ++cx->transactionPhysicalReadsCompleted; } catch(Error&) { @@ -2678,7 +2699,7 @@ void debugAddTags(Transaction *tr) { Transaction::Transaction(Database const& cx) : cx(cx), info(cx->taskID, deterministicRandom()->randomUniqueID()), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), committedVersion(invalidVersion), versionstampPromise(Promise>()), options(cx), numErrors(0), - trLogInfo(createTrLogInfoProbabilistically(cx)), span(info.spanID, "Transaction"_loc) { + trLogInfo(createTrLogInfoProbabilistically(cx)), tr(info.spanID), span(info.spanID, "Transaction"_loc) { if (DatabaseContext::debugUseTags) { debugAddTags(this); } @@ -3450,14 +3471,16 @@ ACTOR static Future tryCommit( Database cx, Reference req.debugID = commitID; state Future reply; if (options.commitOnFirstProxy) { - if(cx->clientInfo->get().firstProxy.present()) { - reply = throwErrorOr ( brokenPromiseToMaybeDelivered ( cx->clientInfo->get().firstProxy.get().commit.tryGetReply(req) ) ); + if (cx->clientInfo->get().firstCommitProxy.present()) { + reply = throwErrorOr(brokenPromiseToMaybeDelivered( + cx->clientInfo->get().firstCommitProxy.get().commit.tryGetReply(req))); } else { - const std::vector& proxies = cx->clientInfo->get().masterProxies; + const std::vector& proxies = cx->clientInfo->get().commitProxies; reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never(); } } else { - reply = basicLoadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, true ); + reply = basicLoadBalance(cx->getCommitProxies(info.useProvisionalProxies), &CommitProxyInterface::commit, + req, TaskPriority::DefaultPromiseEndpoint, true); } choose { @@ -3531,8 +3554,9 @@ ACTOR static Future tryCommit( Database cx, Reference // We don't know if the commit happened, and it might even still be in flight. if (!options.causalWriteRisky) { - // Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the version we submitted with is dead, or by committing a conflicting transaction successfully - //if ( cx->getMasterProxies()->masterGeneration <= originalMasterGeneration ) + // Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the + // version we submitted with is dead, or by committing a conflicting transaction successfully + // if ( cx->getCommitProxies()->masterGeneration <= originalMasterGeneration ) // To ensure the original request is not in flight, we need a key range which intersects its read conflict ranges // We pick a key range which also intersects its write conflict ranges, since that avoids potentially creating conflicts where there otherwise would be none @@ -3879,12 +3903,14 @@ ACTOR Future getConsistentReadVersion(SpanID parentSpan, Da TransactionPriority priority, uint32_t flags, TransactionTagMap tags, Optional debugID) { state Span span("NAPI:getConsistentReadVersion"_loc, parentSpan); - try { - ++cx->transactionReadVersionBatches; - if( debugID.present() ) - g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.Before"); - loop { + + ++cx->transactionReadVersionBatches; + if( debugID.present() ) + g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.Before"); + loop { + try { state GetReadVersionRequest req( span.context, transactionCount, priority, flags, tags, debugID ); + choose { when ( wait( cx->onProxiesChanged() ) ) {} when ( GetReadVersionReply v = wait( basicLoadBalance( cx->getGrvProxies(flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES), &GrvProxyInterface::getConsistentReadVersion, req, cx->taskID ) ) ) { @@ -3913,12 +3939,17 @@ ACTOR Future getConsistentReadVersion(SpanID parentSpan, Da return v; } } + } catch (Error& e) { + if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled) + TraceEvent(SevError, "GetConsistentReadVersionError").error(e); + if(e.code() == error_code_batch_transaction_throttled && !cx->apiVersionAtLeast(630)) { + wait(delayJittered(5.0)); + } else { + throw; + } } - } catch (Error& e) { - if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled) - TraceEvent(SevError, "GetConsistentReadVersionError").error(e); - throw; } + } ACTOR Future readVersionBatcher( DatabaseContext *cx, FutureStream versionStream, TransactionPriority priority, uint32_t flags ) { @@ -4320,8 +4351,11 @@ ACTOR Future>> getReadHotRanges(Da // .detail("KeysEnd", keys.end.printable().c_str()); // } state vector> fReplies(nLocs); + KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { - ReadHotSubRangeRequest req(locations[i].first); + partBegin = (i == 0) ? keys.begin : locations[i].first.begin; + partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end; + ReadHotSubRangeRequest req(KeyRangeRef(partBegin, partEnd)); fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getReadHotRanges, req, TaskPriority::DataDistribution); } @@ -4433,8 +4467,8 @@ ACTOR Future>> waitDataDistributionMetricsLis choose { when(wait(cx->onProxiesChanged())) {} when(ErrorOr rep = - wait(errorOr(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::getDDMetrics, - GetDDMetricsRequest(keys, shardLimit))))) { + wait(errorOr(basicLoadBalance(cx->getCommitProxies(false), &CommitProxyInterface::getDDMetrics, + GetDDMetricsRequest(keys, shardLimit))))) { if (rep.isError()) { throw rep.getError(); } @@ -4449,6 +4483,58 @@ Future>> Transaction::getReadHotRa return ::getReadHotRanges(cx, keys); } +ACTOR Future>> getRangeSplitPoints(Database cx, KeyRange keys, int64_t chunkSize) { + state Span span("NAPI:GetRangeSplitPoints"_loc); + loop { + state vector>> locations = + wait(getKeyRangeLocations(cx, keys, 100, false, &StorageServerInterface::getRangeSplitPoints, + TransactionInfo(TaskPriority::DataDistribution, span.context))); + try { + state int nLocs = locations.size(); + state vector> fReplies(nLocs); + KeyRef partBegin, partEnd; + for (int i = 0; i < nLocs; i++) { + partBegin = (i == 0) ? keys.begin : locations[i].first.begin; + partEnd = (i == nLocs - 1) ? keys.end : locations[i].first.end; + SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize); + fReplies[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::getRangeSplitPoints, req, + TaskPriority::DataDistribution); + } + + wait(waitForAll(fReplies)); + Standalone> results; + + results.push_back_deep(results.arena(), keys.begin); + for (int i = 0; i < nLocs; i++) { + if (i > 0) { + results.push_back_deep(results.arena(), locations[i].first.begin); // Need this shard boundary + } + if (fReplies[i].get().splitPoints.size() > 0) { + results.append(results.arena(), fReplies[i].get().splitPoints.begin(), + fReplies[i].get().splitPoints.size()); + results.arena().dependsOn(fReplies[i].get().splitPoints.arena()); + } + } + if (results.back() != keys.end) { + results.push_back_deep(results.arena(), keys.end); + } + + return results; + } catch (Error& e) { + if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { + TraceEvent(SevError, "GetRangeSplitPoints").error(e); + throw; + } + cx->invalidateCache(keys); + wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution)); + } + } +} + +Future>> Transaction::getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize) { + return ::getRangeSplitPoints(cx, keys, chunkSize); +} + ACTOR Future< Standalone> > splitStorageMetrics( Database cx, KeyRange keys, StorageMetrics limit, StorageMetrics estimated ) { state Span span("NAPI:SplitStorageMetrics"_loc); @@ -4539,7 +4625,9 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn loop { choose { when(wait(cx->onProxiesChanged())) {} - when(wait(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::proxySnapReq, ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, true /*atmostOnce*/ ))) { + when(wait(basicLoadBalance(cx->getCommitProxies(false), &CommitProxyInterface::proxySnapReq, + ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, + true /*atmostOnce*/))) { TraceEvent("SnapCreateExit") .detail("SnapCmd", snapCmd.toString()) .detail("UID", snapUID); @@ -4567,8 +4655,8 @@ ACTOR Future checkSafeExclusions(Database cx, vector exc choose { when(wait(cx->onProxiesChanged())) {} when(ExclusionSafetyCheckReply _ddCheck = - wait(basicLoadBalance(cx->getMasterProxies(false), &MasterProxyInterface::exclusionSafetyCheckReq, - req, cx->taskID))) { + wait(basicLoadBalance(cx->getCommitProxies(false), + &CommitProxyInterface::exclusionSafetyCheckReq, req, cx->taskID))) { ddCheck = _ddCheck.safe; break; } diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 2d35022a4a..323d5af550 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -30,7 +30,7 @@ #include "flow/flow.h" #include "flow/TDMetric.actor.h" #include "fdbclient/FDBTypes.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" @@ -268,6 +268,9 @@ public: Future< Standalone> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated ); Future>> getReadHotRanges(KeyRange const& keys); + // Try to split the given range into equally sized chunks based on estimated size. + // The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end] + Future>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize); // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true ); void atomicOp( const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, bool addConflictRange = true ); diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 5693a48ea9..9c30932c71 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1313,7 +1313,8 @@ Future< Standalone > ReadYourWritesTransaction::getRange( bool reverse ) { if (getDatabase()->apiVersionAtLeast(630)) { - if (specialKeys.contains(begin.getKey()) && end.getKey() <= specialKeys.end) { + if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() && + end.getKey() <= specialKeys.end) { TEST(true); // Special key space get range return getDatabase()->specialKeySpace->getRange(this, begin, end, limits, reverse); } @@ -1338,7 +1339,7 @@ Future< Standalone > ReadYourWritesTransaction::getRange( if(begin.getKey() > maxKey || end.getKey() > maxKey) return key_outside_legal_range(); - //This optimization prevents NULL operations from being added to the conflict range + //This optimization prevents nullptr operations from being added to the conflict range if( limits.isReached() ) { TEST(true); // RYW range read limit 0 return Standalone(); @@ -1401,6 +1402,20 @@ Future ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyR return map(waitOrError(tr.getStorageMetrics(keys, -1), resetPromise.getFuture()), [](const StorageMetrics& m) { return m.bytes; }); } +Future>> ReadYourWritesTransaction::getRangeSplitPoints(const KeyRange& range, + int64_t chunkSize) { + if (checkUsedDuringCommit()) { + return used_during_commit(); + } + if (resetPromise.isSet()) return resetPromise.getFuture().getError(); + + KeyRef maxKey = getMaxReadKey(); + if(range.begin > maxKey || range.end > maxKey) + return key_outside_legal_range(); + + return waitOrError(tr.getRangeSplitPoints(range, chunkSize), resetPromise.getFuture()); +} + void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys ) { if(checkUsedDuringCommit()) { throw used_during_commit(); @@ -2053,9 +2068,6 @@ void ReadYourWritesTransaction::setOptionImpl( FDBTransactionOptions::Option opt case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES: validateOptionValue(value, false); options.specialKeySpaceChangeConfiguration = true; - // By default, it allows to read system keys - // More options will be implicitly enabled if needed when doing set or clear - options.readSystemKeys = true; break; default: break; diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index c95327afcb..bf61ee466d 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -86,6 +86,7 @@ public: } [[nodiscard]] Future>> getAddressesForKey(const Key& key); + Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize); Future getEstimatedRangeSizeBytes(const KeyRange& keys); void addReadConflictRange( KeyRangeRef const& keys ); diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h index dde8bd9059..885993cea2 100644 --- a/fdbclient/RestoreWorkerInterface.actor.h +++ b/fdbclient/RestoreWorkerInterface.actor.h @@ -54,6 +54,7 @@ struct RestoreSysInfo; struct RestoreApplierInterface; struct RestoreFinishRequest; struct RestoreSamplesRequest; +struct RestoreUpdateRateRequest; // RestoreSysInfo includes information each (type of) restore roles should know. // At this moment, it only include appliers. We keep the name for future extension. @@ -174,6 +175,7 @@ struct RestoreApplierInterface : RestoreRoleInterface { RequestStream initVersionBatch; RequestStream collectRestoreRoleInterfaces; RequestStream finishRestore; + RequestStream updateRate; bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); } bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); } @@ -193,12 +195,13 @@ struct RestoreApplierInterface : RestoreRoleInterface { initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint); collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint); finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint); + updateRate.getEndpoint(TaskPriority::LoadBalancedEndpoint); } template void serialize(Ar& ar) { serializer(ar, *(RestoreRoleInterface*)this, heartbeat, sendMutationVector, applyToDB, initVersionBatch, - collectRestoreRoleInterfaces, finishRestore); + collectRestoreRoleInterfaces, finishRestore, updateRate); } std::string toString() const { return nodeID.toString(); } @@ -616,6 +619,50 @@ struct RestoreFinishRequest : TimedRequest { } }; +struct RestoreUpdateRateReply : TimedRequest { + constexpr static FileIdentifier file_identifier = 13018414; + + UID id; + double remainMB; // remaining data in MB to write to DB; + + RestoreUpdateRateReply() = default; + explicit RestoreUpdateRateReply(UID id, double remainMB) : id(id), remainMB(remainMB) {} + + std::string toString() const { + std::stringstream ss; + ss << "RestoreUpdateRateReply NodeID:" << id.toString() << " remainMB:" << remainMB; + return ss.str(); + } + + template + void serialize(Ar& ar) { + serializer(ar, id, remainMB); + } +}; + +struct RestoreUpdateRateRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 13018415; + + int batchIndex; + double writeMB; + + ReplyPromise reply; + + RestoreUpdateRateRequest() = default; + explicit RestoreUpdateRateRequest(int batchIndex, double writeMB) : batchIndex(batchIndex), writeMB(writeMB) {} + + template + void serialize(Ar& ar) { + serializer(ar, batchIndex, writeMB, reply); + } + + std::string toString() const { + std::stringstream ss; + ss << "RestoreUpdateRateRequest batchIndex:" << batchIndex << " writeMB:" << writeMB; + return ss.str(); + } +}; + struct RestoreRequest { constexpr static FileIdentifier file_identifier = 16035338; diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 333887d1f3..f3e72d8efd 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -47,7 +47,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "storage", "transaction", "resolution", - "proxy", + "commit_proxy", "grv_proxy", "master", "test", @@ -84,7 +84,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "role":{ "$enum":[ "master", - "proxy", + "commit_proxy", "grv_proxy", "log", "storage", @@ -278,15 +278,20 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "run_loop_busy":0.2 } }, - "old_logs":[ + "logs":[ { - "logs":[ + "log_interfaces":[ { "id":"7f8d623d0cb9966e", "healthy":true, "address":"1.2.3.4:1234" } ], + "epoch":1, + "current":false, + "begin_version":23, + "end_version":112315141, + "possibly_losing_data":true, "log_replication_factor":3, "log_write_anti_quorum":0, "log_fault_tolerance":2, @@ -485,8 +490,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( )statusSchema" R"statusSchema( "recovery_state":{ + "seconds_since_last_recovered":1, "required_resolvers":1, - "required_proxies":1, + "required_commit_proxies":1, "required_grv_proxies":1, "name":{ "$enum":[ @@ -675,11 +681,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "address":"10.0.4.1" } ], - "auto_proxies":3, + "auto_commit_proxies":3, "auto_grv_proxies":1, "auto_resolvers":1, "auto_logs":3, - "proxies":5, + "commit_proxies":5, "grv_proxies":1, "backup_worker_enabled":1 }, @@ -879,11 +885,11 @@ const KeyRef JSONSchemas::clusterConfigurationSchema = LiteralStringRef(R"config "ssd-2", "memory" ]}, - "auto_proxies":3, + "auto_commit_proxies":3, "auto_grv_proxies":1, "auto_resolvers":1, "auto_logs":3, - "proxies":5 + "commit_proxies":5, "grv_proxies":1 })configSchema"); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 79a18cfa1e..a2d234659e 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -36,18 +36,26 @@ std::unordered_map SpecialKeySpace::moduleToB KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) }, { SpecialKeySpace::MODULE::MANAGEMENT, KeyRangeRef(LiteralStringRef("\xff\xff/management/"), LiteralStringRef("\xff\xff/management0")) }, - { SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) } + { SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) }, + { SpecialKeySpace::MODULE::CONFIGURATION, + KeyRangeRef(LiteralStringRef("\xff\xff/configuration/"), LiteralStringRef("\xff\xff/configuration0")) } }; std::unordered_map SpecialKeySpace::managementApiCommandToRange = { { "exclude", KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0")) .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }, { "failed", KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0")) - .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) } + .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }, + { "lock", singleKeyRange(LiteralStringRef("db_locked")).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }, + { "consistencycheck", singleKeyRange(LiteralStringRef("consistency_check_suspended")) + .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) } }; std::set SpecialKeySpace::options = { "excluded/force", "failed/force" }; +Standalone rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, + const Standalone& res); + // This function will move the given KeySelector as far as possible to the standard form: // orEqual == false && offset == 1 (Standard form) // If the corresponding key is not in the underlying key range, it will move over the range @@ -145,7 +153,7 @@ ACTOR Future normalizeKeySelectorActor(SpecialKeySpace* sks, ReadYourWrite if (!ks->isFirstGreaterOrEqual()) { // The Key Selector clamps up to the legal key space - TraceEvent(SevInfo, "ReadToBoundary") + TraceEvent(SevDebug, "ReadToBoundary") .detail("TerminateKey", ks->getKey()) .detail("TerminateOffset", ks->offset); if (ks->offset < 1) @@ -385,13 +393,33 @@ void SpecialKeySpace::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { return impl->clear(ryw, key); } +bool validateSnakeCaseNaming(const KeyRef& k) { + KeyRef key(k); + // Remove prefix \xff\xff + ASSERT(key.startsWith(specialKeys.begin)); + key = key.removePrefix(specialKeys.begin); + // Suffix can be \xff\xff or \x00 in single key range + if (key.endsWith(specialKeys.begin)) + key = key.removeSuffix(specialKeys.end); + else if (key.endsWith(LiteralStringRef("\x00"))) + key = key.removeSuffix(LiteralStringRef("\x00")); + for (const char& c : key.toString()) { + // only small letters, numbers, '/', '_' is allowed + ASSERT((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '/' || c == '_'); + } + return true; +} + void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type, const KeyRangeRef& kr, SpecialKeyRangeReadImpl* impl) { // module boundary check - if (module == SpecialKeySpace::MODULE::TESTONLY) + if (module == SpecialKeySpace::MODULE::TESTONLY) { ASSERT(normalKeys.contains(kr)); - else + } else { ASSERT(moduleToBoundary.at(module).contains(kr)); + ASSERT(validateSnakeCaseNaming(kr.begin) && + validateSnakeCaseNaming(kr.end)); // validate keys follow snake case naming style + } // make sure the registered range is not overlapping with existing ones // Note: kr.end should not be the same as another range's begin, although it should work even they are the same for (auto iter = readImpls.rangeContaining(kr.begin); true; ++iter) { @@ -456,6 +484,24 @@ Future SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) { return commitActor(this, ryw); } +SKSCTestImpl::SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { + ASSERT(range.contains(kr)); + auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY); + // all keys are written to RYW, since GRV is set, the read should happen locally + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); + auto kvs = resultFuture.getValue(); + return rywGetRange(ryw, kr, kvs); +} + +Future> SKSCTestImpl::commit(ReadYourWritesTransaction* ryw) { + ASSERT(false); + return Optional(); +} + ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} ACTOR static Future> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) { @@ -570,86 +616,82 @@ void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const } } -Key ManagementCommandsOptionsImpl::decode(const KeyRef& key) const { - // Should never be used - ASSERT(false); - return key; -} - -Key ManagementCommandsOptionsImpl::encode(const KeyRef& key) const { - // Should never be used - ASSERT(false); - return key; -} - Future> ManagementCommandsOptionsImpl::commit(ReadYourWritesTransaction* ryw) { // Nothing to do, keys should be used by other impls' commit callback return Optional(); } -// read from rwModule -ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, - const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) { - state KeyRangeRef range = impl->getKeyRange(); - Standalone resultWithoutPrefix = - wait(ryw->getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY)); - ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); +Standalone rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, + const Standalone& res) { + // "res" is the read result regardless of your writes, if ryw disabled, return immediately + if (ryw->readYourWritesDisabled()) return res; + // If ryw enabled, we update it with writes from the transaction Standalone result; - if (ryw->readYourWritesDisabled()) { - for (const KeyValueRef& kv : resultWithoutPrefix) - result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); - } else { - RangeMap>, KeyRangeRef>::Ranges ranges = - ryw->getSpecialKeySpaceWriteMap().containedRanges(range); - RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); - int index = 0; - while (iter != ranges.end()) { - // add all previous entries into result - Key rk = impl->encode(resultWithoutPrefix[index].key); - while (index < resultWithoutPrefix.size() && rk < iter->begin()) { - result.push_back_deep(result.arena(), KeyValueRef(rk, resultWithoutPrefix[index].value)); - ++index; - } + RangeMap>, KeyRangeRef>::Ranges ranges = + ryw->getSpecialKeySpaceWriteMap().containedRanges(kr); + RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); + auto iter2 = res.begin(); + result.arena().dependsOn(res.arena()); + while (iter != ranges.end() || iter2 != res.end()) { + if (iter == ranges.end()) { + result.push_back(result.arena(), KeyValueRef(iter2->key, iter2->value)); + ++iter2; + } else if (iter2 == res.end()) { + // insert if it is a set entry std::pair> entry = iter->value(); + if (entry.first && entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); + } + ++iter; + } else if (iter->range().contains(iter2->key)) { + std::pair> entry = iter->value(); + // if this is a valid range either for set or clear, move iter2 outside the range if (entry.first) { - // add the writen entries if exists - if (entry.second.present()) { + // insert if this is a set entry + if (entry.second.present()) result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); - } - // move index to skip all entries in the iter->range - while (index < resultWithoutPrefix.size() && - iter->range().contains(impl->encode(resultWithoutPrefix[index].key))) - ++index; + // move iter2 outside the range + while (iter2 != res.end() && iter->range().contains(iter2->key)) ++iter2; + } + ++iter; + } else if (iter->begin() > iter2->key) { + result.push_back(result.arena(), KeyValueRef(iter2->key, iter2->value)); + ++iter2; + } else if (iter->end() <= iter2->key) { + // insert if it is a set entry + std::pair> entry = iter->value(); + if (entry.first && entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get())); } ++iter; } - // add all remaining entries into result - while (index < resultWithoutPrefix.size()) { - const KeyValueRef& kv = resultWithoutPrefix[index]; - result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); - ++index; - } } return result; } +// read from those readwrite modules in which special keys have one-to-one mapping with real persisted keys +ACTOR Future> rwModuleWithMappingGetRangeActor(ReadYourWritesTransaction* ryw, + const SpecialKeyRangeRWImpl* impl, + KeyRangeRef kr) { + Standalone resultWithoutPrefix = + wait(ryw->getTransaction().getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY)); + ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); + Standalone result; + for (const KeyValueRef& kv : resultWithoutPrefix) + result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value)); + return rywGetRange(ryw, kr, result); +} + ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} Future> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { - return rwModuleGetRangeActor(ryw, this, kr); + return rwModuleWithMappingGetRangeActor(ryw, this, kr); } void ExcludeServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); -} - -void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); -} - -void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { - ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); + // ignore value + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(ValueRef()))); } Key ExcludeServersRangeImpl::decode(const KeyRef& key) const { @@ -671,7 +713,7 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, bool failed, std:: while (iter != ranges.end()) { auto entry = iter->value(); // only check for exclude(set) operation, include(clear) are not checked - TraceEvent(SevInfo, "ParseNetworkAddress") + TraceEvent(SevDebug, "ParseNetworkAddress") .detail("Valid", entry.first) .detail("Set", entry.second.present()) .detail("Key", iter->begin().toString()); @@ -810,7 +852,6 @@ ACTOR Future checkExclusion(Database db, std::vector* ad } void includeServers(ReadYourWritesTransaction* ryw) { - ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); ryw->setOption(FDBTransactionOptions::LOCK_AWARE); ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); @@ -874,19 +915,12 @@ FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRange Future> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { - return rwModuleGetRangeActor(ryw, this, kr); + return rwModuleWithMappingGetRangeActor(ryw, this, kr); } void FailedServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); -} - -void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { - ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); -} - -void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { - ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); + // ignore value + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(ValueRef()))); } Key FailedServersRangeImpl::decode(const KeyRef& key) const { @@ -943,8 +977,14 @@ ACTOR Future> ExclusionInProgressActor(ReadYourWrites } } + // sort and remove :tls + std::set inProgressAddresses; for (auto const& address : inProgressExclusion) { - Key addrKey = prefix.withSuffix(address.toString()); + inProgressAddresses.insert(formatIpPort(address.ip, address.port)); + } + + for (auto const& address : inProgressAddresses) { + Key addrKey = prefix.withSuffix(address); if (kr.contains(addrKey)) { result.push_back(result.arena(), KeyValueRef(addrKey, ValueRef())); result.arena().dependsOn(addrKey.arena()); @@ -959,3 +999,258 @@ Future> ExclusionInProgressRangeImpl::getRange(ReadYo KeyRangeRef kr) const { return ExclusionInProgressActor(ryw, getKeyRange().begin, kr); } + +ACTOR Future> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, + KeyRangeRef kr) { + vector _workers = wait(getWorkers(&ryw->getTransaction())); + auto workers = _workers; // strip const + // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 + std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { + return formatIpPort(lhs.address.ip, lhs.address.port) < formatIpPort(rhs.address.ip, rhs.address.port); + }); + Standalone result; + for (auto& w : workers) { + // exclude :tls in keys even the network addresss is TLS + KeyRef k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port), result.arena())); + if (kr.contains(k)) { + ValueRef v(result.arena(), w.processClass.toString()); + result.push_back(result.arena(), KeyValueRef(k, v)); + } + } + return rywGetRange(ryw, kr, result); +} + +ACTOR Future> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) { + // enable related options + ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + ryw->setOption(FDBTransactionOptions::LOCK_AWARE); + ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); + vector workers = wait( + getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit() + + auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); + auto iter = ranges.begin(); + while (iter != ranges.end()) { + auto entry = iter->value(); + // only loop through (set) operation, (clear) not exist + if (entry.first && entry.second.present()) { + // parse network address + Key address = iter->begin().removePrefix(range.begin); + AddressExclusion addr = AddressExclusion::parse(address); + // parse class type + ValueRef processClassType = entry.second.get(); + ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource); + // make sure we use the underlying Transaction object to avoid used_during_commit() + bool foundChange = false; + for (int i = 0; i < workers.size(); i++) { + if (addr.excludes(workers[i].address)) { + if (processClass.classType() != ProcessClass::InvalidClass) + ryw->getTransaction().set(processClassKeyFor(workers[i].locality.processId().get()), + processClassValue(processClass)); + else + ryw->getTransaction().clear(processClassKeyFor(workers[i].locality.processId().get())); + foundChange = true; + } + } + if (foundChange) + ryw->getTransaction().set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString()); + } + ++iter; + } + return Optional(); +} + +ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + return getProcessClassActor(ryw, getKeyRange().begin, kr); +} + +Future> ProcessClassRangeImpl::commit(ReadYourWritesTransaction* ryw) { + // Validate network address and process class type + Optional errorMsg; + auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(getKeyRange()); + auto iter = ranges.begin(); + while (iter != ranges.end()) { + auto entry = iter->value(); + // only check for setclass(set) operation, (clear) are forbidden thus not exist + if (entry.first && entry.second.present()) { + // validate network address + Key address = iter->begin().removePrefix(range.begin); + AddressExclusion addr = AddressExclusion::parse(address); + if (!addr.isValid()) { + std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n"; + if (address.toString().find(":tls") != std::string::npos) + error += " Do not include the `:tls' suffix when naming a process\n"; + errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); + return errorMsg; + } + // validate class type + ValueRef processClassType = entry.second.get(); + ProcessClass processClass(processClassType.toString(), ProcessClass::DBSource); + if (processClass.classType() == ProcessClass::InvalidClass && + processClassType != LiteralStringRef("default")) { + std::string error = "ERROR: \'" + processClassType.toString() + "\' is not a valid process class\n"; + errorMsg = ManagementAPIError::toJsonString(false, "setclass", error); + return errorMsg; + } + } + ++iter; + } + return processClassCommitActor(ryw, getKeyRange()); +} + +void throwSpecialKeyApiFailure(ReadYourWritesTransaction* ryw, std::string command, std::string message) { + auto msg = ManagementAPIError::toJsonString(false, command, message); + ryw->setSpecialKeySpaceErrorMsg(msg); + throw special_keys_api_failure(); +} + +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { + return throwSpecialKeyApiFailure(ryw, "setclass", "Clear operation is meaningless thus forbidden for setclass"); +} + +void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { + return throwSpecialKeyApiFailure(ryw, "setclass", + "Clear range operation is meaningless thus forbidden for setclass"); +} + +ACTOR Future> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, + KeyRangeRef kr) { + vector _workers = wait(getWorkers(&ryw->getTransaction())); + auto workers = _workers; // strip const + // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 + std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { + return formatIpPort(lhs.address.ip, lhs.address.port) < formatIpPort(rhs.address.ip, rhs.address.port); + }); + Standalone result; + for (auto& w : workers) { + // exclude :tls in keys even the network addresss is TLS + Key k(prefix.withSuffix(formatIpPort(w.address.ip, w.address.port))); + if (kr.contains(k)) { + Value v(w.processClass.sourceString()); + result.push_back(result.arena(), KeyValueRef(k, v)); + result.arena().dependsOn(k.arena()); + result.arena().dependsOn(v.arena()); + } + } + return result; +} + +ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} + +Future> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + return getProcessClassSourceActor(ryw, getKeyRange().begin, kr); +} + +ACTOR Future> getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) { + ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE); + Optional val = wait(ryw->getTransaction().get(databaseLockedKey)); + Standalone result; + if (val.present()) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, val.get())); + } + return result; +} + +LockDatabaseImpl::LockDatabaseImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { + // single key range, the queried range should always be the same as the underlying range + ASSERT(kr == getKeyRange()); + auto lockEntry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")]; + if (!ryw->readYourWritesDisabled() && lockEntry.first) { + // ryw enabled and we have written to the special key + Standalone result; + if (lockEntry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, lockEntry.second.get())); + } + return result; + } else { + return getLockedKeyActor(ryw, kr); + } +} + +ACTOR Future> lockDatabaseCommitActor(ReadYourWritesTransaction* ryw) { + state Optional msg; + ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE); + Optional val = wait(ryw->getTransaction().get(databaseLockedKey)); + UID uid = deterministicRandom()->randomUniqueID(); + + if (val.present() && BinaryReader::fromStringRef(val.get().substr(10), Unversioned()) != uid) { + // check database not locked + // if locked already, throw error + msg = ManagementAPIError::toJsonString(false, "lock", "Database has already been locked"); + } else if (!val.present()) { + // lock database + ryw->getTransaction().atomicOp(databaseLockedKey, + BinaryWriter::toValue(uid, Unversioned()) + .withPrefix(LiteralStringRef("0123456789")) + .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), + MutationRef::SetVersionstampedValue); + ryw->getTransaction().addWriteConflictRange(normalKeys); + } + + return msg; +} + +ACTOR Future> unlockDatabaseCommitActor(ReadYourWritesTransaction* ryw) { + ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE); + Optional val = wait(ryw->getTransaction().get(databaseLockedKey)); + if (val.present()) { + ryw->getTransaction().clear(singleKeyRange(databaseLockedKey)); + } + return Optional(); +} + +Future> LockDatabaseImpl::commit(ReadYourWritesTransaction* ryw) { + auto lockId = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")].second; + if (lockId.present()) { + return lockDatabaseCommitActor(ryw); + } else { + return unlockDatabaseCommitActor(ryw); + } +} + +ACTOR Future> getConsistencyCheckKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) { + ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE); + ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + Optional val = wait(ryw->getTransaction().get(fdbShouldConsistencyCheckBeSuspended)); + bool ccSuspendSetting = val.present() ? BinaryReader::fromStringRef(val.get(), Unversioned()) : false; + Standalone result; + if (ccSuspendSetting) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, ValueRef())); + } + return result; +} + +ConsistencyCheckImpl::ConsistencyCheckImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> ConsistencyCheckImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + // single key range, the queried range should always be the same as the underlying range + ASSERT(kr == getKeyRange()); + auto entry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")]; + if (!ryw->readYourWritesDisabled() && entry.first) { + // ryw enabled and we have written to the special key + Standalone result; + if (entry.second.present()) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, entry.second.get())); + } + return result; + } else { + return getConsistencyCheckKeyActor(ryw, kr); + } +} + +Future> ConsistencyCheckImpl::commit(ReadYourWritesTransaction* ryw) { + auto entry = + ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")].second; + ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE); + ryw->getTransaction().set(fdbShouldConsistencyCheckBeSuspended, + BinaryWriter::toValue(entry.present(), Unversioned())); + return Optional(); +} diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 4cbc9c5002..1c23f9ad7f 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -67,15 +67,29 @@ private: class SpecialKeyRangeRWImpl : public SpecialKeyRangeReadImpl { public: - virtual void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) = 0; - virtual void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) = 0; - virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) = 0; + virtual void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); + } + virtual void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { + ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); + } + virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); + } virtual Future> commit( ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space // Given the special key to write, return the real key that needs to be modified - virtual Key decode(const KeyRef& key) const = 0; + virtual Key decode(const KeyRef& key) const { + // Default implementation should never be used + ASSERT(false); + return key; + } // Given the read key, return the corresponding special key - virtual Key encode(const KeyRef& key) const = 0; + virtual Key encode(const KeyRef& key) const { + // Default implementation should never be used + ASSERT(false); + return key; + }; explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} @@ -125,6 +139,7 @@ class SpecialKeySpace { public: enum class MODULE { CLUSTERFILEPATH, + CONFIGURATION, // Configuration of the cluster CONNECTIONSTRING, ERRORMSG, // A single key space contains a json string which describes the last error in special-key-space MANAGEMENT, // Management-API @@ -201,6 +216,14 @@ private: void modulesBoundaryInit(); }; +// Used for SpecialKeySpaceCorrectnessWorkload +class SKSCTestImpl : public SpecialKeyRangeRWImpl { +public: + explicit SKSCTestImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future> commit(ReadYourWritesTransaction* ryw) override; +}; + // Use special key prefix "\xff\xff/transaction/conflicting_keys/", // to retrieve keys which caused latest not_committed(conflicting with another transaction) error. // The returned key value pairs are interpretted as : @@ -238,8 +261,6 @@ public: void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; - Key decode(const KeyRef& key) const override; - Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; }; @@ -248,8 +269,6 @@ public: explicit ExcludeServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; @@ -260,8 +279,6 @@ public: explicit FailedServersRangeImpl(KeyRangeRef kr); Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; - void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; Key decode(const KeyRef& key) const override; Key encode(const KeyRef& key) const override; Future> commit(ReadYourWritesTransaction* ryw) override; @@ -273,5 +290,34 @@ public: Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; }; +class ProcessClassRangeImpl : public SpecialKeyRangeRWImpl { +public: + explicit ProcessClassRangeImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future> commit(ReadYourWritesTransaction* ryw) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; +}; + +class ProcessClassSourceRangeImpl : public SpecialKeyRangeReadImpl { +public: + explicit ProcessClassSourceRangeImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; +}; + +class LockDatabaseImpl : public SpecialKeyRangeRWImpl { +public: + explicit LockDatabaseImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future> commit(ReadYourWritesTransaction* ryw) override; +}; + +class ConsistencyCheckImpl : public SpecialKeyRangeRWImpl { +public: + explicit ConsistencyCheckImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future> commit(ReadYourWritesTransaction* ryw) override; +}; + #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index af06453129..bc340727fd 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -72,6 +72,8 @@ struct StorageServerInterface { RequestStream> getKeyValueStoreType; RequestStream watchValue; RequestStream getReadHotRanges; + RequestStream getRangeSplitPoints; + explicit StorageServerInterface(UID uid) : uniqueID( uid ) {} StorageServerInterface() : uniqueID( deterministicRandom()->randomUniqueID() ) {} NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } @@ -98,6 +100,7 @@ struct StorageServerInterface { getKeyValueStoreType = RequestStream>( getValue.getEndpoint().getAdjustedEndpoint(9) ); watchValue = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(10) ); getReadHotRanges = RequestStream( getValue.getEndpoint().getAdjustedEndpoint(11) ); + getRangeSplitPoints = RequestStream(getValue.getEndpoint().getAdjustedEndpoint(12)); } } else { ASSERT(Ar::isDeserializing); @@ -125,6 +128,7 @@ struct StorageServerInterface { streams.push_back(getKeyValueStoreType.getReceiver()); streams.push_back(watchValue.getReceiver()); streams.push_back(getReadHotRanges.getReceiver()); + streams.push_back(getRangeSplitPoints.getReceiver()); FlowTransport::transport().addEndpoints(streams); } }; @@ -479,6 +483,34 @@ struct ReadHotSubRangeRequest { } }; +struct SplitRangeReply { + constexpr static FileIdentifier file_identifier = 11813134; + // If the given range can be divided, contains the split points. + // If the given range cannot be divided(for exmaple its total size is smaller than the chunk size), this would be + // empty + Standalone> splitPoints; + + template + void serialize(Ar& ar) { + serializer(ar, splitPoints); + } +}; +struct SplitRangeRequest { + constexpr static FileIdentifier file_identifier = 10725174; + Arena arena; + KeyRangeRef keys; + int64_t chunkSize; + ReplyPromise reply; + + SplitRangeRequest() {} + SplitRangeRequest(KeyRangeRef const& keys, int64_t chunkSize) : keys(arena, keys), chunkSize(chunkSize) {} + + template + void serialize(Ar& ar) { + serializer(ar, keys, chunkSize, reply, arena); + } +}; + struct GetStorageMetricsReply { constexpr static FileIdentifier file_identifier = 15491478; StorageMetrics load; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index a9bb73fae6..b402ad99a7 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1060,3 +1060,7 @@ const KeyRangeRef testOnlyTxnStateStorePrefixRange( LiteralStringRef("\xff/TESTONLYtxnStateStore/"), LiteralStringRef("\xff/TESTONLYtxnStateStore0") ); + +const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery"); +const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1"); +const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 08bfb6ff88..20091a8045 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -260,10 +260,10 @@ extern const KeyRangeRef logRangesRange; Key logRangesEncodeKey(KeyRef keyBegin, UID logUid); // Returns the start key and optionally the logRange Uid -KeyRef logRangesDecodeKey(KeyRef key, UID* logUid = NULL); +KeyRef logRangesDecodeKey(KeyRef key, UID* logUid = nullptr); // Returns the end key and optionally the key prefix -Key logRangesDecodeValue(KeyRef keyValue, Key* destKeyPrefix = NULL); +Key logRangesDecodeValue(KeyRef keyValue, Key* destKeyPrefix = nullptr); // Returns the encoded key value comprised of the end key and destination prefix Key logRangesEncodeValue(KeyRef keyEnd, KeyRef destPath); @@ -396,6 +396,11 @@ std::pair decodeHealthyZoneValue( ValueRef const& ); // Used to create artifically large txnStateStore instances in testing. extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; +// Snapshot + Incremental Restore +extern const KeyRef writeRecoveryKey; +extern const ValueRef writeRecoveryKeyTrue; +extern const KeyRef snapshotEndVersionKey; + #pragma clang diagnostic pop #endif diff --git a/fdbclient/TagThrottle.actor.cpp b/fdbclient/TagThrottle.actor.cpp index 664e9a352d..224f4839ce 100644 --- a/fdbclient/TagThrottle.actor.cpp +++ b/fdbclient/TagThrottle.actor.cpp @@ -19,7 +19,7 @@ */ #include "fdbclient/TagThrottle.h" -#include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/CommitProxyInterface.h" #include "fdbclient/DatabaseContext.h" #include "flow/actorcompiler.h" // has to be last include diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.actor.cpp index b7cb3bf3fd..592c30dd5d 100644 --- a/fdbclient/TaskBucket.actor.cpp +++ b/fdbclient/TaskBucket.actor.cpp @@ -69,7 +69,7 @@ REGISTER_TASKFUNC(AddTaskFunc); struct IdleTaskFunc : TaskFuncBase { static StringRef name; - static const uint32_t version = 1; + static constexpr uint32_t version = 1; StringRef getName() const { return name; }; Future execute(Database cx, Reference tb, Reference fb, Reference task) { return Void(); }; @@ -1242,6 +1242,6 @@ ACTOR Future getCompletionKey(TaskCompletionKey *self, Future TaskCompletionKey::get(Reference tr, Reference taskBucket) { - ASSERT(key.present() == (joinFuture.getPtr() == NULL)); + ASSERT(key.present() == (joinFuture.getPtr() == nullptr)); return key.present() ? key.get() : getCompletionKey(this, joinFuture->joinedFuture(tr, taskBucket)); } diff --git a/fdbclient/TaskBucket.h b/fdbclient/TaskBucket.h index 8af5f777ba..1470b16678 100644 --- a/fdbclient/TaskBucket.h +++ b/fdbclient/TaskBucket.h @@ -256,7 +256,7 @@ public: return pauseKey; } - Subspace getAvailableSpace(int priority = 0) { + Subspace getAvailableSpace(int priority = 0) const { if(priority == 0) return available; return available_prioritized.get(priority); diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 1a20ff1abd..aa8a00e119 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -91,12 +91,12 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) catch(...) { new (db) DatabaseContext(unknown_error()); } - }, NULL); + }, nullptr); } ThreadSafeDatabase::~ThreadSafeDatabase() { DatabaseContext *db = this->db; - onMainThreadVoid( [db](){ db->delref(); }, NULL ); + onMainThreadVoid( [db](){ db->delref(); }, nullptr ); } ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx) { @@ -114,18 +114,18 @@ ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx) { cx->addref(); new (tr) ReadYourWritesTransaction(Database(cx)); }, - NULL); + nullptr); } ThreadSafeTransaction::~ThreadSafeTransaction() { ReadYourWritesTransaction *tr = this->tr; if (tr) - onMainThreadVoid( [tr](){ tr->delref(); }, NULL ); + onMainThreadVoid( [tr](){ tr->delref(); }, nullptr ); } void ThreadSafeTransaction::cancel() { ReadYourWritesTransaction *tr = this->tr; - onMainThreadVoid( [tr](){ tr->cancel(); }, NULL ); + onMainThreadVoid( [tr](){ tr->cancel(); }, nullptr ); } void ThreadSafeTransaction::setVersion( Version v ) { @@ -171,6 +171,16 @@ ThreadFuture ThreadSafeTransaction::getEstimatedRangeSizeBytes( const K } ); } +ThreadFuture>> ThreadSafeTransaction::getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) { + KeyRange r = range; + + ReadYourWritesTransaction* tr = this->tr; + return onMainThread([tr, r, chunkSize]() -> Future>> { + tr->checkDeferredError(); + return tr->getRangeSplitPoints(r, chunkSize); + }); +} ThreadFuture< Standalone > ThreadSafeTransaction::getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse ) { KeySelector b = begin; @@ -335,17 +345,17 @@ ThreadFuture ThreadSafeTransaction::onError( Error const& e ) { void ThreadSafeTransaction::operator=(ThreadSafeTransaction&& r) noexcept { tr = r.tr; - r.tr = NULL; + r.tr = nullptr; } ThreadSafeTransaction::ThreadSafeTransaction(ThreadSafeTransaction&& r) noexcept { tr = r.tr; - r.tr = NULL; + r.tr = nullptr; } void ThreadSafeTransaction::reset() { ReadYourWritesTransaction *tr = this->tr; - onMainThreadVoid( [tr](){ tr->reset(); }, NULL ); + onMainThreadVoid( [tr](){ tr->reset(); }, nullptr ); } extern const char* getSourceVersion(); diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index add1d1212b..c2acb08355 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -74,6 +74,8 @@ public: ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; + ThreadFuture>> getRangeSplitPoints(const KeyRangeRef& range, + int64_t chunkSize) override; void addReadConflictRange( const KeyRangeRef& keys ) override; void makeSelfConflicting(); @@ -98,7 +100,7 @@ public: ThreadFuture onError( Error const& e ) override; // These are to permit use as state variables in actors: - ThreadSafeTransaction() : tr(NULL) {} + ThreadSafeTransaction() : tr(nullptr) {} void operator=(ThreadSafeTransaction&& r) noexcept; ThreadSafeTransaction(ThreadSafeTransaction&& r) noexcept; diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h index 2208224857..03732d0e34 100644 --- a/fdbclient/VersionedMap.h +++ b/fdbclient/VersionedMap.h @@ -802,7 +802,7 @@ public: void validate() { int count=0, height=0; - PTreeImpl::validate>>( root, at, NULL, NULL, count, height ); + PTreeImpl::validate>>( root, at, nullptr, nullptr, count, height ); if ( height > 100 ) TraceEvent(SevWarnAlways, "DiabolicalPTreeSize").detail("Size", count).detail("Height", height); } diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index f11956d79c..37e57346ee 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -195,7 +195,7 @@ description is not currently required but encouraged.