diff --git a/.gitignore b/.gitignore index bb16f145de..4edd5690c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Build artifacts +/my_build/ /bin/ /lib/ /packages/ @@ -83,6 +84,7 @@ ipch/ compile_commands.json flow/actorcompiler/obj flow/coveragetool/obj +*.code-workspace # IDE indexing (commonly used tools) /compile_commands.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 08df8edfe0..8ff0c2c704 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.13) +if(WIN32) + cmake_minimum_required(VERSION 3.15) +else() + cmake_minimum_required(VERSION 3.13) +endif() + project(foundationdb VERSION 7.1.0 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." @@ -196,9 +201,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CUR if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json - COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py + COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json COMMENT "Build compile commands for IDE" ) add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json) diff --git a/FDBLibTLS/FDBLibTLSPolicy.cpp b/FDBLibTLS/FDBLibTLSPolicy.cpp index 1f6b18e2e9..2e3142165d 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.cpp +++ b/FDBLibTLS/FDBLibTLSPolicy.cpp @@ -42,7 +42,7 @@ FDBLibTLSPolicy::FDBLibTLSPolicy(Reference plugin) key_data_set(false), verify_peers_set(false) { if ((tls_cfg = tls_config_new()) == nullptr) { - TraceEvent(SevError, "FDBLibTLSConfigError"); + TraceEvent(SevError, "FDBLibTLSConfigError").log(); throw std::runtime_error("FDBLibTLSConfigError"); } @@ -67,14 +67,14 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, // servername, since this will be ignored - the servername should be // matched by the verify criteria instead. if (verify_peers_set && servername != nullptr) { - TraceEvent(SevError, "FDBLibTLSVerifyPeersWithServerName"); + TraceEvent(SevError, "FDBLibTLSVerifyPeersWithServerName").log(); return nullptr; } // If verify peers has not been set, then require a server name to // avoid an accidental lack of name validation. if (!verify_peers_set && servername == nullptr) { - TraceEvent(SevError, "FDBLibTLSNoServerName"); + TraceEvent(SevError, "FDBLibTLSNoServerName").log(); return nullptr; } } @@ -123,18 +123,18 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s if (cert_pem_len > INT_MAX) goto err; if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == nullptr) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } if ((certs = sk_X509_new_null()) == nullptr) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } ERR_clear_error(); while ((cert = PEM_read_bio_X509(bio, nullptr, password_cb, nullptr)) != nullptr) { if (!sk_X509_push(certs, cert)) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } } @@ -150,7 +150,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s } if (sk_X509_num(certs) < 1) { - TraceEvent(SevError, "FDBLibTLSNoCerts"); + TraceEvent(SevError, "FDBLibTLSNoCerts").log(); goto err; } @@ -168,11 +168,11 @@ err: bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) { if (ca_data_set) { - TraceEvent(SevError, "FDBLibTLSCAAlreadySet"); + TraceEvent(SevError, "FDBLibTLSCAAlreadySet").log(); return false; } if (session_created) { - TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive"); + TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log(); return false; } @@ -194,11 +194,11 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) { bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) { if (cert_data_set) { - TraceEvent(SevError, "FDBLibTLSCertAlreadySet"); + TraceEvent(SevError, "FDBLibTLSCertAlreadySet").log(); return false; } if (session_created) { - TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive"); + TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log(); return false; } @@ -218,11 +218,11 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c bool rc = false; if (key_data_set) { - TraceEvent(SevError, "FDBLibTLSKeyAlreadySet"); + TraceEvent(SevError, "FDBLibTLSKeyAlreadySet").log(); goto err; } if (session_created) { - TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive"); + TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log(); goto err; } @@ -231,7 +231,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c long len; if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } ERR_clear_error(); @@ -241,7 +241,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) || (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { - TraceEvent(SevError, "FDBLibTLSIncorrectPassword"); + TraceEvent(SevError, "FDBLibTLSIncorrectPassword").log(); } else { ERR_error_string_n(errnum, errbuf, sizeof(errbuf)); TraceEvent(SevError, "FDBLibTLSPrivateKeyError").detail("LibcryptoErrorMessage", errbuf); @@ -250,15 +250,15 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c } BIO_free(bio); if ((bio = BIO_new(BIO_s_mem())) == nullptr) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } if (!PEM_write_bio_PrivateKey(bio, key, nullptr, nullptr, 0, nullptr, nullptr)) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } if ((len = BIO_get_mem_data(bio, &data)) <= 0) { - TraceEvent(SevError, "FDBLibTLSOutOfMemory"); + TraceEvent(SevError, "FDBLibTLSOutOfMemory").log(); goto err; } if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) { @@ -283,16 +283,16 @@ err: bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[], int verify_peers_len[]) { if (verify_peers_set) { - TraceEvent(SevError, "FDBLibTLSVerifyPeersAlreadySet"); + TraceEvent(SevError, "FDBLibTLSVerifyPeersAlreadySet").log(); return false; } if (session_created) { - TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive"); + TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log(); return false; } if (count < 1) { - TraceEvent(SevError, "FDBLibTLSNoVerifyPeers"); + TraceEvent(SevError, "FDBLibTLSNoVerifyPeers").log(); return false; } diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp index 4c4c8e5bfa..75c60dd049 100644 --- a/FDBLibTLS/FDBLibTLSSession.cpp +++ b/FDBLibTLS/FDBLibTLSSession.cpp @@ -73,7 +73,7 @@ FDBLibTLSSession::FDBLibTLSSession(Reference policy, if (is_client) { if ((tls_ctx = tls_client()) == nullptr) { - TraceEvent(SevError, "FDBLibTLSClientError", uid); + TraceEvent(SevError, "FDBLibTLSClientError", uid).log(); throw std::runtime_error("FDBLibTLSClientError"); } if (tls_configure(tls_ctx, policy->tls_cfg) == -1) { @@ -88,7 +88,7 @@ FDBLibTLSSession::FDBLibTLSSession(Reference policy, } } else { if ((tls_sctx = tls_server()) == nullptr) { - TraceEvent(SevError, "FDBLibTLSServerError", uid); + TraceEvent(SevError, "FDBLibTLSServerError", uid).log(); throw std::runtime_error("FDBLibTLSServerError"); } if (tls_configure(tls_sctx, policy->tls_cfg) == -1) { @@ -250,7 +250,7 @@ std::tuple FDBLibTLSSession::check_verify(Referenceparse_cert_pem(cert_pem, cert_pem_len)) == nullptr) @@ -388,14 +388,14 @@ int FDBLibTLSSession::handshake() { int FDBLibTLSSession::read(uint8_t* data, int length) { if (!handshake_completed) { - TraceEvent(SevError, "FDBLibTLSReadHandshakeError"); + TraceEvent(SevError, "FDBLibTLSReadHandshakeError").log(); return FAILED; } ssize_t n = tls_read(tls_ctx, data, length); if (n > 0) { if (n > INT_MAX) { - TraceEvent(SevError, "FDBLibTLSReadOverflow"); + TraceEvent(SevError, "FDBLibTLSReadOverflow").log(); return FAILED; } return (int)n; @@ -415,14 +415,14 @@ int FDBLibTLSSession::read(uint8_t* data, int length) { int FDBLibTLSSession::write(const uint8_t* data, int length) { if (!handshake_completed) { - TraceEvent(SevError, "FDBLibTLSWriteHandshakeError", uid); + TraceEvent(SevError, "FDBLibTLSWriteHandshakeError", uid).log(); return FAILED; } ssize_t n = tls_write(tls_ctx, data, length); if (n > 0) { if (n > INT_MAX) { - TraceEvent(SevError, "FDBLibTLSWriteOverflow", uid); + TraceEvent(SevError, "FDBLibTLSWriteOverflow", uid).log(); return FAILED; } return (int)n; diff --git a/README.md b/README.md index 9e0ddb78a5..053cc4a8e1 100755 --- a/README.md +++ b/README.md @@ -30,7 +30,9 @@ Developers interested in using FoundationDB can get started by downloading and i Developers on an OS for which there is no binary package, or who would like to start hacking on the code, can get started by compiling from source. -The official docker image for building is `foundationdb/foundationdb-build`. It has all dependencies installed. To build outside the official docker image you'll need at least these dependencies: +The official docker image for building is [`foundationdb/build`](https://hub.docker.com/r/foundationdb/build) which has all dependencies installed. The Docker image definitions used by FoundationDB team members can be found in the [dedicated repository.](https://github.com/FoundationDB/fdb-build-support). + +To build outside the official docker image you'll need at least these dependencies: 1. Install cmake Version 3.13 or higher [CMake](https://cmake.org/) 1. Install [Mono](http://www.mono-project.com/download/stable/) @@ -77,7 +79,7 @@ describe the actor compiler source file, not the post-processed output files, and places the output file in the source directory. This file should then be picked up automatically by any tooling. -Note that if building inside of the `foundationdb/foundationdb-build` docker +Note that if building inside of the `foundationdb/build` docker image, the resulting paths will still be incorrect and require manual fixing. One will wish to re-run `cmake` with `-DCMAKE_EXPORT_COMPILE_COMMANDS=OFF` to prevent it from reverting the manual changes. @@ -138,7 +140,7 @@ You should create a second build-directory which you will use for building and d ### Linux There are no special requirements for Linux. A docker image can be pulled from -`foundationdb/foundationdb-build` that has all of FoundationDB's dependencies +`foundationdb/build` that has all of FoundationDB's dependencies pre-installed, and is what the CI uses to build and test PRs. ``` diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index 561ab8d740..be4caf8240 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -79,6 +79,7 @@ if(NOT WIN32) test/unit/fdb_api.hpp) set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp) + set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp) if(OPEN_FOR_IDE) add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h) @@ -88,6 +89,7 @@ if(NOT WIN32) add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp) add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS}) add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS}) + add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) else() add_executable(fdb_c_performance_test test/performance_test.c test/test.h) add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h) @@ -96,6 +98,7 @@ if(NOT WIN32) add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp) add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS}) add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS}) + add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) strip_debug_symbols(fdb_c_performance_test) strip_debug_symbols(fdb_c_ryw_benchmark) strip_debug_symbols(fdb_c_txn_size_test) @@ -106,12 +109,14 @@ if(NOT WIN32) add_dependencies(fdb_c_setup_tests doctest) add_dependencies(fdb_c_unit_tests doctest) + add_dependencies(fdb_c_unit_tests_version_510 doctest) target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR}) target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads) target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads) target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads) + target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads) # do not set RPATH for mako set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE) @@ -146,6 +151,11 @@ if(NOT WIN32) COMMAND $ @CLUSTER_FILE@ fdb) + add_fdbclient_test( + NAME trace_partial_file_suffix_test + COMMAND $ + @CLUSTER_FILE@ + fdb) add_fdbclient_test( NAME fdb_c_external_client_unit_tests COMMAND $ diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index 16fbddf1c9..ecb78e4df7 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -162,7 +162,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) { return TSAVB(f)->isReady(); } -class CAPICallback : public ThreadCallback { +class CAPICallback final : public ThreadCallback { public: CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata) : callbackf(callbackf), f(f), userdata(userdata) {} diff --git a/bindings/c/foundationdb/ClientWorkload.h b/bindings/c/foundationdb/ClientWorkload.h index 0b785e1f31..d04ed68abd 100644 --- a/bindings/c/foundationdb/ClientWorkload.h +++ b/bindings/c/foundationdb/ClientWorkload.h @@ -66,6 +66,7 @@ public: }; struct FDBPromise { + virtual ~FDBPromise() = default; virtual void send(void*) = 0; }; diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index 661b99c5dc..a03f192808 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -1056,12 +1056,12 @@ void* worker_thread(void* thread_args) { } fprintf(debugme, - "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", + "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%lld)\n", worker_id, args->num_processes, thread_id, args->num_threads, - (unsigned int)pthread_self()); + (uint64_t)pthread_self()); if (args->tpsmax) { thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads); diff --git a/bindings/c/test/unit/trace_partial_file_suffix_test.cpp b/bindings/c/test/unit/trace_partial_file_suffix_test.cpp new file mode 100644 index 0000000000..87990ed43c --- /dev/null +++ b/bindings/c/test/unit/trace_partial_file_suffix_test.cpp @@ -0,0 +1,111 @@ +/* + * trace_partial_file_suffix_test.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "flow/Platform.h" + +#define FDB_API_VERSION 710 +#include "foundationdb/fdb_c.h" + +#undef NDEBUG +#include + +void fdb_check(fdb_error_t e) { + if (e) { + std::cerr << fdb_get_error(e) << std::endl; + std::abort(); + } +} + +void set_net_opt(FDBNetworkOption option, const std::string& value) { + fdb_check(fdb_network_set_option(option, reinterpret_cast(value.c_str()), value.size())); +} + +bool file_exists(const char* path) { + FILE* f = fopen(path, "r"); + if (f) { + fclose(f); + return true; + } + return false; +} + +int main(int argc, char** argv) { + fdb_check(fdb_select_api_version(710)); + + std::string file_identifier = "trace_partial_file_suffix_test" + std::to_string(std::random_device{}()); + std::string trace_partial_file_suffix = ".tmp"; + std::string simulated_stray_partial_file = + "trace.127.0.0.1." + file_identifier + ".simulated.xml" + trace_partial_file_suffix; + + // Simulate this process crashing previously by creating a ".tmp" file + { std::ofstream file{ simulated_stray_partial_file }; } + + set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE, ""); + set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_FILE_IDENTIFIER, file_identifier); + set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_PARTIAL_FILE_SUFFIX, trace_partial_file_suffix); + + fdb_check(fdb_setup_network()); + std::thread network_thread{ &fdb_run_network }; + + // Apparently you need to open a database to initialize logging + FDBDatabase* out; + fdb_check(fdb_create_database(nullptr, &out)); + fdb_database_destroy(out); + + // Eventually there's a new trace file for this test ending in .tmp + std::string name; + for (;;) { + for (const auto& path : platform::listFiles(".")) { + if (path.find(file_identifier) != std::string::npos && path.find(".simulated.") == std::string::npos) { + assert(path.substr(path.size() - trace_partial_file_suffix.size()) == trace_partial_file_suffix); + name = path; + break; + } + } + if (!name.empty()) { + break; + } + } + + fdb_check(fdb_stop_network()); + network_thread.join(); + + // After shutting down, the suffix is removed for both the simulated stray file and our new file + if (!trace_partial_file_suffix.empty()) { + assert(!file_exists(name.c_str())); + assert(!file_exists(simulated_stray_partial_file.c_str())); + } + + auto new_name = name.substr(0, name.size() - trace_partial_file_suffix.size()); + auto new_stray_name = + simulated_stray_partial_file.substr(0, simulated_stray_partial_file.size() - trace_partial_file_suffix.size()); + assert(file_exists(new_name.c_str())); + assert(file_exists(new_stray_name.c_str())); + remove(new_name.c_str()); + remove(new_stray_name.c_str()); + assert(!file_exists(new_name.c_str())); + assert(!file_exists(new_stray_name.c_str())); +} diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index 64ed2adddd..fe88e6b96f 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -2177,6 +2177,81 @@ TEST_CASE("monitor_network_busyness") { CHECK(containsGreaterZero); } +// Commit a transaction and confirm it has not been reset +TEST_CASE("commit_does_not_reset") { + fdb::Transaction tr(db); + fdb::Transaction tr2(db); + + // Commit two transactions, one that will fail with conflict and the other + // that will succeed. Ensure both transactions are not reset at the end. + while (1) { + fdb::Int64Future tr1GrvFuture = tr.get_read_version(); + fdb_error_t err = wait_future(tr1GrvFuture); + if (err) { + fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err); + fdb_check(wait_future(tr1OnErrorFuture)); + continue; + } + + int64_t tr1StartVersion; + CHECK(!tr1GrvFuture.get(&tr1StartVersion)); + + fdb::Int64Future tr2GrvFuture = tr2.get_read_version(); + err = wait_future(tr2GrvFuture); + + if (err) { + fdb::EmptyFuture tr2OnErrorFuture = tr2.on_error(err); + fdb_check(wait_future(tr2OnErrorFuture)); + continue; + } + + int64_t tr2StartVersion; + CHECK(!tr2GrvFuture.get(&tr2StartVersion)); + + tr.set(key("foo"), "bar"); + fdb::EmptyFuture tr1CommitFuture = tr.commit(); + err = wait_future(tr1CommitFuture); + if (err) { + fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err); + fdb_check(wait_future(tr1OnErrorFuture)); + continue; + } + + fdb_check(tr2.add_conflict_range(key("foo"), strinc(key("foo")), FDB_CONFLICT_RANGE_TYPE_READ)); + tr2.set(key("foo"), "bar"); + fdb::EmptyFuture tr2CommitFuture = tr2.commit(); + err = wait_future(tr2CommitFuture); + CHECK(err == 1020); // not_committed + + fdb::Int64Future tr1GrvFuture2 = tr.get_read_version(); + err = wait_future(tr1GrvFuture2); + if (err) { + fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err); + fdb_check(wait_future(tr1OnErrorFuture)); + continue; + } + + int64_t tr1EndVersion; + CHECK(!tr1GrvFuture2.get(&tr1EndVersion)); + + fdb::Int64Future tr2GrvFuture2 = tr2.get_read_version(); + err = wait_future(tr2GrvFuture2); + if (err) { + fdb::EmptyFuture tr2OnErrorFuture = tr2.on_error(err); + fdb_check(wait_future(tr2OnErrorFuture)); + continue; + } + + int64_t tr2EndVersion; + CHECK(!tr2GrvFuture2.get(&tr2EndVersion)); + + // If we reset the transaction, then the read version will change + CHECK(tr1StartVersion == tr1EndVersion); + CHECK(tr2StartVersion == tr2EndVersion); + break; + } +} + int main(int argc, char** argv) { if (argc < 3) { std::cout << "Unit tests for the FoundationDB C API.\n" diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp index 3ef201456f..750ba85daf 100644 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ b/bindings/flow/DirectoryLayer.actor.cpp @@ -36,8 +36,8 @@ const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) - : nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), - rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {} + : rootNode(nodeSubspace.get(nodeSubspace.key())), nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), + allocator(rootNode.get(HIGH_CONTENTION_KEY)), allowManualPrefixes(allowManualPrefixes) {} Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { return nodeSubspace.get(prefix); diff --git a/bindings/flow/FDBLoanerTypes.h b/bindings/flow/FDBLoanerTypes.h index 97e4394298..d33a72203c 100644 --- a/bindings/flow/FDBLoanerTypes.h +++ b/bindings/flow/FDBLoanerTypes.h @@ -167,9 +167,9 @@ struct RangeResultRef : VectorRef { RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} RangeResultRef(Arena& p, const RangeResultRef& toCopy) - : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd), + : VectorRef(p, toCopy), more(toCopy.more), readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional()), - VectorRef(p, toCopy) {} + readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd) {} RangeResultRef(const VectorRef& value, bool more, Optional readThrough = Optional()) : VectorRef(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) { } diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp index bef7775cca..337792e508 100644 --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -19,12 +19,11 @@ */ #include "Tuple.h" -#include namespace FDB { // The floating point operations depend on this using the IEEE 754 standard. -BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); -BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +static_assert(std::numeric_limits::is_iec559); +static_assert(std::numeric_limits::is_iec559); const size_t Uuid::SIZE = 16; diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index eb89a9de25..d8696705ef 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -308,7 +308,7 @@ if(NOT OPEN_FOR_IDE) if(RUN_JUNIT_TESTS) # Sets up the JUnit testing structure to run through ctest # - # To add a new junit test, add the class to the JAVA_JUNIT_TESTS variable in `src/tests.cmake`. Note that if you run a Suite, + # To add a new junit test, add the class to the JAVA_JUNIT_TESTS variable in `src/tests.cmake`. Note that if you run a Suite, # ctest will NOT display underlying details of the suite itself, so it's best to avoid junit suites in general. Also, # if you need a different runner other than JUnitCore, you'll have to modify this so be aware. # @@ -316,8 +316,8 @@ if(NOT OPEN_FOR_IDE) # # ctest . # - # from the ${BUILD_DIR}/bindings/java subdirectory. - # + # from the ${BUILD_DIR}/bindings/java subdirectory. + # # Note: if you are running from ${BUILD_DIR}, additional tests of the native logic will be run. To avoid these, use # # ctest . -R java-unit @@ -325,15 +325,15 @@ if(NOT OPEN_FOR_IDE) # ctest has lots of flexible command options, so be sure to refer to its documentation if you want to do something specific(documentation # can be found at https://cmake.org/cmake/help/v3.19/manual/ctest.1.html) - add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} ${JUNIT_RESOURCES} INCLUDE_JARS fdb-java - ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar + add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} ${JUNIT_RESOURCES} INCLUDE_JARS fdb-java + ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-engine-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-params-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar ${CMAKE_BINARY_DIR}/packages/apiguardian-api-1.1.1.jar ) get_property(junit_jar_path TARGET fdb-junit PROPERTY JAR_FILE) - + add_test(NAME java-unit COMMAND ${Java_JAVA_EXECUTABLE} -classpath "${target_jar}:${junit_jar_path}:${JUNIT_CLASSPATH}" @@ -346,12 +346,12 @@ if(NOT OPEN_FOR_IDE) if(RUN_JAVA_INTEGRATION_TESTS) # Set up the integration tests. These tests generally require a running database server to function properly. Most tests # should be written such that they can be run in parallel with other integration tests (e.g. try to use a unique key range for each test - # whenever possible), because it's a reasonable assumption that a single server will be shared among multiple tests, and might do so + # whenever possible), because it's a reasonable assumption that a single server will be shared among multiple tests, and might do so # concurrently. # # Integration tests are run through ctest the same way as unit tests, but their label is prefixed with the entry 'integration-'. - # Note that most java integration tests will fail if they can't quickly connect to a running FDB instance(depending on how the test is written, anyway). - # However, if you want to explicitly skip them, you can run + # Note that most java integration tests will fail if they can't quickly connect to a running FDB instance(depending on how the test is written, anyway). + # However, if you want to explicitly skip them, you can run # # `ctest -E integration` # @@ -368,8 +368,8 @@ if(NOT OPEN_FOR_IDE) # empty, consider generating a random prefix for the keys you write, use # the directory layer with a unique path, etc.) # - add_jar(fdb-integration SOURCES ${JAVA_INTEGRATION_TESTS} ${JAVA_INTEGRATION_RESOURCES} INCLUDE_JARS fdb-java - ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar + add_jar(fdb-integration SOURCES ${JAVA_INTEGRATION_TESTS} ${JAVA_INTEGRATION_RESOURCES} INCLUDE_JARS fdb-java + ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-engine-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-params-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar @@ -382,7 +382,14 @@ if(NOT OPEN_FOR_IDE) COMMAND ${Java_JAVA_EXECUTABLE} -classpath "${target_jar}:${integration_jar_path}:${JUNIT_CLASSPATH}" -Djava.library.path=${CMAKE_BINARY_DIR}/lib - org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" + org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" "-T MultiClient" + ) + + add_multi_fdbclient_test(NAME java-multi-integration + COMMAND ${Java_JAVA_EXECUTABLE} + -classpath "${target_jar}:${integration_jar_path}:${JUNIT_CLASSPATH}" + -Djava.library.path=${CMAKE_BINARY_DIR}/lib + org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" "-t MultiClient" ) endif() diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index b2506965eb..555a6cb434 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -513,7 +513,7 @@ struct JVM { } }; -struct JavaWorkload : FDBWorkload { +struct JavaWorkload final : FDBWorkload { std::shared_ptr jvm; FDBLogger& log; FDBWorkloadContext* context = nullptr; diff --git a/bindings/java/src/README.md b/bindings/java/src/README.md index 6fedba6368..6bd377b85a 100644 --- a/bindings/java/src/README.md +++ b/bindings/java/src/README.md @@ -22,4 +22,19 @@ To skip integration tests, execute `ctest -E integration` from `${BUILD_DIR}/bin To run _only_ integration tests, run `ctest -R integration` from `${BUILD_DIR}/bindings/java`. There are lots of other useful `ctest` commands, which we don't need to get into here. For more information, -see the [https://cmake.org/cmake/help/v3.19/manual/ctest.1.html](ctest documentation). \ No newline at end of file +see the [https://cmake.org/cmake/help/v3.19/manual/ctest.1.html](ctest documentation). + +### Multi-Client tests +Multi-Client tests are integration tests that can only be executed when multiple clusters are running. To write a multi-client +test, do the following: + +1. Tag all tests that require multiple clients with `@Tag("MultiClient")` +2. Ensure that your tests have the `MultiClientHelper` extension present, and Registered as an extension +3. Ensure that your test class is in the the JAVA_INTEGRATION_TESTS list in `test.cmake` + +( see `BasicMultiClientIntegrationTest` for a good reference example) + +It is important to note that it requires significant time to start and stop 3 separate clusters; if the underying test takes a long time to run, +ctest will time out and kill the test. When that happens, there is no guarantee that the FDB clusters will be properly stopped! It is thus +in your best interest to ensure that all tests run in a relatively small amount of time, or have a longer timeout attached. + diff --git a/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java new file mode 100644 index 0000000000..2b02a3656f --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java @@ -0,0 +1,69 @@ +/* + * BasicMultiClientIntegrationTest + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.Collection; +import java.util.Random; + +import com.apple.foundationdb.tuple.Tuple; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Simple class to test multi-client logic. + * + * Note that all Multi-client-only tests _must_ be tagged with "MultiClient", which will ensure that they are excluded + * from non-multi-threaded tests. + */ +public class BasicMultiClientIntegrationTest { + @RegisterExtension public static final MultiClientHelper clientHelper = new MultiClientHelper(); + + @Test + @Tag("MultiClient") + void testMultiClientWritesAndReadsData() throws Exception { + FDB fdb = FDB.selectAPIVersion(630); + fdb.options().setKnob("min_trace_severity=5"); + + Collection dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us + System.out.print("Starting tests."); + Random rand = new Random(); + for (int counter = 0; counter < 25; ++counter) { + for (Database db : dbs) { + String key = Integer.toString(rand.nextInt(100000000)); + String val = Integer.toString(rand.nextInt(100000000)); + + db.run(tr -> { + tr.set(Tuple.from(key).pack(), Tuple.from(val).pack()); + return null; + }); + + String fetchedVal = db.run(tr -> { + byte[] result = tr.get(Tuple.from(key).pack()).join(); + return Tuple.fromBytes(result).getString(0); + }); + Assertions.assertEquals(val, fetchedVal, "Wrong result!"); + } + Thread.sleep(200); + } + } +} diff --git a/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java new file mode 100644 index 0000000000..36df0182a8 --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java @@ -0,0 +1,201 @@ +/* + * CycleMultiClientIntegrationTest + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.Arrays; +import java.util.ArrayList; +import java.util.concurrent.ThreadLocalRandom; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import com.apple.foundationdb.tuple.Tuple; + +import org.junit.jupiter.api.Assertions; + +/** + * Setup: Generating a cycle 0 -> 1 -> 2 -> 3 -> 0, its length is 4 + * Process: randomly choose an element, reverse 2nd and 4rd element, considering the chosen one as the 1st element. + * Check: verify no element is lost or added, and they are still a cycle. + * + * This test is to verify the atomicity of transactions. + */ +public class CycleMultiClientIntegrationTest { + public static final MultiClientHelper clientHelper = new MultiClientHelper(); + + // more write txn than validate txn, as parent thread waits only for validate txn. + private static final int writeTxnCnt = 2000; + private static final int validateTxnCnt = 1000; + private static final int threadPerDB = 5; + + private static final int cycleLength = 4; + private static List expected = new ArrayList<>(Arrays.asList("0", "1", "2", "3")); + + public static void main(String[] args) throws Exception { + FDB fdb = FDB.selectAPIVersion(710); + setupThreads(fdb); + Collection dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us + System.out.println("Starting tests"); + setup(dbs); + System.out.println("Start processing and validating"); + process(dbs); + check(dbs); + System.out.println("Test finished"); + } + + private static synchronized void setupThreads(FDB fdb) { + int clientThreadsPerVersion = clientHelper.readClusterFromEnv().length; + fdb.options().setClientThreadsPerVersion(clientThreadsPerVersion); + System.out.printf("thread per version is %d\n", clientThreadsPerVersion); + fdb.options().setExternalClientDirectory("/var/dynamic-conf/lib"); + fdb.options().setTraceEnable("/tmp"); + fdb.options().setKnob("min_trace_severity=5"); + } + + private static void setup(Collection dbs) { + // 0 -> 1 -> 2 -> 3 -> 0 + for (Database db : dbs) { + db.run(tr -> { + for (int k = 0; k < cycleLength; k++) { + String key = Integer.toString(k); + String value = Integer.toString((k + 1) % cycleLength); + tr.set(Tuple.from(key).pack(), Tuple.from(value).pack()); + } + return null; + }); + } + } + + private static void process(Collection dbs) { + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final Thread thread = new Thread(CycleWorkload.create(db)); + thread.start(); + } + } + } + + private static void check(Collection dbs) throws InterruptedException { + final Map threadsToCheckers = new HashMap<>(); + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final CycleChecker checker = new CycleChecker(db); + final Thread thread = new Thread(checker); + thread.start(); + threadsToCheckers.put(thread, checker); + } + } + + for (Map.Entry entry : threadsToCheckers.entrySet()) { + entry.getKey().join(); + final boolean succeed = entry.getValue().succeed(); + Assertions.assertTrue(succeed, "Cycle test failed"); + } + } + + public static class CycleWorkload implements Runnable { + + private final Database db; + + private CycleWorkload(Database db) { + this.db = db; + } + + public static CycleWorkload create(Database db) { + return new CycleWorkload(db); + } + + @Override + public void run() { + for (int i = 0; i < writeTxnCnt; i++) { + db.run(tr -> { + final int k = ThreadLocalRandom.current().nextInt(cycleLength); + final String key = Integer.toString(k); + byte[] result1 = tr.get(Tuple.from(key).pack()).join(); + String value1 = Tuple.fromBytes(result1).getString(0); + + byte[] result2 = tr.get(Tuple.from(value1).pack()).join(); + String value2 = Tuple.fromBytes(result2).getString(0); + + byte[] result3 = tr.get(Tuple.from(value2).pack()).join(); + String value3 = Tuple.fromBytes(result3).getString(0); + + byte[] result4 = tr.get(Tuple.from(value3).pack()).join(); + + tr.set(Tuple.from(key).pack(), Tuple.from(value2).pack()); + tr.set(Tuple.from(value2).pack(), Tuple.from(value1).pack()); + tr.set(Tuple.from(value1).pack(), Tuple.from(value3).pack()); + return null; + }); + } + } + } + + public static class CycleChecker implements Runnable { + private final Database db; + private boolean succeed; + + public CycleChecker(Database db) { + this.db = db; + this.succeed = true; + } + + public static CycleChecker create(Database db) { + return new CycleChecker(db); + } + + @Override + public void run() { + for (int i = 0; i < validateTxnCnt; i++) { + db.run(tr -> { + final int k = ThreadLocalRandom.current().nextInt(cycleLength); + final String key = Integer.toString(k); + byte[] result1 = tr.get(Tuple.from(key).pack()).join(); + String value1 = Tuple.fromBytes(result1).getString(0); + + byte[] result2 = tr.get(Tuple.from(value1).pack()).join(); + String value2 = Tuple.fromBytes(result2).getString(0); + + byte[] result3 = tr.get(Tuple.from(value2).pack()).join(); + String value3 = Tuple.fromBytes(result3).getString(0); + + byte[] result4 = tr.get(Tuple.from(value3).pack()).join(); + String value4 = Tuple.fromBytes(result4).getString(0); + + if (!key.equals(value4)) { + succeed = false; + } + List actual = new ArrayList<>(Arrays.asList(value1, value2, value3, value4)); + Collections.sort(actual); + if (!expected.equals(actual)) { + succeed = false; + } + return null; + }); + } + } + + public boolean succeed() { + return succeed; + } + } +} diff --git a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java index 5634e7d741..cf8c1bde1d 100644 --- a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java +++ b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java @@ -19,8 +19,6 @@ */ package com.apple.foundationdb; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java b/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java new file mode 100644 index 0000000000..671163955f --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java @@ -0,0 +1,82 @@ +/* + * MultiClientHelper.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.ArrayList; +import java.util.Collection; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * Callback to help define a multi-client scenario and ensure that + * the clients can be configured properly. + */ +public class MultiClientHelper implements BeforeAllCallback,AfterEachCallback{ + private String[] clusterFiles; + private Collection openDatabases; + + public static String[] readClusterFromEnv() { + /* + * Reads the cluster file lists from the ENV variable + * FDB_CLUSTERS. + */ + String clusterFilesProp = System.getenv("FDB_CLUSTERS"); + if (clusterFilesProp == null) { + throw new IllegalStateException("Missing FDB cluster connection file names"); + } + + return clusterFilesProp.split(";"); + } + + Collection openDatabases(FDB fdb){ + if(openDatabases!=null){ + return openDatabases; + } + if(clusterFiles==null){ + clusterFiles = readClusterFromEnv(); + } + Collection dbs = new ArrayList(); + for (String arg : clusterFiles) { + System.out.printf("Opening Cluster: %s\n", arg); + dbs.add(fdb.open(arg)); + } + + this.openDatabases = dbs; + return dbs; + } + + @Override + public void beforeAll(ExtensionContext arg0) throws Exception { + clusterFiles = readClusterFromEnv(); + } + + @Override + public void afterEach(ExtensionContext arg0) throws Exception { + //close any databases that have been opened + if(openDatabases!=null){ + for(Database db : openDatabases){ + db.close(); + } + } + openDatabases = null; + } + +} diff --git a/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java b/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java new file mode 100644 index 0000000000..3358c9e760 --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java @@ -0,0 +1,188 @@ +/* + * RepeatableReadMultiThreadClientTest + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import com.apple.foundationdb.tuple.Tuple; + +import org.junit.jupiter.api.Assertions; + +/** + * This test verify transcations have repeatable read. + * 1 First set initialValue to key. + * 2 Have transactions to read the key and verify the initialValue in a loop, if it does not + * see the initialValue as the value, it set the flag to false. + * + * 3 Then have new transactions set the value and then read to verify the new value is set, + * if it does not read the new value, set the flag to false. + * + * 4 Verify that old transactions have not finished when new transactions have finished, + * then verify old transactions does not have false flag -- it means that old transactions + * are still seeting the initialValue even after new transactions set them to a new value. + */ +public class RepeatableReadMultiThreadClientTest { + public static final MultiClientHelper clientHelper = new MultiClientHelper(); + + private static final int oldValueReadCount = 30; + private static final int threadPerDB = 5; + + private static final String key = "foo"; + private static final String initialValue = "bar"; + private static final String newValue = "cool"; + private static final Map threadToOldValueReaders = new HashMap<>(); + + public static void main(String[] args) throws Exception { + FDB fdb = FDB.selectAPIVersion(710); + setupThreads(fdb); + Collection dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us + System.out.println("Starting tests"); + setup(dbs); + System.out.println("Start processing and validating"); + readOldValue(dbs); + setNewValueAndRead(dbs); + System.out.println("Test finished"); + } + + private static synchronized void setupThreads(FDB fdb) { + int clientThreadsPerVersion = clientHelper.readClusterFromEnv().length; + fdb.options().setClientThreadsPerVersion(clientThreadsPerVersion); + System.out.printf("thread per version is %d\n", clientThreadsPerVersion); + fdb.options().setExternalClientDirectory("/var/dynamic-conf/lib"); + fdb.options().setTraceEnable("/tmp"); + fdb.options().setKnob("min_trace_severity=5"); + } + + private static void setup(Collection dbs) { + // 0 -> 1 -> 2 -> 3 -> 0 + for (Database db : dbs) { + db.run(tr -> { + tr.set(Tuple.from(key).pack(), Tuple.from(initialValue).pack()); + return null; + }); + } + } + + private static void readOldValue(Collection dbs) throws InterruptedException { + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final OldValueReader oldValueReader = new OldValueReader(db); + final Thread thread = new Thread(OldValueReader.create(db)); + thread.start(); + threadToOldValueReaders.put(thread, oldValueReader); + } + } + } + + private static void setNewValueAndRead(Collection dbs) throws InterruptedException { + // threads running NewValueReader need to wait for threads to start first who run OldValueReader + Thread.sleep(1000); + final Map threads = new HashMap<>(); + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final NewValueReader newValueReader = new NewValueReader(db); + final Thread thread = new Thread(NewValueReader.create(db)); + thread.start(); + threads.put(thread, newValueReader); + } + } + + for (Map.Entry entry : threads.entrySet()) { + entry.getKey().join(); + Assertions.assertTrue(entry.getValue().succeed, "new value reader failed to read the correct value"); + } + + for (Map.Entry entry : threadToOldValueReaders.entrySet()) { + Assertions.assertTrue(entry.getKey().isAlive(), "Old value reader finished too soon, cannot verify repeatable read, succeed is " + entry.getValue().succeed); + } + + for (Map.Entry entry : threadToOldValueReaders.entrySet()) { + entry.getKey().join(); + Assertions.assertTrue(entry.getValue().succeed, "old value reader failed to read the correct value"); + } + } + + public static class OldValueReader implements Runnable { + + private final Database db; + private boolean succeed; + + private OldValueReader(Database db) { + this.db = db; + this.succeed = true; + } + + public static OldValueReader create(Database db) { + return new OldValueReader(db); + } + + @Override + public void run() { + db.run(tr -> { + try { + for (int i = 0; i < oldValueReadCount; i++) { + byte[] result = tr.get(Tuple.from(key).pack()).join(); + String value = Tuple.fromBytes(result).getString(0); + if (!initialValue.equals(value)) { + succeed = false; + break; + } + Thread.sleep(100); + } + } + catch (Exception e) { + succeed = false; + } + return null; + }); + } + } + + public static class NewValueReader implements Runnable { + private final Database db; + private boolean succeed; + + public NewValueReader(Database db) { + this.db = db; + this.succeed = true; + } + + public static NewValueReader create(Database db) { + return new NewValueReader(db); + } + + @Override + public void run() { + db.run(tr -> { + tr.set(Tuple.from(key).pack(), Tuple.from(newValue).pack()); + return null; + }); + String value = db.run(tr -> { + byte[] result = tr.get(Tuple.from(key).pack()).join(); + return Tuple.fromBytes(result).getString(0); + }); + if (!newValue.equals(value)) { + succeed = false; + } + } + } +} diff --git a/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java b/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java new file mode 100644 index 0000000000..fb154a20ab --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java @@ -0,0 +1,143 @@ +package com.apple.foundationdb; + +import com.apple.foundationdb.tuple.Tuple; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadLocalRandom; + +import org.junit.jupiter.api.Assertions; + +/** + * Each cluster has a queue, producer writes a key and then send a message to this queue in JVM. + * Consumer would consume the key by checking the existence of the key, if it does not find the key, + * then the test would fail. + * + * This test is to verify the causal consistency of transactions for mutli-threaded client. + */ +public class SidebandMultiThreadClientTest { + public static final MultiClientHelper clientHelper = new MultiClientHelper(); + + private static final Map> db2Queues = new HashMap<>(); + private static final int threadPerDB = 5; + private static final int txnCnt = 1000; + + public static void main(String[] args) throws Exception { + FDB fdb = FDB.selectAPIVersion(710); + setupThreads(fdb); + Collection dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us + for (Database db : dbs) { + db2Queues.put(db, new LinkedBlockingQueue<>()); + } + System.out.println("Start processing and validating"); + process(dbs); + check(dbs); + System.out.println("Test finished"); + } + + private static synchronized void setupThreads(FDB fdb) { + int clientThreadsPerVersion = clientHelper.readClusterFromEnv().length; + fdb.options().setClientThreadsPerVersion(clientThreadsPerVersion); + System.out.printf("thread per version is %d\n", clientThreadsPerVersion); + fdb.options().setExternalClientDirectory("/var/dynamic-conf/lib"); + fdb.options().setTraceEnable("/tmp"); + fdb.options().setKnob("min_trace_severity=5"); + } + + private static void process(Collection dbs) { + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final Thread thread = new Thread(Producer.create(db, db2Queues.get(db))); + thread.start(); + } + } + } + + private static void check(Collection dbs) throws InterruptedException { + final Map threads2Consumers = new HashMap<>(); + for (Database db : dbs) { + for (int i = 0; i < threadPerDB; i++) { + final Consumer consumer = Consumer.create(db, db2Queues.get(db)); + final Thread thread = new Thread(consumer); + thread.start(); + threads2Consumers.put(thread, consumer); + } + } + + for (Map.Entry entry : threads2Consumers.entrySet()) { + entry.getKey().join(); + final boolean succeed = entry.getValue().succeed; + Assertions.assertTrue(succeed, "Sideband test failed"); + } + } + + public static class Producer implements Runnable { + private final Database db; + private final BlockingQueue queue; + + private Producer(Database db, BlockingQueue queue) { + this.db = db; + this.queue = queue; + } + + public static Producer create(Database db, BlockingQueue queue) { + return new Producer(db, queue); + } + + @Override + public void run() { + for (int i = 0; i < txnCnt; i++) { + final long suffix = ThreadLocalRandom.current().nextLong(); + final String key = String.format("Sideband/Multithread/Test/%d", suffix); + db.run(tr -> { + tr.set(Tuple.from(key).pack(), Tuple.from("bar").pack()); + return null; + }); + queue.offer(key); + } + } + } + + public static class Consumer implements Runnable { + private final Database db; + private final BlockingQueue queue; + private boolean succeed; + + private Consumer(Database db, BlockingQueue queue) { + this.db = db; + this.queue = queue; + this.succeed = true; + } + + public static Consumer create(Database db, BlockingQueue queue) { + return new Consumer(db, queue); + } + + @Override + public void run() { + try { + for (int i = 0; i < txnCnt && succeed; i++) { + final String key = queue.take(); + db.run(tr -> { + byte[] result = tr.get(Tuple.from(key).pack()).join(); + if (result == null) { + System.out.println("FAILED to get key " + key + " from DB " + db); + succeed = false; + } + if (!succeed) { + return null; + } + String value = Tuple.fromBytes(result).getString(0); + return null; + }); + } + } catch (InterruptedException e) { + System.out.println("Get Exception in consumer: " + e); + succeed = false; + } + } + } +} diff --git a/bindings/java/src/main/com/apple/foundationdb/KeySelector.java b/bindings/java/src/main/com/apple/foundationdb/KeySelector.java index 789128ce6a..5061b51ba7 100644 --- a/bindings/java/src/main/com/apple/foundationdb/KeySelector.java +++ b/bindings/java/src/main/com/apple/foundationdb/KeySelector.java @@ -165,9 +165,13 @@ public class KeySelector { } /** - * Returns the {@code or-equal} parameter of this {@code KeySelector}. For internal use. + * Returns the orEqual parameter for this {@code KeySelector}. See the + * {@link #KeySelector(byte[], boolean, int)} KeySelector constructor} + * for more details. + * + * @return the {@code or-equal} parameter of this {@code KeySelector}. */ - boolean orEqual() { + public boolean orEqual() { return orEqual; } diff --git a/bindings/java/src/tests.cmake b/bindings/java/src/tests.cmake index 8bed62ecb8..3e9dce6657 100644 --- a/bindings/java/src/tests.cmake +++ b/bindings/java/src/tests.cmake @@ -48,12 +48,17 @@ set(JUNIT_RESOURCES set(JAVA_INTEGRATION_TESTS src/integration/com/apple/foundationdb/DirectoryTest.java src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java + src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java + src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java + src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java + src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java ) # Resources that are used in integration testing, but are not explicitly test files (JUnit rules, # utility classes, and so forth) set(JAVA_INTEGRATION_RESOURCES src/integration/com/apple/foundationdb/RequiresDatabase.java + src/integration/com/apple/foundationdb/MultiClientHelper.java ) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 0893b08aa8..2fe304a0dc 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -81,5 +81,15 @@ if (NOT WIN32 AND NOT OPEN_FOR_IDE) COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py ${CMAKE_BINARY_DIR}/bin/fdbcli @CLUSTER_FILE@ + 1 + ) + add_fdbclient_test( + NAME multi_process_fdbcli_tests + PROCESS_NUMBER 5 + TEST_TIMEOUT 120 # The test can take near to 1 minutes sometime, set timeout to 2 minutes to be safe + COMMAND ${CMAKE_SOURCE_DIR}/bindings/python/tests/fdbcli_tests.py + ${CMAKE_BINARY_DIR}/bin/fdbcli + @CLUSTER_FILE@ + 5 ) endif() diff --git a/bindings/python/tests/fdbcli_tests.py b/bindings/python/tests/fdbcli_tests.py index 71fe5ee6b3..14224a6ca8 100755 --- a/bindings/python/tests/fdbcli_tests.py +++ b/bindings/python/tests/fdbcli_tests.py @@ -332,22 +332,128 @@ def transaction(logger): output7 = run_fdbcli_command('get', 'key') assert output7 == "`key': not found" +def get_fdb_process_addresses(): + # get all processes' network addresses + output = run_fdbcli_command('kill') + # except the first line, each line is one process + addresses = output.split('\n')[1:] + assert len(addresses) == process_number + return addresses + +@enable_logging() +def coordinators(logger): + # we should only have one coordinator for now + output1 = run_fdbcli_command('coordinators') + assert len(output1.split('\n')) > 2 + cluster_description = output1.split('\n')[0].split(': ')[-1] + logger.debug("Cluster description: {}".format(cluster_description)) + coordinators = output1.split('\n')[1].split(': ')[-1] + # verify the coordinator + coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators') + assert len(coordinator_list) == 1 + assert coordinator_list[0]['address'] == coordinators + # verify the cluster description + assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description)) + addresses = get_fdb_process_addresses() + # set all 5 processes as coordinators and update the cluster description + new_cluster_description = 'a_simple_description' + run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description)) + # verify now we have 5 coordinators and the description is updated + output2 = run_fdbcli_command('coordinators') + assert output2.split('\n')[0].split(': ')[-1] == new_cluster_description + assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(5, ','.join(addresses)) + # auto change should go back to 1 coordinator + run_fdbcli_command('coordinators', 'auto') + assert len(get_value_from_status_json(True, 'client', 'coordinators', 'coordinators')) == 1 + +@enable_logging() +def exclude(logger): + # get all processes' network addresses + addresses = get_fdb_process_addresses() + logger.debug("Cluster processes: {}".format(' '.join(addresses))) + # There should be no excluded process for now + no_excluded_process_output = 'There are currently no servers or localities excluded from the database.' + output1 = run_fdbcli_command('exclude') + assert no_excluded_process_output in output1 + # randomly pick one and exclude the process + excluded_address = random.choice(addresses) + # sometimes we need to retry the exclude + while True: + logger.debug("Excluding process: {}".format(excluded_address)) + error_message = run_fdbcli_command_and_get_error('exclude', excluded_address) + if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address): + # exclude coordinator will print the warning, verify the randomly selected process is the coordinator + coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators') + assert len(coordinator_list) == 1 + assert coordinator_list[0]['address'] == excluded_address + break + elif not error_message: + break + else: + logger.debug("Error message: {}\n".format(error_message)) + logger.debug("Retry exclude after 1 second") + time.sleep(1) + output2 = run_fdbcli_command('exclude') + assert 'There are currently 1 servers or localities being excluded from the database' in output2 + assert excluded_address in output2 + run_fdbcli_command('include', excluded_address) + # check the include is successful + output4 = run_fdbcli_command('exclude') + assert no_excluded_process_output in output4 + +# read the system key 'k', need to enable the option first +def read_system_key(k): + output = run_fdbcli_command('option', 'on', 'READ_SYSTEM_KEYS;', 'get', k) + if 'is' not in output: + # key not present + return None + _, value = output.split(' is ') + return value + +@enable_logging() +def throttle(logger): + # no throttled tags at the beginning + no_throttle_tags_output = 'There are no throttled tags' + assert run_fdbcli_command('throttle', 'list') == no_throttle_tags_output + # test 'throttle enable auto' + run_fdbcli_command('throttle', 'enable', 'auto') + # verify the change is applied by reading the system key + # not an elegant way, may change later + enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled') + assert enable_flag == "`1'" + run_fdbcli_command('throttle', 'disable', 'auto') + enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled') + # verify disabled + assert enable_flag == "`0'" + # TODO : test manual throttling, not easy to do now if __name__ == '__main__': - # fdbcli_tests.py - assert len(sys.argv) == 3, "Please pass arguments: " + # fdbcli_tests.py + assert len(sys.argv) == 4, "Please pass arguments: " # shell command template command_template = [sys.argv[1], '-C', sys.argv[2], '--exec'] # tests for fdbcli commands # assertions will fail if fdbcli does not work as expected - advanceversion() - cache_range() - consistencycheck() - datadistribution() - kill() - lockAndUnlock() - maintenance() - setclass() - suspend() - transaction() + process_number = int(sys.argv[3]) + if process_number == 1: + # TODO: disable for now, the change can cause the database unavailable + #advanceversion() + cache_range() + consistencycheck() + datadistribution() + kill() + lockAndUnlock() + maintenance() + setclass() + suspend() + transaction() + throttle() + else: + assert process_number > 1, "Process number should be positive" + # the kill command which used to list processes seems to not work as expected sometime + # which makes the test flaky. + # We need to figure out the reason and then re-enable these tests + #coordinators() + #exclude() + diff --git a/build/artifacts/.gitkeep b/build/artifacts/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/build/cmake/Dockerfile b/build/cmake/Dockerfile deleted file mode 100644 index 0452606a1f..0000000000 --- a/build/cmake/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM centos:6 -LABEL version=0.0.4 - -RUN yum install -y yum-utils -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN yum -y install centos-release-scl -RUN yum install -y devtoolset-7 - -# install cmake -RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz > /tmp/cmake.tar.gz &&\ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\ - sha256sum -c /tmp/cmake-sha.txt &&\ - cd /tmp && tar xf cmake.tar.gz && cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ - -# install boost -RUN curl -L https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 > /tmp/boost.tar.bz2 &&\ - cd /tmp && echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost.tar.bz2" > boost-sha.txt &&\ - sha256sum -c boost-sha.txt && tar xf boost.tar.bz2 && cp -r boost_1_72_0/boost /usr/local/include/ &&\ - rm -rf boost.tar.bz2 boost_1_72_0 - -# install mono (for actorcompiler) -RUN yum install -y epel-release -RUN yum install -y mono-core - -# install Java -RUN yum install -y java-1.8.0-openjdk-devel - -# install LibreSSL -RUN curl https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\ - cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5 libressl.tar.gz" > libressl-sha.txt &&\ - sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\ - cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ - cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- make -j`nproc` install &&\ - rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz - - -# install dependencies for bindings and documentation -# python 2.7 is required for the documentation -RUN yum install -y rh-python36-python-devel rh-ruby24 golang python27 - -# install packaging tools -RUN yum install -y rpm-build debbuild - -CMD scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash diff --git a/build/cmake/build.sh b/build/cmake/build.sh deleted file mode 100644 index ff02e78080..0000000000 --- a/build/cmake/build.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash - -arguments_usage() { - cat </dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/deb.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt b/build/cmake/package_tester/fdb_c_app/CMakeLists.txt deleted file mode 100644 index 60ed25f0ca..0000000000 --- a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 2.8.0) -project(fdb_c_app C) -find_package(FoundationDB-Client REQUIRED) -add_executable(app app.c) -target_link_libraries(app PRIVATE fdb_c) diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c deleted file mode 100644 index 6fe24068f9..0000000000 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ /dev/null @@ -1,7 +0,0 @@ -#define FDB_API_VERSION 710 -#include - -int main(int argc, char* argv[]) { - fdb_select_api_version(710); - return 0; -} diff --git a/build/cmake/package_tester/modules/arguments.sh b/build/cmake/package_tester/modules/arguments.sh deleted file mode 100644 index afe46ee035..0000000000 --- a/build/cmake/package_tester/modules/arguments.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${arguments_sh_included+x} ] -then - arguments_sh_included=1 - - source ${source_dir}/modules/util.sh - - arguments_usage() { - cat <&2 - __res=1 - break - elif [ $docker_parallelism -lt 1 ] - then - echo -e "${RED}Error: -j ${OPTARG} makes no sense" >&2 - __res=1 - break - fi - ;; - P ) - pruning_strategy="${OPTARG}" - if ! [[ "${pruning_strategy}" =~ ^(ALL|FAILED|SUCCEEDED|NONE)$ ]] - then - fail "Unknown pruning strategy ${pruning_strategy}" - fi - ;; - \? ) - curr_index="$((OPTIND-1))" - echo "Unknown option ${@:${curr_index}:1}" - arguments_usage - __res=1 - break - ;; - esac - done - shift $((OPTIND -1)) - commands=("$@") - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/config.sh b/build/cmake/package_tester/modules/config.sh deleted file mode 100644 index 70fc35692c..0000000000 --- a/build/cmake/package_tester/modules/config.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${config_sh_included+x} ] -then - config_sh_included=1 - - source ${source_dir}/modules/util.sh - - config_load_vms() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z "${docker_ini+x}"] - then - docker_file="${source_dir}/../docker.ini" - fi - # parse the ini file and read it into an - # associative array - eval "$(awk -F ' *= *' '{ if ($1 ~ /^\[/) section=$1; else if ($1 !~ /^$/) printf "ini_%s%s=\47%s\47\n", $1, section, $2 }' ${docker_file})" - vms=( "${!ini_name[@]}" ) - if [ $? -ne 0 ] - then - echo "ERROR: Could not parse config-file ${docker_file}" - __res=1 - break - fi - done - exitfun - return ${__res} - } - - config_find_packages() { - local __res=0 - enterfun - for _ in 1 - do - cd ${fdb_build} - while read f - do - if [[ "${f}" =~ .*"clients".* || "${f}" =~ .*"server".* ]] - then - if [ -z ${fdb_packages+x} ] - then - fdb_packages="${f}" - else - fdb_packages="${fdb_packages}:${f}" - fi - fi - done <<< "$(ls *.deb *.rpm)" - if [ $? -ne 0 ] - then - __res=1 - break - fi - done - exitfun - return ${__res} - } - - get_fdb_source() { - local __res=0 - enterfun - cd ${source_dir} - while true - do - if [ -d .git ] - then - # we found the root - pwd - break - fi - if [ `pwd` = "/" ] - then - __res=1 - break - fi - cd .. - done - exitfun - return ${__res} - } - - fdb_build=0 - - config_verify() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z ${fdb_source+x} ] - then - fdb_source=`get_fdb_source` - fi - if [ ! -d "${fdb_build}" ] - then - __res=1 - echo "ERROR: Could not find fdb build dir: ${fdb_build}" - echo " Either set the environment variable fdb_build or" - echo " pass it with -b " - fi - if [ ! -f "${fdb_source}/flow/Net2.actor.cpp" ] - then - __res=1 - echo "ERROR: ${fdb_source} does not appear to be a fdb source" - echo " directory. Either pass it with -s or set" - echo " the environment variable fdb_source." - fi - if [ ${__res} -ne 0 ] - then - break - fi - config_load_vms - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/deb.sh b/build/cmake/package_tester/modules/deb.sh deleted file mode 100644 index fcef96805a..0000000000 --- a/build/cmake/package_tester/modules/deb.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${deb_sh_included}" ] -then - deb_sh_included=1 - - source ${source_dir}/modules/util.sh - - install_build_tools() { - apt-get -y install cmake gcc - } - - install() { - local __res=0 - enterfun - echo "Install FoundationDB" - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_name="$(dpkg -I ${f} | grep Package | sed 's/.*://')" - package_names+=( "${package_name}" ) - done - dpkg -i ${package_files[@]} - apt-get -yf -o Dpkg::Options::="--force-confold" install - __res=$? - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - apt-get -y remove ${package_names[@]} - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/docker.sh b/build/cmake/package_tester/modules/docker.sh deleted file mode 100644 index d38b939957..0000000000 --- a/build/cmake/package_tester/modules/docker.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${docker_sh_included+x}" ] -then - docker_sh_included=1 - source ${source_dir}/modules/util.sh - source ${source_dir}/modules/config.sh - source ${source_dir}/modules/tests.sh - - failed_tests=() - - docker_ids=() - docker_threads=() - docker_logs=() - docker_error_logs=() - - docker_wait_any() { - local __res=0 - enterfun - while [ "${#docker_threads[@]}" -gt 0 ] - do - IFS=";" read -ra res <${pipe_file} - docker_id=${res[0]} - result=${res[1]} - i=0 - for (( idx=0; idx<${#docker_ids[@]}; idx++ )) - do - if [ "${docker_id}" = "${docker_ids[idx]}" ] - then - i=idx - break - fi - done - if [ "${result}" -eq 0 ] - then - echo -e "${GREEN}Test succeeded: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - else - echo -e "${RED}Test FAILED: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - failed_tests+=( "${docker_threads[$i]}" ) - fi - n=$((i+1)) - docker_ids=( "${docker_ids[@]:0:$i}" "${docker_ids[@]:$n}" ) - docker_threads=( "${docker_threads[@]:0:$i}" "${docker_threads[@]:$n}" ) - docker_logs=( "${docker_logs[@]:0:$i}" "${docker_logs[@]:$n}" ) - docker_error_logs=( "${docker_error_logs[@]:0:$i}" "${docker_error_logs[@]:$n}" ) - break - done - exitfun - return "${__res}" - } - - docker_wait_all() { - local __res=0 - while [ "${#docker_threads[@]}" -gt 0 ] - do - docker_wait_any - if [ "$?" -ne 0 ] - then - __res=1 - fi - done - return ${__res} - } - - docker_run() { - local __res=0 - enterfun - for _ in 1 - do - echo "Testing the following:" - echo "======================" - for K in "${vms[@]}" - do - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${K}]} ) ) - echo "Will test the following ${#curr_packages[@]} packages in docker-image ${K}:" - for p in "${curr_packages[@]}" - do - echo " ${p}" - done - echo - done - log_dir="${fdb_build}/pkg_tester" - pipe_file="${fdb_build}/pkg_tester.pipe" - lock_file="${fdb_build}/pkg_tester.lock" - if [ -p "${pipe_file}" ] - then - rm "${pipe_file}" - successOr "Could not delete old pipe file" - fi - if [ -f "${lock_file}" ] - then - rm "${lock_file}" - successOr "Could not delete old pipe file" - fi - touch "${lock_file}" - successOr "Could not create lock file" - mkfifo "${pipe_file}" - successOr "Could not create pipe file" - mkdir -p "${log_dir}" - # setup the containers - # TODO: shall we make this parallel as well? - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_location="${ini_location[$vm]}" - if [[ "$curr_location" = /* ]] - then - cd "${curr_location}" - else - cd ${source_dir}/../${curr_location} - fi - docker_buid_logs="${log_dir}/docker_build_${curr_name}" - docker build . -t ${curr_name} 1> "${docker_buid_logs}.log" 2> "${docker_buid_logs}.err" - successOr "Building Docker image ${curr_name} failed - see ${docker_buid_logs}.log and ${docker_buid_logs}.err" - done - if [ ! -z "${tests_to_run+x}"] - then - tests=() - IFS=';' read -ra tests <<< "${tests_to_run}" - fi - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_format="${ini_format[$vm]}" - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${vm}]} ) ) - for curr_test in "${tests[@]}" - do - if [ "${#docker_ids[@]}" -ge "${docker_parallelism}" ] - then - docker_wait_any - fi - log_file="${log_dir}/${curr_name}_${curr_test}.log" - err_file="${log_dir}/${curr_name}_${curr_test}.err" - docker_id=$( docker run -d -v "${fdb_source}:/foundationdb"\ - -v "${fdb_build}:/build"\ - ${curr_name} /sbin/init ) - echo "Starting Test ${curr_name}/${curr_test} Docker-ID: ${docker_id}" - { - docker exec "${docker_id}" bash \ - /foundationdb/build/cmake/package_tester/${curr_format}_tests.sh -n ${curr_test} ${curr_packages[@]}\ - 2> ${err_file} 1> ${log_file} - res=$? - if [ "${pruning_strategy}" = "ALL" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -eq 0 ] && [ "${pruning_strategy}" = "SUCCEEDED" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -ne 0 ] && [ "${pruning_strategy}" = "FAILED" ] - then - docker kill "${docker_id}" > /dev/null - fi - flock "${lock_file}" echo "${docker_id};${res}" >> "${pipe_file}" - } & - docker_ids+=( "${docker_id}" ) - docker_threads+=( "${curr_name}/${curr_test}" ) - docker_logs+=( "${log_file}" ) - docker_error_logs+=( "${err_file}" ) - done - done - docker_wait_all - rm ${pipe_file} - if [ "${#failed_tests[@]}" -eq 0 ] - then - echo -e "${GREEN}SUCCESS${NC}" - else - echo -e "${RED}FAILURE" - echo "The following tests failed:" - for t in "${failed_tests[@]}" - do - echo " - ${t}" - done - echo -e "${NC}" - __res=1 - fi - done - exitfun - return "${__res}" - } -fi diff --git a/build/cmake/package_tester/modules/globals.sh b/build/cmake/package_tester/modules/globals.sh deleted file mode 100644 index 795a4adc66..0000000000 --- a/build/cmake/package_tester/modules/globals.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This module has to be included first and only once. -# This is because of a limitation of older bash versions -# that doesn't allow us to declare associative arrays -# globally. - -if [ -z "${global_sh_included+x}"] -then - global_sh_included=1 -else - echo "global.sh can only be included once" - exit 1 -fi - -declare -A ini_name -declare -A ini_location -declare -A ini_packages -declare -A ini_format -declare -A test_start_state -declare -A test_exit_state -declare -a tests -declare -a vms diff --git a/build/cmake/package_tester/modules/rpm.sh b/build/cmake/package_tester/modules/rpm.sh deleted file mode 100644 index 866bde558e..0000000000 --- a/build/cmake/package_tester/modules/rpm.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${rpm_sh_included}" ] -then - rpm_sh_included=1 - - source ${source_dir}/modules/util.sh - - conf_save_extension=".rpmsave" - - install_build_tools() { - yum -y install cmake gcc - } - - install() { - local __res=0 - enterfun - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_names+=( "$(rpm -qp ${f})" ) - done - yum install -y ${package_files[@]} - __res=$? - # give the server some time to come up - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - if [ "$1" == "purge" ] - then - yum remove --purge -y ${package_names[@]} - else - yum remove -y ${package_names[@]} - fi - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/test_args.sh b/build/cmake/package_tester/modules/test_args.sh deleted file mode 100644 index bb88da945f..0000000000 --- a/build/cmake/package_tester/modules/test_args.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${test_args_sh_included+x} ] -then - test_args_sh_included=1 - - source ${source_dir}/modules/util.sh - - test_args_usage() { - me=`basename "$0"` - echo "usage: ${me} [-h] files..." - cat < /tmp/fdb.cluster - successOr "Could not create fdb.cluster file" - sed '/\[fdbserver.4500\]/a \[fdbserver.4501\]' /foundationdb/packaging/foundationdb.conf > /tmp/foundationdb.conf - successOr "Could not change foundationdb.conf file" - # we need to keep these files around for testing that the install didn't change them - cp /tmp/fdb.cluster /etc/foundationdb/fdb.cluster - cp /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf - - install - successOr "FoundationDB install failed" - # make sure we are not in build directory as there is a fdbc.cluster file there - echo "Configure new database - Install isn't supposed to do this for us" - echo "as there was an existing configuration" - cd / - timeout 2 fdbcli --exec 'configure new single ssd' - successOr "Couldn't configure new database" - tests_healthy - num_processes="$(timeout 2 fdbcli --exec 'status' | grep "FoundationDB processes" | sed -e 's/.*- //')" - if [ "${num_processes}" -ne 2 ] - then - fail Number of processes incorrect after config change - fi - - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - differences="$(diff /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - - uninstall - # make sure config didn't get deleted - # RPM, however, renames the file on remove, so we need to check for this - conffile="/etc/foundationdb/foundationdb.conf${conf_save_extension}" - if [ ! -f /etc/foundationdb/fdb.cluster ] || [ ! -f "${conffile}" ] - then - fail "Uninstall removed configuration" - fi - differences="$(diff /tmp/foundationdb.conf ${conffile})" - if [ -n "${differences}" ] - then - fail "${conffile} changed during remove" - fi - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail "/etc/foundationdb/fdb.cluster changed during remove" - fi - - return 0 - } -fi diff --git a/build/cmake/package_tester/modules/util.sh b/build/cmake/package_tester/modules/util.sh deleted file mode 100644 index c3d643bdfc..0000000000 --- a/build/cmake/package_tester/modules/util.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${util_sh_included+x} ] -then - util_sh_included=1 - - # for colored output - RED='\033[0;31m' - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - NC='\033[0m' # No Color - - - enterfun() { - pushd . > /dev/null - } - - exitfun() { - popd > /dev/null - } - - fail() { - false - successOr ${@:1} - } - - successOr() { - local __res=$? - if [ ${__res} -ne 0 ] - then - if [ "$#" -gt 1 ] - then - >&2 echo -e "${RED}${@:1} ${NC}" - fi - exit ${__res} - fi - return 0 - } - -fi diff --git a/build/cmake/package_tester/rpm_tests.sh b/build/cmake/package_tester/rpm_tests.sh deleted file mode 100755 index a88bfb4f15..0000000000 --- a/build/cmake/package_tester/rpm_tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/rpm.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/test_packages.sh b/build/cmake/package_tester/test_packages.sh deleted file mode 100755 index 05642073d8..0000000000 --- a/build/cmake/package_tester/test_packages.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/config.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/arguments.sh -source ${source_dir}/modules/docker.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - arguments_parse "$@" - if [ $? -ne 0 ] - then - __res=1 - break - fi - config_verify - if [ $? -ne 0 ] - then - __res=1 - break - fi - docker_run - __res=$? - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml deleted file mode 100644 index 4dcc30c683..0000000000 --- a/build/docker-compose.yaml +++ /dev/null @@ -1,105 +0,0 @@ -version: "3" - -services: - common: &common - image: foundationdb/foundationdb-build:0.1.24 - - build-setup: &build-setup - <<: *common - depends_on: [common] - volumes: - - ..:/__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - working_dir: /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - BUILD_DIR=./work - - release-setup: &release-setup - <<: *build-setup - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - RELEASE=true - - BUILD_DIR=./work - - snapshot-setup: &snapshot-setup - <<: *build-setup - - build-docs: - <<: *build-setup - volumes: - - ..:/foundationdb - working_dir: /foundationdb - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage' - - - release-packages: &release-packages - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - snapshot-packages: &snapshot-packages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - prb-packages: - <<: *snapshot-packages - - - release-bindings: &release-bindings - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - snapshot-bindings: &snapshot-bindings - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - prb-bindings: - <<: *snapshot-bindings - - - snapshot-cmake: &snapshot-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' - - prb-cmake: - <<: *snapshot-cmake - - - snapshot-bindings-cmake: &snapshot-bindings-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "bindings/all"' - - prb-bindings-cmake: - <<: *snapshot-bindings-cmake - - - snapshot-cmake: &snapshot-testpackages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}"' - - prb-testpackages: - <<: *snapshot-testpackages - - - snapshot-ctest: &snapshot-ctest - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-ctest: - <<: *snapshot-ctest - - - snapshot-correctness: &snapshot-correctness - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-correctness: - <<: *snapshot-correctness - - - shell: - <<: *build-setup - volumes: - - ..:/foundationdb - entrypoint: /bin/bash diff --git a/build/docker/centos6/build/Dockerfile b/build/docker/centos6/build/Dockerfile deleted file mode 100644 index 0a1fbbd70a..0000000000 --- a/build/docker/centos6/build/Dockerfile +++ /dev/null @@ -1,290 +0,0 @@ -FROM centos:6 - -WORKDIR /tmp - -RUN sed -i -e '/enabled/d' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -e '/gpgcheck=1/a enabled=0' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e "s/6\.0/$(cut -d\ -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --enable rhel-server-rhscl-7-rpms && \ - sed -i -e 's/#baseurl=/baseurl=/g' \ - -e 's/mirror.centos.org/vault.centos.org/g' \ - -e 's/mirrorlist=/#mirrorlist=/g' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo && \ - yum install -y \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-valgrind-devel \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-1.8.0-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby24 && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install autoconf -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz -o autoconf.tar.gz && \ - echo "954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 autoconf.tar.gz" > autoconf-sha.txt && \ - sha256sum -c autoconf-sha.txt && \ - mkdir autoconf && \ - tar --strip-components 1 --no-same-owner --directory autoconf -xf autoconf.tar.gz && \ - cd autoconf && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install automake -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/automake/automake-1.13.4.tar.gz -o automake.tar.gz && \ - echo "4c93abc0bff54b296f41f92dd3aa1e73e554265a6f719df465574983ef6f878c automake.tar.gz" > automake-sha.txt && \ - sha256sum -c automake-sha.txt && \ - mkdir automake && \ - tar --strip-components 1 --no-same-owner --directory automake -xf automake.tar.gz && \ - cd automake && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz && \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-project-10.0.0.tar.xz -o llvm.tar.xz && \ - echo "6287a85f4a6aeb07dbffe27847117fe311ada48005f2b00241b523fe7b60716e llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl - -# # Setting this environment variable switches from OpenSSL to BoringSSL -# ENV OPENSSL_ROOT_DIR=/opt/boringssl -# -# # install BoringSSL: TODO: They don't seem to have releases(?) I picked today's master SHA. -# RUN cd /opt &&\ -# git clone https://boringssl.googlesource.com/boringssl &&\ -# cd boringssl &&\ -# git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\ -# mkdir build -# -# # ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS -# # also, enable -fPIC. -# # this is moderately uglier than creating a patchfile, but easier to maintain. -# RUN cd /opt/boringssl &&\ -# for f in crypto/fipsmodule/rand/fork_detect_test.cc \ -# include/openssl/bn.h \ -# ssl/test/bssl_shim.cc ; do \ -# perl -p -i -e 's/#include /#define __STDC_FORMAT_MACROS 1\n#include /g;' $f ; \ -# done &&\ -# perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\ -# git diff -# -# RUN cd /opt/boringssl/build &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\ -# ./ssl/ssl_test &&\ -# mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib -# -# # Localize time zone -# ARG TIMEZONEINFO=America/Los_Angeles -# RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -# -# LABEL version=${IMAGE_TAG} -# ENV DOCKER_IMAGEVER=${IMAGE_TAG} -# ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 -# ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc -# ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ -# -# ENV CCACHE_NOHASHDIR=true -# ENV CCACHE_UMASK=0000 -# ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches" -# -# CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/docker/centos6/devel/Dockerfile b/build/docker/centos6/devel/Dockerfile deleted file mode 100644 index c5c9db2914..0000000000 --- a/build/docker/centos6/devel/Dockerfile +++ /dev/null @@ -1,84 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - jq \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - >> .bashrc diff --git a/build/docker/centos6/distcc/Dockerfile b/build/docker/centos6/distcc/Dockerfile deleted file mode 100644 index a96e67dff2..0000000000 --- a/build/docker/centos6/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/build/Dockerfile b/build/docker/centos7/build/Dockerfile deleted file mode 100644 index de376d2557..0000000000 --- a/build/docker/centos7/build/Dockerfile +++ /dev/null @@ -1,247 +0,0 @@ -FROM centos:7 - -WORKDIR /tmp -COPY mono-project.com.rpmkey.pgp ./ -RUN rpmkeys --import mono-project.com.rpmkey.pgp && \ - curl -Ls https://download.mono-project.com/repo/centos7-stable.repo -o /etc/yum.repos.d/mono-centos7-stable.repo && \ - yum repolist && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ - yum install -y \ - autoconf \ - automake \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-systemtap-sdt-devel \ - docker-ce \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-11-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - yum install -y \ - rh-python38 \ - rh-python38-python-devel \ - rh-ruby27; \ - else \ - yum install -y \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby26; \ - fi && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN if [ "$(uname -p)" == "aarch64" ]; then \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-aarch64.tar.gz -o cmake.tar.gz; \ - echo "69ec045c6993907a4f4a77349d0a0668f1bd3ce8bc5f6fbab6dc7a7e2ffc4f80 cmake.tar.gz" > cmake-sha.txt; \ - else \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz; \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt; \ - fi && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz -o llvm.tar.xz && \ - echo "b7b639fc675fa1c86dd6d0bc32267be9eb34451748d2efd03f674b773000e92b llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - source /opt/rh/rh-python38/enable; \ - else \ - source /opt/rh/rh-python36/enable; \ - fi && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# valgrind -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.17.0.tar.bz2 -o valgrind-3.17.0.tar.bz2 && \ - echo "ad3aec668e813e40f238995f60796d9590eee64a16dff88421430630e69285a2 valgrind-3.17.0.tar.bz2" > valgrind-sha.txt && \ - sha256sum -c valgrind-sha.txt && \ - mkdir valgrind && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory valgrind -xjf valgrind-3.17.0.tar.bz2 && \ - cd valgrind && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl diff --git a/build/docker/centos7/build/mono-project.com.rpmkey.pgp b/build/docker/centos7/build/mono-project.com.rpmkey.pgp deleted file mode 100644 index 4d7f8726d4..0000000000 --- a/build/docker/centos7/build/mono-project.com.rpmkey.pgp +++ /dev/null @@ -1,40 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 -Comment: Hostname: sks.pod01.fleetstreetops.com - -mQENBFPfqCcBCADctOzyTxfWvf40Nlb+AMkcJyb505WSbzhWU8yPmBNAJOnbwueMsTkNMHEO -u8fGRNxRWj5o/Db1N7EoSQtK3OgFnBef8xquUyrzA1nJ2aPfUWX+bhTG1TwyrtLaOssFRz6z -/h/ChUIFvt2VZCw+Yx4BiKi+tvgwrHTYB/Yf2J9+R/1O6949n6veFFRBfgPOL0djhvRqXzhv -FjJkh4xhTaGVeOnRR3+YQkblmti2n6KYl0n2kNB40ujSqpTloSfnR5tmJpz00WoOA9MJBdvH -txTTn8l6rVzXbm4mW9ZmB1kht/BgWaNLaIisW5AZSkQKer35wOWf0G7Gw+cWHq+I7W9pABEB -AAG0OlhhbWFyaW4gUHVibGljIEplbmtpbnMgKGF1dG8tc2lnbmluZykgPHJlbGVuZ0B4YW1h -cmluLmNvbT6JARwEEAECAAYFAlQIhKQACgkQyQ+cuQ4frQyc1wf+MCusJK4ANLWikbgiSSx1 -qMBveBlLKLEdCxYY+B9rc/pRDw448iBdd+nuSVdbRoqLgoN8gHbClboP+i22yw+mga0KASD7 -b1mpdYB0npR3H73zbYArn3qTV8s/yUXkIAEFUtj0yoEuv8KjO8P7nZJh8OuqqAupUVN0s3Kj -ONqXqi6Ro3fvVEZWOUFZl/FmY5KmXlpcw+YwE5CaNhJ2WunrjFTDqynRU/LeoPEKuwyYvfo9 -37zJFCrpAUMTr/9QpEKmV61H7fEHA9oHq97FBwWfjOU0l2mrXt1zJ97xVd2DXxrZodlkiY6B -76rhaT4ZhltY1E7WB2Z9WPfTe1Y6jz4fZ4kBHAQQAQgABgUCWEyoiAAKCRABFQplW72BAn/P -CAC0GkRBR3JTmG8WGeQMLb/o6Gon9cxpLnKv1GgFbHSM7XYMe7ySh5zxORwFuECuJ5+qcA6c -Ve/kJAV8rewLULL9yvHK3oK7R8zoVGbFVm+lyoxiaXpkkWg21Mb8IubiO+tA/dJc7hKQSpoI -0+dmJNaNrTVwqj0tQ8e0OL9KvBOYwFbSe06bocSNPVmKCt0EOvpGcQfzFw5UEjJVkqFn/moU -rSxj0YsJpwRXB1pOsBaQC6r9oCgUvxPf4H77U07+ImXzxRWInVPYFSXSiBA7p+hzvsikmZEl -iIAia8mTteUF1GeK4kafUk6iZZUfBlCIb9sV4O9Vvv8W0VjK4Vg6O2UAiQE4BBMBAgAiBQJT -36gnAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCmoZs409gx75DoB/9h5p8u1cUS -y6Mp2PjjW398LJZaqWwaa2W/lcLEKN7oWTC5Yf5BEuVsO9270pVln9Cv7hiqcbC8kywk+sZv -RsYO3uoTRwsmImc/7uaK382hey1A2hvkH5fYHmY/5Z/Z0bm/A0k0chhG2ycjWjZXYLZ96I0V -U3ZBQBHoh3qRtgWq4yWTsCJBX+FKPBdmkIpgcPXQw+hak0mj2sILqjScRZT1Oe+WJsMNMaLa -8dSdw+pPm8NM/VGLmO9iTTDApuAsRixpCYLdJY+ThGNrKe6xDswQo8gr3gbBkJi0wLRDP2Rz -q7rD0TC2PxOaWOZ7hmyz+EhjLcjZhHNJTaa+NV0k8YAwuQENBFPfqCcBCACtc7HssC9S3PxJ -m1youvGfYLhm+KzMO+gIoy7R32VXIZNxrkMYzaeerqSsMwxdhEjyOscT+rJbRGZ+9iPOGeh4 -AqZlzzOuxQ/Lg5h+2mGVXe0Avb+A2zC56mLSQCL3W8NjABUZdknnc1YIf9Dz05fy4jPEttNS -y+Rzte0ITLH1Hy/PKBrlF5n+G1/86f3L5n1ZZXmV3vi+rXT/OyEh9xRS4usmR6kVh4o2XGlI -zUrUjhZvb4lxrHfWgzKlWFoUSydaZDk7eikTKF692RiSSpLbDLW2sNOdzT2eqv2B8CJRF5sL -bD6BB3dAbH7KfqKiCT3xcCZhNEZw+M+GcRO/HNbnABEBAAGJAR8EGAECAAkFAlPfqCcCGwwA -CgkQpqGbONPYMe+sNQgAwjm9PJ45t7NBNTXn1zadoQQbPqz9qAlWiII0k+zzJCTTVqgyIXJY -I6zdNiB/Oh1Xajs/T9z9tL54+LLqgtZKa0lzDOmcxn6Iujf3a1MFdYxKgaQtT2ADxAimuBoz -3Y1ohxXgAs2+VISWYoPBI+UWhYqg11zq3uwpFIYQBRgkVydCxefCxY19okNp9FPC7KJPpJkO -NgDAK693Y9mOZXSq+XeGhjy3Sxesl0PYLIfV33z+vCpc2o1dDA5wuycgfqupNQITkQm6gPOH -1jLu8Vttm4fdEtVMcqkn8dJFomo3JW3qxI7IWwjbVRg10G8LGAuBbD6CA0dGSf8PkHFYv2Xs -dQ== -=MWcF ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/build/docker/centos7/devel/Dockerfile b/build/docker/centos7/devel/Dockerfile deleted file mode 100644 index 98f1923c17..0000000000 --- a/build/docker/centos7/devel/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum-config-manager --add-repo=https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/repo/epel-7/carlwgeorge-ripgrep-epel-7.repo && \ - yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - fish \ - jq \ - ripgrep \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN curl -Ls https://update.code.visualstudio.com/latest/server-linux-x64/stable -o /tmp/vscode-server-linux-x64.tar.gz && \ - mkdir -p .vscode-server/bin/latest && \ - tar --strip-components 1 --no-same-owner --directory .vscode-server/bin/latest -xf /tmp/vscode-server-linux-x64.tar.gz && \ - touch .vscode-server/bin/latest/0 && \ - rm -rf /tmp/* -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -Eeuo pipefail' \ - '' \ - 'mkdir -p ~/.docker' \ - 'cat > ~/.docker/config.json << EOF' \ - '{' \ - ' "proxies":' \ - ' {' \ - ' "default":' \ - ' {' \ - ' "httpProxy": "${HTTP_PROXY}",' \ - ' "httpsProxy": "${HTTPS_PROXY}",' \ - ' "noProxy": "${NO_PROXY}"' \ - ' }' \ - ' }' \ - '}' \ - 'EOF' \ - > docker_proxy.sh && \ - chmod 755 docker_proxy.sh && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - 'bash ${HOME}/docker_proxy.sh' \ - >> .bashrc diff --git a/build/docker/centos7/distcc/Dockerfile b/build/docker/centos7/distcc/Dockerfile deleted file mode 100644 index 785e6bee93..0000000000 --- a/build/docker/centos7/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/ycsb/Dockerfile b/build/docker/centos7/ycsb/Dockerfile deleted file mode 100644 index a8b60230b3..0000000000 --- a/build/docker/centos7/ycsb/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -ENV YCSB_VERSION=ycsb-foundationdb-binding-0.17.0 \ - PATH=${PATH}:/usr/bin - -RUN cd /opt \ - && eval curl "-Ls https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-foundationdb-binding-0.17.0.tar.gz" \ - | tar -xzvf - - -RUN rm -Rf /opt/${YCSB_VERSION}/lib/fdb-java-5.2.5.jar - -# COPY The Appropriate fdb-java-.jar Aaron from packages -# COPY binary RPM for foundationd-db -# Install Binary - -WORKDIR "/opt/${YCSB_VERSION}" - -ENTRYPOINT ["bin/ycsb.sh"] \ No newline at end of file diff --git a/build/gen_dev_docker.sh b/build/gen_dev_docker.sh deleted file mode 100755 index 89129d5a86..0000000000 --- a/build/gen_dev_docker.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# we first check whether the user is in the group docker -user=$(id -un) -DIR_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) -group=$(id -gn) -uid=$(id -u) -gid=$(id -g) -gids=( $(id -G) ) -groups=( $(id -Gn) ) -tmpdir="/tmp/fdb-docker-${DIR_UUID}" -image=fdb-dev - -pushd . -mkdir ${tmpdir} -cd ${tmpdir} - -echo - -cat <> Dockerfile -FROM foundationdb/foundationdb-dev:0.11.1 -RUN yum install -y sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN groupadd -g 1100 sudo -EOF - -num_groups=${#gids[@]} -additional_groups="-G sudo" -for ((i=0;i> Dockerfile - if [ ${gids[i]} -ne ${gid} ] - then - additional_groups="${additional_groups},${gids[$i]}" - fi -done - -cat <> Dockerfile -RUN useradd -u ${uid} -g ${gid} ${additional_groups} -m ${user} - -USER ${user} -CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash - -EOF - -echo "Created ${tmpdir}" -echo "Buidling Docker container ${image}" -sudo docker build -t ${image} . - -popd - -echo "Writing startup script" -mkdir -p $HOME/bin -cat < $HOME/bin/fdb-dev -#!/usr/bin/bash - -if [ -d "\${CCACHE_DIR}" ] -then - args="-v \${CCACHE_DIR}:\${CCACHE_DIR}" - args="\${args} -e CCACHE_DIR=\${CCACHE_DIR}" - args="\${args} -e CCACHE_UMASK=\${CCACHE_UMASK}" - ccache_args=\$args -fi - -if [ -t 1 ] ; then - TERMINAL_ARGS=-it `# Run in interactive mode and simulate a TTY` -else - TERMINAL_ARGS=-i `# Run in interactive mode` -fi - -sudo docker run --rm `# delete (temporary) image after return` \\ - \${TERMINAL_ARGS} \\ - --privileged=true `# Run in privileged mode ` \\ - --cap-add=SYS_PTRACE \\ - --security-opt seccomp=unconfined \\ - -v "${HOME}:${HOME}" `# Mount home directory` \\ - -w="\$(pwd)" \\ - \${ccache_args} \\ - ${image} "\$@" -EOF - -cat < $HOME/bin/clangd -#!/usr/bin/bash - -fdb-dev scl enable devtoolset-8 rh-python36 rh-ruby24 -- clangd -EOF - -if [[ ":$PATH:" != *":$HOME/bin:"* ]] -then - echo "WARNING: $HOME/bin is not in your PATH!" - echo -e "\tThis can cause problems with some scripts (like fdb-clangd)" -fi -chmod +x $HOME/bin/fdb-dev -chmod +x $HOME/bin/clangd -echo "To start the dev docker image run $HOME/bin/fdb-dev" -echo "$HOME/bin/clangd can be used for IDE integration" -echo "You can edit these files but be aware that this script will overwrite your changes if you rerun it" diff --git a/build/get_package_name.sh b/build/get_package_name.sh deleted file mode 100755 index c2c94d126b..0000000000 --- a/build/get_package_name.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' diff --git a/build/get_version.sh b/build/get_version.sh deleted file mode 100755 index a7a2a179f2..0000000000 --- a/build/get_version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' - diff --git a/build/txt-to-toml.py b/build/txt-to-toml.py deleted file mode 100755 index 68d1dcdbb5..0000000000 --- a/build/txt-to-toml.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import sys - - -def main(): - if len(sys.argv) != 2: - print("Usage: txt-to-toml.py [src.txt]") - return 1 - - filename = sys.argv[1] - - indent = " " - in_workload = False - first_test = False - keys_before_test = False - - for line in open(filename): - k = "" - v = "" - - if line.strip().startswith(";"): - print((indent if in_workload else "") + line.strip().replace(";", "#")) - continue - - if "=" in line: - (k, v) = line.strip().split("=") - (k, v) = (k.strip(), v.strip()) - - if k == "testTitle": - first_test = True - if in_workload: - print("") - in_workload = False - if keys_before_test: - print("") - keys_before_test = False - print("[[test]]") - - if k == "testName": - in_workload = True - print("") - print(indent + "[[test.workload]]") - - if not first_test: - keys_before_test = True - - if v.startswith("."): - v = "0" + v - - if any(c.isalpha() or c in ["/", "!"] for c in v): - if v != "true" and v != "false": - v = "'" + v + "'" - - if k == "buggify": - print("buggify = " + ("true" if v == "'on'" else "false")) - elif k: - print((indent if in_workload else "") + k + " = " + v) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 7e0502c52e..8a4f638380 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -39,6 +39,9 @@ function(configure_testing) endfunction() function(verify_testing) + if(NOT ENABLE_SIMULATION_TESTS) + return() + endif() foreach(test_file IN LISTS fdb_test_files) message(SEND_ERROR "${test_file} found but it is not associated with a test") endforeach() @@ -119,27 +122,30 @@ function(add_fdb_test) set(VALGRIND_OPTION "--use-valgrind") endif() list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") - add_test(NAME ${test_name} - COMMAND $ ${TestRunner} - -n ${test_name} - -b ${PROJECT_BINARY_DIR} - -t ${test_type} - -O ${OLD_FDBSERVER_BINARY} - --crash - --aggregate-traces ${TEST_AGGREGATE_TRACES} - --log-format ${TEST_LOG_FORMAT} - --keep-logs ${TEST_KEEP_LOGS} - --keep-simdirs ${TEST_KEEP_SIMDIR} - --seed ${SEED} - --test-number ${assigned_id} - ${BUGGIFY_OPTION} - ${VALGRIND_OPTION} - ${ADD_FDB_TEST_TEST_FILES} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - get_filename_component(test_dir_full ${first_file} DIRECTORY) - if(NOT ${test_dir_full} STREQUAL "") - get_filename_component(test_dir ${test_dir_full} NAME) - set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}") + if (ENABLE_SIMULATION_TESTS) + add_test(NAME ${test_name} + COMMAND $ ${TestRunner} + -n ${test_name} + -b ${PROJECT_BINARY_DIR} + -t ${test_type} + -O ${OLD_FDBSERVER_BINARY} + --crash + --aggregate-traces ${TEST_AGGREGATE_TRACES} + --log-format ${TEST_LOG_FORMAT} + --keep-logs ${TEST_KEEP_LOGS} + --keep-simdirs ${TEST_KEEP_SIMDIR} + --seed ${SEED} + --test-number ${assigned_id} + ${BUGGIFY_OPTION} + ${VALGRIND_OPTION} + ${ADD_FDB_TEST_TEST_FILES} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) + get_filename_component(test_dir_full ${first_file} DIRECTORY) + if(NOT ${test_dir_full} STREQUAL "") + get_filename_component(test_dir ${test_dir_full} NAME) + set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}") + endif() endif() # set variables used for generating test packages set(TEST_NAMES ${TEST_NAMES} ${test_name} PARENT_SCOPE) @@ -261,6 +267,14 @@ function(create_correctness_package) ) add_custom_target(package_tests ALL DEPENDS ${tar_file}) add_dependencies(package_tests strip_only_fdbserver TestHarness) + set(unversioned_tar_file "${CMAKE_BINARY_DIR}/packages/correctness.tar.gz") + add_custom_command( + OUTPUT "${unversioned_tar_file}" + DEPENDS "${tar_file}" + COMMAND ${CMAKE_COMMAND} -E copy "${tar_file}" "${unversioned_tar_file}" + COMMENT "Copy correctness package to ${unversioned_tar_file}") + add_custom_target(package_tests_u DEPENDS "${unversioned_tar_file}") + add_dependencies(package_tests_u package_tests) endfunction() function(create_valgrind_correctness_package) @@ -288,6 +302,14 @@ function(create_valgrind_correctness_package) ) add_custom_target(package_valgrind_tests ALL DEPENDS ${tar_file}) add_dependencies(package_valgrind_tests strip_only_fdbserver TestHarness) + set(unversioned_tar_file "${CMAKE_BINARY_DIR}/packages/valgrind.tar.gz") + add_custom_command( + OUTPUT "${unversioned_tar_file}" + DEPENDS "${tar_file}" + COMMAND ${CMAKE_COMMAND} -E copy "${tar_file}" "${unversioned_tar_file}" + COMMENT "Copy valgrind package to ${unversioned_tar_file}") + add_custom_target(package_valgrind_tests_u DEPENDS "${unversioned_tar_file}") + add_dependencies(package_valgrind_tests_u package_valgrind_tests) endif() endfunction() @@ -378,9 +400,10 @@ function(package_bindingtester) add_dependencies(bindingtester copy_bindingtester_binaries) endfunction() +# Creates a single cluster before running the specified command (usually a ctest test) function(add_fdbclient_test) set(options DISABLED ENABLED) - set(oneValueArgs NAME) + set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT) set(multiValueArgs COMMAND) cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") if(OPEN_FOR_IDE) @@ -396,12 +419,57 @@ function(add_fdbclient_test) message(FATAL_ERROR "COMMAND is a required argument for add_fdbclient_test") endif() message(STATUS "Adding Client test ${T_NAME}") - add_test(NAME "${T_NAME}" - COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py + if (T_PROCESS_NUMBER) + add_test(NAME "${T_NAME}" + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py + --build-dir ${CMAKE_BINARY_DIR} + --process-number ${T_PROCESS_NUMBER} + -- + ${T_COMMAND}) + else() + add_test(NAME "${T_NAME}" + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py --build-dir ${CMAKE_BINARY_DIR} -- ${T_COMMAND}) - set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) + endif() + if (T_TEST_TIMEOUT) + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT ${T_TEST_TIMEOUT}) + else() + # default timeout + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) + endif() + set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) +endfunction() + +# Creates 3 distinct clusters before running the specified command. +# This is useful for testing features that require multiple clusters (like the +# multi-cluster FDB client) +function(add_multi_fdbclient_test) + set(options DISABLED ENABLED) + set(oneValueArgs NAME) + set(multiValueArgs COMMAND) + cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + if(OPEN_FOR_IDE) + return() + endif() + if(NOT T_ENABLED AND T_DISABLED) + return() + endif() + if(NOT T_NAME) + message(FATAL_ERROR "NAME is a required argument for add_multi_fdbclient_test") + endif() + if(NOT T_COMMAND) + message(FATAL_ERROR "COMMAND is a required argument for add_multi_fdbclient_test") + endif() + message(STATUS "Adding Client test ${T_NAME}") + add_test(NAME "${T_NAME}" + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py + --build-dir ${CMAKE_BINARY_DIR} + --clusters 3 + -- + ${T_COMMAND}) + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) endfunction() function(add_java_test) diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake index 687c266f0b..57ddbd41df 100644 --- a/cmake/CompileBoost.cmake +++ b/cmake/CompileBoost.cmake @@ -1,61 +1,73 @@ function(compile_boost) + + # Initialize function incoming parameters set(options) set(oneValueArgs TARGET) set(multiValueArgs BUILD_ARGS CXXFLAGS LDFLAGS) - cmake_parse_arguments(MY "${options}" "${oneValueArgs}" + cmake_parse_arguments(COMPILE_BOOST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - # Configure the boost toolset to use - set(BOOTSTRAP_ARGS "--with-libraries=context") - set(B2_COMMAND "./b2") - set(BOOST_COMPILER_FLAGS -fvisibility=hidden -fPIC -std=c++14 -w) + + # Configure bootstrap command + set(BOOTSTRAP_COMMAND "./bootstrap.sh") + set(BOOTSTRAP_LIBRARIES "context") + set(BOOST_CXX_COMPILER "${CMAKE_CXX_COMPILER}") - if(APPLE) - set(BOOST_TOOLSET "clang-darwin") - # this is to fix a weird macOS issue -- by default - # cmake would otherwise pass a compiler that can't - # compile boost - set(BOOST_CXX_COMPILER "/usr/bin/clang++") - elseif(CLANG) + if(CLANG) set(BOOST_TOOLSET "clang") - list(APPEND BOOTSTRAP_ARGS "${BOOTSTRAP_COMMAND} --with-toolset=clang") + if(APPLE) + # this is to fix a weird macOS issue -- by default + # cmake would otherwise pass a compiler that can't + # compile boost + set(BOOST_CXX_COMPILER "/usr/bin/clang++") + endif() else() set(BOOST_TOOLSET "gcc") endif() - if(APPLE OR USE_LIBCXX) - list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++) - endif() - set(BOOST_ADDITIONAL_COMPILE_OPTIOINS "") - foreach(flag IN LISTS BOOST_COMPILER_FLAGS MY_CXXFLAGS) - string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIOINS "${flag} ") - endforeach() - foreach(flag IN LISTS MY_LDFLAGS) - string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIOINS "${flag} ") - endforeach() - configure_file(${CMAKE_SOURCE_DIR}/cmake/user-config.jam.cmake ${CMAKE_BINARY_DIR}/user-config.jam) + message(STATUS "Use ${BOOST_TOOLSET} to build boost") + # Configure b2 command + set(B2_COMMAND "./b2") + set(BOOST_COMPILER_FLAGS -fvisibility=hidden -fPIC -std=c++17 -w) + set(BOOST_LINK_FLAGS "") + if(APPLE OR CLANG OR USE_LIBCXX) + list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++ -nostdlib++) + list(APPEND BOOST_LINK_FLAGS -static-libgcc -lc++ -lc++abi) + endif() + + # Update the user-config.jam + set(BOOST_ADDITIONAL_COMPILE_OPTIOINS "") + foreach(flag IN LISTS BOOST_COMPILER_FLAGS COMPILE_BOOST_CXXFLAGS) + string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "${flag} ") + endforeach() + #foreach(flag IN LISTS BOOST_LINK_FLAGS COMPILE_BOOST_LDFLAGS) + # string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "${flag} ") + #endforeach() + configure_file(${CMAKE_SOURCE_DIR}/cmake/user-config.jam.cmake ${CMAKE_BINARY_DIR}/user-config.jam) set(USER_CONFIG_FLAG --user-config=${CMAKE_BINARY_DIR}/user-config.jam) + # Build boost include(ExternalProject) set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install") - ExternalProject_add("${MY_TARGET}Project" + ExternalProject_add("${COMPILE_BOOST_TARGET}Project" URL "https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2" URL_HASH SHA256=59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 - CONFIGURE_COMMAND ./bootstrap.sh ${BOOTSTRAP_ARGS} - BUILD_COMMAND ${B2_COMMAND} link=static ${MY_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install + CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET} + BUILD_COMMAND ${B2_COMMAND} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install BUILD_IN_SOURCE ON INSTALL_COMMAND "" UPDATE_COMMAND "" BUILD_BYPRODUCTS "${BOOST_INSTALL_DIR}/boost/config.hpp" "${BOOST_INSTALL_DIR}/lib/libboost_context.a") - add_library(${MY_TARGET}_context STATIC IMPORTED) - add_dependencies(${MY_TARGET}_context ${MY_TARGET}Project) - set_target_properties(${MY_TARGET}_context PROPERTIES IMPORTED_LOCATION "${BOOST_INSTALL_DIR}/lib/libboost_context.a") + add_library(${COMPILE_BOOST_TARGET}_context STATIC IMPORTED) + add_dependencies(${COMPILE_BOOST_TARGET}_context ${COMPILE_BOOST_TARGET}Project) + set_target_properties(${COMPILE_BOOST_TARGET}_context PROPERTIES IMPORTED_LOCATION "${BOOST_INSTALL_DIR}/lib/libboost_context.a") - add_library(${MY_TARGET} INTERFACE) - target_include_directories(${MY_TARGET} SYSTEM INTERFACE ${BOOST_INSTALL_DIR}/include) - target_link_libraries(${MY_TARGET} INTERFACE ${MY_TARGET}_context) -endfunction() + add_library(${COMPILE_BOOST_TARGET} INTERFACE) + target_include_directories(${COMPILE_BOOST_TARGET} SYSTEM INTERFACE ${BOOST_INSTALL_DIR}/include) + target_link_libraries(${COMPILE_BOOST_TARGET} INTERFACE ${COMPILE_BOOST_TARGET}_context) + +endfunction(compile_boost) if(USE_SANITIZER) if(WIN32) @@ -72,10 +84,20 @@ if(USE_SANITIZER) return() endif() -list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0) # since boost 1.72 boost installs cmake configs. We will enforce config mode set(Boost_USE_STATIC_LIBS ON) -set(BOOST_HINT_PATHS /opt/boost_1_72_0) + +# Clang and Gcc will have different name mangling to std::call_once, etc. +if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$") + list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0_clang) + set(BOOST_HINT_PATHS /opt/boost_1_72_0_clang) + message(STATUS "Using Clang version of boost::context") +else () + list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0) + set(BOOST_HINT_PATHS /opt/boost_1_72_0) + message(STATUS "Using g++ version of boost::context") +endif () + if(BOOST_ROOT) list(APPEND BOOST_HINT_PATHS ${BOOST_ROOT}) endif() diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake index 4fcf78a334..6d6e959fd5 100644 --- a/cmake/CompileRocksDB.cmake +++ b/cmake/CompileRocksDB.cmake @@ -36,8 +36,8 @@ if (RocksDB_FOUND) ${BINARY_DIR}/librocksdb.a) else() ExternalProject_Add(rocksdb - URL https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz - URL_HASH SHA256=d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee + URL https://github.com/facebook/rocksdb/archive/v6.22.1.tar.gz + URL_HASH SHA256=2df8f34a44eda182e22cf84dee7a14f17f55d305ff79c06fb3cd1e5f8831e00d CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB} -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index c14c5011c5..6379f7bf14 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -100,8 +100,7 @@ if(WIN32) endif() add_compile_options(/W0 /EHsc /bigobj $<$:/Zi> /MP /FC /Gm-) add_compile_definitions(NOMINMAX) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") else() set(GCC NO) set(CLANG NO) @@ -262,10 +261,6 @@ else() if (CLANG) add_compile_options() - # Clang has link errors unless `atomic` is specifically requested. - if(NOT APPLE) - #add_link_options(-latomic) - endif() if (APPLE OR USE_LIBCXX) add_compile_options($<$:-stdlib=libc++>) if (NOT APPLE) @@ -286,25 +281,20 @@ else() -Wpessimizing-move -Woverloaded-virtual -Wshift-sign-overflow - # Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 10 + # Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 11 -Wno-comment - -Wno-dangling-else -Wno-delete-non-virtual-dtor -Wno-format -Wno-mismatched-tags -Wno-missing-field-initializers - -Wno-reorder - -Wno-reorder-ctor -Wno-sign-compare -Wno-tautological-pointer-compare -Wno-undefined-var-template - -Wno-tautological-pointer-compare -Wno-unknown-pragmas -Wno-unknown-warning-option -Wno-unused-function -Wno-unused-local-typedef -Wno-unused-parameter - -Wno-self-assign ) if (USE_CCACHE) add_compile_options( diff --git a/cmake/user-config.jam.cmake b/cmake/user-config.jam.cmake index 2978c670a5..6d2883cc95 100644 --- a/cmake/user-config.jam.cmake +++ b/cmake/user-config.jam.cmake @@ -1 +1 @@ -using @BOOST_TOOLSET@ : : @BOOST_CXX_COMPILER@ : @BOOST_ADDITIONAL_COMPILE_OPTIOINS@ ; +using @BOOST_TOOLSET@ : : @BOOST_CXX_COMPILER@ : @BOOST_ADDITIONAL_COMPILE_OPTIONS@ ; diff --git a/contrib/TestHarness/Program.cs.cmake b/contrib/TestHarness/Program.cs.cmake index ab5d557dfd..0b907fcbb2 100644 --- a/contrib/TestHarness/Program.cs.cmake +++ b/contrib/TestHarness/Program.cs.cmake @@ -144,7 +144,9 @@ namespace SummarizeTest string oldBinaryFolder = (args.Length > 1) ? args[1] : Path.Combine("/opt", "joshua", "global_data", "oldBinaries"); bool useValgrind = args.Length > 2 && args[2].ToLower() == "true"; int maxTries = (args.Length > 3) ? int.Parse(args[3]) : 3; - return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN)); + bool buggifyEnabled = (args.Length > 4) ? bool.Parse(args[4]) : true; + bool faultInjectionEnabled = (args.Length > 5) ? bool.Parse(args[5]) : true; + return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN), buggifyEnabled, faultInjectionEnabled); } catch(Exception e) { @@ -240,10 +242,10 @@ namespace SummarizeTest } } - static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "") + static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "", bool buggifyEnabled = true, bool faultInjectionEnabled = true) { int seed = random.Next(1000000000); - bool buggify = random.NextDouble() < buggifyOnRatio; + bool buggify = buggifyEnabled ? (random.NextDouble() < buggifyOnRatio) : false; string testFile = null; string testDir = ""; string oldServerName = ""; @@ -353,11 +355,11 @@ namespace SummarizeTest bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0"); bool useToml = File.Exists(testFile + "-1.toml"); string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt"; - result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim); + result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled); if (result == 0) { string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt"; - result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim); + result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled); } } else @@ -365,13 +367,13 @@ namespace SummarizeTest int expectedUnseed = -1; if (!useValgrind && unseedCheck) { - result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout, noSim); + result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout, noSim, faultInjectionEnabled); } if (!retryableError) { int unseed; - result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout, noSim); + result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout, noSim, faultInjectionEnabled); } } @@ -386,7 +388,7 @@ namespace SummarizeTest private static int RunTest(string fdbserverName, string tlsPluginFile, string summaryFileName, string errorFileName, int seed, bool buggify, string testFile, string runDir, string uid, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, bool useValgrind, bool restarting = false, - bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false, bool noSim = false) + bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false, bool noSim = false, bool faultInjectionEnabled = true) { unseed = -1; @@ -407,7 +409,7 @@ namespace SummarizeTest Directory.CreateDirectory(tempPath); Directory.SetCurrentDirectory(tempPath); - if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, oldBinaryName); + if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, faultInjectionEnabled, oldBinaryName); string valgrindOutputFile = null; using (var process = new System.Diagnostics.Process()) @@ -422,15 +424,16 @@ namespace SummarizeTest process.StartInfo.RedirectStandardOutput = true; string role = (noSim) ? "test" : "simulation"; var args = ""; + string faultInjectionArg = string.IsNullOrEmpty(oldBinaryName) ? string.Format("-fi {0}", faultInjectionEnabled ? "on" : "off") : ""; if (willRestart && oldBinaryName.EndsWith("alpha6")) { - args = string.Format("-Rs 1000000000 -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} --crash", - role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg); + args = string.Format("-Rs 1000000000 -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} {6} --crash", + role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg); } else { - args = string.Format("-Rs 1GB -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} --crash", - role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg); + args = string.Format("-Rs 1GB -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} {6} --crash", + role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg); } if (restarting) args = args + " --restarting"; if (useValgrind && !willRestart) @@ -524,7 +527,7 @@ namespace SummarizeTest var xout = new XElement("UnableToKillProcess", new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways)); - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled); return 104; } } @@ -536,7 +539,8 @@ namespace SummarizeTest consoleThread.Join(); var traceFiles = Directory.GetFiles(tempPath, "trace*.*").Where(s => s.EndsWith(".xml") || s.EndsWith(".json")).ToArray(); - if (traceFiles.Length == 0) + // if no traces caused by the process failed then the result will include its stderr + if (process.ExitCode == 0 && traceFiles.Length == 0) { if (!traceToStdout) { @@ -548,7 +552,7 @@ namespace SummarizeTest new XAttribute("Plugin", tlsPluginFile), new XAttribute("MachineName", System.Environment.MachineName)); - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled); ok = useValgrind ? 0 : 103; } else @@ -587,7 +591,7 @@ namespace SummarizeTest new XAttribute("Severity", (int)Magnesium.Severity.SevError), new XAttribute("ErrorMessage", e.Message)); - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled); return 101; } finally @@ -637,6 +641,15 @@ namespace SummarizeTest { if(!String.IsNullOrEmpty(errLine.Data)) { + if (errLine.Data.EndsWith("WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!")) { + // When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives. + return; + } + if (errLine.Data.EndsWith("Warning: unimplemented fcntl command: 1036")) { + // Valgrind produces this warning when F_SET_RW_HINT is used + return; + } + hasError = true; if(Errors.Count < maxErrors) { if(errLine.Data.Length > maxErrorLength) { @@ -694,13 +707,14 @@ namespace SummarizeTest } } - static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, string oldBinary="") + static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, bool faultInjectionEnabled, string oldBinary="") { var xout = new XElement("TestPlan", new XAttribute("TestUID", uid), new XAttribute("RandomSeed", randomSeed), new XAttribute("TestFile", testFileName), new XAttribute("BuggifyEnabled", buggify ? "1" : "0"), + new XAttribute("FaultInjectionEnabled", faultInjectionEnabled ? "1" : "0"), new XAttribute("DeterminismCheck", testDeterminism ? "1" : "0"), new XAttribute("OldBinary", Path.GetFileName(oldBinary))); AppendToSummary(summaryFileName, xout); @@ -790,6 +804,8 @@ namespace SummarizeTest new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"), new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); testBeginFound = true; + if (ev.DDetails.ContainsKey("FaultInjectionEnabled")) + xout.Add(new XAttribute("FaultInjectionEnabled", ev.Details.FaultInjectionEnabled)); } if (ev.Type == "Simulation") { @@ -961,10 +977,6 @@ namespace SummarizeTest int stderrBytes = 0; foreach (string err in outputErrors) { - if (err.EndsWith("WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!")) { - // When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives. - continue; - } if (stderrSeverity == (int)Magnesium.Severity.SevError) { error = true; @@ -1229,7 +1241,7 @@ namespace SummarizeTest } private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null, - int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null) + int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null, bool? faultInjectionEnabled = null) { var test = new XElement("Test", xout); if(testFile != null) @@ -1238,6 +1250,8 @@ namespace SummarizeTest test.Add(new XAttribute("RandomSeed", seed)); if(buggify != null) test.Add(new XAttribute("BuggifyEnabled", buggify.Value ? "1" : "0")); + if(faultInjectionEnabled != null) + test.Add(new XAttribute("FaultInjectionEnabled", faultInjectionEnabled.Value ? "1" : "0")); if(determinismCheck != null) test.Add(new XAttribute("DeterminismCheck", determinismCheck.Value ? "1" : "0")); if(oldBinaryName != null) diff --git a/build/gen_compile_db.py b/contrib/gen_compile_db.py old mode 100755 new mode 100644 similarity index 100% rename from build/gen_compile_db.py rename to contrib/gen_compile_db.py diff --git a/design/tlog-spilling.md.html b/design/tlog-spilling.md.html index 58bbde503e..da72f8eccf 100644 --- a/design/tlog-spilling.md.html +++ b/design/tlog-spilling.md.html @@ -352,7 +352,7 @@ API for random reads to the DiskQueue. That ability is now required for peeking, and thus, `IDiskQueue`'s API has been enhanced correspondingly: ``` CPP -enum class CheckHashes { NO, YES }; +BOOLEAN_PARAM(CheckHashes); class IDiskQueue { // ... @@ -369,9 +369,9 @@ and not `(start, length)`. Spilled data, when using spill-by-value, was resistant to bitrot via data being checksummed interally within SQLite's B-tree. Now that reads can be done directly, the responsibility for verifying data integrity falls upon the -DiskQueue. `CheckHashes::YES` will cause the DiskQueue to use the checksum in +DiskQueue. `CheckHashes::TRUE` will cause the DiskQueue to use the checksum in each DiskQueue page to verify data integrity. If an externally maintained -checksums exists to verify the returned data, then `CheckHashes::NO` can be +checksums exists to verify the returned data, then `CheckHashes::FALSE` can be used to elide the checksumming. A page failing its checksum will cause the transaction log to die with an `io_error()`. diff --git a/documentation/CMakeLists.txt b/documentation/CMakeLists.txt index ccd60a2bbd..e734e28e91 100644 --- a/documentation/CMakeLists.txt +++ b/documentation/CMakeLists.txt @@ -1,4 +1,8 @@ add_subdirectory(tutorial) +if(WIN32) + return() +endif() + # build a virtualenv set(sphinx_dir ${CMAKE_CURRENT_SOURCE_DIR}/sphinx) set(venv_dir ${CMAKE_CURRENT_BINARY_DIR}/venv) diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst index 707d8e3246..46e5aa6fcc 100644 --- a/documentation/sphinx/source/api-version-upgrade-guide.rst +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -25,6 +25,8 @@ API version 700 General ------- +* Committing a transaction will no longer partially reset it. In particular, getting the read version from a transaction that has committed or failed to commit with an error will return the original read version. + Python bindings --------------- diff --git a/documentation/sphinx/source/ha-write-path.rst b/documentation/sphinx/source/ha-write-path.rst new file mode 100644 index 0000000000..e5d20a2694 --- /dev/null +++ b/documentation/sphinx/source/ha-write-path.rst @@ -0,0 +1,195 @@ +################################################### +FDB HA Write Path: How a mutation travels in FDB HA +################################################### + +| Author: Meng Xu +| Reviewer: Alex Miller, Jingyu Zhou, Lukas Joswiak, Trevor Clinkenbeard +| Audience: FDB developers, SREs and expert users. + +This document describes how a mutation is replicated and moved from proxy to storage servers (SS) in a FDB High Availability (HA) cluster. Historically, FDB HA is also called Fearless DR or Multi-region configuration. + +To simplify the description, we assume the HA cluster has the following configuration: + +* Replication factor = 3 for storage servers. It means each mutation is replicated to 3 storage servers in the primary datacenter (DC) and 3 SSes in the secondary DC. + +* Replication factor = 3 for transaction logs (tLogs). It means each mutation is synchronously replicated to 3 primary tLogs and 1 satellite tLog. + +* Satellite replication factor = 1 satellite single replication. It means each mutation must be synchronously replicated to 1 satellite tLog before it can be committed. + + * The satellite replication factor can be configured with one or two satellites and single, double or triple replicas as described here. We typically use only 1 satellite single replica config. + +* Only 1 satellite is configured in the primary DC. + +We describe the background knowledge -- Sharding and Tag structure -- before we discuss how a mutation travels in a FDB HA cluster. + +Sharding: Which shard goes to which servers? +============================================ + +A shard is a continuous key range. FDB divides the entire keyspace to thousands of shards. A mutation’s key decides which shard it belongs to. + +Shard-to-SS mapping is determined by the \xff/keyServers/ system keyspace. In the system keyspace, a shard’s begin key is used as the key, the shard’s end key is the next key, and the shard’s SSes are the value. For example, we have the following key-values in the system keyspace: \xff/keyServers/a=(SS1,SS2,SS3) , \xff/keyServers/b=(SS2,SS4,SS7) . It indicates: shard [a,b) will be saved on storage servers whose IDs are SS1, SS2, SS3; and shard [b, \xff\xff) will be saved on storage servers SS2, SS4, SS7. + +SS-to-tag mapping is decided by the \xff/serverTag/ system keyspace. A tag is a whole number (i.e., natural number and 0). Each SS is mapped to a tag and vice versa. We use tags to represent SSes in the transaction system to save space and speed up search, because tags are continuous and small numbers (described in Tag structure section) while SS IDs are 64 bit random UID. + +Shard-to-tLog mapping is decided by shard-to-SS mapping and tLog’s replication policy. We use an example to explain how it works. Assume a mutation is mapped to SS1, SS2, and SS5, whose tags are respectively 1, 2, 5. The system has four tlogs. We use a function to transfer tag to tLog index: f(tag) = tLogIndex, where f(tag) is a modular function in FDB 6.2 and 6.3 implementation. In the example, the mutation’s assigned tLogs will be 1, 2, 1, which is calculated as the shard’s tag % 4. As you may notice, the three tags produces only two unique tLog indexes, which does not satisfy tLog’s replication policy that requires 3 tLog replicas. The proxy will call the replication policy engine, selectReplicas(), to choose another tLog for the mutation. + + +Tag structure +============= + +Tag is an overloaded term in FDB. In the early history of FDB, a tag is a number used in SS-to-tag mapping. As FDB evolves, tags are used by different components for different purposes: + +* As FDB evolves to HA, tags are used by not only primary tLogs but also satellite tLogs, log router, and remote tLogs; + +* As FDB scales and we work to reduce the recovery time, a special tag for transaction state store (txnStateStore) is introduced; + +* FDB also have transaction tags which are used for transaction throttling, not for the tag-partitioned log system mentioned in this article. See :ref:`transaction-tagging` + +To distinguish the types of tags used for different purposes at different locations (primary DC or remote DC), we introduce Tag structure, which has two fields: + +* locality (int8_t): When it is non-negative value, it decides which DC id the tag is used in. For example, if it is 0, it means the tag is used in primary DC and the tag’s id represents a storage server and is used for primary tLogs to index by storage servers. When it is negative, it decides which types of tags the tag belongs to. For example, if it is -2, it is a log router tag, and its id is used to decide which log router the tagged mutation should be sent to. The definition of all localities are in FDBTypes.h and you can easily find it if you search tagLocalitySpecial in the file. + +* id (uint16_t): Once locality decides which FDB components will the tag be applied to, id decides which process in the component type will be used for the tagged mutation. + + * FDB components in this context means (i) which DC of tLogs, and (ii) which types of tLogs. + +To simplify our discussion in the document, we use “tag.id” to represent a tag’s id, and tag as the Tag structure that has both locality and id. We represent a Tag as (locality, id). + + + +How does a mutation travel in FDB? +================================== + +To simplify the description, we ignore the batching mechanisms happening in each component in the data path that are used to improve the system’s performance. + +Figure 1 illustrates how a mutation is routed inside FDB. The solid lines are asynchronous pull operations, while the dotted lines are synchronous push operations. + +.. image:: images/FDB_ha_write_path.png + +At Client +--------- + +When an application creates a transaction and writes mutations, its FDB client sends the set of mutations to a proxy, say proxy 0. Now let’s focus on one of the normal mutations, say m1, whose key is in the normal keyspace. + +At Proxy +-------- + +**Sequencing.** *It first asks the master for the commit version of this transaction batch*. The master acts like a sequencer for FDB transactions to determine the order of transactions to commit by assigning a new commit version and the last assigned commit version as the previous commit version. The transaction log system will use the [previous commit version, commit version] pair to determine its commit order, i.e., only make this transaction durable after the transaction with the previous commit version is made durable. + +**Conflict checking.** *Proxy then checks if the transaction has conflicts* with others by sending mutations to resolvers. Resolvers check if there are conflicts among mutations in different transactions from different proxies. Suppose the mutation m1’s transaction passes conflict check and can be committed. + +**Commit mutation messages.** *Proxy then commits the mutations to tLogs*. Each proxy has the shard-to-tag mapping. It assigns Tags (which has locality and id) to the mutation m1. In the HA cluster in FDB 6.2, the mutation has the following Tags: + +* 3 tags for primary DC. Assume they are (0, 1), (0, 2), and (0,5). The tag ids are decided by which primary SSes will eventually save the mutation; + +* 3 tags for remote DC. Assume they are (1, 3), (1, 6), (1, 10). The tag ids are decided by which remote SSes will eventually save the mutation; + +* 1 tag for log router. Assume it is (-2, 3), where -2 is the locality value for all log router tags. The tag id is randomly chosen by proxy as well. + +* No tag for satellite tLog. The "satellite TLog locality" -5 in the code is used when recruiting a satellite TLog to tell it that it is a satellite TLog. This causes the satellite TLog to only index log router tags (-2) and not bother indexing any of the >0 tags. + +Why do we need log routers? Why cannot we let remote tLog directly pull data from primary tLogs? + +The main reason is to avoid shipping the mutation across WAN multiple times. If you attach remote SS's tags, the same mutation will cross WAN 3 times. In contrast, the router tag reduces it to only 1 time. + +Why do we randomly assign tag id for satellite tLogs and log routers? + +Another alternative is to use remote SSes’ tags to decide which satellite tLog and log routers a shard should always go to. We tried that approach before and compared its performance with randomly assigned tags. Evaluation showed that randomly assigning a mutation to satellite tLogs and log routers provide lower latency and higher throughput for these two types of logs. This is somewhat expected: When we randomly assign a mutation to a satellite tlog (and log router), we may assign mutations in the same shard to different satellite tLogs (and log routers). The randomness happens to balance load on the logs. + +Proxy groups mutations with the same tag as messages. Proxy then synchronously pushes these mutation messages to tLogs based on the tags. Proxy cannot acknowledge that the transaction is committed until the message has been durable on all primary and satellite tLogs. + +**Commit empty messages to tLogs.** When a proxy commits a tagged mutation message at version V1 to tLogs, it also has to commit an empty message at the same version V1 to the rest of tLogs. This makes sure every tLog has the same versions of messages, even though some messages are empty. This is a trick used in FDB to let all tLogs march at the same versions. The reason why FDB does the trick is that the master hands out segments of versions as 'from v1 to v2', and the TLogs need to be able to piece all of them back together into one consistent timeline. It may or may not be a good design decision, because a slow tLog can delay other tLogs of the same kind. We may want to revisit the design later. + + +At primary tLogs and satellite tLogs +------------------------------------ + +Once it receives mutations pushed by proxies, it builds indexes for each tag’s mutations. Primary TLogs index both log router tags and the primary DC's SS tags. Satellite tLogs only index log router tags. + +If tLogs’ mutations cannot be peeked and popped by its consumers (i.e., SSes and log routers) quickly enough, tLogs’ memory usage will increase. When buffered mutations exceed 1.5GB (configurable by knob), their in-memory index will be spilled into a “Tag,version->disk location” B-tree. + +tLogs also maintain two properties: + +* It will not make a mutation at version V1 durable until mutations before V1 has been made durable; + +* It will not pop (i.e., delete) mutations at version V2, until mutations before V2 have been popped. + + +At primary SS +------------- + +**Primary tLog of a SS.** Since a SS’s tag is identically mapped to one tLog. The tLog has all mutations for the SS and is the primary tLog for the SS. When the SS peeks data from tLogs, it will prefer to peek data from its primary tLog. If the primary tLog crashes, it will contact the rest of tLogs, ask for mutations with the SS’s tag, and merge them together. This complex merge operation is abstracted in the TagPartitionedLogSystem interface. + +**Pulling data from tLogs.** Each SS in the primary DC keeps pulling mutations, whose tag is the SS’s tag, from tLogs. Once mutations before a version V1 are made durable on a SS, the SS pops the tag upto the version V1 from *all* tLogs. The pop operation is an RPC to tLogs through the TagPartitionedLogSystem interface. + +Since the mutation m1 has three tags for primary SSes, the mutation will be made durable on three primary SSes. This marks the end of the mutation’s journey in the primary DC. + +Now let’s look at how the mutation m1 is routed to the remote DC. + + +At log router +------------- + +Log routers are consumers of satellite tLogs or primary tLogs, controlled by a knob LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED. By default, the knob is configured for log routers to use satellite tLogs. This relationship is similar to primary SSes to primary tLogs. + +Each log router tag is mapped to one log router. Each log router keeps pulling mutations, which have the log router’s tag, from satellite tLogs. The number of log router tags is always the same as the number of log routers, which is always some N multiple of the number of satellite logs. Each log router has a preferred satellite TLog that has all of its mutations, so in the normal steady state, each satellite should have N log routers peeking from it (and only it). + +Log router buffers its mutations in memory and waits for the remote tLogs to peek and pop its data. If the buffered data cannot be popped by remote tLog quickly enough, log router’s memory usage will increase. To avoid out of memory (OOM), a log router only buffers 5 seconds of mutations in memory. It pauses peeking data from satellite tLogs until its excessive buffered mutations have been popped by remote tLogs. + + +At remote tLogs +--------------- + +Remote tLogs are consumers of log routers. Each remote tLog keeps pulling mutations, which have the remote tLog’s tag, from log routers. Because log router tags are randomly chosen for mutations, a remote tLog’s mutations can spread across all log routers. So each remote tLog must contact all log routers for its data and merge these mutations in increasing order of versions on the remote tLog. + +Once a remote tLog collects and merge mutations from all log routers, it makes them durable on disk, index them based on their tags, and pop the mutations from log routers. + +Now the mutation m1 has arrived at the remote tLog, which is similar as when it arrives at the primary tLog. + + +At remote SSes +-------------- + +Similar to how primary SSes pull mutations from primary tLogs, each remote SS keeps pulling mutations, which have its tag, from remote tLogs. Once a remote SS makes mutations up to a version V1 durable, the SS pops its tag to the version V1 from all remote tLogs. + + +Implementation +============== + +* proxy assigns tags to a mutation: + +https://github.com/apple/foundationdb/blob/7eabdf784a21bca102f84e7eaf14bafc54605dff/fdbserver/MasterProxyServer.actor.cpp#L1410 + + +Mutation Serialization (WiP) +============================ + +This section will go into detail on how mutations are serialized as preparation for ingestion into the TagPartitionedLogSystem. This has also been covered at: + +https://drive.google.com/file/d/1OaP5bqH2kst1VxD6RWj8h2cdr9rhhBHy/view + +The proxy handles splitting transactions into their individual mutations. These mutations are then serialized and synchronously sent to multiple transaction logs. + +The process starts in *commitBatch*. Eventually, *assignMutationsToStorageServers* is called to assign mutations to storage servers and serialize them. This function loops over each mutation in each transaction, determining the set of tags for the mutation (which storage servers it will be sent to), and then calling *LogPushData.writeTypedMessage* on the mutation. + +The *LogPushData* class is used to hold serialized mutations on a per transaction log basis. It’s *messagesWriter* field holds one *BinaryWriter* per transaction log. + +*LogPushData.writeTypedMessage* is the function that serializes each mutation and writes it to the correct binary stream to be sent to the corresponding transaction log. Each serialized mutation contains additional metadata about the message, with the format: + +.. image:: /images/serialized_mutation_metadata_format.png + +* Message size: size of the message, in bytes, excluding the four bytes used for the message size + +* Subsequence: integer value used for message ordering + +* # of tags: integer value used to indicate the number of tags following + +* Tag: serialized *Tag* object, repeated # of tags times for each location + +Metadata takes up (10 + 3 * number_of_tags) bytes of each serialized mutation. + +There is an additional metadata message prepended to the list of mutations in certain circumstances. To assist with visibility efforts, transaction logs and storage servers need to be able to associate a mutation with the transaction it was part of. This allows individual transactions to be tracked as they travel throughout FDB. Thus, at the beginning of each transaction, a *SpanProtocolMessage* will be written to the message stream before the first mutation for each location. A *SpanProtocolMessage* is a separate message, similar to the *LogProtocolMessage*, which holds metadata about the transaction itself. + +An example may work best to illustrate the serialization process. Assume a client submits a transaction consisting of two mutations, m1 and m2. The proxy determines that m1 should be sent to tlogs 1, 2, and 3, while m2 should be sent to tlogs 2, 3, and 4. When m1 is serialized, a *LogProtocolMessage* will be written to the message stream for tlogs 1, 2, and 3 before the serialized m1 is written. Next, when m2 is serialized, a *LogProtocolMessage* will only be written to tlog 4, because tlogs 2 and 3 have already had a *LogProtocolMessage* written to them *for the transaction*. When all mutations in a transaction have been written, the process starts over for the next transaction. + +This allows all transaction logs to receive information about the transaction each mutation is a part of. Storage servers will pull this information when pulling mutations, allowing them to track transaction info as well. diff --git a/documentation/sphinx/source/images/FDB_ha_write_path.png b/documentation/sphinx/source/images/FDB_ha_write_path.png new file mode 100644 index 0000000000..8c085f166b Binary files /dev/null and b/documentation/sphinx/source/images/FDB_ha_write_path.png differ diff --git a/documentation/sphinx/source/images/serialized_mutation_metadata_format.png b/documentation/sphinx/source/images/serialized_mutation_metadata_format.png new file mode 100644 index 0000000000..416f4a495c Binary files /dev/null and b/documentation/sphinx/source/images/serialized_mutation_metadata_format.png differ diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 44db8d8a77..14c29f3f8a 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -2,6 +2,28 @@ Release Notes ############# +6.3.19 +====== +* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) `_ + +6.3.18 +====== +* The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5221) `_ +* Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5221) `_ +* Added Mako latency measurements. `(PR #5255) `_ +* Fixed a bug introduced when porting restoring an inconsistent snapshot feature from 7.0 branch to 6.3 branch. The parameter that controls whether to perform an inconsistent snapshot restore may instead be used to lock the database during restore. `(PR #5228) `_ +* Added SidebandMultiThreadClientTest, which validates causal consistency for multi-threaded client. `(PR #5173) `_ + +6.3.17 +====== +* Made readValuePrefix consistent regarding error messages. `(PR #5160) `_ +* Added ``TLogPopDetails`` trace event to tLog pop. `(PR #5134) `_ +* Added ``CommitBatchingEmptyMessageRatio`` metric to track the ratio of empty messages to tlogs. `(PR #5087) `_ +* Observability improvements in ProxyStats. `(PR #5046) `_ +* Added ``RecoveryInternal`` and ``ProxyReplies`` trace events to recovery_transaction step in recovery. `(PR #5038) `_ +* Multi-threaded client documentation improvements. `(PR #5033) `_ +* Added ``ClusterControllerWorkerFailed`` trace event when a worker is removed from cluster controller. `(PR #5035) `_ +* Added histograms for storage server write path components. `(PR #5019) `_ 6.3.15 ====== diff --git a/documentation/sphinx/source/release-notes/release-notes-700.rst b/documentation/sphinx/source/release-notes/release-notes-700.rst index ddb9e11ed1..072154e5a6 100644 --- a/documentation/sphinx/source/release-notes/release-notes-700.rst +++ b/documentation/sphinx/source/release-notes/release-notes-700.rst @@ -28,6 +28,8 @@ Features * Added the Testing Storage Server (TSS), which allows FoundationDB to run an "untrusted" storage engine with identical workload to the current storage engine, with zero impact on durability or correctness, and minimal impact on performance. `(Documentation) `_ `(PR #4556) `_ * Added perpetual storage wiggle that supports less impactful B-trees recreation and data migration. These will also be used for deploying the Testing Storage Server which compares 2 storage engines' results. See :ref:`Documentation ` for details. `(PR #4838) `_ * Improved the efficiency with which storage servers replicate data between themselves. `(PR #5017) `_ +* Added support to ``exclude command`` to exclude based on locality match. `(PR #5113) `_ +* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5328) `_ Performance ----------- @@ -60,6 +62,8 @@ Fixes * Added a new pre-backup action when creating a backup. Backups can now either verify the range data is being saved to is empty before the backup begins (current behavior) or clear the range where data is being saved to. Fixes a ``restore_destination_not_empty`` failure after a backup retry due to ``commit_unknown_failure``. `(PR #4595) `_ * When configured with ``usable_regions=2``, a cluster would not fail over to a region which contained only storage class processes. `(PR #4599) `_ * If a restore is done using a prefix to remove and specific key ranges to restore, the key range boundaries must begin with the prefix to remove. `(PR #4684) `_ +* The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5220) `_ +* Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5220) `_ Status ------ @@ -89,6 +93,8 @@ Other Changes * The ``foundationdb`` service installed by the RPM packages will now automatically restart ``fdbmonitor`` after 60 seconds when it fails. `(PR #3841) `_ * Capture output of forked snapshot processes in trace events. `(PR #4254) `_ * Add ErrorKind field to Severity 40 trace events. `(PR #4741) `_ +* Added histograms for the storage server write path components. `(PR #5021) `_ +* Committing a transaction will no longer partially reset it as of API version 700. `(PR #5271) `_ Earlier release notes --------------------- diff --git a/documentation/sphinx/source/request-tracing.rst b/documentation/sphinx/source/request-tracing.rst index cca170ad24..03d2d35c50 100644 --- a/documentation/sphinx/source/request-tracing.rst +++ b/documentation/sphinx/source/request-tracing.rst @@ -1,7 +1,97 @@ .. _request-tracing: -######################### -Request Tracing Framework -######################### +############### +Request Tracing +############### -.. include:: guide-common.rst.inc +The request tracing framework adds the ability to monitor transactions as they +move through FoundationDB. Tracing provides a detailed view into where +transactions spend time with data exported in near real-time, enabling fast +performance debugging. The FoundationDB tracing framework is based off the +`OpenTracing `_ specification. + +*Disambiguation:* :ref:`Trace files ` are +local log files containing debug and error output from a local ``fdbserver`` +binary. Request tracing produces similarly named *traces* which record the +amount of time a transaction spent in a part of the system. This document uses +the term tracing (or trace) to refer to these request traces, not local debug +information, unless otherwise specified. + +*Note*: Full request tracing capability requires at least ``TLogVersion::V6``. + +============== +Recording data +============== + +The request tracing framework produces no data by default. To enable collection +of traces, specify the collection type using the ``--tracer`` command line +option for ``fdbserver`` and the ``DISTRIBUTED_CLIENT_TRACER`` :ref:`network +option ` for clients. Both client +and server must have the same trace value set to perform correctly. + +========================= =============== +**Option** **Description** +------------------------- --------------- +none No tracing data is collected. +file, logfile, log_file Write tracing data to FDB trace files, specified with ``--logdir``. +network_lossy Send tracing data as UDP packets. Data is sent to ``localhost:8889``, but the default port can be changed by setting the ``TRACING_UDP_LISTENER_PORT`` knob. This option is useful if you have a log aggregation program to collect trace data. +========================= =============== + +----------- +Data format +----------- + +Spans are the building blocks of traces. A span represents an operation in the +life of a transaction, including the start and end timestamp and an operation. +A collection of spans make up a trace, representing a single transaction. The +tracing framework outputs individual spans, which can be reconstructed into +traces through their parent relationships. + +Trace data sent as UDP packets when using the ``network_lossy`` option is +serialized using `MessagePack `_. To save on the amount of +data sent, spans are serialized as an array of length 8 (if the span has one or +more parents), or length 7 (if the span has no parents). + +The fields of a span are specified below. The index at which the field appears +in the serialized msgpack array is also specified, for those using the UDP +collection format. + +================== ========= ======== =============== +**Field** **Index** **Type** **Description** +------------------ --------- -------- --------------- +Source IP:port 0 string The IP and port of the machine where the span originated. +Trace ID 1 uint64 The 64-bit identifier of the trace. All spans in a trace share the same trace ID. +Span ID 2 uint64 The 64-bit identifier of the span. All spans have a unique identifier. +Start timestamp 3 double The timestamp when the operation represented by the span began. +End timestamp 4 double The timestamp when the operation represented by the span ended. +Operation name 5 string The name of the operation the span represents. +Tags 6 map User defined tags, added manually to specify additional information. +Parent span IDs 7 vector (Optional) A list of span IDs representing parents of this span. +================== ========= ======== =============== + +^^^^^^^^^^^^^^^^^^^^^ +Multiple parent spans +^^^^^^^^^^^^^^^^^^^^^ + +Unlike traditional distributed tracing frameworks, FoundationDB spans can have +multiple parents. Because many FDB transactions are batched into a single +transaction, to continue tracing the request, the batched transaction must +treat all its component transactions as parents. + +--------------- +Control options +--------------- + +In addition to the command line parameter described above, tracing can be set +at a database and transaction level. + +Tracing can be globally disabled by setting the +``distributed_transaction_trace_disable`` database option. It can be enabled by +setting the ``distributed_transaction_trace_enable`` database option. If +neither option is specified but a tracer option is set as described above, +tracing will be enabled. + +Tracing can be enabled or disabled for individual transactions. The special key +space exposes an API to set a custom trace ID for a transaction, or to disable +tracing for the transaction. See the special key space :ref:`tracing module +documentation ` to learn more. diff --git a/documentation/sphinx/source/special-keys.rst b/documentation/sphinx/source/special-keys.rst index 4d80169f6f..a0a6d7b3da 100644 --- a/documentation/sphinx/source/special-keys.rst +++ b/documentation/sphinx/source/special-keys.rst @@ -250,6 +250,8 @@ use the global configuration functions. #. ``\xff\xff/global_config/ := `` Read/write. Reading keys in the range will return a tuple decoded string representation of the value for the given key. Writing a value will update all processes in the cluster with the new key-value pair. Values must be written using the :ref:`api-python-tuple-layer`. +.. _special-key-space-tracing-module: + Tracing module -------------- diff --git a/documentation/sphinx/source/technical-overview.rst b/documentation/sphinx/source/technical-overview.rst index f66dfb3311..af0021f3f9 100644 --- a/documentation/sphinx/source/technical-overview.rst +++ b/documentation/sphinx/source/technical-overview.rst @@ -30,6 +30,8 @@ These documents explain the engineering design of FoundationDB, with detailed in * :doc:`read-write-path` describes how FDB read and write path works. +* :doc:`ha-write-path` describes how FDB write path works in HA setting. + .. toctree:: :maxdepth: 1 :titlesonly: @@ -48,3 +50,4 @@ These documents explain the engineering design of FoundationDB, with detailed in testing kv-architecture read-write-path + ha-write-path diff --git a/documentation/sphinx/source/visibility.rst b/documentation/sphinx/source/visibility.rst index de16800ce0..200dea0447 100644 --- a/documentation/sphinx/source/visibility.rst +++ b/documentation/sphinx/source/visibility.rst @@ -6,7 +6,7 @@ Visibility Documents Curation of documents related to Visibility into FDB. -* :doc:`request-tracing` walks you through request-tracing framework. +* :doc:`request-tracing` provides fine-grained visibility into the flow of transactions through the system. .. toctree:: :maxdepth: 2 diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp index 50d6fbb7ac..87fee7f2ce 100644 --- a/documentation/tutorial/tutorial.actor.cpp +++ b/documentation/tutorial/tutorial.actor.cpp @@ -183,6 +183,7 @@ ACTOR Future echoServer() { req.reply.send(std::string(req.message.rbegin(), req.message.rend())); } when(state StreamRequest req = waitNext(echoServer.stream.getFuture())) { + req.reply.setByteLimit(1024); state int i = 0; for (; i < 100; ++i) { wait(req.reply.onReady()); diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp index 4f102a31de..cf05cf95eb 100644 --- a/fdbbackup/FileConverter.actor.cpp +++ b/fdbbackup/FileConverter.actor.cpp @@ -598,7 +598,7 @@ int main(int argc, char** argv) { Error::init(); StringRef url(param.container_url); - setupNetwork(0, true); + setupNetwork(0, UseMetrics::True); TraceEvent::setNetworkThread(); openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group); diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h index e3890cb476..9bb1036a2f 100644 --- a/fdbbackup/FileConverter.h +++ b/fdbbackup/FileConverter.h @@ -41,6 +41,11 @@ enum { OPT_TRACE_LOG_GROUP, OPT_INPUT_FILE, OPT_BUILD_FLAGS, + OPT_LIST_ONLY, + OPT_KEY_PREFIX, + OPT_HEX_KEY_PREFIX, + OPT_BEGIN_VERSION_FILTER, + OPT_END_VERSION_FILTER, OPT_HELP }; @@ -62,6 +67,11 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP }, TLS_OPTION_FLAGS #endif { OPT_BUILD_FLAGS, "--build_flags", SO_NONE }, + { OPT_LIST_ONLY, "--list_only", SO_NONE }, + { OPT_KEY_PREFIX, "-k", SO_REQ_SEP }, + { OPT_HEX_KEY_PREFIX, "--hex_prefix", SO_REQ_SEP }, + { OPT_BEGIN_VERSION_FILTER, "--begin_version_filter", SO_REQ_SEP }, + { OPT_END_VERSION_FILTER, "--end_version_filter", SO_REQ_SEP }, { OPT_HELP, "-?", SO_NONE }, { OPT_HELP, "-h", SO_NONE }, { OPT_HELP, "--help", SO_NONE }, diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index 193564d905..19dbaf4f80 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -19,14 +19,20 @@ */ #include +#include #include +#include +#include #include #include "fdbbackup/BackupTLSConfig.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" #include "fdbbackup/FileConverter.h" +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" #include "fdbclient/MutationList.h" +#include "flow/IRandom.h" #include "flow/Trace.h" #include "flow/flow.h" #include "flow/serialize.h" @@ -65,6 +71,14 @@ void printDecodeUsage() { TLS_HELP #endif " --build_flags Print build information and exit.\n" + " --list_only Print file list and exit.\n" + " -k KEY_PREFIX Use the prefix for filtering mutations\n" + " --hex_prefix HEX_PREFIX\n" + " The prefix specified in HEX format, e.g., \\x05\\x01.\n" + " --begin_version_filter BEGIN_VERSION\n" + " The version range's begin version (inclusive) for filtering.\n" + " --end_version_filter END_VERSION\n" + " The version range's end version (exclusive) for filtering.\n" "\n"; return; } @@ -76,9 +90,19 @@ void printBuildInformation() { struct DecodeParams { std::string container_url; std::string fileFilter; // only files match the filter will be decoded - bool log_enabled = false; + bool log_enabled = true; std::string log_dir, trace_format, trace_log_group; BackupTLSConfig tlsConfig; + bool list_only = false; + std::string prefix; // Key prefix for filtering + Version beginVersionFilter = 0; + Version endVersionFilter = std::numeric_limits::max(); + + // Returns if [begin, end) overlap with the filter range + bool overlap(Version begin, Version end) const { + // Filter [100, 200), [50,75) [200, 300) + return !(begin >= endVersionFilter || end <= beginVersionFilter); + } std::string toString() { std::string s; @@ -97,12 +121,69 @@ struct DecodeParams { s.append(" LogGroup:").append(trace_log_group); } } + s.append(", list_only: ").append(list_only ? "true" : "false"); + if (beginVersionFilter != 0) { + s.append(", beginVersionFilter: ").append(std::to_string(beginVersionFilter)); + } + if (endVersionFilter < std::numeric_limits::max()) { + s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter)); + } + if (!prefix.empty()) { + s.append(", KeyPrefix: ").append(printable(KeyRef(prefix))); + } return s; } - - }; +// Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a", +// into the binary string. +std::string decode_hex_string(std::string line) { + size_t i = 0; + std::string ret; + + while (i <= line.length()) { + switch (line[i]) { + case '\\': + if (i + 2 > line.length()) { + std::cerr << "Invalid hex string at: " << i << "\n"; + return ret; + } + switch (line[i + 1]) { + char ent, save; + case '"': + case '\\': + case ' ': + case ';': + line.erase(i, 1); + break; + case 'x': + if (i + 4 > line.length()) { + std::cerr << "Invalid hex string at: " << i << "\n"; + return ret; + } + char* pEnd; + save = line[i + 4]; + line[i + 4] = 0; + ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); + if (*pEnd) { + std::cerr << "Invalid hex string at: " << i << "\n"; + return ret; + } + line[i + 4] = save; + line.replace(i, 4, 1, ent); + break; + default: + std::cerr << "Invalid hex string at: " << i << "\n"; + return ret; + } + default: + i++; + } + } + + return line.substr(0, i); +} + int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) { while (args->Next()) { auto lastError = args->LastError(); @@ -124,6 +205,26 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) { param->container_url = args->OptionArg(); break; + case OPT_LIST_ONLY: + param->list_only = true; + break; + + case OPT_KEY_PREFIX: + param->prefix = args->OptionArg(); + break; + + case OPT_HEX_KEY_PREFIX: + param->prefix = decode_hex_string(args->OptionArg()); + break; + + case OPT_BEGIN_VERSION_FILTER: + param->beginVersionFilter = std::atoll(args->OptionArg()); + break; + + case OPT_END_VERSION_FILTER: + param->endVersionFilter = std::atoll(args->OptionArg()); + break; + case OPT_CRASHONERROR: g_crashOnError = true; break; @@ -141,7 +242,7 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) { break; case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args->OptionArg())) { + if (!selectTraceFormatter(args->OptionArg())) { std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; return FDB_EXIT_ERROR; } @@ -202,78 +303,18 @@ void printLogFiles(std::string msg, const std::vector& files) { std::vector getRelevantLogFiles(const std::vector& files, const DecodeParams& params) { std::vector filtered; for (const auto& file : files) { - if (file.fileName.find(params.fileFilter) != std::string::npos) { + if (file.fileName.find(params.fileFilter) != std::string::npos && + params.overlap(file.beginVersion, file.endVersion + 1)) { filtered.push_back(file); } } return filtered; } -std::pair decode_key(const StringRef& key) { - ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t)); - - uint8_t hash; - Version version; - int32_t part; - BinaryReader rd(key, Unversioned()); - rd >> hash >> version >> part; - version = bigEndian64(version); - part = bigEndian32(part); - - int32_t v = version / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; - ASSERT(((uint8_t)hashlittle(&v, sizeof(v), 0)) == hash); - - return std::make_pair(version, part); -} - -// Decodes an encoded list of mutations in the format of: -// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k], -// where a mutation is encoded as: -// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][key][value] -std::vector decode_value(const StringRef& value) { - StringRefReader reader(value, restore_corrupted_data()); - - reader.consume(); // Consume the includeVersion - uint32_t val_length = reader.consume(); - if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) { - TraceEvent(SevError, "ValueError") - .detail("ValueLen", val_length) - .detail("ValueSize", value.size()) - .detail("Value", printable(value)); - } - - std::vector mutations; - while (1) { - if (reader.eof()) - break; - - // Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep() - uint32_t type, p1len, p2len; - type = reader.consume(); - p1len = reader.consume(); - p2len = reader.consume(); - - const uint8_t* key = reader.consume(p1len); - const uint8_t* val = reader.consume(p2len); - - mutations.emplace_back((MutationRef::Type)type, StringRef(key, p1len), StringRef(val, p2len)); - } - return mutations; -} - struct VersionedMutations { Version version; std::vector mutations; - Arena arena; // The arena that contains the mutations. -}; - -struct VersionedKVPart { - Arena arena; - Version version; - int32_t part; - StringRef kv; - VersionedKVPart(Arena arena, Version version, int32_t part, StringRef kv) - : arena(arena), version(version), part(part), kv(kv) {} + std::string serializedMutations; // buffer that contains mutations }; /* @@ -293,174 +334,66 @@ struct VersionedKVPart { * at any time this object might have two blocks of data in memory. */ class DecodeProgress { - std::vector keyValues; + std::vector>> blocks; + std::unordered_map mutationBlocksByVersion; public: DecodeProgress() = default; - template - DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward(values)) {} + DecodeProgress(const LogFile& file) : file(file) {} // If there are no more mutations to pull from the file. - // However, we could have unfinished version in the buffer when EOF is true, - // which means we should look for data in the next file. The caller - // should call getUnfinishedBuffer() to get these left data. - bool finished() const { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); } - - std::vector&& getUnfinishedBuffer() && { return std::move(keyValues); } - - // Returns all mutations of the next version in a batch. - Future getNextBatch() { return getNextBatchImpl(this); } + bool finished() const { return done; } + // Open and loads file into memory Future openFile(Reference container) { return openFileImpl(this, container); } // The following are private APIs: - // Returns true if value contains complete data. - static bool isValueComplete(StringRef value) { - StringRefReader reader(value, restore_corrupted_data()); - - reader.consume(); // Consume the includeVersion - uint32_t val_length = reader.consume(); - return val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t); - } - // PRECONDITION: finished() must return false before calling this function. // Returns the next batch of mutations along with the arena backing it. // Note the returned batch can be empty when the file has unfinished // version batch data that are in the next file. - ACTOR static Future getNextBatchImpl(DecodeProgress* self) { - ASSERT(!self->finished()); + VersionedMutations getNextBatch() { + ASSERT(!finished()); - loop { - if (self->keyValues.size() <= 1) { - // Try to decode another block when less than one left - wait(readAndDecodeFile(self)); - } - - const auto& kv = self->keyValues[0]; - ASSERT(kv.part == 0); - - // decode next versions, check if they are continuous parts - int idx = 1; // next kv pair in "keyValues" - int bufSize = kv.kv.size(); - for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) { - if (idx == self->keyValues.size()) - break; - - const auto& nextKV = self->keyValues[idx]; - if (kv.version != nextKV.version) { - break; - } - - if (lastPart + 1 != nextKV.part) { - TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", nextKV.part); - throw restore_corrupted_data(); - } - bufSize += nextKV.kv.size(); - } - - VersionedMutations m; - m.version = kv.version; - TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size()); - StringRef value = kv.kv; - if (idx > 1) { - // Stitch parts into one and then decode one by one - Standalone buf = self->combineValues(idx, bufSize); - value = buf; - m.arena = buf.arena(); - } - if (isValueComplete(value)) { - m.mutations = decode_value(value); - if (m.arena.getSize() == 0) { - m.arena = kv.arena; - } - self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx); - return m; - } else if (!self->eof) { - // Read one more block, hopefully the missing part of the value can be found. - wait(readAndDecodeFile(self)); - } else { - TraceEvent(SevWarn, "MissingValue").detail("Version", m.version); - self->leftover = true; - return m; // Empty mutations + VersionedMutations vms; + for (auto& [version, m] : mutationBlocksByVersion) { + if (m.isComplete()) { + vms.version = version; + std::vector mutations = fileBackup::decodeMutationLogValue(m.serializedMutations); + TraceEvent("Decode").detail("Version", vms.version).detail("N", mutations.size()); + vms.mutations.insert(vms.mutations.end(), mutations.begin(), mutations.end()); + vms.serializedMutations = m.serializedMutations; + mutationBlocksByVersion.erase(version); + return vms; } } - } - // Returns a buffer which stitches first "idx" values into one. - // "len" MUST equal the summation of these values. - Standalone combineValues(const int idx, const int len) { - ASSERT(idx <= keyValues.size() && idx > 1); - - Standalone buf = makeString(len); - int n = 0; - for (int i = 0; i < idx; i++) { - const auto& value = keyValues[i].kv; - memcpy(mutateString(buf) + n, value.begin(), value.size()); - n += value.size(); - } - - ASSERT(n == len); - return buf; - } - - // Decodes a block into KeyValueRef stored in "keyValues". - void decode_block(const Standalone& buf, int len) { - StringRef block(buf.begin(), len); - StringRefReader reader(block, restore_corrupted_data()); - - try { - // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION - if (reader.consume() != BACKUP_AGENT_MLOG_VERSION) - throw restore_unsupported_file_version(); - - // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. - while (1) { - // If eof reached or first key len bytes is 0xFF then end of block was reached. - if (reader.eof() || *reader.rptr == 0xFF) - break; - - // Read key and value. If anything throws then there is a problem. - uint32_t kLen = reader.consumeNetworkUInt32(); - const uint8_t* k = reader.consume(kLen); - std::pair version_part = decode_key(StringRef(k, kLen)); - uint32_t vLen = reader.consumeNetworkUInt32(); - const uint8_t* v = reader.consume(vLen); - TraceEvent(SevDecodeInfo, "Block") - .detail("KeySize", kLen) - .detail("valueSize", vLen) - .detail("Offset", reader.rptr - buf.begin()) - .detail("Version", version_part.first) - .detail("Part", version_part.second); - keyValues.emplace_back(buf.arena(), version_part.first, version_part.second, StringRef(v, vLen)); - } - - // Make sure any remaining bytes in the block are 0xFF - for (auto b : reader.remainder()) { - if (b != 0xFF) - throw restore_corrupted_data_padding(); - } - - // The (version, part) in a block can be out of order, i.e., (3, 0) - // can be followed by (4, 0), and then (3, 1). So we need to sort them - // first by version, and then by part number. - std::sort(keyValues.begin(), keyValues.end(), [](const VersionedKVPart& a, const VersionedKVPart& b) { - return a.version == b.version ? a.part < b.part : a.version < b.version; - }); - return; - } catch (Error& e) { - TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin()); - throw; + // No complete versions + if (!mutationBlocksByVersion.empty()) { + TraceEvent(SevWarn, "UnfishedBlocks").detail("NumberOfVersions", mutationBlocksByVersion.size()); } + done = true; + return vms; } ACTOR static Future openFileImpl(DecodeProgress* self, Reference container) { Reference fd = wait(container->readFile(self->file.fileName)); self->fd = fd; - wait(readAndDecodeFile(self)); + while (!self->eof) { + wait(readAndDecodeFile(self)); + } return Void(); } + // Add chunks to mutationBlocksByVersion + void addBlockKVPairs(VectorRef chunks) { + for (auto& kv : chunks) { + auto versionAndChunkNumber = fileBackup::decodeMutationLogKey(kv.key); + mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv); + } + } + // Reads a file block, decodes it into key/value pairs, and stores these pairs. ACTOR static Future readAndDecodeFile(DecodeProgress* self) { try { @@ -470,17 +403,18 @@ public: return Void(); } - state Standalone buf = makeString(len); - state int rLen = wait(self->fd->read(mutateString(buf), len, self->offset)); + // Decode a file block into log_key and log_value chunks + Standalone> chunks = + wait(fileBackup::decodeMutationLogFileBlock(self->fd, self->offset, len)); + self->blocks.push_back(chunks); + TraceEvent("ReadFile") .detail("Name", self->file.fileName) - .detail("Len", rLen) + .detail("Len", len) .detail("Offset", self->offset); - if (rLen != len) { - throw restore_corrupted_data(); - } - self->decode_block(buf, rLen); - self->offset += rLen; + self->addBlockKVPairs(chunks); + self->offset += len; + return Void(); } catch (Error& e) { TraceEvent(SevWarn, "CorruptLogFileBlock") @@ -496,12 +430,55 @@ public: Reference fd; int64_t offset = 0; bool eof = false; - bool leftover = false; // Done but has unfinished version batch data left + bool done = false; }; +ACTOR Future process_file(Reference container, LogFile file, UID uid, DecodeParams params) { + if (file.fileSize == 0) { + TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName); + return Void(); + } + + state DecodeProgress progress(file); + wait(progress.openFile(container)); + while (!progress.finished()) { + VersionedMutations vms = progress.getNextBatch(); + if (vms.version < params.beginVersionFilter || vms.version >= params.endVersionFilter) { + TraceEvent("SkipVersion").detail("Version", vms.version); + continue; + } + + int sub = 0; + for (const auto& m : vms.mutations) { + sub++; // sub sequence number starts at 1 + bool print = params.prefix.empty(); // no filtering + + if (!print) { + if (isSingleKeyMutation((MutationRef::Type)m.type)) { + print = m.param1.startsWith(StringRef(params.prefix)); + } else if (m.type == MutationRef::ClearRange) { + KeyRange range(KeyRangeRef(m.param1, m.param2)); + print = range.contains(StringRef(params.prefix)); + } else { + ASSERT(false); + } + } + if (print) { + TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid) + .detail("Version", vms.version) + .setMaxFieldLength(10000) + .detail("M", m.toString()); + std::cout << vms.version << " " << m.toString() << "\n"; + } + } + } + TraceEvent("ProcessFileDone", uid).detail("File", file.fileName); + return Void(); +} + ACTOR Future decode_logs(DecodeParams params) { state Reference container = IBackupContainer::openContainer(params.container_url); - + state UID uid = deterministicRandom()->randomUniqueID(); state BackupFileList listing = wait(container->dumpFileList()); // remove partitioned logs listing.logs.erase(std::remove_if(listing.logs.begin(), @@ -512,7 +489,8 @@ ACTOR Future decode_logs(DecodeParams params) { }), listing.logs.end()); std::sort(listing.logs.begin(), listing.logs.end()); - TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); + TraceEvent("Container", uid).detail("URL", params.container_url).detail("Logs", listing.logs.size()); + TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params.toString()); BackupDescription desc = wait(container->describeBackup()); std::cout << "\n" << desc.toString() << "\n"; @@ -520,26 +498,15 @@ ACTOR Future decode_logs(DecodeParams params) { state std::vector logs = getRelevantLogFiles(listing.logs, params); printLogFiles("Relevant files are: ", logs); - state int i = 0; - // Previous file's unfinished version data - state std::vector left; - for (; i < logs.size(); i++) { - if (logs[i].fileSize == 0) - continue; + if (params.list_only) return Void(); - state DecodeProgress progress(logs[i], std::move(left)); - wait(progress.openFile(container)); - while (!progress.finished()) { - VersionedMutations vms = wait(progress.getNextBatch()); - for (const auto& m : vms.mutations) { - std::cout << vms.version << " " << m.toString() << "\n"; - } - } - left = std::move(progress).getUnfinishedBuffer(); - if (!left.empty()) { - TraceEvent("UnfinishedFile").detail("File", logs[i].fileName).detail("Q", left.size()); - } + state int idx = 0; + while (idx < logs.size()) { + TraceEvent("ProcessFile").detail("Name", logs[idx].fileName).detail("I", idx); + wait(process_file(container, logs[idx], uid, params)); + idx++; } + TraceEvent("DecodeDone", uid); return Void(); } @@ -564,6 +531,8 @@ int main(int argc, char** argv) { } if (!param.trace_format.empty()) { setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format)); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr); } if (!param.trace_log_group.empty()) { setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group)); @@ -571,7 +540,7 @@ int main(int argc, char** argv) { } if (!param.tlsConfig.setupTLS()) { - TraceEvent(SevError, "TLSError"); + TraceEvent(SevError, "TLSError").log(); throw tls_error(); } @@ -579,15 +548,20 @@ int main(int argc, char** argv) { Error::init(); StringRef url(param.container_url); - setupNetwork(0, true); + setupNetwork(0, UseMetrics::True); TraceEvent::setNetworkThread(); - openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "decode", param.trace_log_group); + openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group); param.tlsConfig.setupBlobCredentials(); auto f = stopAfter(decode_logs(param)); runNetwork(); + + flushTraceFileVoid(); + fflush(stdout); + closeTraceFile(); + return status; } catch (Error& e) { std::cerr << "ERROR: " << e.what() << "\n"; diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 04fc42cded..caab2918b3 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -133,6 +133,7 @@ enum { OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, OPT_INCREMENTALONLY, + OPT_ENCRYPTION_KEY_FILE, // Backup Modify OPT_MOD_ACTIVE_INTERVAL, @@ -259,6 +260,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -697,6 +699,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, { OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP }, { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent_snapshot_only", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -1089,6 +1092,8 @@ static void printBackupUsage(bool devhelp) { " Performs incremental backup without the base backup.\n" " This option indicates to the backup agent that it will only need to record the log files, " "and ignore the range files.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for encrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1162,6 +1167,8 @@ static void printRestoreUsage(bool devhelp) { " To be used in conjunction with incremental restore.\n" " Indicates to the backup agent to only begin replaying log files from a certain version, " "instead of the entire set.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for decrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1463,7 +1470,7 @@ ACTOR Future getLayerStatus(Reference tr std::string id, ProgramExe exe, Database dest, - bool snapshot = false) { + Snapshot snapshot = Snapshot::False) { // This process will write a document that looks like this: // { backup : { $expires : {}, version: } // so that the value under 'backup' will eventually expire to null and thus be ignored by @@ -1639,7 +1646,7 @@ ACTOR Future cleanupStatus(Reference tr, std::string name, std::string id, int limit = 1) { - state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); + state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True)); state bool readMore = false; state int i; for (i = 0; i < docs.size(); ++i) { @@ -1668,7 +1675,7 @@ ACTOR Future cleanupStatus(Reference tr, } if (readMore) { limit = 10000; - RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); + RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True)); docs = std::move(docs2); readMore = false; } @@ -1705,7 +1712,10 @@ ACTOR Future getLayerStatus(Database src, std::string root // Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay // based on that and BACKUP_AGGREGATE_POLL_RATE -ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double* pollDelay) { +ACTOR Future updateAgentPollRate(Database src, + std::string rootKey, + std::string name, + std::shared_ptr pollDelay) { loop { try { json_spirit::mObject status = wait(getLayerStatus(src, rootKey)); @@ -1727,7 +1737,7 @@ ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::s ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name, ProgramExe exe, - double* pollDelay, + std::shared_ptr pollDelay, Database taskDest = Database(), std::string id = nondeterministicRandom()->randomUniqueID().toString()) { state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; @@ -1757,7 +1767,8 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Future futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest, true); + state Future futureStatusDoc = + getLayerStatus(tr, name, id, exe, taskDest, Snapshot::True); wait(cleanupStatus(tr, rootKey, name, id)); std::string statusdoc = wait(futureStatusDoc); tr->set(instanceKey, statusdoc); @@ -1774,7 +1785,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, // Now that status was written at least once by this process (and hopefully others), start the poll rate // control updater if it wasn't started yet - if (!pollRateUpdater.isValid() && pollDelay != nullptr) + if (!pollRateUpdater.isValid()) pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); } catch (Error& e) { TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e); @@ -1784,17 +1795,17 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, } ACTOR Future runDBAgent(Database src, Database dest) { - state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; + state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); std::string id = nondeterministicRandom()->randomUniqueID().toString(); - state Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, &pollDelay, dest, id); + state Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id); state Future status_other = - statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, &pollDelay, dest, id); + statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id); state DatabaseBackupAgent backupAgent(src); loop { try { - wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); + wait(backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; } catch (Error& e) { if (e.code() == error_code_operation_cancelled) @@ -1811,14 +1822,14 @@ ACTOR Future runDBAgent(Database src, Database dest) { } ACTOR Future runAgent(Database db) { - state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - state Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, &pollDelay); + state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); + state Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay); state FileBackupAgent backupAgent; loop { try { - wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); + wait(backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; } catch (Error& e) { if (e.code() == error_code_operation_cancelled) @@ -1846,7 +1857,8 @@ ACTOR Future submitDBBackup(Database src, backupRanges.push_back_deep(backupRanges.arena(), normalKeys); } - wait(backupAgent.submitBackup(dest, KeyRef(tagName), backupRanges, false, StringRef(), StringRef(), true)); + wait(backupAgent.submitBackup( + dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True)); // Check if a backup agent is running bool agentRunning = wait(backupAgent.checkActive(dest)); @@ -1890,10 +1902,10 @@ ACTOR Future submitBackup(Database db, Standalone> backupRanges, std::string tagName, bool dryRun, - bool waitForCompletion, - bool stopWhenDone, - bool usePartitionedLog, - bool incrementalBackupOnly) { + WaitForComplete waitForCompletion, + StopWhenDone stopWhenDone, + UsePartitionedLog usePartitionedLog, + IncrementalBackupOnly incrementalBackupOnly) { try { state FileBackupAgent backupAgent; @@ -1996,7 +2008,7 @@ ACTOR Future switchDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName, - bool forceAction) { + ForceAction forceAction) { try { state DatabaseBackupAgent backupAgent(src); @@ -2046,7 +2058,7 @@ ACTOR Future statusDBBackup(Database src, Database dest, std::string tagNa return Void(); } -ACTOR Future statusBackup(Database db, std::string tagName, bool showErrors, bool json) { +ACTOR Future statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) { try { state FileBackupAgent backupAgent; @@ -2063,11 +2075,15 @@ ACTOR Future statusBackup(Database db, std::string tagName, bool showError return Void(); } -ACTOR Future abortDBBackup(Database src, Database dest, std::string tagName, bool partial, bool dstOnly) { +ACTOR Future abortDBBackup(Database src, + Database dest, + std::string tagName, + PartialBackup partial, + DstOnly dstOnly) { try { state DatabaseBackupAgent backupAgent(src); - wait(backupAgent.abortBackup(dest, Key(tagName), partial, false, dstOnly)); + wait(backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::False, dstOnly)); wait(backupAgent.unlockBackup(dest, Key(tagName))); printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); @@ -2118,7 +2134,7 @@ ACTOR Future abortBackup(Database db, std::string tagName) { return Void(); } -ACTOR Future cleanupMutations(Database db, bool deleteData) { +ACTOR Future cleanupMutations(Database db, DeleteData deleteData) { try { wait(cleanupBackup(db, deleteData)); } catch (Error& e) { @@ -2131,7 +2147,7 @@ ACTOR Future cleanupMutations(Database db, bool deleteData) { return Void(); } -ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDone) { +ACTOR Future waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) { try { state FileBackupAgent backupAgent; @@ -2150,7 +2166,7 @@ ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDon return Void(); } -ACTOR Future discontinueBackup(Database db, std::string tagName, bool waitForCompletion) { +ACTOR Future discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) { try { state FileBackupAgent backupAgent; @@ -2220,7 +2236,9 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause return Void(); } -Reference openBackupContainer(const char* name, std::string destinationContainer) { +Reference openBackupContainer(const char* name, + std::string destinationContainer, + Optional const& encryptionKeyFile = {}) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2230,7 +2248,7 @@ Reference openBackupContainer(const char* name, std::string de Reference c; try { - c = IBackupContainer::openContainer(destinationContainer); + c = IBackupContainer::openContainer(destinationContainer, encryptionKeyFile); } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { @@ -2255,12 +2273,13 @@ ACTOR Future runRestore(Database db, Version targetVersion, std::string targetTimestamp, bool performRestore, - bool verbose, - bool waitForDone, + Verbose verbose, + WaitForComplete waitForDone, std::string addPrefix, std::string removePrefix, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly) { + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, + Optional encryptionKeyFile) { if (ranges.empty()) { ranges.push_back_deep(ranges.arena(), normalKeys); } @@ -2296,7 +2315,8 @@ ACTOR Future runRestore(Database db, try { state FileBackupAgent backupAgent; - state Reference bc = openBackupContainer(exeRestore.toString().c_str(), container); + state Reference bc = + openBackupContainer(exeRestore.toString().c_str(), container, encryptionKeyFile); // If targetVersion is unset then use the maximum restorable version from the backup description if (targetVersion == invalidVersion) { @@ -2306,7 +2326,7 @@ ACTOR Future runRestore(Database db, BackupDescription desc = wait(bc->describeBackup()); - if (onlyAppyMutationLogs && desc.contiguousLogEnd.present()) { + if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { targetVersion = desc.contiguousLogEnd.get() - 1; } else if (desc.maxRestorableVersion.present()) { targetVersion = desc.maxRestorableVersion.get(); @@ -2330,10 +2350,11 @@ ACTOR Future runRestore(Database db, verbose, KeyRef(addPrefix), KeyRef(removePrefix), - true, - onlyAppyMutationLogs, + LockDB::True, + onlyApplyMutationLogs, inconsistentSnapshotOnly, - beginVersion)); + beginVersion, + encryptionKeyFile)); if (waitForDone && verbose) { // If restore is now complete then report version restored @@ -2369,8 +2390,8 @@ ACTOR Future runFastRestoreTool(Database db, Standalone> ranges, Version dbVersion, bool performRestore, - bool verbose, - bool waitForDone) { + Verbose verbose, + WaitForComplete waitForDone) { try { state FileBackupAgent backupAgent; state Version restoreVersion = invalidVersion; @@ -2413,7 +2434,7 @@ ACTOR Future runFastRestoreTool(Database db, ranges, KeyRef(container), dbVersion, - true, + LockDB::True, randomUID, LiteralStringRef(""), LiteralStringRef(""))); @@ -2512,7 +2533,8 @@ ACTOR Future expireBackupData(const char* name, Database db, bool force, Version restorableAfterVersion, - std::string restorableAfterDatetime) { + std::string restorableAfterDatetime, + Optional encryptionKeyFile) { if (!endDatetime.empty()) { Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); endVersion = v; @@ -2531,7 +2553,7 @@ ACTOR Future expireBackupData(const char* name, } try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state IBackupContainer::ExpireProgress progress; state std::string lastProgress; @@ -2613,9 +2635,10 @@ ACTOR Future describeBackup(const char* name, std::string destinationContainer, bool deep, Optional cx, - bool json) { + bool json, + Optional encryptionKeyFile) { try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state BackupDescription desc = wait(c->describeBackup(deep)); if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); @@ -2645,7 +2668,7 @@ ACTOR Future queryBackup(const char* name, Version restoreVersion, std::string originalClusterFile, std::string restoreTimestamp, - bool verbose) { + Verbose verbose) { state UID operationId = deterministicRandom()->randomUniqueID(); state JsonBuilderObject result; state std::string errorMessage; @@ -2838,7 +2861,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp } state BackupConfig config(uidFlag.get().first); - EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info())); + EBackupState s = wait(config.stateEnum().getOrThrow(tr, Snapshot::False, backup_invalid_info())); if (!FileBackupAgent::isRunnable(s)) { fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); throw backup_error(); @@ -2858,7 +2881,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp } if (options.activeSnapshotIntervalSeconds.present()) { - Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error())); + Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, Snapshot::False, backup_error())); config.snapshotTargetEndVersion().set(tr, begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); @@ -3244,13 +3267,13 @@ int main(int argc, char* argv[]) { Version beginVersion = invalidVersion; Version restoreVersion = invalidVersion; std::string restoreTimestamp; - bool waitForDone = false; - bool stopWhenDone = true; - bool usePartitionedLog = false; // Set to true to use new backup system - bool incrementalBackupOnly = false; - bool onlyAppyMutationLogs = false; - bool inconsistentSnapshotOnly = false; - bool forceAction = false; + WaitForComplete waitForDone{ false }; + StopWhenDone stopWhenDone{ true }; + UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system + IncrementalBackupOnly incrementalBackupOnly{ false }; + OnlyApplyMutationLogs onlyApplyMutationLogs{ false }; + InconsistentSnapshotOnly inconsistentSnapshotOnly{ false }; + ForceAction forceAction{ false }; bool trace = false; bool quietDisplay = false; bool dryRun = false; @@ -3260,8 +3283,8 @@ int main(int argc, char* argv[]) { uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE; uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE; ESOError lastError; - bool partial = true; - bool dstOnly = false; + PartialBackup partial{ true }; + DstOnly dstOnly{ false }; LocalityData localities; uint64_t memLimit = 8LL << 30; Optional ti; @@ -3271,7 +3294,8 @@ int main(int argc, char* argv[]) { std::string restoreClusterFileDest; std::string restoreClusterFileOrig; bool jsonOutput = false; - bool deleteData = false; + DeleteData deleteData{ false }; + Optional encryptionKeyFile; BackupModifyOptions modifyOptions; @@ -3355,13 +3379,13 @@ int main(int argc, char* argv[]) { dryRun = true; break; case OPT_DELETE_DATA: - deleteData = true; + deleteData.set(true); break; case OPT_MIN_CLEANUP_SECONDS: knobs.emplace_back("min_cleanup_seconds", args->OptionArg()); break; case OPT_FORCE: - forceAction = true; + forceAction.set(true); break; case OPT_TRACE: trace = true; @@ -3441,10 +3465,10 @@ int main(int argc, char* argv[]) { sourceClusterFile = args->OptionArg(); break; case OPT_CLEANUP: - partial = false; + partial.set(false); break; case OPT_DSTONLY: - dstOnly = true; + dstOnly.set(true); break; case OPT_KNOB: { std::string syn = args->OptionSyntax(); @@ -3503,17 +3527,20 @@ int main(int argc, char* argv[]) { modifyOptions.verifyUID = args->OptionArg(); break; case OPT_WAITFORDONE: - waitForDone = true; + waitForDone.set(true); break; case OPT_NOSTOPWHENDONE: - stopWhenDone = false; + stopWhenDone.set(false); break; case OPT_USE_PARTITIONED_LOG: - usePartitionedLog = true; + usePartitionedLog.set(true); break; case OPT_INCREMENTALONLY: - incrementalBackupOnly = true; - onlyAppyMutationLogs = true; + incrementalBackupOnly.set(true); + onlyApplyMutationLogs.set(true); + break; + case OPT_ENCRYPTION_KEY_FILE: + encryptionKeyFile = args->OptionArg(); break; case OPT_RESTORECONTAINER: restoreContainer = args->OptionArg(); @@ -3565,7 +3592,7 @@ int main(int argc, char* argv[]) { break; } case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: { - inconsistentSnapshotOnly = true; + inconsistentSnapshotOnly.set(true); break; } #ifdef _WIN32 @@ -3704,7 +3731,7 @@ int main(int argc, char* argv[]) { } } - IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::False, IsSimulated::False); auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection(); for (const auto& [knobName, knobValueString] : knobs) { try { @@ -3731,7 +3758,7 @@ int main(int argc, char* argv[]) { } // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - g_knobs.initialize(Randomize::NO, IsSimulated::NO); + g_knobs.initialize(Randomize::False, IsSimulated::False); if (trace) { if (!traceLogGroup.empty()) @@ -3769,7 +3796,7 @@ int main(int argc, char* argv[]) { Reference c; try { - setupNetwork(0, true); + setupNetwork(0, UseMetrics::True); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); return FDB_EXIT_ERROR; @@ -3813,7 +3840,7 @@ int main(int argc, char* argv[]) { } try { - db = Database::createDatabase(ccf, -1, true, localities); + db = Database::createDatabase(ccf, -1, IsInternal::True, localities); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str()); @@ -3833,7 +3860,7 @@ int main(int argc, char* argv[]) { } try { - sourceDb = Database::createDatabase(sourceCcf, -1, true, localities); + sourceDb = Database::createDatabase(sourceCcf, -1, IsInternal::True, localities); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str()); @@ -3853,7 +3880,7 @@ int main(int argc, char* argv[]) { if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. - openBackupContainer(argv[0], destinationContainer); + openBackupContainer(argv[0], destinationContainer, encryptionKeyFile); f = stopAfter(submitBackup(db, destinationContainer, initialSnapshotIntervalSeconds, @@ -3879,7 +3906,7 @@ int main(int argc, char* argv[]) { case BackupType::STATUS: if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter(statusBackup(db, tagName, true, jsonOutput)); + f = stopAfter(statusBackup(db, tagName, ShowErrors::True, jsonOutput)); break; case BackupType::ABORT: @@ -3932,7 +3959,8 @@ int main(int argc, char* argv[]) { db, forceAction, expireRestorableAfterVersion, - expireRestorableAfterDatetime)); + expireRestorableAfterDatetime, + encryptionKeyFile)); break; case BackupType::DELETE_BACKUP: @@ -3952,7 +3980,8 @@ int main(int argc, char* argv[]) { destinationContainer, describeDeep, describeTimestamps ? Optional(db) : Optional(), - jsonOutput)); + jsonOutput, + encryptionKeyFile)); break; case BackupType::LIST: @@ -3968,7 +3997,7 @@ int main(int argc, char* argv[]) { restoreVersion, restoreClusterFileOrig, restoreTimestamp, - !quietDisplay)); + Verbose{ !quietDisplay })); break; case BackupType::DUMP: @@ -4029,15 +4058,16 @@ int main(int argc, char* argv[]) { restoreVersion, restoreTimestamp, !dryRun, - !quietDisplay, + Verbose{ !quietDisplay }, waitForDone, addPrefix, removePrefix, - onlyAppyMutationLogs, - inconsistentSnapshotOnly)); + onlyApplyMutationLogs, + inconsistentSnapshotOnly, + encryptionKeyFile)); break; case RestoreType::WAIT: - f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true))); + f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::True))); break; case RestoreType::ABORT: f = stopAfter( @@ -4097,8 +4127,14 @@ int main(int argc, char* argv[]) { // TODO: We have not implemented the code commented out in this case switch (restoreType) { case RestoreType::START: - f = stopAfter(runFastRestoreTool( - db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, !quietDisplay, waitForDone)); + f = stopAfter(runFastRestoreTool(db, + tagName, + restoreContainer, + backupKeys, + restoreVersion, + !dryRun, + Verbose{ !quietDisplay }, + waitForDone)); break; case RestoreType::WAIT: printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n"); diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt index 7b14ebd6a9..7e43f57c31 100644 --- a/fdbcli/CMakeLists.txt +++ b/fdbcli/CMakeLists.txt @@ -8,6 +8,7 @@ set(FDBCLI_SRCS ForceRecoveryWithDataLossCommand.actor.cpp MaintenanceCommand.actor.cpp SnapshotCommand.actor.cpp + ThrottleCommand.actor.cpp Util.cpp linenoise/linenoise.h) diff --git a/fdbcli/ThrottleCommand.actor.cpp b/fdbcli/ThrottleCommand.actor.cpp new file mode 100644 index 0000000000..7692c17b69 --- /dev/null +++ b/fdbcli/ThrottleCommand.actor.cpp @@ -0,0 +1,645 @@ +/* + * ThrottleCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/IClientApi.h" +#include "fdbclient/TagThrottle.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/CommitTransaction.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/genericactors.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace { + +// Helper functions copied from TagThrottle.actor.cpp +// The only difference is transactions are changed to go through MultiversionTransaction, +// instead of the native Transaction(i.e., RYWTransaction) + +ACTOR Future getValidAutoEnabled(Reference tr) { + state bool result; + loop { + Optional value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey))); + if (!value.present()) { + tr->reset(); + wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF)); + continue; + } else if (value.get() == LiteralStringRef("1")) { + result = true; + } else if (value.get() == LiteralStringRef("0")) { + result = false; + } else { + TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue").detail("Value", value.get()); + tr->reset(); + wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF)); + continue; + } + return result; + }; +} + +ACTOR Future> getThrottledTags(Reference db, + int limit, + bool containsRecommend = false) { + state Reference tr = db->createTransaction(); + state bool reportAuto = containsRecommend; + loop { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + try { + if (!containsRecommend) { + wait(store(reportAuto, getValidAutoEnabled(tr))); + } + state ThreadFuture f = tr->getRange( + reportAuto ? tagThrottleKeys : KeyRangeRef(tagThrottleKeysPrefix, tagThrottleAutoKeysPrefix), limit); + RangeResult throttles = wait(safeThreadFutureToFuture(f)); + std::vector results; + for (auto throttle : throttles) { + results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key), + TagThrottleValue::fromValue(throttle.value))); + } + return results; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future> getRecommendedTags(Reference db, int limit) { + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + try { + bool enableAuto = wait(getValidAutoEnabled(tr)); + if (enableAuto) { + return std::vector(); + } + state ThreadFuture f = + tr->getRange(KeyRangeRef(tagThrottleAutoKeysPrefix, tagThrottleKeys.end), limit); + RangeResult throttles = wait(safeThreadFutureToFuture(f)); + std::vector results; + for (auto throttle : throttles) { + results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key), + TagThrottleValue::fromValue(throttle.value))); + } + return results; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future updateThrottleCount(Reference tr, int64_t delta) { + state ThreadFuture> countVal = tr->get(tagThrottleCountKey); + state ThreadFuture> limitVal = tr->get(tagThrottleLimitKey); + + wait(success(safeThreadFutureToFuture(countVal)) && success(safeThreadFutureToFuture(limitVal))); + + int64_t count = 0; + int64_t limit = 0; + + if (countVal.get().present()) { + BinaryReader reader(countVal.get().get(), Unversioned()); + reader >> count; + } + + if (limitVal.get().present()) { + BinaryReader reader(limitVal.get().get(), Unversioned()); + reader >> limit; + } + + count += delta; + + if (count > limit) { + throw too_many_tag_throttles(); + } + + BinaryWriter writer(Unversioned()); + writer << count; + + tr->set(tagThrottleCountKey, writer.toValue()); + return Void(); +} + +void signalThrottleChange(Reference tr) { + tr->atomicOp( + tagThrottleSignalKey, LiteralStringRef("XXXXXXXXXX\x00\x00\x00\x00"), MutationRef::SetVersionstampedValue); +} + +ACTOR Future throttleTags(Reference db, + TagSet tags, + double tpsRate, + double initialDuration, + TagThrottleType throttleType, + TransactionPriority priority, + Optional expirationTime = Optional(), + Optional reason = Optional()) { + state Reference tr = db->createTransaction(); + state Key key = TagThrottleKey(tags, throttleType, priority).toKey(); + + ASSERT(initialDuration > 0); + + if (throttleType == TagThrottleType::MANUAL) { + reason = TagThrottledReason::MANUAL; + } + TagThrottleValue throttle(tpsRate, + expirationTime.present() ? expirationTime.get() : 0, + initialDuration, + reason.present() ? reason.get() : TagThrottledReason::UNSET); + BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason())); + wr << throttle; + state Value value = wr.toValue(); + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + try { + if (throttleType == TagThrottleType::MANUAL) { + Optional oldThrottle = wait(safeThreadFutureToFuture(tr->get(key))); + if (!oldThrottle.present()) { + wait(updateThrottleCount(tr, 1)); + } + } + + tr->set(key, value); + + if (throttleType == TagThrottleType::MANUAL) { + signalThrottleChange(tr); + } + + wait(safeThreadFutureToFuture(tr->commit())); + return Void(); + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future unthrottleTags(Reference db, + TagSet tags, + Optional throttleType, + Optional priority) { + state Reference tr = db->createTransaction(); + + state std::vector keys; + for (auto p : allTransactionPriorities) { + if (!priority.present() || priority.get() == p) { + if (!throttleType.present() || throttleType.get() == TagThrottleType::AUTO) { + keys.push_back(TagThrottleKey(tags, TagThrottleType::AUTO, p).toKey()); + } + if (!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) { + keys.push_back(TagThrottleKey(tags, TagThrottleType::MANUAL, p).toKey()); + } + } + } + + state bool removed = false; + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + try { + state std::vector>> values; + values.reserve(keys.size()); + for (auto key : keys) { + values.push_back(safeThreadFutureToFuture(tr->get(key))); + } + + wait(waitForAll(values)); + + int delta = 0; + for (int i = 0; i < values.size(); ++i) { + if (values[i].get().present()) { + if (TagThrottleKey::fromKey(keys[i]).throttleType == TagThrottleType::MANUAL) { + delta -= 1; + } + + tr->clear(keys[i]); + + // Report that we are removing this tag if we ever see it present. + // This protects us from getting confused if the transaction is maybe committed. + // It's ok if someone else actually ends up removing this tag at the same time + // and we aren't the ones to actually do it. + removed = true; + } + } + + if (delta != 0) { + wait(updateThrottleCount(tr, delta)); + } + if (removed) { + signalThrottleChange(tr); + wait(safeThreadFutureToFuture(tr->commit())); + } + + return removed; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future enableAuto(Reference db, bool enabled) { + state Reference tr = db->createTransaction(); + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + try { + Optional value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey))); + if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) || + (!enabled && value.get() != LiteralStringRef("0"))) { + tr->set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0")); + signalThrottleChange(tr); + + wait(safeThreadFutureToFuture(tr->commit())); + } + return Void(); + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future unthrottleMatchingThrottles(Reference db, + KeyRef beginKey, + KeyRef endKey, + Optional priority, + bool onlyExpiredThrottles) { + state Reference tr = db->createTransaction(); + + state KeySelector begin = firstGreaterOrEqual(beginKey); + state KeySelector end = firstGreaterOrEqual(endKey); + + state bool removed = false; + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + try { + // holds memory of the RangeResult + state ThreadFuture f = tr->getRange(begin, end, 1000); + state RangeResult tags = wait(safeThreadFutureToFuture(f)); + state uint64_t unthrottledTags = 0; + uint64_t manualUnthrottledTags = 0; + for (auto tag : tags) { + if (onlyExpiredThrottles) { + double expirationTime = TagThrottleValue::fromValue(tag.value).expirationTime; + if (expirationTime == 0 || expirationTime > now()) { + continue; + } + } + + TagThrottleKey key = TagThrottleKey::fromKey(tag.key); + if (priority.present() && key.priority != priority.get()) { + continue; + } + + if (key.throttleType == TagThrottleType::MANUAL) { + ++manualUnthrottledTags; + } + + removed = true; + tr->clear(tag.key); + unthrottledTags++; + } + + if (manualUnthrottledTags > 0) { + wait(updateThrottleCount(tr, -manualUnthrottledTags)); + } + + if (unthrottledTags > 0) { + signalThrottleChange(tr); + } + + wait(safeThreadFutureToFuture(tr->commit())); + + if (!tags.more) { + return removed; + } + + ASSERT(tags.size() > 0); + begin = KeySelector(firstGreaterThan(tags[tags.size() - 1].key), tags.arena()); + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +Future unthrottleAll(Reference db, + Optional tagThrottleType, + Optional priority) { + KeyRef begin = tagThrottleKeys.begin; + KeyRef end = tagThrottleKeys.end; + + if (tagThrottleType.present() && tagThrottleType == TagThrottleType::AUTO) { + begin = tagThrottleAutoKeysPrefix; + } else if (tagThrottleType.present() && tagThrottleType == TagThrottleType::MANUAL) { + end = tagThrottleAutoKeysPrefix; + } + + return unthrottleMatchingThrottles(db, begin, end, priority, false); +} + +} // namespace + +namespace fdb_cli { + +ACTOR Future throttleCommandActor(Reference db, std::vector tokens) { + + if (tokens.size() == 1) { + printUsage(tokens[0]); + return false; + } else if (tokencmp(tokens[1], "list")) { + if (tokens.size() > 4) { + printf("Usage: throttle list [throttled|recommended|all] [LIMIT]\n"); + printf("\n"); + printf("Lists tags that are currently throttled.\n"); + printf("The default LIMIT is 100 tags.\n"); + return false; + } + + state bool reportThrottled = true; + state bool reportRecommended = false; + if (tokens.size() >= 3) { + if (tokencmp(tokens[2], "recommended")) { + reportThrottled = false; + reportRecommended = true; + } else if (tokencmp(tokens[2], "all")) { + reportThrottled = true; + reportRecommended = true; + } else if (!tokencmp(tokens[2], "throttled")) { + printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str()); + return false; + } + } + + state int throttleListLimit = 100; + if (tokens.size() >= 4) { + char* end; + throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10); + if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) { + fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str()); + return false; + } + } + + state std::vector tags; + if (reportThrottled && reportRecommended) { + wait(store(tags, getThrottledTags(db, throttleListLimit, true))); + } else if (reportThrottled) { + wait(store(tags, getThrottledTags(db, throttleListLimit))); + } else if (reportRecommended) { + wait(store(tags, getRecommendedTags(db, throttleListLimit))); + } + + bool anyLogged = false; + for (auto itr = tags.begin(); itr != tags.end(); ++itr) { + if (itr->expirationTime > now()) { + if (!anyLogged) { + printf("Throttled tags:\n\n"); + printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n"); + printf(" --------------+----------------+-----------+--------+------------+------\n"); + + anyLogged = true; + } + + std::string reasonStr = "unset"; + if (itr->reason == TagThrottledReason::MANUAL) { + reasonStr = "manual"; + } else if (itr->reason == TagThrottledReason::BUSY_WRITE) { + reasonStr = "busy write"; + } else if (itr->reason == TagThrottledReason::BUSY_READ) { + reasonStr = "busy read"; + } + + printf(" %12d | %13ds | %9s | %6s | %10s |%s\n", + (int)(itr->tpsRate), + std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)), + transactionPriorityToString(itr->priority, false), + itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual", + reasonStr.c_str(), + itr->tag.toString().c_str()); + } + } + + if (tags.size() == throttleListLimit) { + printf("\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n", + throttleListLimit); + printf("Usage: throttle list [LIMIT]\n"); + } + if (!anyLogged) { + printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended"); + } + } else if (tokencmp(tokens[1], "on")) { + if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) { + printf("Usage: throttle on tag [RATE] [DURATION] [PRIORITY]\n"); + printf("\n"); + printf("Enables throttling for transactions with the specified tag.\n"); + printf("An optional transactions per second rate can be specified (default 0).\n"); + printf("An optional duration can be specified, which must include a time suffix (s, m, h, " + "d) (default 1h).\n"); + printf("An optional priority can be specified. Choices are `default', `immediate', and " + "`batch' (default `default').\n"); + return false; + } + + double tpsRate = 0.0; + uint64_t duration = 3600; + TransactionPriority priority = TransactionPriority::DEFAULT; + + if (tokens.size() >= 5) { + char* end; + tpsRate = std::strtod((const char*)tokens[4].begin(), &end); + if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) { + fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str()); + return false; + } + if (tpsRate < 0) { + fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate); + return false; + } + } + if (tokens.size() == 6) { + Optional parsedDuration = parseDuration(tokens[5].toString()); + if (!parsedDuration.present()) { + fprintf(stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str()); + return false; + } + duration = parsedDuration.get(); + + if (duration == 0) { + fprintf(stderr, "ERROR: throttle duration cannot be 0\n"); + return false; + } + } + if (tokens.size() == 7) { + if (tokens[6] == LiteralStringRef("default")) { + priority = TransactionPriority::DEFAULT; + } else if (tokens[6] == LiteralStringRef("immediate")) { + priority = TransactionPriority::IMMEDIATE; + } else if (tokens[6] == LiteralStringRef("batch")) { + priority = TransactionPriority::BATCH; + } else { + fprintf(stderr, + "ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', " + "or `batch'.\n", + tokens[6].toString().c_str()); + return false; + } + } + + TagSet tags; + tags.addTag(tokens[3]); + + wait(throttleTags(db, tags, tpsRate, duration, TagThrottleType::MANUAL, priority)); + printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str()); + } else if (tokencmp(tokens[1], "off")) { + int nextIndex = 2; + TagSet tags; + bool throttleTypeSpecified = false; + bool is_error = false; + Optional throttleType = TagThrottleType::MANUAL; + Optional priority; + + if (tokens.size() == 2) { + is_error = true; + } + + while (nextIndex < tokens.size() && !is_error) { + if (tokencmp(tokens[nextIndex], "all")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = Optional(); + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "auto")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::AUTO; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "manual")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::MANUAL; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "default")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::DEFAULT; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "immediate")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::IMMEDIATE; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "batch")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::BATCH; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "tag")) { + if (tags.size() > 0 || nextIndex == tokens.size() - 1) { + is_error = true; + continue; + } + tags.addTag(tokens[nextIndex + 1]); + nextIndex += 2; + } + } + + if (!is_error) { + state const char* throttleTypeString = + !throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually "); + state std::string priorityString = + priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : ""; + + if (tags.size() > 0) { + bool success = wait(unthrottleTags(db, tags, throttleType, priority)); + if (success) { + printf("Unthrottled tag `%s'%s\n", tokens[3].toString().c_str(), priorityString.c_str()); + } else { + printf("Tag `%s' was not %sthrottled%s\n", + tokens[3].toString().c_str(), + throttleTypeString, + priorityString.c_str()); + } + } else { + bool unthrottled = wait(unthrottleAll(db, throttleType, priority)); + if (unthrottled) { + printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str()); + } else { + printf("There were no tags being %sthrottled%s\n", throttleTypeString, priorityString.c_str()); + } + } + } else { + printf("Usage: throttle off [all|auto|manual] [tag ] [PRIORITY]\n"); + printf("\n"); + printf("Disables throttling for throttles matching the specified filters. At least one " + "filter must be used.\n\n"); + printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type " + "of throttle\n"); + printf("affected. `all' targets all throttles, `auto' targets those created by the " + "cluster, and\n"); + printf("`manual' targets those created manually (default `manual').\n\n"); + printf("The `tag' filter can be use to turn off only a specific tag.\n\n"); + printf("The priority filter can be used to turn off only throttles at specific priorities. " + "Choices are\n"); + printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n"); + } + } else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { + if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) { + printf("Usage: throttle auto\n"); + printf("\n"); + printf("Enables or disable automatic tag throttling.\n"); + return false; + } + state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable"); + wait(enableAuto(db, autoTagThrottlingEnabled)); + printf("Automatic tag throttling has been %s\n", autoTagThrottlingEnabled ? "enabled" : "disabled"); + } else { + printUsage(tokens[0]); + return false; + } + + return true; +} + +CommandFactory throttleFactory( + "throttle", + CommandHelp("throttle [ARGS]", + "view and control throttled tags", + "Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' " + "to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n")); +} // namespace fdb_cli diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 7e7abd2e3c..6af3a49b17 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -648,11 +648,6 @@ void initHelp() { "namespace for all the profiling-related commands.", "Different types support different actions. Run `profile` to get a list of " "types, and iteratively explore the help.\n"); - helpMap["throttle"] = - CommandHelp("throttle [ARGS]", - "view and control throttled tags", - "Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' " - "to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n"); helpMap["cache_range"] = CommandHelp( "cache_range ", "Mark a key range to add to or remove from storage caches.", @@ -3151,7 +3146,7 @@ struct CLIOptions { } // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - g_knobs.initialize(Randomize::NO, IsSimulated::NO); + g_knobs.initialize(Randomize::False, IsSimulated::False); } int processArg(CSimpleOpt& args) { @@ -3322,7 +3317,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { TraceEvent::setNetworkThread(); try { - db = Database::createDatabase(ccf, -1, false); + db = Database::createDatabase(ccf, -1, IsInternal::False); if (!opt.exec.present()) { printf("Using cluster file `%s'.\n", ccf->getFilename().c_str()); } @@ -3960,6 +3955,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } + wait(makeInterruptable(GlobalConfig::globalConfig().onInitialized())); if (tokencmp(tokens[2], "get")) { if (tokens.size() != 3) { fprintf(stderr, "ERROR: Addtional arguments to `get` are not supported.\n"); @@ -4494,300 +4490,12 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "throttle")) { - if (tokens.size() == 1) { - printUsage(tokens[0]); + bool _result = wait(throttleCommandActor(db2, tokens)); + if (!_result) is_error = true; - continue; - } else if (tokencmp(tokens[1], "list")) { - if (tokens.size() > 4) { - printf("Usage: throttle list [throttled|recommended|all] [LIMIT]\n"); - printf("\n"); - printf("Lists tags that are currently throttled.\n"); - printf("The default LIMIT is 100 tags.\n"); - is_error = true; - continue; - } - - state bool reportThrottled = true; - state bool reportRecommended = false; - if (tokens.size() >= 3) { - if (tokencmp(tokens[2], "recommended")) { - reportThrottled = false; - reportRecommended = true; - } else if (tokencmp(tokens[2], "all")) { - reportThrottled = true; - reportRecommended = true; - } else if (!tokencmp(tokens[2], "throttled")) { - printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str()); - is_error = true; - continue; - } - } - - state int throttleListLimit = 100; - if (tokens.size() >= 4) { - char* end; - throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10); - if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) { - fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str()); - is_error = true; - continue; - } - } - - state std::vector tags; - if (reportThrottled && reportRecommended) { - wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit, true))); - } else if (reportThrottled) { - wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit))); - } else if (reportRecommended) { - wait(store(tags, ThrottleApi::getRecommendedTags(db, throttleListLimit))); - } - - bool anyLogged = false; - for (auto itr = tags.begin(); itr != tags.end(); ++itr) { - if (itr->expirationTime > now()) { - if (!anyLogged) { - printf("Throttled tags:\n\n"); - printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n"); - printf( - " --------------+----------------+-----------+--------+------------+------\n"); - - anyLogged = true; - } - - std::string reasonStr = "unset"; - if (itr->reason == TagThrottledReason::MANUAL) { - reasonStr = "manual"; - } else if (itr->reason == TagThrottledReason::BUSY_WRITE) { - reasonStr = "busy write"; - } else if (itr->reason == TagThrottledReason::BUSY_READ) { - reasonStr = "busy read"; - } - - printf(" %12d | %13ds | %9s | %6s | %10s |%s\n", - (int)(itr->tpsRate), - std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)), - transactionPriorityToString(itr->priority, false), - itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual", - reasonStr.c_str(), - itr->tag.toString().c_str()); - } - } - - if (tags.size() == throttleListLimit) { - printf( - "\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n", - throttleListLimit); - printf("Usage: throttle list [LIMIT]\n"); - } - if (!anyLogged) { - printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended"); - } - } else if (tokencmp(tokens[1], "on")) { - if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) { - printf("Usage: throttle on tag [RATE] [DURATION] [PRIORITY]\n"); - printf("\n"); - printf("Enables throttling for transactions with the specified tag.\n"); - printf("An optional transactions per second rate can be specified (default 0).\n"); - printf("An optional duration can be specified, which must include a time suffix (s, m, h, " - "d) (default 1h).\n"); - printf("An optional priority can be specified. Choices are `default', `immediate', and " - "`batch' (default `default').\n"); - is_error = true; - continue; - } - - double tpsRate = 0.0; - uint64_t duration = 3600; - TransactionPriority priority = TransactionPriority::DEFAULT; - - if (tokens.size() >= 5) { - char* end; - tpsRate = std::strtod((const char*)tokens[4].begin(), &end); - if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) { - fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str()); - is_error = true; - continue; - } - if (tpsRate < 0) { - fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate); - is_error = true; - continue; - } - } - if (tokens.size() == 6) { - Optional parsedDuration = parseDuration(tokens[5].toString()); - if (!parsedDuration.present()) { - fprintf( - stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str()); - is_error = true; - continue; - } - duration = parsedDuration.get(); - - if (duration == 0) { - fprintf(stderr, "ERROR: throttle duration cannot be 0\n"); - is_error = true; - continue; - } - } - if (tokens.size() == 7) { - if (tokens[6] == LiteralStringRef("default")) { - priority = TransactionPriority::DEFAULT; - } else if (tokens[6] == LiteralStringRef("immediate")) { - priority = TransactionPriority::IMMEDIATE; - } else if (tokens[6] == LiteralStringRef("batch")) { - priority = TransactionPriority::BATCH; - } else { - fprintf(stderr, - "ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', " - "or `batch'.\n", - tokens[6].toString().c_str()); - is_error = true; - continue; - } - } - - TagSet tags; - tags.addTag(tokens[3]); - - wait(ThrottleApi::throttleTags(db, tags, tpsRate, duration, TagThrottleType::MANUAL, priority)); - printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str()); - } else if (tokencmp(tokens[1], "off")) { - int nextIndex = 2; - TagSet tags; - bool throttleTypeSpecified = false; - Optional throttleType = TagThrottleType::MANUAL; - Optional priority; - - if (tokens.size() == 2) { - is_error = true; - } - - while (nextIndex < tokens.size() && !is_error) { - if (tokencmp(tokens[nextIndex], "all")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = Optional(); - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "auto")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = TagThrottleType::AUTO; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "manual")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = TagThrottleType::MANUAL; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "default")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::DEFAULT; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "immediate")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::IMMEDIATE; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "batch")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::BATCH; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "tag")) { - if (tags.size() > 0 || nextIndex == tokens.size() - 1) { - is_error = true; - continue; - } - tags.addTag(tokens[nextIndex + 1]); - nextIndex += 2; - } - } - - if (!is_error) { - state const char* throttleTypeString = - !throttleType.present() - ? "" - : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually "); - state std::string priorityString = - priority.present() - ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) - : ""; - - if (tags.size() > 0) { - bool success = wait(ThrottleApi::unthrottleTags(db, tags, throttleType, priority)); - if (success) { - printf("Unthrottled tag `%s'%s\n", - tokens[3].toString().c_str(), - priorityString.c_str()); - } else { - printf("Tag `%s' was not %sthrottled%s\n", - tokens[3].toString().c_str(), - throttleTypeString, - priorityString.c_str()); - } - } else { - bool unthrottled = wait(ThrottleApi::unthrottleAll(db, throttleType, priority)); - if (unthrottled) { - printf("Unthrottled all %sthrottled tags%s\n", - throttleTypeString, - priorityString.c_str()); - } else { - printf("There were no tags being %sthrottled%s\n", - throttleTypeString, - priorityString.c_str()); - } - } - } else { - printf("Usage: throttle off [all|auto|manual] [tag ] [PRIORITY]\n"); - printf("\n"); - printf("Disables throttling for throttles matching the specified filters. At least one " - "filter must be used.\n\n"); - printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type " - "of throttle\n"); - printf("affected. `all' targets all throttles, `auto' targets those created by the " - "cluster, and\n"); - printf("`manual' targets those created manually (default `manual').\n\n"); - printf("The `tag' filter can be use to turn off only a specific tag.\n\n"); - printf("The priority filter can be used to turn off only throttles at specific priorities. " - "Choices are\n"); - printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n"); - } - } else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { - if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) { - printf("Usage: throttle auto\n"); - printf("\n"); - printf("Enables or disable automatic tag throttling.\n"); - is_error = true; - continue; - } - state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable"); - wait(ThrottleApi::enableAuto(db, autoTagThrottlingEnabled)); - printf("Automatic tag throttling has been %s\n", - autoTagThrottlingEnabled ? "enabled" : "disabled"); - } else { - printUsage(tokens[0]); - is_error = true; - } continue; } + if (tokencmp(tokens[0], "cache_range")) { if (tokens.size() != 4) { printUsage(tokens[0]); @@ -4924,7 +4632,7 @@ int main(int argc, char** argv) { registerCrashHandler(); - IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::False, IsSimulated::False); #ifdef __unixish__ struct sigaction act; diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h index 6d69f2879e..8ab228ea6d 100644 --- a/fdbcli/fdbcli.actor.h +++ b/fdbcli/fdbcli.actor.h @@ -83,6 +83,8 @@ ACTOR Future forceRecoveryWithDataLossCommandActor(Reference db ACTOR Future maintenanceCommandActor(Reference db, std::vector tokens); // snapshot command ACTOR Future snapshotCommandActor(Reference db, std::vector tokens); +// throttle command +ACTOR Future throttleCommandActor(Reference db, std::vector tokens); } // namespace fdb_cli diff --git a/fdbclient/AsyncFileS3BlobStore.actor.h b/fdbclient/AsyncFileS3BlobStore.actor.h index bc520bda90..db436755b3 100644 --- a/fdbclient/AsyncFileS3BlobStore.actor.h +++ b/fdbclient/AsyncFileS3BlobStore.actor.h @@ -256,7 +256,7 @@ public: m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) { // Add first part - m_parts.push_back(Reference(new Part(1, m_bstore->knobs.multipart_min_part_size))); + m_parts.push_back(makeReference(1, m_bstore->knobs.multipart_min_part_size)); } }; diff --git a/fdbclient/AsyncTaskThread.actor.cpp b/fdbclient/AsyncTaskThread.actor.cpp index 2e7c6e3596..b63a731045 100644 --- a/fdbclient/AsyncTaskThread.actor.cpp +++ b/fdbclient/AsyncTaskThread.actor.cpp @@ -18,6 +18,8 @@ * limitations under the License. */ +#include + #include "fdbclient/AsyncTaskThread.h" #include "flow/UnitTest.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -30,13 +32,22 @@ public: bool isTerminate() const override { return true; } }; -ACTOR Future asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, int* sum, int count) { +ACTOR Future asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, std::atomic *sum, int count, int clientId, double meanSleep) { state int i = 0; + state double randomSleep = 0.0; for (; i < count; ++i) { + randomSleep = deterministicRandom()->random01() * 2 * meanSleep; + wait(delay(randomSleep)); wait(asyncTaskThread->execAsync([sum = sum] { - ++(*sum); + sum->fetch_add(1); return Void(); })); + TraceEvent("AsyncTaskThreadIncrementedSum") + .detail("Index", i) + .detail("Sum", sum->load()) + .detail("ClientId", clientId) + .detail("RandomSleep", randomSleep) + .detail("MeanSleep", meanSleep); } return Void(); } @@ -51,7 +62,7 @@ AsyncTaskThread::~AsyncTaskThread() { bool wakeUp = false; { std::lock_guard g(m); - wakeUp = queue.push(std::make_shared()); + wakeUp = queue.push(std::make_unique()); } if (wakeUp) { cv.notify_one(); @@ -61,7 +72,7 @@ AsyncTaskThread::~AsyncTaskThread() { void AsyncTaskThread::run(AsyncTaskThread* self) { while (true) { - std::shared_ptr task; + std::unique_ptr task; { std::unique_lock lk(self->m); self->cv.wait(lk, [self] { return !self->queue.canSleep(); }); @@ -75,14 +86,30 @@ void AsyncTaskThread::run(AsyncTaskThread* self) { } TEST_CASE("/asynctaskthread/add") { - state int sum = 0; + state std::atomic sum = 0; state AsyncTaskThread asyncTaskThread; + state int numClients = 10; + state int incrementsPerClient = 100; std::vector> clients; - clients.reserve(10); - for (int i = 0; i < 10; ++i) { - clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100)); + clients.reserve(numClients); + for (int clientId = 0; clientId < numClients; ++clientId) { + clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, incrementsPerClient, clientId, deterministicRandom()->random01() * 0.01)); } wait(waitForAll(clients)); - ASSERT(sum == 1000); + ASSERT_EQ(sum.load(), numClients * incrementsPerClient); + return Void(); +} + +TEST_CASE("/asynctaskthread/error") { + state AsyncTaskThread asyncTaskThread; + try { + wait(asyncTaskThread.execAsync([]{ + throw operation_failed(); + return Void(); + })); + ASSERT(false); + } catch (Error &e) { + ASSERT_EQ(e.code(), error_code_operation_failed); + } return Void(); } diff --git a/fdbclient/AsyncTaskThread.h b/fdbclient/AsyncTaskThread.h index e7ea8b3cf2..223a434257 100644 --- a/fdbclient/AsyncTaskThread.h +++ b/fdbclient/AsyncTaskThread.h @@ -48,7 +48,7 @@ public: }; class AsyncTaskThread { - ThreadSafeQueue> queue; + ThreadSafeQueue> queue; std::condition_variable cv; std::mutex m; std::thread thread; @@ -60,7 +60,7 @@ class AsyncTaskThread { bool wakeUp = false; { std::lock_guard g(m); - wakeUp = queue.push(std::make_shared>(func)); + wakeUp = queue.push(std::make_unique>(func)); } if (wakeUp) { cv.notify_one(); @@ -88,6 +88,7 @@ public: auto funcResult = func(); onMainThreadVoid([promise, funcResult] { promise.send(funcResult); }, nullptr, priority); } catch (Error& e) { + TraceEvent("ErrorExecutingAsyncTask").error(e); onMainThreadVoid([promise, e] { promise.sendError(e); }, nullptr, priority); } }); diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index c8903b9fe4..450bc30c63 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -36,6 +36,26 @@ #include "fdbclient/BackupContainer.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DECLARE_BOOLEAN_PARAM(LockDB); +FDB_DECLARE_BOOLEAN_PARAM(UnlockDB); +FDB_DECLARE_BOOLEAN_PARAM(StopWhenDone); +FDB_DECLARE_BOOLEAN_PARAM(Verbose); +FDB_DECLARE_BOOLEAN_PARAM(WaitForComplete); +FDB_DECLARE_BOOLEAN_PARAM(ForceAction); +FDB_DECLARE_BOOLEAN_PARAM(Terminator); +FDB_DECLARE_BOOLEAN_PARAM(IncrementalBackupOnly); +FDB_DECLARE_BOOLEAN_PARAM(UsePartitionedLog); +FDB_DECLARE_BOOLEAN_PARAM(OnlyApplyMutationLogs); +FDB_DECLARE_BOOLEAN_PARAM(InconsistentSnapshotOnly); +FDB_DECLARE_BOOLEAN_PARAM(ShowErrors); +FDB_DECLARE_BOOLEAN_PARAM(AbortOldBackup); +FDB_DECLARE_BOOLEAN_PARAM(DstOnly); // TODO: More descriptive name? +FDB_DECLARE_BOOLEAN_PARAM(WaitForDestUID); +FDB_DECLARE_BOOLEAN_PARAM(CheckBackupUID); +FDB_DECLARE_BOOLEAN_PARAM(DeleteData); +FDB_DECLARE_BOOLEAN_PARAM(SetValidation); +FDB_DECLARE_BOOLEAN_PARAM(PartialBackup); + class BackupAgentBase : NonCopyable { public: // Time formatter for anything backup or restore related @@ -65,6 +85,7 @@ public: static const Key keyConfigStopWhenDoneKey; static const Key keyStateStatus; static const Key keyStateStop; + static const Key keyStateLogBeginVersion; static const Key keyLastUid; static const Key keyBeginKey; static const Key keyEndKey; @@ -82,151 +103,26 @@ public: static const Key keySourceStates; static const Key keySourceTagName; - static const int logHeaderSize; + static constexpr int logHeaderSize = 12; // Convert the status text to an enumerated value - static EnumState getState(std::string stateText) { - auto enState = EnumState::STATE_ERRORED; - - if (stateText.empty()) { - enState = EnumState::STATE_NEVERRAN; - } - - else if (!stateText.compare("has been submitted")) { - enState = EnumState::STATE_SUBMITTED; - } - - else if (!stateText.compare("has been started")) { - enState = EnumState::STATE_RUNNING; - } - - else if (!stateText.compare("is differential")) { - enState = EnumState::STATE_RUNNING_DIFFERENTIAL; - } - - else if (!stateText.compare("has been completed")) { - enState = EnumState::STATE_COMPLETED; - } - - else if (!stateText.compare("has been aborted")) { - enState = EnumState::STATE_ABORTED; - } - - else if (!stateText.compare("has been partially aborted")) { - enState = EnumState::STATE_PARTIALLY_ABORTED; - } - - return enState; - } + static EnumState getState(std::string const& stateText); // Convert the status enum to a text description - static const char* getStateText(EnumState enState) { - const char* stateText; - - switch (enState) { - case EnumState::STATE_ERRORED: - stateText = "has errored"; - break; - case EnumState::STATE_NEVERRAN: - stateText = "has never been started"; - break; - case EnumState::STATE_SUBMITTED: - stateText = "has been submitted"; - break; - case EnumState::STATE_RUNNING: - stateText = "has been started"; - break; - case EnumState::STATE_RUNNING_DIFFERENTIAL: - stateText = "is differential"; - break; - case EnumState::STATE_COMPLETED: - stateText = "has been completed"; - break; - case EnumState::STATE_ABORTED: - stateText = "has been aborted"; - break; - case EnumState::STATE_PARTIALLY_ABORTED: - stateText = "has been partially aborted"; - break; - default: - stateText = ""; - break; - } - - return stateText; - } + static const char* getStateText(EnumState enState); // Convert the status enum to a name - static const char* getStateName(EnumState enState) { - const char* s; - - switch (enState) { - case EnumState::STATE_ERRORED: - s = "Errored"; - break; - case EnumState::STATE_NEVERRAN: - s = "NeverRan"; - break; - case EnumState::STATE_SUBMITTED: - s = "Submitted"; - break; - case EnumState::STATE_RUNNING: - s = "Running"; - break; - case EnumState::STATE_RUNNING_DIFFERENTIAL: - s = "RunningDifferentially"; - break; - case EnumState::STATE_COMPLETED: - s = "Completed"; - break; - case EnumState::STATE_ABORTED: - s = "Aborted"; - break; - case EnumState::STATE_PARTIALLY_ABORTED: - s = "Aborting"; - break; - default: - s = ""; - break; - } - - return s; - } + static const char* getStateName(EnumState enState); // Determine if the specified state is runnable - static bool isRunnable(EnumState enState) { - bool isRunnable = false; + static bool isRunnable(EnumState enState); - switch (enState) { - case EnumState::STATE_SUBMITTED: - case EnumState::STATE_RUNNING: - case EnumState::STATE_RUNNING_DIFFERENTIAL: - case EnumState::STATE_PARTIALLY_ABORTED: - isRunnable = true; - break; - default: - break; - } + static KeyRef getDefaultTag() { return StringRef(defaultTagName); } - return isRunnable; - } - - static const KeyRef getDefaultTag() { return StringRef(defaultTagName); } - - static const std::string getDefaultTagName() { return defaultTagName; } + static std::string getDefaultTagName() { return defaultTagName; } // This is only used for automatic backup name generation - static Standalone getCurrentTime() { - double t = now(); - time_t curTime = t; - char buffer[128]; - struct tm* timeinfo; - timeinfo = localtime(&curTime); - strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo); - - std::string time(buffer); - return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime)))); - } + static Standalone getCurrentTime(); protected: static const std::string defaultTagName; @@ -249,7 +145,11 @@ public: KeyBackedProperty lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); } - Future run(Database cx, double* pollDelay, int maxConcurrentTasks) { + Future run(Database cx, double pollDelay, int maxConcurrentTasks) { + return taskBucket->run(cx, futureBucket, std::make_shared(pollDelay), maxConcurrentTasks); + } + + Future run(Database cx, std::shared_ptr pollDelay, int maxConcurrentTasks) { return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); } @@ -260,13 +160,13 @@ public: static Key getPauseKey(); // parallel restore - Future parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true); + Future parallelRestoreFinish(Database cx, UID randomUID, UnlockDB = UnlockDB::True); Future submitParallelRestore(Database cx, Key backupTag, Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix); @@ -288,29 +188,31 @@ public: Key tagName, Key url, Standalone> ranges, - bool waitForComplete = true, - Version targetVersion = -1, - bool verbose = true, + WaitForComplete = WaitForComplete::True, + Version targetVersion = ::invalidVersion, + Verbose = Verbose::True, Key addPrefix = Key(), Key removePrefix = Key(), - bool lockDB = true, - bool onlyAppyMutationLogs = false, - bool inconsistentSnapshotOnly = false, - Version beginVersion = -1); + LockDB = LockDB::True, + OnlyApplyMutationLogs = OnlyApplyMutationLogs::False, + InconsistentSnapshotOnly = InconsistentSnapshotOnly::False, + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}); Future restore(Database cx, Optional cxOrig, Key tagName, Key url, - bool waitForComplete = true, - Version targetVersion = -1, - bool verbose = true, + WaitForComplete waitForComplete = WaitForComplete::True, + Version targetVersion = ::invalidVersion, + Verbose verbose = Verbose::True, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key(), - bool lockDB = true, - bool onlyAppyMutationLogs = false, - bool inconsistentSnapshotOnly = false, - Version beginVersion = -1) { + LockDB lockDB = LockDB::True, + OnlyApplyMutationLogs onlyApplyMutationLogs = OnlyApplyMutationLogs::False, + InconsistentSnapshotOnly inconsistentSnapshotOnly = InconsistentSnapshotOnly::False, + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}) { Standalone> rangeRef; rangeRef.push_back_deep(rangeRef.arena(), range); return restore(cx, @@ -324,9 +226,10 @@ public: addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, - beginVersion); + beginVersion, + encryptionKeyFileName); } Future atomicRestore(Database cx, Key tagName, @@ -347,7 +250,7 @@ public: Future abortRestore(Database cx, Key tagName); // Waits for a restore tag to reach a final (stable) state. - Future waitRestore(Database cx, Key tagName, bool verbose); + Future waitRestore(Database cx, Key tagName, Verbose); // Get a string describing the status of a tag Future restoreStatus(Reference tr, Key tagName); @@ -362,20 +265,22 @@ public: Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone = true, - bool partitionedLog = false, - bool incrementalBackupOnly = false); + StopWhenDone = StopWhenDone::True, + UsePartitionedLog = UsePartitionedLog::False, + IncrementalBackupOnly = IncrementalBackupOnly::False, + Optional const& encryptionKeyFileName = {}); Future submitBackup(Database cx, Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone = true, - bool partitionedLog = false, - bool incrementalBackupOnly = false) { + StopWhenDone stopWhenDone = StopWhenDone::True, + UsePartitionedLog partitionedLog = UsePartitionedLog::False, + IncrementalBackupOnly incrementalBackupOnly = IncrementalBackupOnly::False, + Optional const& encryptionKeyFileName = {}) { return runRYWTransactionFailIfLocked(cx, [=](Reference tr) { return submitBackup(tr, outContainer, @@ -385,7 +290,8 @@ public: backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); }); } @@ -407,19 +313,19 @@ public: return runRYWTransaction(cx, [=](Reference tr) { return abortBackup(tr, tagName); }); } - Future getStatus(Database cx, bool showErrors, std::string tagName); + Future getStatus(Database cx, ShowErrors, std::string tagName); Future getStatusJSON(Database cx, std::string tagName); Future> getLastRestorable(Reference tr, Key tagName, - bool snapshot = false); + Snapshot = Snapshot::False); void setLastRestorable(Reference tr, Key tagName, Version version); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. Future waitBackup(Database cx, std::string tagName, - bool stopWhenDone = true, + StopWhenDone = StopWhenDone::True, Reference* pContainer = nullptr, UID* pUID = nullptr); @@ -462,8 +368,8 @@ public: DatabaseBackupAgent(DatabaseBackupAgent&& r) noexcept : subspace(std::move(r.subspace)), states(std::move(r.states)), config(std::move(r.config)), errors(std::move(r.errors)), ranges(std::move(r.ranges)), tagNames(std::move(r.tagNames)), - taskBucket(std::move(r.taskBucket)), futureBucket(std::move(r.futureBucket)), - sourceStates(std::move(r.sourceStates)), sourceTagNames(std::move(r.sourceTagNames)) {} + sourceStates(std::move(r.sourceStates)), sourceTagNames(std::move(r.sourceTagNames)), + taskBucket(std::move(r.taskBucket)), futureBucket(std::move(r.futureBucket)) {} void operator=(DatabaseBackupAgent&& r) noexcept { subspace = std::move(r.subspace); @@ -478,7 +384,11 @@ public: sourceTagNames = std::move(r.sourceTagNames); } - Future run(Database cx, double* pollDelay, int maxConcurrentTasks) { + Future run(Database cx, double pollDelay, int maxConcurrentTasks) { + return taskBucket->run(cx, futureBucket, std::make_shared(pollDelay), maxConcurrentTasks); + } + + Future run(Database cx, std::shared_ptr pollDelay, int maxConcurrentTasks) { return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); } @@ -487,7 +397,7 @@ public: Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction = false); + ForceAction = ForceAction::False); Future unlockBackup(Reference tr, Key tagName); Future unlockBackup(Database cx, Key tagName) { @@ -506,18 +416,18 @@ public: Future submitBackup(Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone = true, + StopWhenDone = StopWhenDone::True, Key addPrefix = StringRef(), Key removePrefix = StringRef(), - bool lockDatabase = false, + LockDB lockDatabase = LockDB::False, PreBackupAction backupAction = PreBackupAction::VERIFY); Future submitBackup(Database cx, Key tagName, Standalone> backupRanges, - bool stopWhenDone = true, + StopWhenDone stopWhenDone = StopWhenDone::True, Key addPrefix = StringRef(), Key removePrefix = StringRef(), - bool lockDatabase = false, + LockDB lockDatabase = LockDB::False, PreBackupAction backupAction = PreBackupAction::VERIFY) { return runRYWTransaction(cx, [=](Reference tr) { return submitBackup( @@ -533,35 +443,36 @@ public: Future abortBackup(Database cx, Key tagName, - bool partial = false, - bool abortOldBackup = false, - bool dstOnly = false, - bool waitForDestUID = false); + PartialBackup = PartialBackup::False, + AbortOldBackup = AbortOldBackup::False, + DstOnly = DstOnly::False, + WaitForDestUID = WaitForDestUID::False); Future getStatus(Database cx, int errorLimit, Key tagName); - Future getStateValue(Reference tr, UID logUid, bool snapshot = false); + Future getStateValue(Reference tr, UID logUid, Snapshot = Snapshot::False); Future getStateValue(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr) { return getStateValue(tr, logUid); }); } - Future getDestUid(Reference tr, UID logUid, bool snapshot = false); + Future getDestUid(Reference tr, UID logUid, Snapshot = Snapshot::False); Future getDestUid(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr) { return getDestUid(tr, logUid); }); } - Future getLogUid(Reference tr, Key tagName, bool snapshot = false); + Future getLogUid(Reference tr, Key tagName, Snapshot = Snapshot::False); Future getLogUid(Database cx, Key tagName) { return runRYWTransaction(cx, [=](Reference tr) { return getLogUid(tr, tagName); }); } - Future getRangeBytesWritten(Reference tr, UID logUid, bool snapshot = false); - Future getLogBytesWritten(Reference tr, UID logUid, bool snapshot = false); - + Future getRangeBytesWritten(Reference tr, + UID logUid, + Snapshot = Snapshot::False); + Future getLogBytesWritten(Reference tr, UID logUid, Snapshot = Snapshot::False); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. - Future waitBackup(Database cx, Key tagName, bool stopWhenDone = true); + Future waitBackup(Database cx, Key tagName, StopWhenDone = StopWhenDone::True); Future waitSubmitted(Database cx, Key tagName); Future waitUpgradeToLatestDrVersion(Database cx, Key tagName); @@ -619,7 +530,7 @@ Future eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion = Optional(), - bool checkBackupUid = false, + CheckBackupUID = CheckBackupUID::False, Version backupUid = 0); Key getApplyKey(Version version, Key backupUid); Version getLogKeyVersion(Key key); @@ -631,18 +542,18 @@ ACTOR Future readCommitted(Database cx, PromiseStream results, Reference lock, KeyRangeRef range, - bool terminator = true, - bool systemAccess = false, - bool lockAware = false); + Terminator terminator = Terminator::True, + AccessSystemKeys systemAccess = AccessSystemKeys::False, + LockAware lockAware = LockAware::False); ACTOR Future readCommitted(Database cx, PromiseStream results, Future active, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy, - bool terminator = true, - bool systemAccess = false, - bool lockAware = false); + Terminator terminator = Terminator::True, + AccessSystemKeys systemAccess = AccessSystemKeys::False, + LockAware lockAware = LockAware::False); ACTOR Future applyMutations(Database cx, Key uid, Key addPrefix, @@ -652,7 +563,7 @@ ACTOR Future applyMutations(Database cx, RequestStream commit, NotifiedVersion* committedVersion, Reference> keyVersion); -ACTOR Future cleanupBackup(Database cx, bool deleteData); +ACTOR Future cleanupBackup(Database cx, DeleteData deleteData); using EBackupState = BackupAgentBase::EnumState; template <> @@ -695,14 +606,15 @@ public: typedef KeyBackedMap TagMap; // Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace. class TagUidMap : public KeyBackedMap { + ACTOR static Future> getAll_impl(TagUidMap* tagsMap, + Reference tr, + Snapshot snapshot); + public: TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {} - ACTOR static Future> getAll_impl(TagUidMap* tagsMap, - Reference tr, - bool snapshot); - - Future> getAll(Reference tr, bool snapshot = false) { + Future> getAll(Reference tr, + Snapshot snapshot = Snapshot::False) { return getAll_impl(this, tr, snapshot); } @@ -718,12 +630,12 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) { } static inline Future> getAllRestoreTags(Reference tr, - bool snapshot = false) { + Snapshot snapshot = Snapshot::False) { return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot); } static inline Future> getAllBackupTags(Reference tr, - bool snapshot = false) { + Snapshot snapshot = Snapshot::False) { return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot); } @@ -738,7 +650,9 @@ public: KeyBackedConfig(StringRef prefix, Reference task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {} - Future toTask(Reference tr, Reference task, bool setValidation = true) { + Future toTask(Reference tr, + Reference task, + SetValidation setValidation = SetValidation::True) { // Set the uid task parameter TaskParams.uid().set(task, uid); @@ -803,11 +717,22 @@ protected: template <> inline Tuple Codec>::pack(Reference const& bc) { - return Tuple().append(StringRef(bc->getURL())); + Tuple tuple; + tuple.append(StringRef(bc->getURL())); + if (bc->getEncryptionKeyFileName().present()) { + tuple.append(bc->getEncryptionKeyFileName().get()); + } + return tuple; } template <> inline Reference Codec>::unpack(Tuple const& val) { - return IBackupContainer::openContainer(val.getString(0).toString()); + ASSERT(val.size() == 1 || val.size() == 2); + auto url = val.getString(0).toString(); + Optional encryptionKeyFileName; + if (val.size() == 2) { + encryptionKeyFileName = val.getString(1).toString(); + } + return IBackupContainer::openContainer(url, encryptionKeyFileName); } class BackupConfig : public KeyBackedConfig { @@ -1056,6 +981,11 @@ ACTOR Future>> decodeRangeFileBlock(Reference< int64_t offset, int len); +// Reads a mutation log block from file and parses into batch mutation blocks for further parsing. +ACTOR Future>> decodeMutationLogFileBlock(Reference file, + int64_t offset, + int len); + // Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed. Value makePadding(int size); } // namespace fileBackup diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp index 4b00857503..774f606344 100644 --- a/fdbclient/BackupAgentBase.actor.cpp +++ b/fdbclient/BackupAgentBase.actor.cpp @@ -26,6 +26,24 @@ #include "flow/ActorCollection.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DEFINE_BOOLEAN_PARAM(LockDB); +FDB_DEFINE_BOOLEAN_PARAM(UnlockDB); +FDB_DEFINE_BOOLEAN_PARAM(StopWhenDone); +FDB_DEFINE_BOOLEAN_PARAM(Verbose); +FDB_DEFINE_BOOLEAN_PARAM(WaitForComplete); +FDB_DEFINE_BOOLEAN_PARAM(ForceAction); +FDB_DEFINE_BOOLEAN_PARAM(Terminator); +FDB_DEFINE_BOOLEAN_PARAM(UsePartitionedLog); +FDB_DEFINE_BOOLEAN_PARAM(InconsistentSnapshotOnly); +FDB_DEFINE_BOOLEAN_PARAM(ShowErrors); +FDB_DEFINE_BOOLEAN_PARAM(AbortOldBackup); +FDB_DEFINE_BOOLEAN_PARAM(DstOnly); +FDB_DEFINE_BOOLEAN_PARAM(WaitForDestUID); +FDB_DEFINE_BOOLEAN_PARAM(CheckBackupUID); +FDB_DEFINE_BOOLEAN_PARAM(DeleteData); +FDB_DEFINE_BOOLEAN_PARAM(SetValidation); +FDB_DEFINE_BOOLEAN_PARAM(PartialBackup); + std::string BackupAgentBase::formatTime(int64_t epochs) { time_t curTime = (time_t)epochs; char buffer[30]; @@ -95,32 +113,33 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) { return ts; } -const Key BackupAgentBase::keyFolderId = LiteralStringRef("config_folderid"); -const Key BackupAgentBase::keyBeginVersion = LiteralStringRef("beginVersion"); -const Key BackupAgentBase::keyEndVersion = LiteralStringRef("endVersion"); -const Key BackupAgentBase::keyPrevBeginVersion = LiteralStringRef("prevBeginVersion"); -const Key BackupAgentBase::keyConfigBackupTag = LiteralStringRef("config_backup_tag"); -const Key BackupAgentBase::keyConfigLogUid = LiteralStringRef("config_log_uid"); -const Key BackupAgentBase::keyConfigBackupRanges = LiteralStringRef("config_backup_ranges"); -const Key BackupAgentBase::keyConfigStopWhenDoneKey = LiteralStringRef("config_stop_when_done"); -const Key BackupAgentBase::keyStateStop = LiteralStringRef("state_stop"); -const Key BackupAgentBase::keyStateStatus = LiteralStringRef("state_status"); -const Key BackupAgentBase::keyLastUid = LiteralStringRef("last_uid"); -const Key BackupAgentBase::keyBeginKey = LiteralStringRef("beginKey"); -const Key BackupAgentBase::keyEndKey = LiteralStringRef("endKey"); -const Key BackupAgentBase::keyDrVersion = LiteralStringRef("drVersion"); -const Key BackupAgentBase::destUid = LiteralStringRef("destUid"); -const Key BackupAgentBase::backupStartVersion = LiteralStringRef("backupStartVersion"); +const Key BackupAgentBase::keyFolderId = "config_folderid"_sr; +const Key BackupAgentBase::keyBeginVersion = "beginVersion"_sr; +const Key BackupAgentBase::keyEndVersion = "endVersion"_sr; +const Key BackupAgentBase::keyPrevBeginVersion = "prevBeginVersion"_sr; +const Key BackupAgentBase::keyConfigBackupTag = "config_backup_tag"_sr; +const Key BackupAgentBase::keyConfigLogUid = "config_log_uid"_sr; +const Key BackupAgentBase::keyConfigBackupRanges = "config_backup_ranges"_sr; +const Key BackupAgentBase::keyConfigStopWhenDoneKey = "config_stop_when_done"_sr; +const Key BackupAgentBase::keyStateStop = "state_stop"_sr; +const Key BackupAgentBase::keyStateStatus = "state_status"_sr; +const Key BackupAgentBase::keyStateLogBeginVersion = "last_begin_version"_sr; +const Key BackupAgentBase::keyLastUid = "last_uid"_sr; +const Key BackupAgentBase::keyBeginKey = "beginKey"_sr; +const Key BackupAgentBase::keyEndKey = "endKey"_sr; +const Key BackupAgentBase::keyDrVersion = "drVersion"_sr; +const Key BackupAgentBase::destUid = "destUid"_sr; +const Key BackupAgentBase::backupStartVersion = "backupStartVersion"_sr; -const Key BackupAgentBase::keyTagName = LiteralStringRef("tagname"); -const Key BackupAgentBase::keyStates = LiteralStringRef("state"); -const Key BackupAgentBase::keyConfig = LiteralStringRef("config"); -const Key BackupAgentBase::keyErrors = LiteralStringRef("errors"); -const Key BackupAgentBase::keyRanges = LiteralStringRef("ranges"); -const Key BackupAgentBase::keyTasks = LiteralStringRef("tasks"); -const Key BackupAgentBase::keyFutures = LiteralStringRef("futures"); -const Key BackupAgentBase::keySourceStates = LiteralStringRef("source_states"); -const Key BackupAgentBase::keySourceTagName = LiteralStringRef("source_tagname"); +const Key BackupAgentBase::keyTagName = "tagname"_sr; +const Key BackupAgentBase::keyStates = "state"_sr; +const Key BackupAgentBase::keyConfig = "config"_sr; +const Key BackupAgentBase::keyErrors = "errors"_sr; +const Key BackupAgentBase::keyRanges = "ranges"_sr; +const Key BackupAgentBase::keyTasks = "tasks"_sr; +const Key BackupAgentBase::keyFutures = "futures"_sr; +const Key BackupAgentBase::keySourceStates = "source_states"_sr; +const Key BackupAgentBase::keySourceTagName = "source_tagname"_sr; bool copyParameter(Reference source, Reference dest, Key key) { if (source) { @@ -374,9 +393,9 @@ ACTOR Future readCommitted(Database cx, PromiseStream results, Reference lock, KeyRangeRef range, - bool terminator, - bool systemAccess, - bool lockAware) { + Terminator terminator, + AccessSystemKeys systemAccess, + LockAware lockAware) { state KeySelector begin = firstGreaterOrEqual(range.begin); state KeySelector end = firstGreaterOrEqual(range.end); state Transaction tr(cx); @@ -450,9 +469,9 @@ ACTOR Future readCommitted(Database cx, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy, - bool terminator, - bool systemAccess, - bool lockAware) { + Terminator terminator, + AccessSystemKeys systemAccess, + LockAware lockAware) { state KeySelector nextKey = firstGreaterOrEqual(range.begin); state KeySelector end = firstGreaterOrEqual(range.end); @@ -559,7 +578,8 @@ Future readCommitted(Database cx, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy) { - return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true); + return readCommitted( + cx, results, Void(), lock, range, groupBy, Terminator::True, AccessSystemKeys::True, LockAware::True); } ACTOR Future dumpData(Database cx, @@ -770,7 +790,7 @@ ACTOR static Future _eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion, - bool checkBackupUid, + CheckBackupUID checkBackupUid, Version backupUid) { state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix); state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath); @@ -898,7 +918,7 @@ Future eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion, - bool checkBackupUid, + CheckBackupUID checkBackupUid, Version backupUid) { return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid); } @@ -995,7 +1015,7 @@ ACTOR Future cleanupLogMutations(Database cx, Value destUidValue, bool del } } -ACTOR Future cleanupBackup(Database cx, bool deleteData) { +ACTOR Future cleanupBackup(Database cx, DeleteData deleteData) { state Reference tr(new ReadYourWritesTransaction(cx)); loop { try { @@ -1014,3 +1034,124 @@ ACTOR Future cleanupBackup(Database cx, bool deleteData) { } } } + +// Convert the status text to an enumerated value +BackupAgentBase::EnumState BackupAgentBase::getState(std::string const& stateText) { + auto enState = EnumState::STATE_ERRORED; + + if (stateText.empty()) { + enState = EnumState::STATE_NEVERRAN; + } + + else if (!stateText.compare("has been submitted")) { + enState = EnumState::STATE_SUBMITTED; + } + + else if (!stateText.compare("has been started")) { + enState = EnumState::STATE_RUNNING; + } + + else if (!stateText.compare("is differential")) { + enState = EnumState::STATE_RUNNING_DIFFERENTIAL; + } + + else if (!stateText.compare("has been completed")) { + enState = EnumState::STATE_COMPLETED; + } + + else if (!stateText.compare("has been aborted")) { + enState = EnumState::STATE_ABORTED; + } + + else if (!stateText.compare("has been partially aborted")) { + enState = EnumState::STATE_PARTIALLY_ABORTED; + } + + return enState; +} + +const char* BackupAgentBase::getStateText(EnumState enState) { + const char* stateText; + + switch (enState) { + case EnumState::STATE_ERRORED: + stateText = "has errored"; + break; + case EnumState::STATE_NEVERRAN: + stateText = "has never been started"; + break; + case EnumState::STATE_SUBMITTED: + stateText = "has been submitted"; + break; + case EnumState::STATE_RUNNING: + stateText = "has been started"; + break; + case EnumState::STATE_RUNNING_DIFFERENTIAL: + stateText = "is differential"; + break; + case EnumState::STATE_COMPLETED: + stateText = "has been completed"; + break; + case EnumState::STATE_ABORTED: + stateText = "has been aborted"; + break; + case EnumState::STATE_PARTIALLY_ABORTED: + stateText = "has been partially aborted"; + break; + default: + stateText = ""; + break; + } + + return stateText; +} + +const char* BackupAgentBase::getStateName(EnumState enState) { + switch (enState) { + case EnumState::STATE_ERRORED: + return "Errored"; + case EnumState::STATE_NEVERRAN: + return "NeverRan"; + case EnumState::STATE_SUBMITTED: + return "Submitted"; + break; + case EnumState::STATE_RUNNING: + return "Running"; + case EnumState::STATE_RUNNING_DIFFERENTIAL: + return "RunningDifferentially"; + case EnumState::STATE_COMPLETED: + return "Completed"; + case EnumState::STATE_ABORTED: + return "Aborted"; + case EnumState::STATE_PARTIALLY_ABORTED: + return "Aborting"; + default: + return ""; + } +} + +bool BackupAgentBase::isRunnable(EnumState enState) { + switch (enState) { + case EnumState::STATE_SUBMITTED: + case EnumState::STATE_RUNNING: + case EnumState::STATE_RUNNING_DIFFERENTIAL: + case EnumState::STATE_PARTIALLY_ABORTED: + return true; + default: + return false; + } +} + +Standalone BackupAgentBase::getCurrentTime() { + double t = now(); + time_t curTime = t; + char buffer[128]; + struct tm* timeinfo; + timeinfo = localtime(&curTime); + strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo); + + std::string time(buffer); + return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime)))); +} + +std::string const BackupAgentBase::defaultTagName = "default"; diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a71ab0c6ff..8e304184ba 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -58,6 +58,7 @@ ACTOR Future appendStringRefWithLen(Reference file, Standalon wait(file->append(s.begin(), s.size())); return Void(); } + } // namespace IBackupFile_impl Future IBackupFile::appendStringRefWithLen(Standalone s) { @@ -253,7 +254,8 @@ std::vector IBackupContainer::getURLFormats() { } // Get an IBackupContainer based on a container URL string -Reference IBackupContainer::openContainer(const std::string& url) { +Reference IBackupContainer::openContainer(const std::string& url, + Optional const& encryptionKeyFileName) { static std::map> m_cache; Reference& r = m_cache[url]; @@ -262,9 +264,9 @@ Reference IBackupContainer::openContainer(const std::string& u try { StringRef u(url); - if (u.startsWith(LiteralStringRef("file://"))) { - r = Reference(new BackupContainerLocalDirectory(url)); - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + if (u.startsWith("file://"_sr)) { + r = makeReference(url, encryptionKeyFileName); + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; // The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options. @@ -277,15 +279,16 @@ Reference IBackupContainer::openContainer(const std::string& u for (auto c : resource) if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); - r = Reference(new BackupContainerS3BlobStore(bstore, resource, backupParams)); + r = makeReference(bstore, resource, backupParams, encryptionKeyFileName); } #ifdef BUILD_AZURE_BACKUP - else if (u.startsWith(LiteralStringRef("azure://"))) { - u.eat(LiteralStringRef("azure://")); - auto address = NetworkAddress::parse(u.eat(LiteralStringRef("/")).toString()); - auto containerName = u.eat(LiteralStringRef("/")).toString(); - auto accountName = u.eat(LiteralStringRef("/")).toString(); - r = Reference(new BackupContainerAzureBlobStore(address, containerName, accountName)); + else if (u.startsWith("azure://"_sr)) { + u.eat("azure://"_sr); + auto accountName = u.eat("@"_sr).toString(); + auto endpoint = u.eat("/"_sr).toString(); + auto containerName = u.eat("/"_sr).toString(); + r = makeReference( + endpoint, accountName, containerName, encryptionKeyFileName); } #endif else { @@ -293,6 +296,7 @@ Reference IBackupContainer::openContainer(const std::string& u throw backup_invalid_url(); } + r->encryptionKeyFileName = encryptionKeyFileName; r->URL = url; return r; } catch (Error& e) { @@ -315,10 +319,10 @@ Reference IBackupContainer::openContainer(const std::string& u ACTOR Future> listContainers_impl(std::string baseURL) { try { StringRef u(baseURL); - if (u.startsWith(LiteralStringRef("file://"))) { + if (u.startsWith("file://"_sr)) { std::vector results = wait(BackupContainerLocalDirectory::listURLs(baseURL)); return results; - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; S3BlobStoreEndpoint::ParametersT backupParams; @@ -333,14 +337,14 @@ ACTOR Future> listContainers_impl(std::string baseURL) } // Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name - BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams); + BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {}); std::vector results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket())); return results; } // TODO: Enable this when Azure backups are ready /* - else if (u.startsWith(LiteralStringRef("azure://"))) { + else if (u.startsWith("azure://"_sr)) { std::vector results = wait(BackupContainerAzureBlobStore::listURLs(baseURL)); return results; } @@ -386,7 +390,7 @@ ACTOR Future timeKeeperVersionFromDatetime(std::string datetime, Databa tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state std::vector> results = - wait(versionMap.getRange(tr, 0, time, 1, false, true)); + wait(versionMap.getRange(tr, 0, time, 1, Snapshot::False, Reverse::True)); if (results.size() != 1) { // No key less than time was found in the database // Look for a key >= time. @@ -425,7 +429,7 @@ ACTOR Future> timeKeeperEpochsFromVersion(Version v, Reference // Find the highest time < mid state std::vector> results = - wait(versionMap.getRange(tr, min, mid, 1, false, true)); + wait(versionMap.getRange(tr, min, mid, 1, Snapshot::False, Reverse::True)); if (results.size() != 1) { if (mid == min) { diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 2da1e50985..3491f825e7 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -293,16 +293,58 @@ public: Version beginVersion = -1) = 0; // Get an IBackupContainer based on a container spec string - static Reference openContainer(const std::string& url); + static Reference openContainer(const std::string& url, + const Optional& encryptionKeyFileName = {}); static std::vector getURLFormats(); static Future> listContainers(const std::string& baseURL); - std::string getURL() const { return URL; } + std::string const &getURL() const { return URL; } + Optional const &getEncryptionKeyFileName() const { return encryptionKeyFileName; } static std::string lastOpenError; private: std::string URL; + Optional encryptionKeyFileName; }; +namespace fileBackup { +// Accumulates mutation log value chunks, as both a vector of chunks and as a combined chunk, +// in chunk order, and can check the chunk set for completion or intersection with a set +// of ranges. +struct AccumulatedMutations { + AccumulatedMutations() : lastChunkNumber(-1) {} + + // Add a KV pair for this mutation chunk set + // It will be accumulated onto serializedMutations if the chunk number is + // the next expected value. + void addChunk(int chunkNumber, const KeyValueRef& kv); + + // Returns true if both + // - 1 or more chunks were added to this set + // - The header of the first chunk contains a valid protocol version and a length + // that matches the bytes after the header in the combined value in serializedMutations + bool isComplete() const; + + // Returns true if a complete chunk contains any MutationRefs which intersect with any + // range in ranges. + // It is undefined behavior to run this if isComplete() does not return true. + bool matchesAnyRange(const std::vector& ranges) const; + + std::vector kvs; + std::string serializedMutations; + int lastChunkNumber; +}; + +// Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and +// returns (commitVersion, chunkNumber) +std::pair decodeMutationLogKey(const StringRef& key); + +// Decodes an encoded list of mutations in the format of: +// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k], +// where a mutation is encoded as: +// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][param1][param2] +std::vector decodeMutationLogValue(const StringRef& value); +} // namespace fileBackup + #endif diff --git a/fdbclient/BackupContainerAzureBlobStore.actor.cpp b/fdbclient/BackupContainerAzureBlobStore.actor.cpp index dea07c382e..763104dc3f 100644 --- a/fdbclient/BackupContainerAzureBlobStore.actor.cpp +++ b/fdbclient/BackupContainerAzureBlobStore.actor.cpp @@ -19,37 +19,70 @@ */ #include "fdbclient/BackupContainerAzureBlobStore.h" +#include "fdbrpc/AsyncFileEncrypted.h" +#include #include "flow/actorcompiler.h" // This must be the last #include. +namespace { + +std::string const notFoundErrorCode = "404"; + +void printAzureError(std::string const& operationName, azure::storage_lite::storage_error const& err) { + printf("(%s) : Error from Azure SDK : %s (%s) : %s", + operationName.c_str(), + err.code_name.c_str(), + err.code.c_str(), + err.message.c_str()); +} + +template +T waitAzureFuture(std::future>&& f, std::string const& operationName) { + auto outcome = f.get(); + if (outcome.success()) { + return outcome.response(); + } else { + printAzureError(operationName, outcome.error()); + throw backup_error(); + } +} + +} // namespace + class BackupContainerAzureBlobStoreImpl { public: using AzureClient = azure::storage_lite::blob_client; class ReadFile final : public IAsyncFile, ReferenceCounted { - AsyncTaskThread& asyncTaskThread; + AsyncTaskThread* asyncTaskThread; std::string containerName; std::string blobName; - AzureClient* client; + std::shared_ptr client; public: ReadFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, - AzureClient* client) - : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} + std::shared_ptr const& client) + : asyncTaskThread(&asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } - Future read(void* data, int length, int64_t offset) { - return asyncTaskThread.execAsync([client = this->client, - containerName = this->containerName, - blobName = this->blobName, - data, - length, - offset] { + Future read(void* data, int length, int64_t offset) override { + TraceEvent(SevDebug, "BCAzureBlobStoreRead") + .detail("Length", length) + .detail("Offset", offset) + .detail("ContainerName", containerName) + .detail("BlobName", blobName); + return asyncTaskThread->execAsync([client = this->client, + containerName = this->containerName, + blobName = this->blobName, + data, + length, + offset] { std::ostringstream oss(std::ios::out | std::ios::binary); - client->download_blob_to_stream(containerName, blobName, offset, length, oss); + waitAzureFuture(client->download_blob_to_stream(containerName, blobName, offset, length, oss), + "download_blob_to_stream"); auto str = std::move(oss).str(); memcpy(data, str.c_str(), str.size()); return static_cast(str.size()); @@ -60,19 +93,23 @@ public: Future truncate(int64_t size) override { throw file_not_writable(); } Future sync() override { throw file_not_writable(); } Future size() const override { - return asyncTaskThread.execAsync([client = this->client, - containerName = this->containerName, - blobName = this->blobName] { - return static_cast(client->get_blob_properties(containerName, blobName).get().response().size); - }); + TraceEvent(SevDebug, "BCAzureBlobStoreReadFileSize") + .detail("ContainerName", containerName) + .detail("BlobName", blobName); + return asyncTaskThread->execAsync( + [client = this->client, containerName = this->containerName, blobName = this->blobName] { + auto resp = + waitAzureFuture(client->get_blob_properties(containerName, blobName), "get_blob_properties"); + return static_cast(resp.size); + }); } std::string getFilename() const override { return blobName; } int64_t debugFD() const override { return 0; } }; class WriteFile final : public IAsyncFile, ReferenceCounted { - AsyncTaskThread& asyncTaskThread; - AzureClient* client; + AsyncTaskThread* asyncTaskThread; + std::shared_ptr client; std::string containerName; std::string blobName; int64_t m_cursor{ 0 }; @@ -87,8 +124,8 @@ public: WriteFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, - AzureClient* client) - : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} + std::shared_ptr const& client) + : asyncTaskThread(&asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} void addref() override { ReferenceCounted::addref(); } void delref() override { ReferenceCounted::delref(); } @@ -113,22 +150,33 @@ public: return Void(); } Future sync() override { + TraceEvent(SevDebug, "BCAzureBlobStoreSync") + .detail("Length", buffer.size()) + .detail("ContainerName", containerName) + .detail("BlobName", blobName); auto movedBuffer = std::move(buffer); - buffer.clear(); - return asyncTaskThread.execAsync([client = this->client, - containerName = this->containerName, - blobName = this->blobName, - buffer = std::move(movedBuffer)] { - std::istringstream iss(std::move(buffer)); - auto resp = client->append_block_from_stream(containerName, blobName, iss).get(); - return Void(); - }); + buffer = {}; + if (!movedBuffer.empty()) { + return asyncTaskThread->execAsync([client = this->client, + containerName = this->containerName, + blobName = this->blobName, + buffer = std::move(movedBuffer)] { + std::istringstream iss(std::move(buffer)); + waitAzureFuture(client->append_block_from_stream(containerName, blobName, iss), + "append_block_from_stream"); + return Void(); + }); + } + return Void(); } Future size() const override { - return asyncTaskThread.execAsync( + TraceEvent(SevDebug, "BCAzureBlobStoreSize") + .detail("ContainerName", containerName) + .detail("BlobName", blobName); + return asyncTaskThread->execAsync( [client = this->client, containerName = this->containerName, blobName = this->blobName] { - auto resp = client->get_blob_properties(containerName, blobName).get().response(); - ASSERT(resp.valid()); // TODO: Should instead throw here + auto resp = + waitAzureFuture(client->get_blob_properties(containerName, blobName), "get_blob_properties"); return static_cast(resp.size); }); } @@ -162,35 +210,53 @@ public: static bool isDirectory(const std::string& blobName) { return blobName.size() && blobName.back() == '/'; } + // Hack to get around the fact that macros don't work inside actor functions + static Reference encryptFile(Reference const& f, AsyncFileEncrypted::Mode mode) { + Reference result = f; +#if ENCRYPTION_ENABLED + result = makeReference(result, mode); +#endif + return result; + } + ACTOR static Future> readFile(BackupContainerAzureBlobStore* self, std::string fileName) { bool exists = wait(self->blobExists(fileName)); if (!exists) { throw file_not_found(); } - return Reference( - new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get())); + Reference f = + makeReference(self->asyncTaskThread, self->containerName, fileName, self->client); + if (self->usesEncryption()) { + f = encryptFile(f, AsyncFileEncrypted::Mode::READ_ONLY); + } + return f; } ACTOR static Future> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) { + TraceEvent(SevDebug, "BCAzureBlobStoreCreateWriteFile") + .detail("ContainerName", self->containerName) + .detail("FileName", fileName); wait(self->asyncTaskThread.execAsync( - [client = self->client.get(), containerName = self->containerName, fileName = fileName] { - auto outcome = client->create_append_blob(containerName, fileName).get(); + [client = self->client, containerName = self->containerName, fileName = fileName] { + waitAzureFuture(client->create_append_blob(containerName, fileName), "create_append_blob"); return Void(); })); - return Reference( - new BackupFile(fileName, - Reference(new WriteFile( - self->asyncTaskThread, self->containerName, fileName, self->client.get())))); + Reference f = + makeReference(self->asyncTaskThread, self->containerName, fileName, self->client); + if (self->usesEncryption()) { + f = encryptFile(f, AsyncFileEncrypted::Mode::APPEND_ONLY); + } + return makeReference(fileName, f); } - static void listFiles(AzureClient* client, + static void listFiles(std::shared_ptr const& client, const std::string& containerName, const std::string& path, std::function folderPathFilter, BackupContainerFileSystem::FilesAndSizesT& result) { - auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response(); + auto resp = waitAzureFuture(client->list_blobs_segmented(containerName, "/", "", path), "list_blobs_segmented"); for (const auto& blob : resp.blobs) { - if (isDirectory(blob.name) && folderPathFilter(blob.name)) { + if (isDirectory(blob.name) && (!folderPathFilter || folderPathFilter(blob.name))) { listFiles(client, containerName, blob.name, folderPathFilter, result); } else { result.emplace_back(blob.name, blob.content_length); @@ -204,8 +270,12 @@ public: BackupContainerFileSystem::FilesAndSizesT files = wait(self->listFiles()); filesToDelete = files.size(); } - wait(self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] { - client->delete_container(containerName).wait(); + TraceEvent(SevDebug, "BCAzureBlobStoreDeleteContainer") + .detail("FilesToDelete", filesToDelete) + .detail("ContainerName", self->containerName) + .detail("TrackNumDeleted", pNumDeleted != nullptr); + wait(self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client] { + waitAzureFuture(client->delete_container(containerName), "delete_container"); return Void(); })); if (pNumDeleted) { @@ -213,26 +283,45 @@ public: } return Void(); } + }; Future BackupContainerAzureBlobStore::blobExists(const std::string& fileName) { - return asyncTaskThread.execAsync( - [client = this->client.get(), containerName = this->containerName, fileName = fileName] { - auto resp = client->get_blob_properties(containerName, fileName).get().response(); - return resp.valid(); - }); + TraceEvent(SevDebug, "BCAzureBlobStoreCheckExists") + .detail("FileName", fileName) + .detail("ContainerName", containerName); + return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, fileName = fileName] { + auto outcome = client->get_blob_properties(containerName, fileName).get(); + if (outcome.success()) { + return true; + } else { + auto const& err = outcome.error(); + if (err.code == notFoundErrorCode) { + return false; + } else { + printAzureError("get_blob_properties", err); + throw backup_error(); + } + } + }); } -BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address, +BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const std::string& endpoint, const std::string& accountName, - const std::string& containerName) + const std::string& containerName, + const Optional& encryptionKeyFileName) : containerName(containerName) { - std::string accountKey = std::getenv("AZURE_KEY"); - + setEncryptionKey(encryptionKeyFileName); + const char* _accountKey = std::getenv("AZURE_KEY"); + if (!_accountKey) { + TraceEvent(SevError, "EnvironmentVariableNotFound").detail("EnvVariable", "AZURE_KEY"); + // TODO: More descriptive error? + throw backup_error(); + } + std::string accountKey = _accountKey; auto credential = std::make_shared(accountName, accountKey); auto storageAccount = std::make_shared( - accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str())); - + accountName, credential, true, format("https://%s", endpoint.c_str())); client = std::make_unique(storageAccount, 1); } @@ -244,15 +333,30 @@ void BackupContainerAzureBlobStore::delref() { } Future BackupContainerAzureBlobStore::create() { - return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { - client->create_container(containerName).wait(); - return Void(); - }); + TraceEvent(SevDebug, "BCAzureBlobStoreCreateContainer").detail("ContainerName", containerName); + Future createContainerFuture = + asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] { + waitAzureFuture(client->create_container(containerName), "create_container"); + return Void(); + }); + Future encryptionSetupFuture = usesEncryption() ? encryptionSetupComplete() : Void(); + return createContainerFuture && encryptionSetupFuture; } Future BackupContainerAzureBlobStore::exists() { - return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { - auto resp = client->get_container_properties(containerName).get().response(); - return resp.valid(); + TraceEvent(SevDebug, "BCAzureBlobStoreCheckContainerExists").detail("ContainerName", containerName); + return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] { + auto outcome = client->get_container_properties(containerName).get(); + if (outcome.success()) { + return true; + } else { + auto const& err = outcome.error(); + if (err.code == notFoundErrorCode) { + return false; + } else { + printAzureError("got_container_properties", err); + throw backup_error(); + } + } }); } @@ -267,22 +371,23 @@ Future> BackupContainerAzureBlobStore::writeFile(const st Future BackupContainerAzureBlobStore::listFiles( const std::string& path, std::function folderPathFilter) { - return asyncTaskThread.execAsync([client = this->client.get(), - containerName = this->containerName, - path = path, - folderPathFilter = folderPathFilter] { - FilesAndSizesT result; - BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result); - return result; - }); + TraceEvent(SevDebug, "BCAzureBlobStoreListFiles").detail("ContainerName", containerName).detail("Path", path); + return asyncTaskThread.execAsync( + [client = this->client, containerName = this->containerName, path = path, folderPathFilter = folderPathFilter] { + FilesAndSizesT result; + BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result); + return result; + }); } Future BackupContainerAzureBlobStore::deleteFile(const std::string& fileName) { - return asyncTaskThread.execAsync( - [containerName = this->containerName, fileName = fileName, client = client.get()]() { - client->delete_blob(containerName, fileName).wait(); - return Void(); - }); + TraceEvent(SevDebug, "BCAzureBlobStoreDeleteFile") + .detail("ContainerName", containerName) + .detail("FileName", fileName); + return asyncTaskThread.execAsync([containerName = this->containerName, fileName = fileName, client = client]() { + client->delete_blob(containerName, fileName).wait(); + return Void(); + }); } Future BackupContainerAzureBlobStore::deleteContainer(int* pNumDeleted) { @@ -295,5 +400,5 @@ Future> BackupContainerAzureBlobStore::listURLs(const s } std::string BackupContainerAzureBlobStore::getURLFormat() { - return "azure://:///"; + return "azure://@//"; } diff --git a/fdbclient/BackupContainerAzureBlobStore.h b/fdbclient/BackupContainerAzureBlobStore.h index 193fe4a301..3e860e8116 100644 --- a/fdbclient/BackupContainerAzureBlobStore.h +++ b/fdbclient/BackupContainerAzureBlobStore.h @@ -33,7 +33,7 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem, ReferenceCounted { using AzureClient = azure::storage_lite::blob_client; - std::unique_ptr client; + std::shared_ptr client; std::string containerName; AsyncTaskThread asyncTaskThread; @@ -42,9 +42,10 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem, friend class BackupContainerAzureBlobStoreImpl; public: - BackupContainerAzureBlobStore(const NetworkAddress& address, + BackupContainerAzureBlobStore(const std::string& endpoint, const std::string& accountName, - const std::string& containerName); + const std::string& containerName, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 87e4ffcbf0..31d9260084 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -23,6 +23,7 @@ #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BackupContainerLocalDirectory.h" #include "fdbclient/JsonBuilder.h" +#include "flow/StreamCipher.h" #include "flow/UnitTest.h" #include @@ -162,7 +163,6 @@ public: state Version maxVer = 0; state RangeFile rf; state json_spirit::mArray fileArray; - state int i; // Validate each filename, update version range for (const auto& f : fileNames) { @@ -290,13 +290,13 @@ public: std::map> tagIndices; // tagId -> indices in files for (int i = 0; i < logs.size(); i++) { - ASSERT(logs[i].tagId >= 0); - ASSERT(logs[i].tagId < logs[i].totalTags); + ASSERT_GE(logs[i].tagId, 0); + ASSERT_LT(logs[i].tagId, logs[i].totalTags); auto& indices = tagIndices[logs[i].tagId]; // filter out if indices.back() is subset of files[i] or vice versa if (!indices.empty()) { if (logs[indices.back()].isSubset(logs[i])) { - ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize); indices.back() = i; } else if (!logs[i].isSubset(logs[indices.back()])) { indices.push_back(i); @@ -864,7 +864,7 @@ public: int i = 0; for (int j = 1; j < logs.size(); j++) { if (logs[j].isSubset(logs[i])) { - ASSERT(logs[j].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[j].fileSize, logs[i].fileSize); continue; } @@ -1032,10 +1032,10 @@ public: } static std::string versionFolderString(Version v, int smallestBucket) { - ASSERT(smallestBucket < 14); + ASSERT_LT(smallestBucket, 14); // Get a 0-padded fixed size representation of v std::string vFixedPrecision = format("%019lld", v); - ASSERT(vFixedPrecision.size() == 19); + ASSERT_EQ(vFixedPrecision.size(), 19); // Truncate smallestBucket from the fixed length representation vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket); @@ -1126,6 +1126,45 @@ public: return false; } +#if ENCRYPTION_ENABLED + ACTOR static Future createTestEncryptionKeyFile(std::string filename) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( + filename, + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, + 0600)); + StreamCipher::Key::RawKeyType testKey; + generateRandomData(testKey.data(), testKey.size()); + keyFile->write(testKey.data(), testKey.size(), 0); + wait(keyFile->sync()); + return Void(); + } + + ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { + state Reference keyFile; + state StreamCipher::Key::RawKeyType key; + try { + Reference _keyFile = + wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + keyFile = _keyFile; + } catch (Error& e) { + TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") + .detail("FileName", encryptionKeyFileName) + .error(e); + throw e; + } + int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); + if (bytesRead != key.size()) { + TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") + .detail("ExpectedSize", key.size()) + .detail("ActualSize", bytesRead); + throw invalid_encryption_key_file(); + } + ASSERT_EQ(bytesRead, key.size()); + StreamCipher::Key::initializeKey(std::move(key)); + return Void(); + } +#endif // ENCRYPTION_ENABLED + }; // class BackupContainerFileSystemImpl Future> BackupContainerFileSystem::writeLogFile(Version beginVersion, @@ -1432,6 +1471,29 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() { return { Reference::addRef(this), "mutation_log_type" }; } +bool BackupContainerFileSystem::usesEncryption() const { + return encryptionSetupFuture.isValid(); +} +Future BackupContainerFileSystem::encryptionSetupComplete() const { + return encryptionSetupFuture; +} + +void BackupContainerFileSystem::setEncryptionKey(Optional const& encryptionKeyFileName) { + if (encryptionKeyFileName.present()) { +#if ENCRYPTION_ENABLED + encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get()); +#else + encryptionSetupFuture = Void(); +#endif + } +} +Future BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const& filename) { +#if ENCRYPTION_ENABLED + return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename); +#else + return Void(); +#endif +} namespace backup_test { @@ -1444,13 +1506,16 @@ int chooseFileSize(std::vector& sizes) { return deterministicRandom()->randomInt(0, 2e6); } -ACTOR Future writeAndVerifyFile(Reference c, Reference f, int size, FlowLock* lock) { +ACTOR Future writeAndVerifyFile(Reference c, + Reference f, + int size, + FlowLock* lock) { state Standalone> content; wait(lock->take(TaskPriority::DefaultYield, size)); - state FlowLock::Releaser releaser(*lock, size); + state FlowLock::Releaser releaser(*lock, size); - printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str()); + printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str()); content.resize(content.arena(), size); for (int i = 0; i < content.size(); ++i) { content[i] = (uint8_t)deterministicRandom()->randomInt(0, 256); @@ -1466,12 +1531,12 @@ ACTOR Future writeAndVerifyFile(Reference c, Reference inputFile = wait(c->readFile(f->getFileName())); int64_t fileSize = wait(inputFile->size()); - ASSERT(size == fileSize); + ASSERT_EQ(size, fileSize); if (size > 0) { state Standalone> buf; buf.resize(buf.arena(), fileSize); int b = wait(inputFile->read(buf.begin(), buf.size(), 0)); - ASSERT(b == buf.size()); + ASSERT_EQ(b, buf.size()); ASSERT(buf == content); } return Void(); @@ -1485,7 +1550,7 @@ Version nextVersion(Version v) { // Write a snapshot file with only begin & end key ACTOR static Future testWriteSnapshotFile(Reference file, Key begin, Key end, uint32_t blockSize) { - ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size()); + ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size()); uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION; // write Header @@ -1506,12 +1571,16 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR static Future testBackupContainer(std::string url) { +ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { state FlowLock lock(100e6); + if (encryptionKeyFileName.present()) { + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get())); + } + printf("BackupContainerTest URL %s\n", url.c_str()); - state Reference c = IBackupContainer::openContainer(url); + state Reference c = IBackupContainer::openContainer(url, encryptionKeyFileName); // Make sure container doesn't exist, then create it. try { @@ -1534,9 +1603,9 @@ ACTOR static Future testBackupContainer(std::string url) { // List of sizes to use to test edge cases on underlying file implementations state std::vector fileSizes = { 0 }; if (StringRef(url).startsWith(LiteralStringRef("blob"))) { - fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE); - fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10); - } + fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE); + fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10); + } loop { state Version logStart = v; @@ -1597,9 +1666,9 @@ ACTOR static Future testBackupContainer(std::string url) { wait(waitForAll(writes)); state BackupFileList listing = wait(c->dumpFileList()); - ASSERT(listing.ranges.size() == nRangeFiles); - ASSERT(listing.logs.size() == logs.size()); - ASSERT(listing.snapshots.size() == snapshots.size()); + ASSERT_EQ(listing.ranges.size(), nRangeFiles); + ASSERT_EQ(listing.logs.size(), logs.size()); + ASSERT_EQ(listing.snapshots.size(), snapshots.size()); state BackupDescription desc = wait(c->describeBackup()); printf("\n%s\n", desc.toString().c_str()); @@ -1629,8 +1698,8 @@ ACTOR static Future testBackupContainer(std::string url) { // If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot if (f.isError()) { - ASSERT(f.getError().code() == error_code_backup_cannot_expire); - ASSERT(i == listing.snapshots.size() - 1); + ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire); + ASSERT_EQ(i, listing.snapshots.size() - 1); wait(c->expireData(expireVersion, true)); } @@ -1646,31 +1715,34 @@ ACTOR static Future testBackupContainer(std::string url) { ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist); BackupFileList empty = wait(c->dumpFileList()); - ASSERT(empty.ranges.size() == 0); - ASSERT(empty.logs.size() == 0); - ASSERT(empty.snapshots.size() == 0); + ASSERT_EQ(empty.ranges.size(), 0); + ASSERT_EQ(empty.logs.size(), 0); + ASSERT_EQ(empty.snapshots.size(), 0); printf("BackupContainerTest URL=%s PASSED.\n", url.c_str()); return Void(); } -TEST_CASE("/backup/containers/localdir") { - if (g_network->isSimulated()) - wait(testBackupContainer(format("file://simfdb/backups/%llx", timer_int()))); - else - wait(testBackupContainer(format("file:///private/tmp/fdb_backups/%llx", timer_int()))); +TEST_CASE("/backup/containers/localdir/unencrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {})); return Void(); -}; +} + +TEST_CASE("/backup/containers/localdir/encrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), + format("%s/test_encryption_key", params.getDataDir().c_str()))); + return Void(); +} TEST_CASE("/backup/containers/url") { if (!g_network->isSimulated()) { const char* url = getenv("FDB_TEST_BACKUP_URL"); ASSERT(url != nullptr); - wait(testBackupContainer(url)); + wait(testBackupContainer(url, {})); } return Void(); -}; +} TEST_CASE("/backup/containers_list") { if (!g_network->isSimulated()) { @@ -1683,7 +1755,7 @@ TEST_CASE("/backup/containers_list") { } } return Void(); -}; +} TEST_CASE("/backup/time") { // test formatTime() diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index cd0ddf4435..292fc67abb 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -152,6 +152,12 @@ public: VectorRef keyRangesFilter, bool logsOnly, Version beginVersion) final; + static Future createTestEncryptionKeyFile(std::string const& filename); + +protected: + bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); + Future encryptionSetupComplete() const; private: struct VersionProperty { @@ -186,6 +192,8 @@ private: Future> old_listRangeFiles(Version beginVersion, Version endVersion); friend class BackupContainerFileSystemImpl; + + Future encryptionSetupFuture; }; #endif diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index e0c78a31bf..f3082e9d81 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -31,7 +31,8 @@ namespace { class BackupFile : public IBackupFile, ReferenceCounted { public: BackupFile(const std::string& fileName, Reference file, const std::string& finalFullPath) - : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0), m_blockSize(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) { + : IBackupFile(fileName), m_file(file), m_writeOffset(0), m_finalFullPath(finalFullPath), + m_blockSize(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) { if (BUGGIFY) { m_blockSize = deterministicRandom()->randomInt(100, 20000); } @@ -131,7 +132,10 @@ std::string BackupContainerLocalDirectory::getURLFormat() { return "file://"; } -BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url) { +BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url, + const Optional& encryptionKeyFileName) { + setEncryptionKey(encryptionKeyFileName); + std::string path; if (url.find("file://") != 0) { TraceEvent(SevWarn, "BackupContainerLocalDirectory") @@ -193,7 +197,10 @@ Future> BackupContainerLocalDirectory::listURLs(const s } Future BackupContainerLocalDirectory::create() { - // Nothing should be done here because create() can be called by any process working with the container URL, + if (usesEncryption()) { + return encryptionSetupComplete(); + } + // No directory should be created here because create() can be called by any process working with the container URL, // such as fdbbackup. Since "local directory" containers are by definition local to the machine they are // accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to // every file creation, which is done in openFile(). Creating the directory here will result in unnecessary @@ -207,6 +214,9 @@ Future BackupContainerLocalDirectory::exists() { Future> BackupContainerLocalDirectory::readFile(const std::string& path) { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } // Simulation does not properly handle opening the same file from multiple machines using a shared filesystem, // so create a symbolic link to make each file opening appear to be unique. This could also work in production // but only if the source directory is writeable which shouldn't be required for a restore. @@ -218,7 +228,7 @@ Future> BackupContainerLocalDirectory::readFile(const std: } if (g_simulator.getCurrentProcess()->uid == UID()) { - TraceEvent(SevError, "BackupContainerReadFileOnUnsetProcessID"); + TraceEvent(SevError, "BackupContainerReadFileOnUnsetProcessID").log(); } std::string uniquePath = fullPath + "." + g_simulator.getCurrentProcess()->uid.toString() + ".lnk"; unlink(uniquePath.c_str()); @@ -258,8 +268,11 @@ Future> BackupContainerLocalDirectory::readFile(const std: } Future> BackupContainerLocalDirectory::writeFile(const std::string& path) { - int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | - IAsyncFile::OPEN_READWRITE; + int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath)); std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp"; diff --git a/fdbclient/BackupContainerLocalDirectory.h b/fdbclient/BackupContainerLocalDirectory.h index 9db8e07aef..f7c77e4636 100644 --- a/fdbclient/BackupContainerLocalDirectory.h +++ b/fdbclient/BackupContainerLocalDirectory.h @@ -33,7 +33,7 @@ public: static std::string getURLFormat(); - BackupContainerLocalDirectory(const std::string& url); + BackupContainerLocalDirectory(const std::string& url, Optional const& encryptionKeyFileName); static Future> listURLs(const std::string& url); diff --git a/fdbclient/BackupContainerS3BlobStore.actor.cpp b/fdbclient/BackupContainerS3BlobStore.actor.cpp index 4e89402ae0..b915701a3f 100644 --- a/fdbclient/BackupContainerS3BlobStore.actor.cpp +++ b/fdbclient/BackupContainerS3BlobStore.actor.cpp @@ -20,6 +20,9 @@ #include "fdbclient/AsyncFileS3BlobStore.actor.h" #include "fdbclient/BackupContainerS3BlobStore.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +#include "fdbrpc/AsyncFileEncrypted.h" +#endif #include "fdbrpc/AsyncFileReadAhead.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -103,6 +106,10 @@ public: wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), "")); } + if (bc->usesEncryption()) { + wait(bc->encryptionSetupComplete()); + } + return Void(); } @@ -137,9 +144,10 @@ std::string BackupContainerS3BlobStore::indexEntry() { BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params) + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName) : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") { - + setEncryptionKey(encryptionKeyFileName); // Currently only one parameter is supported, "bucket" for (const auto& [name, value] : params) { if (name == "bucket") { @@ -164,12 +172,19 @@ std::string BackupContainerS3BlobStore::getURLFormat() { } Future> BackupContainerS3BlobStore::readFile(const std::string& path) { - return Reference(new AsyncFileReadAheadCache( - Reference(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))), - m_bstore->knobs.read_block_size, - m_bstore->knobs.read_ahead_blocks, - m_bstore->knobs.concurrent_reads_per_file, - m_bstore->knobs.read_cache_blocks_per_file)); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); + +#if ENCRYPTION_ENABLED + if (usesEncryption()) { + f = makeReference(f, AsyncFileEncrypted::Mode::READ_ONLY); + } +#endif + f = makeReference(f, + m_bstore->knobs.read_block_size, + m_bstore->knobs.read_ahead_blocks, + m_bstore->knobs.concurrent_reads_per_file, + m_bstore->knobs.read_cache_blocks_per_file); + return f; } Future> BackupContainerS3BlobStore::listURLs(Reference bstore, @@ -178,8 +193,13 @@ Future> BackupContainerS3BlobStore::listURLs(Reference< } Future> BackupContainerS3BlobStore::writeFile(const std::string& path) { - return Reference(new BackupContainerS3BlobStoreImpl::BackupFile( - path, Reference(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path))))); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); +#if ENCRYPTION_ENABLED + if (usesEncryption()) { + f = makeReference(f, AsyncFileEncrypted::Mode::APPEND_ONLY); + } +#endif + return Future>(makeReference(path, f)); } Future BackupContainerS3BlobStore::deleteFile(const std::string& path) { diff --git a/fdbclient/BackupContainerS3BlobStore.h b/fdbclient/BackupContainerS3BlobStore.h index 57199fcb85..9e47483adf 100644 --- a/fdbclient/BackupContainerS3BlobStore.h +++ b/fdbclient/BackupContainerS3BlobStore.h @@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem, public: BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params); + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 57bd2d895b..512e253657 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -15,6 +15,8 @@ set(FDBCLIENT_SRCS BackupContainerLocalDirectory.h BackupContainerS3BlobStore.actor.cpp BackupContainerS3BlobStore.h + ClientBooleanParams.cpp + ClientBooleanParams.h ClientKnobCollection.cpp ClientKnobCollection.h ClientKnobs.cpp @@ -170,7 +172,7 @@ if(BUILD_AZURE_BACKUP) endif() add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs}) -add_dependencies(fdbclient fdboptions) +add_dependencies(fdbclient fdboptions fdb_c_options) if(BUILD_AZURE_BACKUP) target_link_libraries(fdbclient PUBLIC fdbrpc PRIVATE curl uuid azure-storage-lite) else() diff --git a/fdbserver/IConfigDatabaseNode.cpp b/fdbclient/ClientBooleanParams.cpp similarity index 59% rename from fdbserver/IConfigDatabaseNode.cpp rename to fdbclient/ClientBooleanParams.cpp index d5b0a84bd6..1027fdece6 100644 --- a/fdbserver/IConfigDatabaseNode.cpp +++ b/fdbclient/ClientBooleanParams.cpp @@ -1,5 +1,5 @@ /* - * IConfigDatabaseNode.actor.cpp + * ClientBooleanParams.cpp * * This source file is part of the FoundationDB open source project * @@ -18,14 +18,13 @@ * limitations under the License. */ -#include "fdbserver/IConfigDatabaseNode.h" -#include "fdbserver/PaxosConfigDatabaseNode.h" -#include "fdbserver/SimpleConfigDatabaseNode.h" +#include "fdbclient/ClientBooleanParams.h" -Reference IConfigDatabaseNode::createSimple(std::string const& folder) { - return makeReference(folder); -} - -Reference IConfigDatabaseNode::createPaxos(std::string const& folder) { - return makeReference(folder); -} +FDB_DEFINE_BOOLEAN_PARAM(EnableLocalityLoadBalance); +FDB_DEFINE_BOOLEAN_PARAM(LockAware); +FDB_DEFINE_BOOLEAN_PARAM(Reverse); +FDB_DEFINE_BOOLEAN_PARAM(Snapshot); +FDB_DEFINE_BOOLEAN_PARAM(IsInternal); +FDB_DEFINE_BOOLEAN_PARAM(AddConflictRange); +FDB_DEFINE_BOOLEAN_PARAM(UseMetrics); +FDB_DEFINE_BOOLEAN_PARAM(IsSwitchable); diff --git a/fdbserver/PaxosConfigDatabaseNode.h b/fdbclient/ClientBooleanParams.h similarity index 60% rename from fdbserver/PaxosConfigDatabaseNode.h rename to fdbclient/ClientBooleanParams.h index 062ab809de..c078c6575e 100644 --- a/fdbserver/PaxosConfigDatabaseNode.h +++ b/fdbclient/ClientBooleanParams.h @@ -1,5 +1,5 @@ /* - * PaxosConfigDatabaseNode.h + * ClientBooleanParams.h * * This source file is part of the FoundationDB open source project * @@ -20,17 +20,13 @@ #pragma once -#include "fdbserver/IConfigDatabaseNode.h" +#include "flow/BooleanParam.h" -/* - * Fault-tolerant configuration database node implementation - */ -class PaxosConfigDatabaseNode : public IConfigDatabaseNode { - std::unique_ptr impl; - -public: - PaxosConfigDatabaseNode(std::string const& folder); - ~PaxosConfigDatabaseNode(); - Future serve(ConfigTransactionInterface const&) override; - Future serve(ConfigFollowerInterface const&) override; -}; +FDB_DECLARE_BOOLEAN_PARAM(EnableLocalityLoadBalance); +FDB_DECLARE_BOOLEAN_PARAM(LockAware); +FDB_DECLARE_BOOLEAN_PARAM(Reverse); +FDB_DECLARE_BOOLEAN_PARAM(Snapshot); +FDB_DECLARE_BOOLEAN_PARAM(IsInternal); +FDB_DECLARE_BOOLEAN_PARAM(AddConflictRange); +FDB_DECLARE_BOOLEAN_PARAM(UseMetrics); +FDB_DECLARE_BOOLEAN_PARAM(IsSwitchable); diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index c7090a374d..238b99bfc3 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -29,8 +29,7 @@ ClientKnobs::ClientKnobs(Randomize randomize) { initialize(randomize); } -void ClientKnobs::initialize(Randomize _randomize) { - bool const randomize = (_randomize == Randomize::YES); +void ClientKnobs::initialize(Randomize randomize) { // clang-format off init( TOO_MANY, 1000000 ); @@ -253,13 +252,13 @@ void ClientKnobs::initialize(Randomize _randomize) { TEST_CASE("/fdbclient/knobs/initialize") { // This test depends on TASKBUCKET_TIMEOUT_VERSIONS being defined as a constant multiple of CORE_VERSIONSPERSECOND - ClientKnobs clientKnobs(Randomize::NO); + ClientKnobs clientKnobs(Randomize::False); int64_t initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND; int initialTaskBucketTimeoutVersions = clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS; clientKnobs.setKnob("core_versionspersecond", initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions); - clientKnobs.initialize(Randomize::NO); + clientKnobs.initialize(Randomize::False); ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions * 2); return Void(); diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h index d8a26deb4a..08c00fc9fa 100644 --- a/fdbclient/ClientKnobs.h +++ b/fdbclient/ClientKnobs.h @@ -22,9 +22,13 @@ #define FDBCLIENT_KNOBS_H #pragma once +#include "flow/BooleanParam.h" #include "flow/Knobs.h" #include "flow/flow.h" +FDB_DECLARE_BOOLEAN_PARAM(Randomize); +FDB_DECLARE_BOOLEAN_PARAM(IsSimulated); + class ClientKnobs : public KnobsImpl { public: int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index 1f58616a87..2623d540b8 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -65,6 +65,7 @@ struct CommitProxyInterface { bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); } bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); } + NetworkAddressList addresses() const { return commit.getEndpoint().addresses; } template void serialize(Archive& ar) { @@ -250,7 +251,7 @@ struct GetReadVersionRequest : TimedRequest { uint32_t flags = 0, TransactionTagMap tags = TransactionTagMap(), Optional debugID = Optional()) - : spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), + : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags), debugID(debugID), maxVersion(maxVersion) { flags = flags & ~FLAG_PRIORITY_MASK; switch (priority) { @@ -326,7 +327,7 @@ struct GetKeyServerLocationsRequest { int limit, bool reverse, Arena const& arena) - : spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {} + : arena(arena), spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse) {} template void serialize(Ar& ar) { diff --git a/fdbclient/ConfigTransactionInterface.cpp b/fdbclient/ConfigTransactionInterface.cpp index c912668aff..66618e01d7 100644 --- a/fdbclient/ConfigTransactionInterface.cpp +++ b/fdbclient/ConfigTransactionInterface.cpp @@ -20,12 +20,13 @@ #include "fdbclient/ConfigTransactionInterface.h" #include "fdbclient/CoordinationInterface.h" +#include "fdbclient/SystemData.h" #include "flow/IRandom.h" ConfigTransactionInterface::ConfigTransactionInterface() : _id(deterministicRandom()->randomUniqueID()) {} void ConfigTransactionInterface::setupWellKnownEndpoints() { - getVersion.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETVERSION, TaskPriority::Coordination); + getGeneration.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETGENERATION, TaskPriority::Coordination); get.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GET, TaskPriority::Coordination); getClasses.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETCLASSES, TaskPriority::Coordination); getKnobs.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETKNOBS, TaskPriority::Coordination); @@ -33,8 +34,8 @@ void ConfigTransactionInterface::setupWellKnownEndpoints() { } ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote) - : getVersion(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETVERSION)), get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)), - getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)), + : getGeneration(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)), + get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)), getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)), getKnobs(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), commit(Endpoint({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) { } @@ -45,3 +46,30 @@ bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rh bool ConfigTransactionInterface::operator!=(ConfigTransactionInterface const& rhs) const { return !(*this == rhs); } + +bool ConfigGeneration::operator==(ConfigGeneration const& rhs) const { + return liveVersion == rhs.liveVersion && committedVersion == rhs.committedVersion; +} + +bool ConfigGeneration::operator!=(ConfigGeneration const& rhs) const { + return !(*this == rhs); +} + +void ConfigTransactionCommitRequest::set(KeyRef key, ValueRef value) { + if (key == configTransactionDescriptionKey) { + annotation.description = KeyRef(arena, value); + } else { + ConfigKey configKey = ConfigKeyRef::decodeKey(key); + auto knobValue = IKnobCollection::parseKnobValue( + configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST); + mutations.emplace_back_deep(arena, configKey, knobValue.contents()); + } +} + +void ConfigTransactionCommitRequest::clear(KeyRef key) { + if (key == configTransactionDescriptionKey) { + annotation.description = ""_sr; + } else { + mutations.emplace_back_deep(arena, ConfigKeyRef::decodeKey(key), Optional{}); + } +} diff --git a/fdbclient/ConfigTransactionInterface.h b/fdbclient/ConfigTransactionInterface.h index 6b6173cada..ff85760a3f 100644 --- a/fdbclient/ConfigTransactionInterface.h +++ b/fdbclient/ConfigTransactionInterface.h @@ -27,22 +27,38 @@ #include "fdbrpc/fdbrpc.h" #include "flow/flow.h" -struct ConfigTransactionGetVersionReply { - static constexpr FileIdentifier file_identifier = 2934851; - ConfigTransactionGetVersionReply() = default; - explicit ConfigTransactionGetVersionReply(Version version) : version(version) {} - Version version; +struct ConfigGeneration { + // The live version of each node is monotonically increasing + Version liveVersion{ 0 }; + // The committedVersion of each node is the version of the last commit made durable. + // Each committedVersion was previously given to clients as a liveVersion, prior to commit. + Version committedVersion{ 0 }; + + bool operator==(ConfigGeneration const&) const; + bool operator!=(ConfigGeneration const&) const; template void serialize(Ar& ar) { - serializer(ar, version); + serializer(ar, liveVersion, committedVersion); } }; -struct ConfigTransactionGetVersionRequest { +struct ConfigTransactionGetGenerationReply { + static constexpr FileIdentifier file_identifier = 2934851; + ConfigTransactionGetGenerationReply() = default; + explicit ConfigTransactionGetGenerationReply(ConfigGeneration generation) : generation(generation) {} + ConfigGeneration generation; + + template + void serialize(Ar& ar) { + serializer(ar, generation); + } +}; + +struct ConfigTransactionGetGenerationRequest { static constexpr FileIdentifier file_identifier = 138941; - ReplyPromise reply; - ConfigTransactionGetVersionRequest() = default; + ReplyPromise reply; + ConfigTransactionGetGenerationRequest() = default; template void serialize(Ar& ar) { @@ -64,45 +80,36 @@ struct ConfigTransactionGetReply { struct ConfigTransactionGetRequest { static constexpr FileIdentifier file_identifier = 923040; - Version version; + ConfigGeneration generation; ConfigKey key; ReplyPromise reply; ConfigTransactionGetRequest() = default; - explicit ConfigTransactionGetRequest(Version version, ConfigKey key) : version(version), key(key) {} + explicit ConfigTransactionGetRequest(ConfigGeneration generation, ConfigKey key) + : generation(generation), key(key) {} template void serialize(Ar& ar) { - serializer(ar, version, key, reply); + serializer(ar, generation, key, reply); } }; struct ConfigTransactionCommitRequest { static constexpr FileIdentifier file_identifier = 103841; Arena arena; - Version version{ ::invalidVersion }; + ConfigGeneration generation{ ::invalidVersion, ::invalidVersion }; VectorRef mutations; ConfigCommitAnnotationRef annotation; ReplyPromise reply; size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); } - template - void serialize(Ar& ar) { - serializer(ar, arena, version, mutations, annotation, reply); - } -}; - -struct ConfigTransactionGetRangeReply { - static constexpr FileIdentifier file_identifier = 430263; - Standalone range; - - ConfigTransactionGetRangeReply() = default; - explicit ConfigTransactionGetRangeReply(Standalone range) : range(range) {} + void set(KeyRef key, ValueRef value); + void clear(KeyRef key); template void serialize(Ar& ar) { - serializer(ar, range); + serializer(ar, arena, generation, mutations, annotation, reply); } }; @@ -122,15 +129,15 @@ struct ConfigTransactionGetConfigClassesReply { struct ConfigTransactionGetConfigClassesRequest { static constexpr FileIdentifier file_identifier = 7163400; - Version version; + ConfigGeneration generation; ReplyPromise reply; ConfigTransactionGetConfigClassesRequest() = default; - explicit ConfigTransactionGetConfigClassesRequest(Version version) : version(version) {} + explicit ConfigTransactionGetConfigClassesRequest(ConfigGeneration generation) : generation(generation) {} template void serialize(Ar& ar) { - serializer(ar, version); + serializer(ar, generation); } }; @@ -149,17 +156,17 @@ struct ConfigTransactionGetKnobsReply { struct ConfigTransactionGetKnobsRequest { static constexpr FileIdentifier file_identifier = 987410; - Version version; + ConfigGeneration generation; Optional configClass; ReplyPromise reply; ConfigTransactionGetKnobsRequest() = default; - explicit ConfigTransactionGetKnobsRequest(Version version, Optional configClass) - : version(version), configClass(configClass) {} + explicit ConfigTransactionGetKnobsRequest(ConfigGeneration generation, Optional configClass) + : generation(generation), configClass(configClass) {} template void serialize(Ar& ar) { - serializer(ar, version, configClass, reply); + serializer(ar, generation, configClass, reply); } }; @@ -172,7 +179,7 @@ struct ConfigTransactionInterface { public: static constexpr FileIdentifier file_identifier = 982485; - struct RequestStream getVersion; + struct RequestStream getGeneration; struct RequestStream get; struct RequestStream getClasses; struct RequestStream getKnobs; @@ -188,6 +195,6 @@ public: template void serialize(Ar& ar) { - serializer(ar, getVersion, get, getClasses, getKnobs, commit); + serializer(ar, getGeneration, get, getClasses, getKnobs, commit); } }; diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h index 2c80899aae..a2abfa87db 100644 --- a/fdbclient/CoordinationInterface.h +++ b/fdbclient/CoordinationInterface.h @@ -38,7 +38,7 @@ constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3); constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10); constexpr UID WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE(-1, 11); -constexpr UID WLTOKEN_CONFIGTXN_GETVERSION(-1, 12); +constexpr UID WLTOKEN_CONFIGTXN_GETGENERATION(-1, 12); constexpr UID WLTOKEN_CONFIGTXN_GET(-1, 13); constexpr UID WLTOKEN_CONFIGTXN_GETCLASSES(-1, 14); constexpr UID WLTOKEN_CONFIGTXN_GETKNOBS(-1, 15); diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp index 20f9c6bcf2..8ddc6d6be0 100644 --- a/fdbclient/DatabaseBackupAgent.actor.cpp +++ b/fdbclient/DatabaseBackupAgent.actor.cpp @@ -44,22 +44,29 @@ const Key DatabaseBackupAgent::keyDatabasesInSync = LiteralStringRef("databases_ const int DatabaseBackupAgent::LATEST_DR_VERSION = 1; DatabaseBackupAgent::DatabaseBackupAgent() - : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)), - states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)), - errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)), + : subspace(Subspace(databaseBackupPrefixRange.begin)), states(subspace.get(BackupAgentBase::keyStates)), + config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)), + ranges(subspace.get(BackupAgentBase::keyRanges)), tagNames(subspace.get(BackupAgentBase::keyTagName)), sourceStates(subspace.get(BackupAgentBase::keySourceStates)), - sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {} + sourceTagNames(subspace.get(BackupAgentBase::keyTagName)), + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::True, + PriorityBatch::False, + LockAware::True)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) { +} DatabaseBackupAgent::DatabaseBackupAgent(Database src) - : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)), - states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)), - errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)), + : subspace(Subspace(databaseBackupPrefixRange.begin)), states(subspace.get(BackupAgentBase::keyStates)), + config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)), + ranges(subspace.get(BackupAgentBase::keyRanges)), tagNames(subspace.get(BackupAgentBase::keyTagName)), sourceStates(subspace.get(BackupAgentBase::keySourceStates)), - sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) { + sourceTagNames(subspace.get(BackupAgentBase::keyTagName)), + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::True, + PriorityBatch::False, + LockAware::True)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) { taskBucket->src = src; } @@ -234,7 +241,8 @@ struct BackupRangeTaskFunc : TaskFuncBase { // retrieve kvData state PromiseStream results; - state Future rc = readCommitted(taskBucket->src, results, lock, range, true, true, true); + state Future rc = readCommitted( + taskBucket->src, results, lock, range, Terminator::True, AccessSystemKeys::True, LockAware::True); state Key rangeBegin = range.begin; state Key rangeEnd; state bool endOfStream = false; @@ -316,16 +324,20 @@ struct BackupRangeTaskFunc : TaskFuncBase { applyMutationsKeyVersionCountRange.begin); state Future backupVersions = krmGetRanges(tr, prefix, KeyRangeRef(rangeBegin, rangeEnd), BUGGIFY ? 2 : 2000, 1e5); - state Future> logVersionValue = tr->get( - task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin), true); - state Future> rangeCountValue = tr->get(rangeCountKey, true); - state Future prevRange = tr->getRange( - firstGreaterOrEqual(prefix), lastLessOrEqual(rangeBegin.withPrefix(prefix)), 1, true, true); + state Future> logVersionValue = + tr->get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin), + Snapshot::True); + state Future> rangeCountValue = tr->get(rangeCountKey, Snapshot::True); + state Future prevRange = tr->getRange(firstGreaterOrEqual(prefix), + lastLessOrEqual(rangeBegin.withPrefix(prefix)), + 1, + Snapshot::True, + Reverse::True); state Future nextRange = tr->getRange(firstGreaterOrEqual(rangeEnd.withPrefix(prefix)), firstGreaterOrEqual(strinc(prefix)), 1, - true, - false); + Snapshot::True, + Reverse::False); state Future verified = taskBucket->keepRunning(tr, task); wait(checkDatabaseLock(tr, @@ -353,7 +365,7 @@ struct BackupRangeTaskFunc : TaskFuncBase { TEST(true); // range insert delayed because too versionMap is too large if (rangeCount > CLIENT_KNOBS->BACKUP_MAP_KEY_UPPER_LIMIT) - TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge"); + TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge").log(); wait(delay(1)); task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin; @@ -363,7 +375,7 @@ struct BackupRangeTaskFunc : TaskFuncBase { Version logVersion = logVersionValue.get().present() ? BinaryReader::fromStringRef(logVersionValue.get().get(), Unversioned()) - : -1; + : ::invalidVersion; if (logVersion >= values.second) { task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin; return Void(); @@ -633,7 +645,7 @@ struct EraseLogRangeTaskFunc : TaskFuncBase { task->params[BackupAgentBase::keyConfigLogUid], task->params[BackupAgentBase::destUid], Optional(endVersion), - true, + CheckBackupUID::True, BinaryReader::fromStringRef(task->params[BackupAgentBase::keyFolderId], Unversioned()))); wait(tr->commit()); return Void(); @@ -886,9 +898,9 @@ struct CopyLogRangeTaskFunc : TaskFuncBase { locks[j], ranges[j], decodeBKMutationLogKey, - true, - true, - true)); + Terminator::True, + AccessSystemKeys::True, + LockAware::True)); } // copy the range @@ -1191,7 +1203,7 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase { task->params[DatabaseBackupAgent::keyFolderId], Unversioned())) return Void(); - wait(eraseLogData(tr, logUidValue, destUidValue, Optional(), true, backupUid)); + wait(eraseLogData(tr, logUidValue, destUidValue, Optional(), CheckBackupUID::True, backupUid)); wait(tr->commit()); return Void(); } catch (Error& e) { @@ -1321,6 +1333,10 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase { .detail("LogUID", task->params[BackupAgentBase::keyConfigLogUid]); } + // set the log version to the state + tr->set(StringRef(states.pack(DatabaseBackupAgent::keyStateLogBeginVersion)), + BinaryWriter::toValue(beginVersion, Unversioned())); + if (!stopWhenDone.present()) { state Reference allPartsDone = futureBucket->future(tr); std::vector> addTaskVector; @@ -1592,9 +1608,9 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase { lock, ranges[i], decodeBKMutationLogKey, - true, - true, - true)); + Terminator::True, + AccessSystemKeys::True, + LockAware::True)); dump.push_back(dumpData(cx, task, results[i], lock.getPtr(), taskBucket)); } @@ -1701,7 +1717,7 @@ struct AbortOldBackupTaskFunc : TaskFuncBase { } TraceEvent("DBA_AbortOldBackup").detail("TagName", tagNameKey.printable()); - wait(srcDrAgent.abortBackup(cx, tagNameKey, false, true)); + wait(srcDrAgent.abortBackup(cx, tagNameKey, PartialBackup::False, AbortOldBackup::True)); return Void(); } @@ -1867,7 +1883,7 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase { state Reference onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]); if (task->params[BackupAgentBase::destUid].size() == 0) { - TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade"); + TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade").log(); wait(success(AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone)))); } else { Version beginVersion = @@ -2362,11 +2378,11 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade try { // Check if src is unlocked and dest is locked if (getLockedStatus(srcStatus) != false) { - TraceEvent(SevWarn, "DBA_AtomicSwitchOverSrcLocked"); + TraceEvent(SevWarn, "DBA_AtomicSwitchOverSrcLocked").log(); throw backup_error(); } if (getLockedStatus(destStatus) != true) { - TraceEvent(SevWarn, "DBA_AtomicSwitchOverDestUnlocked"); + TraceEvent(SevWarn, "DBA_AtomicSwitchOverDestUnlocked").log(); throw backup_error(); } // Check if mutation-stream-id matches @@ -2387,7 +2403,7 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade destDRAgents.end(), std::inserter(intersectingAgents, intersectingAgents.begin())); if (intersectingAgents.empty()) { - TraceEvent(SevWarn, "DBA_SwitchOverPossibleDRAgentsIncorrectSetup"); + TraceEvent(SevWarn, "DBA_SwitchOverPossibleDRAgentsIncorrectSetup").log(); throw backup_error(); } } catch (std::runtime_error& e) { @@ -2445,7 +2461,7 @@ public: ACTOR static Future waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, - bool stopWhenDone) { + StopWhenDone stopWhenDone) { state std::string backTrace; state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -2510,10 +2526,10 @@ public: Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone, + StopWhenDone stopWhenDone, Key addPrefix, Key removePrefix, - bool lockDB, + LockDB lockDB, DatabaseBackupAgent::PreBackupAction backupAction) { state UID logUid = deterministicRandom()->randomUniqueID(); state Key logUidValue = BinaryWriter::toValue(logUid, Unversioned()); @@ -2667,7 +2683,7 @@ public: Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction) { + ForceAction forceAction) { state DatabaseBackupAgent drAgent(dest); state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName)); state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid)); @@ -2742,7 +2758,7 @@ public: } } - TraceEvent("DBA_SwitchoverReady"); + TraceEvent("DBA_SwitchoverReady").log(); try { wait(backupAgent->discontinueBackup(dest, tagName)); @@ -2751,9 +2767,9 @@ public: throw; } - wait(success(backupAgent->waitBackup(dest, tagName, true))); + wait(success(backupAgent->waitBackup(dest, tagName, StopWhenDone::True))); - TraceEvent("DBA_SwitchoverStopped"); + TraceEvent("DBA_SwitchoverStopped").log(); state ReadYourWritesTransaction tr3(dest); loop { @@ -2774,31 +2790,31 @@ public: } } - TraceEvent("DBA_SwitchoverVersionUpgraded"); + TraceEvent("DBA_SwitchoverVersionUpgraded").log(); try { wait(drAgent.submitBackup(backupAgent->taskBucket->src, tagName, backupRanges, - false, + StopWhenDone::False, addPrefix, removePrefix, - true, + LockDB::True, DatabaseBackupAgent::PreBackupAction::NONE)); } catch (Error& e) { if (e.code() != error_code_backup_duplicate) throw; } - TraceEvent("DBA_SwitchoverSubmitted"); + TraceEvent("DBA_SwitchoverSubmitted").log(); wait(success(drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName))); - TraceEvent("DBA_SwitchoverStarted"); + TraceEvent("DBA_SwitchoverStarted").log(); wait(backupAgent->unlockBackup(dest, tagName)); - TraceEvent("DBA_SwitchoverUnlocked"); + TraceEvent("DBA_SwitchoverUnlocked").log(); return Void(); } @@ -2835,10 +2851,10 @@ public: ACTOR static Future abortBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, - bool partial, - bool abortOldBackup, - bool dstOnly, - bool waitForDestUID) { + PartialBackup partial, + AbortOldBackup abortOldBackup, + DstOnly dstOnly, + WaitForDestUID waitForDestUID) { state Reference tr(new ReadYourWritesTransaction(cx)); state Key logUidValue, destUidValue; state UID logUid, destUid; @@ -3063,8 +3079,8 @@ public: errorLimit > 0 ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, - false, - true) + Snapshot::False, + Reverse::True) : Future(); state Future> fBackupUid = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -3080,6 +3096,9 @@ public: state Future> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())) .pack(BackupAgentBase::keyConfigBackupRanges)); + state Future> flogVersionKey = + tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) + .pack(BackupAgentBase::keyStateLogBeginVersion)); state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid)); @@ -3095,7 +3114,14 @@ public: } state Optional stopVersionKey = wait(fStopVersionKey); - + Optional logVersionKey = wait(flogVersionKey); + state std::string logVersionText + = ". Last log version is " + + ( + logVersionKey.present() + ? format("%lld", BinaryReader::fromStringRef(logVersionKey.get(), Unversioned())) + : "unset" + ); Optional backupKeysPacked = wait(fBackupKeysPacked); state Standalone> backupRanges; @@ -3115,7 +3141,7 @@ public: break; case EBackupState::STATE_RUNNING_DIFFERENTIAL: statusText += - "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database.\n"; + "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database" + logVersionText + ".\n"; break; case EBackupState::STATE_COMPLETED: { Version stopVersion = @@ -3127,13 +3153,13 @@ public: } break; case EBackupState::STATE_PARTIALLY_ABORTED: { statusText += "The previous DR on tag `" + tagNameDisplay + "' " + - BackupAgentBase::getStateText(backupState) + ".\n"; + BackupAgentBase::getStateText(backupState) + logVersionText + ".\n"; statusText += "Abort the DR with --cleanup before starting a new DR.\n"; break; } default: statusText += "The previous DR on tag `" + tagNameDisplay + "' " + - BackupAgentBase::getStateText(backupState) + ".\n"; + BackupAgentBase::getStateText(backupState) + logVersionText + ".\n"; break; } } @@ -3191,7 +3217,7 @@ public: ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -3204,7 +3230,7 @@ public: ACTOR static Future getDestUid(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key destUidKey = @@ -3217,7 +3243,7 @@ public: ACTOR static Future getLogUid(DatabaseBackupAgent* backupAgent, Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Optional logUid = wait(tr->get(backupAgent->tagNames.pack(tagName), snapshot)); @@ -3235,7 +3261,7 @@ Future DatabaseBackupAgent::atomicSwitchover(Database dest, Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction) { + ForceAction forceAction) { return DatabaseBackupAgentImpl::atomicSwitchover( this, dest, tagName, backupRanges, addPrefix, removePrefix, forceAction); } @@ -3243,10 +3269,10 @@ Future DatabaseBackupAgent::atomicSwitchover(Database dest, Future DatabaseBackupAgent::submitBackup(Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone, + StopWhenDone stopWhenDone, Key addPrefix, Key removePrefix, - bool lockDatabase, + LockDB lockDatabase, PreBackupAction backupAction) { return DatabaseBackupAgentImpl::submitBackup( this, tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, backupAction); @@ -3258,10 +3284,10 @@ Future DatabaseBackupAgent::discontinueBackup(Reference DatabaseBackupAgent::abortBackup(Database cx, Key tagName, - bool partial, - bool abortOldBackup, - bool dstOnly, - bool waitForDestUID) { + PartialBackup partial, + AbortOldBackup abortOldBackup, + DstOnly dstOnly, + WaitForDestUID waitForDestUID) { return DatabaseBackupAgentImpl::abortBackup(this, cx, tagName, partial, abortOldBackup, dstOnly, waitForDestUID); } @@ -3271,15 +3297,15 @@ Future DatabaseBackupAgent::getStatus(Database cx, int errorLimit, Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid, bool snapshot) { +Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid, Snapshot snapshot) { return DatabaseBackupAgentImpl::getDestUid(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName, bool snapshot) { +Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName, Snapshot snapshot) { return DatabaseBackupAgentImpl::getLogUid(this, tr, tagName, snapshot); } @@ -3287,7 +3313,7 @@ Future DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key return DatabaseBackupAgentImpl::waitUpgradeToLatestDrVersion(this, cx, tagName); } -Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) { +Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, StopWhenDone stopWhenDone) { return DatabaseBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone); } @@ -3297,12 +3323,12 @@ Future DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName Future DatabaseBackupAgent::getRangeBytesWritten(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DRConfig(logUid).rangeBytesWritten().getD(tr, snapshot); } Future DatabaseBackupAgent::getLogBytesWritten(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DRConfig(logUid).logBytesWritten().getD(tr, snapshot); } diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 35b0ee25ce..a693b65291 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -52,7 +52,7 @@ public: private: DatabaseContext* cx; StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality) - : cx(cx), ReferencedInterface(interf, locality) {} + : ReferencedInterface(interf, locality), cx(cx) {} }; struct LocationInfo : MultiInterface>, FastAllocated { @@ -158,11 +158,11 @@ public: static Database create(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, - bool enableLocalityLoadBalance, + EnableLocalityLoadBalance, TaskPriority taskID = TaskPriority::DefaultEndpoint, - bool lockAware = false, + LockAware = LockAware::False, int apiVersion = Database::API_VERSION_LATEST, - bool switchable = false); + IsSwitchable = IsSwitchable::False); ~DatabaseContext(); @@ -181,13 +181,13 @@ public: switchable)); } - std::pair> getCachedLocation(const KeyRef&, bool isBackward = false); + std::pair> getCachedLocation(const KeyRef&, Reverse isBackward = Reverse::False); bool getCachedLocations(const KeyRangeRef&, vector>>&, int limit, - bool reverse); + Reverse reverse); Reference setCachedLocation(const KeyRangeRef&, const vector&); - void invalidateCache(const KeyRef&, bool isBackward = false); + void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::False); void invalidateCache(const KeyRangeRef&); bool sampleReadTags() const; @@ -197,7 +197,7 @@ public: Reference getCommitProxies(bool useProvisionalProxies); Future> getCommitProxiesFuture(bool useProvisionalProxies); Reference getGrvProxies(bool useProvisionalProxies); - Future onProxiesChanged(); + Future onProxiesChanged() const; Future getHealthMetrics(bool detailed); // Returns the protocol version reported by the coordinator this client is connected to @@ -218,7 +218,7 @@ public: void setOption(FDBDatabaseOptions::Option option, Optional value); Error deferredError; - bool lockAware; + LockAware lockAware{ LockAware::False }; bool isError() const { return deferredError.code() != invalid_error_code; } @@ -243,7 +243,7 @@ public: // new cluster. Future switchConnectionFile(Reference standby); Future connectionFileChanged(); - bool switchable = false; + IsSwitchable switchable{ false }; // Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure Future rebootWorker(StringRef address, bool check = false, int duration = 0); @@ -256,15 +256,15 @@ public: // private: explicit DatabaseContext(Reference>> connectionFile, Reference> clientDBInfo, - Reference>> coordinator, + Reference> const> coordinator, Future clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, - bool enableLocalityLoadBalance, - bool lockAware, - bool internal = true, + EnableLocalityLoadBalance, + LockAware, + IsInternal = IsInternal::True, int apiVersion = Database::API_VERSION_LATEST, - bool switchable = false); + IsSwitchable = IsSwitchable::False); explicit DatabaseContext(const Error& err); @@ -276,14 +276,14 @@ public: Future monitorProxiesInfoChange; Future monitorTssInfoChange; Future tssMismatchHandler; - PromiseStream tssMismatchStream; + PromiseStream>> tssMismatchStream; Reference commitProxies; Reference grvProxies; bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time. UID proxiesLastChange; LocalityData clientLocality; QueueModel queueModel; - bool enableLocalityLoadBalance; + EnableLocalityLoadBalance enableLocalityLoadBalance{ EnableLocalityLoadBalance::False }; struct VersionRequest { SpanID spanContext; @@ -308,7 +308,7 @@ public: // trust that the read version (possibly set manually by the application) is actually from the correct cluster. // Updated everytime we get a GRV response Version minAcceptableReadVersion = std::numeric_limits::max(); - void validateVersion(Version); + void validateVersion(Version) const; // Client status updater struct ClientStatusUpdater { @@ -336,7 +336,7 @@ public: std::unordered_map ssidTagMapping; UID dbId; - bool internal; // Only contexts created through the C client and fdbcli are non-internal + IsInternal internal; // Only contexts created through the C client and fdbcli are non-internal PrioritizedTransactionTagMap throttledTags; @@ -406,7 +406,7 @@ public: Future connected; // An AsyncVar that reports the coordinator this DatabaseContext is interacting with - Reference>> coordinator; + Reference> const> coordinator; Reference>> statusClusterInterface; Future statusLeaderMon; @@ -435,7 +435,6 @@ public: static bool debugUseTags; static const std::vector debugTransactionTagChoices; - std::unordered_map> watchMap; // Cache of the latest commit versions of storage servers. VersionVector ssVersionVectorCache; @@ -457,6 +456,8 @@ public: void getLatestCommitVersions(const Reference& locationInfo, Version readVersion, VersionVector& latestCommitVersions); +private: + std::unordered_map> watchMap; }; #endif diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index a60b8ca743..a4d3b35276 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -670,9 +670,9 @@ struct RangeResultRef : VectorRef { RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} RangeResultRef(Arena& p, const RangeResultRef& toCopy) - : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd), + : VectorRef(p, toCopy), more(toCopy.more), readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional()), - VectorRef(p, toCopy) {} + readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd) {} RangeResultRef(const VectorRef& value, bool more, Optional readThrough = Optional()) : VectorRef(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) { } diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 35d6743821..a487c527ac 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -42,6 +42,9 @@ #include "flow/actorcompiler.h" // This must be the last #include. +FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly); +FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs); + #define SevFRTestInfo SevVerbose //#define SevFRTestInfo SevInfo @@ -117,7 +120,7 @@ Key FileBackupAgent::getPauseKey() { ACTOR Future> TagUidMap::getAll_impl(TagUidMap* tagsMap, Reference tr, - bool snapshot) { + Snapshot snapshot) { state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot)); std::vector results; @@ -142,7 +145,7 @@ public: } KeyBackedProperty addPrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } KeyBackedProperty removePrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } - KeyBackedProperty onlyAppyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } + KeyBackedProperty onlyApplyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } KeyBackedProperty inconsistentSnapshotOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } // XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges KeyBackedProperty restoreRange() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } @@ -248,9 +251,9 @@ public: Key applyMutationsMapPrefix() { return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid); } ACTOR static Future getApplyVersionLag_impl(Reference tr, UID uid) { - // Both of these are snapshot reads - state Future> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), true); - state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true); + state Future> beginVal = + tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True); + state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True); wait(success(beginVal) && success(endVal)); if (!beginVal.get().present() || !endVal.get().present()) @@ -440,8 +443,12 @@ FileBackupAgent::FileBackupAgent() // The other subspaces have logUID -> value , config(subspace.get(BackupAgentBase::keyConfig)), lastRestorable(subspace.get(FileBackupAgent::keyLastRestorable)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)) {} + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::True, + PriorityBatch::False, + LockAware::True)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) { +} namespace fileBackup { @@ -684,9 +691,9 @@ private: int64_t blockEnd; }; -ACTOR Future>> decodeLogFileBlock(Reference file, - int64_t offset, - int len) { +ACTOR Future>> decodeMutationLogFileBlock(Reference file, + int64_t offset, + int len) { state Standalone buf = makeString(len); int rLen = wait(file->read(mutateString(buf), len, offset)); if (rLen != len) @@ -863,10 +870,10 @@ ACTOR static Future abortFiveOneBackup(FileBackupAgent* backupAgent, tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded())); state BackupConfig config(current.first); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); @@ -952,7 +959,7 @@ ACTOR static Future addBackupTask(StringRef name, Reference waitFor = Reference(), std::function)> setupTaskFn = NOP_SETUP_TASK_FN, int priority = 0, - bool setValidation = true) { + SetValidation setValidation = SetValidation::True) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1107,7 +1114,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase { Params.beginKey().set(task, range.end); // Save and extend the task with the new begin parameter - state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, true)); + state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::True)); // Update the range bytes written in the backup config backup.rangeBytesWritten().atomicOp(tr, file->size(), MutationRef::AddValue); @@ -1201,7 +1208,13 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase { // retrieve kvData state PromiseStream results; - state Future rc = readCommitted(cx, results, lock, KeyRangeRef(beginKey, endKey), true, true, true); + state Future rc = readCommitted(cx, + results, + lock, + KeyRangeRef(beginKey, endKey), + Terminator::True, + AccessSystemKeys::True, + LockAware::True); state RangeFileWriter rangeFile; state BackupConfig backup(task); @@ -2044,7 +2057,8 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase { state std::vector> rc; for (auto& range : ranges) { - rc.push_back(readCommitted(cx, results, lock, range, false, true, true)); + rc.push_back( + readCommitted(cx, results, lock, range, Terminator::False, AccessSystemKeys::True, LockAware::True)); } state Future sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr const& result) { @@ -2222,7 +2236,7 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase { Params.destUidValue().set(task, destUidValue); }, 0, - false)); + SetValidation::False)); return key; } @@ -3230,7 +3244,7 @@ REGISTER_TASKFUNC(RestoreRangeTaskFunc); // Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and // returns (commitVersion, chunkNumber) -std::pair decodeLogKey(const StringRef& key) { +std::pair decodeMutationLogKey(const StringRef& key) { ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t)); uint8_t hash; @@ -3251,7 +3265,7 @@ std::pair decodeLogKey(const StringRef& key) { // [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k], // where a mutation is encoded as: // [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][param1][param2] -std::vector decodeLogValue(const StringRef& value) { +std::vector decodeMutationLogValue(const StringRef& value) { StringRefReader reader(value, restore_corrupted_data()); Version protocolVersion = reader.consume(); @@ -3286,72 +3300,54 @@ std::vector decodeLogValue(const StringRef& value) { return mutations; } -// Accumulates mutation log value chunks, as both a vector of chunks and as a combined chunk, -// in chunk order, and can check the chunk set for completion or intersection with a set -// of ranges. -struct AccumulatedMutations { - AccumulatedMutations() : lastChunkNumber(-1) {} - - // Add a KV pair for this mutation chunk set - // It will be accumulated onto serializedMutations if the chunk number is - // the next expected value. - void addChunk(int chunkNumber, const KeyValueRef& kv) { - if (chunkNumber == lastChunkNumber + 1) { - lastChunkNumber = chunkNumber; - serializedMutations += kv.value.toString(); - } else { - lastChunkNumber = -2; - serializedMutations.clear(); - } - kvs.push_back(kv); +void AccumulatedMutations::addChunk(int chunkNumber, const KeyValueRef& kv) { + if (chunkNumber == lastChunkNumber + 1) { + lastChunkNumber = chunkNumber; + serializedMutations += kv.value.toString(); + } else { + lastChunkNumber = -2; + serializedMutations.clear(); } + kvs.push_back(kv); +} - // Returns true if both - // - 1 or more chunks were added to this set - // - The header of the first chunk contains a valid protocol version and a length - // that matches the bytes after the header in the combined value in serializedMutations - bool isComplete() const { - if (lastChunkNumber >= 0) { - StringRefReader reader(serializedMutations, restore_corrupted_data()); +bool AccumulatedMutations::isComplete() const { + if (lastChunkNumber >= 0) { + StringRefReader reader(serializedMutations, restore_corrupted_data()); - Version protocolVersion = reader.consume(); - if (protocolVersion <= 0x0FDB00A200090001) { - throw incompatible_protocol_version(); - } - - uint32_t vLen = reader.consume(); - return vLen == reader.remainder().size(); + Version protocolVersion = reader.consume(); + if (protocolVersion <= 0x0FDB00A200090001) { + throw incompatible_protocol_version(); } - return false; + uint32_t vLen = reader.consume(); + return vLen == reader.remainder().size(); } - // Returns true if a complete chunk contains any MutationRefs which intersect with any - // range in ranges. - // It is undefined behavior to run this if isComplete() does not return true. - bool matchesAnyRange(const std::vector& ranges) const { - std::vector mutations = decodeLogValue(serializedMutations); - for (auto& m : mutations) { - for (auto& r : ranges) { - if (m.type == MutationRef::ClearRange) { - if (r.intersects(KeyRangeRef(m.param1, m.param2))) { - return true; - } - } else { - if (r.contains(m.param1)) { - return true; - } + return false; +} + +// Returns true if a complete chunk contains any MutationRefs which intersect with any +// range in ranges. +// It is undefined behavior to run this if isComplete() does not return true. +bool AccumulatedMutations::matchesAnyRange(const std::vector& ranges) const { + std::vector mutations = decodeMutationLogValue(serializedMutations); + for (auto& m : mutations) { + for (auto& r : ranges) { + if (m.type == MutationRef::ClearRange) { + if (r.intersects(KeyRangeRef(m.param1, m.param2))) { + return true; + } + } else { + if (r.contains(m.param1)) { + return true; } } } - - return false; } - std::vector kvs; - std::string serializedMutations; - int lastChunkNumber; -}; + return false; +} // Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete // and have data relevant to one of the KV ranges in ranges @@ -3359,7 +3355,7 @@ std::vector filterLogMutationKVPairs(VectorRef data, c std::unordered_map mutationBlocksByVersion; for (auto& kv : data) { - auto versionAndChunkNumber = decodeLogKey(kv.key); + auto versionAndChunkNumber = decodeMutationLogKey(kv.key); mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv); } @@ -3430,7 +3426,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase { state Key mutationLogPrefix = restore.mutationLogPrefix(); state Reference inFile = wait(bc->readFile(logFile.fileName)); - state Standalone> dataOriginal = wait(decodeLogFileBlock(inFile, readOffset, readLen)); + state Standalone> dataOriginal = wait(decodeMutationLogFileBlock(inFile, readOffset, readLen)); // Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this // restore based on the restore range set. @@ -3580,9 +3576,9 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase { state int64_t remainingInBatch = Params.remainingInBatch().get(task); state bool addingToExistingBatch = remainingInBatch > 0; state Version restoreVersion; - state Future> onlyAppyMutationLogs = restore.onlyAppyMutationLogs().get(tr); + state Future> onlyApplyMutationLogs = restore.onlyApplyMutationLogs().get(tr); - wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyAppyMutationLogs) && + wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyApplyMutationLogs) && checkTaskVersion(tr->getDatabase(), task, name, version)); // If not adding to an existing batch then update the apply mutations end version so the mutations from the @@ -4058,12 +4054,13 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase { tr->setOption(FDBTransactionOptions::LOCK_AWARE); wait(checkTaskVersion(tr->getDatabase(), task, name, version)); - wait(store(beginVersion, restore.beginVersion().getD(tr, false, invalidVersion))); + wait(store(beginVersion, restore.beginVersion().getD(tr, Snapshot::False, ::invalidVersion))); wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr))); wait(store(ranges, restore.getRestoreRangesOrDefault(tr))); - wait(store(logsOnly, restore.onlyAppyMutationLogs().getD(tr, false, false))); - wait(store(inconsistentSnapshotOnly, restore.inconsistentSnapshotOnly().getD(tr, false, false))); + wait(store(logsOnly, restore.onlyApplyMutationLogs().getD(tr, Snapshot::False, false))); + wait(store(inconsistentSnapshotOnly, + restore.inconsistentSnapshotOnly().getD(tr, Snapshot::False, false))); wait(taskBucket->keepRunning(tr, task)); @@ -4245,7 +4242,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase { tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE))); wait(taskBucket->finish(tr, task)); - state Future> logsOnly = restore.onlyAppyMutationLogs().get(tr); + state Future> logsOnly = restore.onlyApplyMutationLogs().get(tr); wait(success(logsOnly)); if (logsOnly.get().present() && logsOnly.get().get()) { // If this is an incremental restore, we need to set the applyMutationsMapPrefix @@ -4314,7 +4311,7 @@ public: static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; // Parallel restore - ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true) { + ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB = UnlockDB::True) { state ReadYourWritesTransaction tr(cx); state Optional restoreRequestDoneKeyValue; TraceEvent("FastRestoreToolWaitForRestoreToFinish").detail("DBLock", randomUID); @@ -4365,7 +4362,7 @@ public: Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { @@ -4458,7 +4455,7 @@ public: ACTOR static Future waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, - bool stopWhenDone, + StopWhenDone stopWhenDone, Reference* pContainer = nullptr, UID* pUID = nullptr) { state std::string backTrace; @@ -4476,7 +4473,8 @@ public: } state BackupConfig config(oldUidAndAborted.get().first); - state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + state EBackupState status = + wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); // Break, if one of the following is true // - no longer runnable @@ -4486,7 +4484,7 @@ public: if (pContainer != nullptr) { Reference c = - wait(config.backupContainer().getOrThrow(tr, false, backup_invalid_info())); + wait(config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info())); *pContainer = c; } @@ -4506,6 +4504,7 @@ public: } } + // TODO: Get rid of all of these confusing boolean flags ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, Key outContainer, @@ -4513,9 +4512,10 @@ public: int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, - bool stopWhenDone, - bool partitionedLog, - bool incrementalBackupOnly) { + StopWhenDone stopWhenDone, + UsePartitionedLog partitionedLog, + IncrementalBackupOnly incrementalBackupOnly, + Optional encryptionKeyFileName) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); @@ -4531,7 +4531,7 @@ public: if (uidAndAbortedFlag.present()) { state BackupConfig prevConfig(uidAndAbortedFlag.get().first); state EBackupState prevBackupStatus = - wait(prevConfig.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + wait(prevConfig.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); if (FileBackupAgent::isRunnable(prevBackupStatus)) { throw backup_duplicate(); } @@ -4553,7 +4553,7 @@ public: backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString()); } - state Reference bc = IBackupContainer::openContainer(backupContainer); + state Reference bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName); try { wait(timeoutError(bc->create(), 30)); } catch (Error& e) { @@ -4644,9 +4644,9 @@ public: Version restoreVersion, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, Version beginVersion, UID uid) { KeyRangeMap restoreRangeSet; @@ -4698,7 +4698,7 @@ public: .removePrefix(removePrefix) .withPrefix(addPrefix); RangeResult existingRows = wait(tr->getRange(restoreIntoRange, 1)); - if (existingRows.size() > 0 && !onlyAppyMutationLogs) { + if (existingRows.size() > 0 && !onlyApplyMutationLogs) { throw restore_destination_not_empty(); } } @@ -4715,7 +4715,7 @@ public: restore.sourceContainer().set(tr, bc); restore.stateEnum().set(tr, ERestoreState::QUEUED); restore.restoreVersion().set(tr, restoreVersion); - restore.onlyAppyMutationLogs().set(tr, onlyAppyMutationLogs); + restore.onlyApplyMutationLogs().set(tr, onlyApplyMutationLogs); restore.inconsistentSnapshotOnly().set(tr, inconsistentSnapshotOnly); restore.beginVersion().set(tr, beginVersion); if (BUGGIFY && restoreRanges.size() == 1) { @@ -4738,7 +4738,7 @@ public: } // This method will return the final status of the backup - ACTOR static Future waitRestore(Database cx, Key tagName, bool verbose) { + ACTOR static Future waitRestore(Database cx, Key tagName, Verbose verbose) { state ERestoreState status; loop { state Reference tr(new ReadYourWritesTransaction(cx)); @@ -4794,9 +4794,9 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName.toString()); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded())); state BackupConfig config(current.first); - state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + state EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); if (!FileBackupAgent::isRunnable(status)) { throw backup_unneeded(); @@ -4845,11 +4845,11 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded())); state BackupConfig config(current.first); state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); @@ -4951,7 +4951,7 @@ public: state BackupConfig config(uidAndAbortedFlag.get().first); state EBackupState backupState = - wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); JsonBuilderObject statusDoc; statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState)); statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState)); @@ -5075,7 +5075,7 @@ public: ACTOR static Future getStatus(FileBackupAgent* backupAgent, Database cx, - bool showErrors, + ShowErrors showErrors, std::string tagName) { state Reference tr(new ReadYourWritesTransaction(cx)); state std::string statusText; @@ -5095,7 +5095,8 @@ public: state Future> fPaused = tr->get(backupAgent->taskBucket->getPauseKey()); if (uidAndAbortedFlag.present()) { config = BackupConfig(uidAndAbortedFlag.get().first); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = + wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)); backupState = status; } @@ -5257,7 +5258,7 @@ public: ACTOR static Future> getLastRestorable(FileBackupAgent* backupAgent, Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Optional version = wait(tr->get(backupAgent->lastRestorable.pack(tagName), snapshot)); @@ -5290,7 +5291,7 @@ public: // removePrefix: for each key to be restored, remove this prefix first. // lockDB: if set lock the database with randomUid before performing restore; // otherwise, check database is locked with the randomUid - // onlyAppyMutationLogs: only perform incremental restore, by only applying mutation logs + // onlyApplyMutationLogs: only perform incremental restore, by only applying mutation logs // inconsistentSnapshotOnly: Ignore mutation log files during the restore to speedup the process. // When set to true, gives an inconsistent snapshot, thus not recommended // beginVersion: restore's begin version @@ -5301,15 +5302,16 @@ public: Key tagName, Key url, Standalone> ranges, - bool waitForComplete, + WaitForComplete waitForComplete, Version targetVersion, - bool verbose, + Verbose verbose, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, Version beginVersion, + Optional encryptionKeyFileName, UID randomUid) { // The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might. if (ranges.empty()) { @@ -5327,12 +5329,12 @@ public: if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) targetVersion = desc.maxRestorableVersion.get(); - if (targetVersion == invalidVersion && onlyAppyMutationLogs && desc.contiguousLogEnd.present()) { + if (targetVersion == invalidVersion && onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { targetVersion = desc.contiguousLogEnd.get() - 1; } Optional restoreSet = - wait(bc->getRestoreSet(targetVersion, ranges, onlyAppyMutationLogs, beginVersion)); + wait(bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion)); if (!restoreSet.present()) { TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible") @@ -5364,7 +5366,7 @@ public: addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, beginVersion, randomUid)); @@ -5395,7 +5397,7 @@ public: Standalone> ranges, Key addPrefix, Key removePrefix, - bool fastRestore) { + UsePartitionedLog fastRestore) { state Reference ryw_tr = Reference(new ReadYourWritesTransaction(cx)); state BackupConfig backupConfig; @@ -5458,7 +5460,7 @@ public: try { wait(discontinueBackup(backupAgent, ryw_tr, tagName)); wait(ryw_tr->commit()); - TraceEvent("AS_DiscontinuedBackup"); + TraceEvent("AS_DiscontinuedBackup").log(); break; } catch (Error& e) { if (e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate) { @@ -5468,8 +5470,8 @@ public: } } - wait(success(waitBackup(backupAgent, cx, tagName.toString(), true))); - TraceEvent("AS_BackupStopped"); + wait(success(waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::True))); + TraceEvent("AS_BackupStopped").log(); ryw_tr->reset(); loop { @@ -5482,7 +5484,7 @@ public: ryw_tr->clear(range); } wait(ryw_tr->commit()); - TraceEvent("AS_ClearedRange"); + TraceEvent("AS_ClearedRange").log(); break; } catch (Error& e) { wait(ryw_tr->onError(e)); @@ -5492,14 +5494,20 @@ public: Reference bc = wait(backupConfig.backupContainer().getOrThrow(cx)); if (fastRestore) { - TraceEvent("AtomicParallelRestoreStartRestore"); - Version targetVersion = -1; - bool lockDB = true; - wait(submitParallelRestore( - cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid, addPrefix, removePrefix)); + TraceEvent("AtomicParallelRestoreStartRestore").log(); + Version targetVersion = ::invalidVersion; + wait(submitParallelRestore(cx, + tagName, + ranges, + KeyRef(bc->getURL()), + targetVersion, + LockDB::True, + randomUid, + addPrefix, + removePrefix)); state bool hasPrefix = (addPrefix.size() > 0 || removePrefix.size() > 0); TraceEvent("AtomicParallelRestoreWaitForRestoreFinish").detail("HasPrefix", hasPrefix); - wait(parallelRestoreFinish(cx, randomUid, !hasPrefix)); + wait(parallelRestoreFinish(cx, randomUid, UnlockDB{ !hasPrefix })); // If addPrefix or removePrefix set, we want to transform the effect by copying data if (hasPrefix) { wait(transformRestoredDatabase(cx, ranges, addPrefix, removePrefix)); @@ -5507,22 +5515,23 @@ public: } return -1; } else { - TraceEvent("AS_StartRestore"); + TraceEvent("AS_StartRestore").log(); Version ver = wait(restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, - true, - -1, - true, + WaitForComplete::True, + ::invalidVersion, + Verbose::True, addPrefix, removePrefix, - true, - false, - false, - invalidVersion, + LockDB::True, + OnlyApplyMutationLogs::False, + InconsistentSnapshotOnly::False, + ::invalidVersion, + {}, randomUid)); return ver; } @@ -5537,16 +5546,15 @@ public: Standalone> ranges, Key addPrefix, Key removePrefix) { - return success(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true)); + return success( + atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::True)); } }; -const std::string BackupAgentBase::defaultTagName = "default"; -const int BackupAgentBase::logHeaderSize = 12; const int FileBackupAgent::dataFooterSize = 20; // Return if parallel restore has finished -Future FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB) { +Future FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB) { return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID, unlockDB); } @@ -5555,7 +5563,7 @@ Future FileBackupAgent::submitParallelRestore(Database cx, Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { @@ -5576,15 +5584,16 @@ Future FileBackupAgent::restore(Database cx, Key tagName, Key url, Standalone> ranges, - bool waitForComplete, + WaitForComplete waitForComplete, Version targetVersion, - bool verbose, + Verbose verbose, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, - Version beginVersion) { + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, + Version beginVersion, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::restore(this, cx, cxOrig, @@ -5597,9 +5606,10 @@ Future FileBackupAgent::restore(Database cx, addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, beginVersion, + encryptionKeyFileName, deterministicRandom()->randomUniqueID()); } @@ -5608,7 +5618,8 @@ Future FileBackupAgent::atomicRestore(Database cx, Standalone> ranges, Key addPrefix, Key removePrefix) { - return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix, false); + return FileBackupAgentImpl::atomicRestore( + this, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::False); } Future FileBackupAgent::abortRestore(Reference tr, Key tagName) { @@ -5623,7 +5634,7 @@ Future FileBackupAgent::restoreStatus(Reference FileBackupAgent::waitRestore(Database cx, Key tagName, bool verbose) { +Future FileBackupAgent::waitRestore(Database cx, Key tagName, Verbose verbose) { return FileBackupAgentImpl::waitRestore(cx, tagName, verbose); }; @@ -5631,11 +5642,12 @@ Future FileBackupAgent::submitBackup(Reference Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone, - bool partitionedLog, - bool incrementalBackupOnly) { + StopWhenDone stopWhenDone, + UsePartitionedLog partitionedLog, + IncrementalBackupOnly incrementalBackupOnly, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::submitBackup(this, tr, outContainer, @@ -5645,7 +5657,8 @@ Future FileBackupAgent::submitBackup(Reference backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); } Future FileBackupAgent::discontinueBackup(Reference tr, Key tagName) { @@ -5656,7 +5669,7 @@ Future FileBackupAgent::abortBackup(Reference t return FileBackupAgentImpl::abortBackup(this, tr, tagName); } -Future FileBackupAgent::getStatus(Database cx, bool showErrors, std::string tagName) { +Future FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) { return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName); } @@ -5666,7 +5679,7 @@ Future FileBackupAgent::getStatusJSON(Database cx, std::string tagN Future> FileBackupAgent::getLastRestorable(Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot); } @@ -5678,7 +5691,7 @@ void FileBackupAgent::setLastRestorable(Reference tr, Future FileBackupAgent::waitBackup(Database cx, std::string tagName, - bool stopWhenDone, + StopWhenDone stopWhenDone, Reference* pContainer, UID* pUID) { return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID); @@ -5739,8 +5752,8 @@ ACTOR static Future writeKVs(Database cx, Standalonefirst)) { @@ -165,7 +167,7 @@ ACTOR Future GlobalConfig::migrate(GlobalConfig* self) { // attempt this migration at the same time, sometimes resulting in // aborts due to conflicts. Purposefully avoid retrying, making this // migration best-effort. - TraceEvent(SevInfo, "GlobalConfigMigrationError").detail("What", e.what()); + TraceEvent(SevInfo, "GlobalConfig_MigrationError").detail("What", e.what()); } return Void(); @@ -174,6 +176,7 @@ ACTOR Future GlobalConfig::migrate(GlobalConfig* self) { // Updates local copy of global configuration by reading the entire key-range // from storage. ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { + // TraceEvent trace(SevInfo, "GlobalConfig_Refresh"); self->erase(KeyRangeRef(""_sr, "\xff"_sr)); Transaction tr(self->cx); diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 2d63d8de60..444f1ab697 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -72,7 +72,7 @@ public: // to allow global configuration to run transactions on the latest // database. template - static void create(Database& cx, Reference> db, const ClientDBInfo* dbInfo) { + static void create(Database& cx, Reference const> db, const ClientDBInfo* dbInfo) { if (g_network->global(INetwork::enGlobalConfig) == nullptr) { auto config = new GlobalConfig{ cx }; g_network->setGlobal(INetwork::enGlobalConfig, config); diff --git a/fdbclient/GrvProxyInterface.h b/fdbclient/GrvProxyInterface.h index 21c8d06589..85ad4d16bc 100644 --- a/fdbclient/GrvProxyInterface.h +++ b/fdbclient/GrvProxyInterface.h @@ -46,6 +46,7 @@ struct GrvProxyInterface { bool operator==(GrvProxyInterface const& r) const { return id() == r.id(); } bool operator!=(GrvProxyInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return getConsistentReadVersion.getEndpoint().getPrimaryAddress(); } + NetworkAddressList addresses() const { return getConsistentReadVersion.getEndpoint().addresses; } template void serialize(Archive& ar) { diff --git a/fdbclient/IConfigTransaction.cpp b/fdbclient/IConfigTransaction.cpp index d37276fb04..f91483eb76 100644 --- a/fdbclient/IConfigTransaction.cpp +++ b/fdbclient/IConfigTransaction.cpp @@ -18,6 +18,8 @@ * limitations under the License. */ +#include + #include "fdbclient/IConfigTransaction.h" #include "fdbclient/SimpleConfigTransaction.h" #include "fdbclient/PaxosConfigTransaction.h" @@ -26,10 +28,6 @@ Reference IConfigTransaction::createTestSimple(ConfigTransac return makeReference(cti); } -Reference IConfigTransaction::createSimple(Database const& cx) { - return makeReference(cx); -} - -Reference IConfigTransaction::createPaxos(Database const& cx) { - return makeReference(cx); +Reference IConfigTransaction::createTestPaxos(std::vector const& ctis) { + return makeReference(ctis); } diff --git a/fdbclient/IConfigTransaction.h b/fdbclient/IConfigTransaction.h index 007dd9e2d3..42d5769c51 100644 --- a/fdbclient/IConfigTransaction.h +++ b/fdbclient/IConfigTransaction.h @@ -40,12 +40,13 @@ public: virtual ~IConfigTransaction() = default; static Reference createTestSimple(ConfigTransactionInterface const&); - static Reference createSimple(Database const&); - static Reference createPaxos(Database const&); + static Reference createTestPaxos(std::vector const&); // Not implemented: void setVersion(Version) override { throw client_invalid_operation(); } - Future getKey(KeySelector const& key, bool snapshot = false) override { throw client_invalid_operation(); } + Future getKey(KeySelector const& key, Snapshot snapshot = Snapshot::False) override { + throw client_invalid_operation(); + } Future>> getAddressesForKey(Key const& key) override { throw client_invalid_operation(); } diff --git a/fdbclient/IKnobCollection.cpp b/fdbclient/IKnobCollection.cpp index 7f3a595763..26a0dcb22d 100644 --- a/fdbclient/IKnobCollection.cpp +++ b/fdbclient/IKnobCollection.cpp @@ -56,17 +56,17 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri static std::unique_ptr clientKnobCollection, serverKnobCollection, testKnobCollection; if (type == Type::CLIENT) { if (!clientKnobCollection) { - clientKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + clientKnobCollection = create(type, Randomize::False, IsSimulated::False); } return clientKnobCollection->parseKnobValue(knobName, knobValue); } else if (type == Type::SERVER) { if (!serverKnobCollection) { - serverKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + serverKnobCollection = create(type, Randomize::False, IsSimulated::False); } return serverKnobCollection->parseKnobValue(knobName, knobValue); } else if (type == Type::TEST) { if (!testKnobCollection) { - testKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + testKnobCollection = create(type, Randomize::False, IsSimulated::False); } return testKnobCollection->parseKnobValue(knobName, knobValue); } @@ -74,7 +74,7 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri } std::unique_ptr IKnobCollection::globalKnobCollection = - IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::False, IsSimulated::False); void IKnobCollection::setGlobalKnobCollection(Type type, Randomize randomize, IsSimulated isSimulated) { globalKnobCollection = create(type, randomize, isSimulated); diff --git a/fdbclient/ISingleThreadTransaction.cpp b/fdbclient/ISingleThreadTransaction.cpp index 0b9a8f5abb..c8fcf828d5 100644 --- a/fdbclient/ISingleThreadTransaction.cpp +++ b/fdbclient/ISingleThreadTransaction.cpp @@ -26,33 +26,15 @@ ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type type) { if (type == Type::RYW) { - auto tr = - (ReadYourWritesTransaction*)(ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction))); - tr->preinitializeOnForeignThread(); + auto tr = new ReadYourWritesTransaction; return tr; } else if (type == Type::SIMPLE_CONFIG) { - auto tr = (SimpleConfigTransaction*)(SimpleConfigTransaction::operator new(sizeof(SimpleConfigTransaction))); + auto tr = new SimpleConfigTransaction; return tr; } else if (type == Type::PAXOS_CONFIG) { - auto tr = (PaxosConfigTransaction*)(PaxosConfigTransaction::operator new(sizeof(PaxosConfigTransaction))); + auto tr = new PaxosConfigTransaction; return tr; } ASSERT(false); return nullptr; } - -void ISingleThreadTransaction::create(ISingleThreadTransaction* tr, Type type, Database db) { - switch (type) { - case Type::RYW: - new (tr) ReadYourWritesTransaction(db); - break; - case Type::SIMPLE_CONFIG: - new (tr) SimpleConfigTransaction(db); - break; - case Type::PAXOS_CONFIG: - new (tr) PaxosConfigTransaction(db); - break; - default: - ASSERT(false); - } -} diff --git a/fdbclient/ISingleThreadTransaction.h b/fdbclient/ISingleThreadTransaction.h index 80dd184e74..e1c9aa9575 100644 --- a/fdbclient/ISingleThreadTransaction.h +++ b/fdbclient/ISingleThreadTransaction.h @@ -23,6 +23,7 @@ #include "fdbclient/FDBOptions.g.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/KeyRangeMap.h" +#include "fdbclient/NativeAPI.actor.h" #include "flow/Error.h" #include "flow/FastRef.h" @@ -44,23 +45,23 @@ public: }; static ISingleThreadTransaction* allocateOnForeignThread(Type type); - static void create(ISingleThreadTransaction* tr, Type type, Database db); + virtual void setDatabase(Database const&) = 0; virtual void setVersion(Version v) = 0; virtual Future getReadVersion() = 0; virtual Optional getCachedReadVersion() const = 0; - virtual Future> get(const Key& key, bool snapshot = false) = 0; - virtual Future getKey(const KeySelector& key, bool snapshot = false) = 0; - virtual Future> getRange(const KeySelector& begin, - const KeySelector& end, - int limit, - bool snapshot = false, - bool reverse = false) = 0; - virtual Future> getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) = 0; + virtual Future> get(const Key& key, Snapshot = Snapshot::False) = 0; + virtual Future getKey(const KeySelector& key, Snapshot = Snapshot::False) = 0; + virtual Future getRange(const KeySelector& begin, + const KeySelector& end, + int limit, + Snapshot = Snapshot::False, + Reverse = Reverse::False) = 0; + virtual Future getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot = Snapshot::False, + Reverse = Reverse::False) = 0; virtual Future>> getAddressesForKey(Key const& key) = 0; virtual Future>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0; virtual Future getEstimatedRangeSizeBytes(KeyRange const& keys) = 0; diff --git a/fdbclient/KeyBackedTypes.h b/fdbclient/KeyBackedTypes.h index f92324e4ab..b82b5c7357 100644 --- a/fdbclient/KeyBackedTypes.h +++ b/fdbclient/KeyBackedTypes.h @@ -150,7 +150,7 @@ template class KeyBackedProperty { public: KeyBackedProperty(KeyRef key) : key(key) {} - Future> get(Reference tr, bool snapshot = false) const { + Future> get(Reference tr, Snapshot snapshot = Snapshot::False) const { return map(tr->get(key, snapshot), [](Optional const& val) -> Optional { if (val.present()) return Codec::unpack(Tuple::unpack(val.get())); @@ -158,12 +158,14 @@ public: }); } // Get property's value or defaultValue if it doesn't exist - Future getD(Reference tr, bool snapshot = false, T defaultValue = T()) const { + Future getD(Reference tr, + Snapshot snapshot = Snapshot::False, + T defaultValue = T()) const { return map(get(tr, snapshot), [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); } // Get property's value or throw error if it doesn't exist Future getOrThrow(Reference tr, - bool snapshot = false, + Snapshot snapshot = Snapshot::False, Error err = key_not_found()) const { auto keyCopy = key; auto backtrace = platform::get_backtrace(); @@ -180,7 +182,7 @@ public: }); } - Future> get(Database cx, bool snapshot = false) const { + Future> get(Database cx, Snapshot snapshot = Snapshot::False) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -190,7 +192,7 @@ public: }); } - Future getD(Database cx, bool snapshot = false, T defaultValue = T()) const { + Future getD(Database cx, Snapshot snapshot = Snapshot::False, T defaultValue = T()) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -200,7 +202,7 @@ public: }); } - Future getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const { + Future getOrThrow(Database cx, Snapshot snapshot = Snapshot::False, Error err = key_not_found()) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -235,7 +237,7 @@ template class KeyBackedBinaryValue { public: KeyBackedBinaryValue(KeyRef key) : key(key) {} - Future> get(Reference tr, bool snapshot = false) const { + Future> get(Reference tr, Snapshot snapshot = Snapshot::False) const { return map(tr->get(key, snapshot), [](Optional const& val) -> Optional { if (val.present()) return BinaryReader::fromStringRef(val.get(), Unversioned()); @@ -243,8 +245,11 @@ public: }); } // Get property's value or defaultValue if it doesn't exist - Future getD(Reference tr, bool snapshot = false, T defaultValue = T()) const { - return map(get(tr, false), [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); + Future getD(Reference tr, + Snapshot snapshot = Snapshot::False, + T defaultValue = T()) const { + return map(get(tr, Snapshot::False), + [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); } void set(Reference tr, T const& val) { return tr->set(key, BinaryWriter::toValue(val, Unversioned())); @@ -273,8 +278,8 @@ public: KeyType const& begin, Optional const& end, int limit, - bool snapshot = false, - bool reverse = false) const { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) const { Subspace s = space; // 'this' could be invalid inside lambda Key endKey = end.present() ? s.pack(Codec::pack(end.get())) : space.range().end; return map( @@ -293,7 +298,7 @@ public: Future> get(Reference tr, KeyType const& key, - bool snapshot = false) const { + Snapshot snapshot = Snapshot::False) const { return map(tr->get(space.pack(Codec::pack(key)), snapshot), [](Optional const& val) -> Optional { if (val.present()) @@ -339,7 +344,7 @@ public: ValueType const& begin, Optional const& end, int limit, - bool snapshot = false) const { + Snapshot snapshot = Snapshot::False) const { Subspace s = space; // 'this' could be invalid inside lambda Key endKey = end.present() ? s.pack(Codec::pack(end.get())) : space.range().end; return map( @@ -353,7 +358,9 @@ public: }); } - Future exists(Reference tr, ValueType const& val, bool snapshot = false) const { + Future exists(Reference tr, + ValueType const& val, + Snapshot snapshot = Snapshot::False) const { return map(tr->get(space.pack(Codec::pack(val)), snapshot), [](Optional const& val) -> bool { return val.present(); }); } diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.actor.cpp index 7b7dcdf1e3..607bc56d97 100644 --- a/fdbclient/KeyRangeMap.actor.cpp +++ b/fdbclient/KeyRangeMap.actor.cpp @@ -119,7 +119,8 @@ void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr, ACTOR Future krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) { state KeyRange withPrefix = KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString()); - RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); + RangeResult old = + wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True)); Value oldValue; bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); @@ -140,7 +141,8 @@ ACTOR Future krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, V ACTOR Future krmSetRange(Reference tr, Key mapPrefix, KeyRange range, Value value) { state KeyRange withPrefix = KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString()); - RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); + RangeResult old = + wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True)); Value oldValue; bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); @@ -175,8 +177,10 @@ static Future krmSetRangeCoalescing_(Transaction* tr, KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString()); state vector> keys; - keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true)); - keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true)); + keys.push_back( + tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::True)); + keys.push_back( + tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, Snapshot::True)); wait(waitForAll(keys)); // Determine how far to extend this range at the beginning diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index 18de758b78..7015b6fa92 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -143,7 +143,7 @@ std::map configForToken(std::string const& mode) { } if (key == "perpetual_storage_wiggle" && isInteger(value)) { - int ppWiggle = atoi(value.c_str()); + int ppWiggle = std::stoi(value); if (ppWiggle >= 2 || ppWiggle < 0) { printf("Error: Only 0 and 1 are valid values of perpetual_storage_wiggle at present.\n"); return out; @@ -1956,7 +1956,8 @@ ACTOR Future> getExcludedLocalities(Database cx) { // Decodes the locality string to a pair of locality prefix and its value. // The prefix could be dcid, processid, machineid, processid. std::pair decodeLocality(const std::string& locality) { - StringRef localityRef(locality.c_str()); + StringRef localityRef((const uint8_t*)(locality.c_str()), locality.size()); + std::string localityKeyValue = localityRef.removePrefix(LocalityData::ExcludeLocalityPrefix).toString(); int split = localityKeyValue.find(':'); if (split != std::string::npos) { @@ -2472,7 +2473,8 @@ ACTOR Future changeCachedRange(Database cx, KeyRangeRef range, bool add) { tr.clear(sysRangeClear); tr.clear(privateRange); tr.addReadConflictRange(privateRange); - RangeResult previous = wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, true)); + RangeResult previous = + wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, Snapshot::True)); bool prevIsCached = false; if (!previous.empty()) { std::vector prevVal; @@ -2488,7 +2490,7 @@ ACTOR Future changeCachedRange(Database cx, KeyRangeRef range, bool add) { tr.set(sysRange.begin, trueValue); tr.set(privateRange.begin, serverKeysTrue); } - RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, false)); + RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, Snapshot::False)); bool afterIsCached = false; if (!after.empty()) { std::vector afterVal; diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index 22bbbfc4b7..f8c1769384 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -26,6 +26,28 @@ #include "flow/Platform.h" #include "flow/actorcompiler.h" // has to be last include +namespace { + +std::string trim(std::string const& connectionString) { + // Strip out whitespace + // Strip out characters between a # and a newline + std::string trimmed; + auto end = connectionString.end(); + for (auto c = connectionString.begin(); c != end; ++c) { + if (*c == '#') { + ++c; + while (c != end && *c != '\n' && *c != '\r') + ++c; + if (c == end) + break; + } else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t') + trimmed += *c; + } + return trimmed; +} + +} // namespace + std::pair ClusterConnectionFile::lookupClusterFileName(std::string const& filename) { if (filename.length()) return std::make_pair(filename, false); @@ -154,24 +176,6 @@ std::string ClusterConnectionString::getErrorString(std::string const& source, E } } -std::string trim(std::string const& connectionString) { - // Strip out whitespace - // Strip out characters between a # and a newline - std::string trimmed; - auto end = connectionString.end(); - for (auto c = connectionString.begin(); c != end; ++c) { - if (*c == '#') { - ++c; - while (c != end && *c != '\n' && *c != '\r') - ++c; - if (c == end) - break; - } else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t') - trimmed += *c; - } - return trimmed; -} - ClusterConnectionString::ClusterConnectionString(std::string const& connectionString) { auto trimmed = trim(connectionString); @@ -838,6 +842,7 @@ ACTOR Future monitorProxiesOneGeneration( clientInfo->set(ni); successIdx = idx; } else { + TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cannot talk to cluster controller idx = (idx + 1) % addrs.size(); if (idx == successIdx) { wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY)); diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index b9b195a9da..f57e1ccb4f 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -49,7 +49,7 @@ struct ClientData { OpenDatabaseRequest getRequest(); - ClientData() : clientInfo(new AsyncVar>(CachedSerialization())) {} + ClientData() : clientInfo(makeReference>>()) {} }; struct MonitorLeaderInfo { @@ -58,7 +58,7 @@ struct MonitorLeaderInfo { MonitorLeaderInfo() : hasConnected(false) {} explicit MonitorLeaderInfo(Reference intermediateConnFile) - : intermediateConnFile(intermediateConnFile), hasConnected(false) {} + : hasConnected(false), intermediateConnFile(intermediateConnFile) {} }; // Monitors the given coordination group's leader election process and provides a best current guess diff --git a/fdbclient/MultiVersionAssignmentVars.h b/fdbclient/MultiVersionAssignmentVars.h index c21af9f96d..58b68713de 100644 --- a/fdbclient/MultiVersionAssignmentVars.h +++ b/fdbclient/MultiVersionAssignmentVars.h @@ -281,7 +281,7 @@ template class FlatMapSingleAssignmentVar final : public ThreadSingleAssignmentVar, ThreadCallback { public: FlatMapSingleAssignmentVar(ThreadFuture source, std::function>(ErrorOr)> mapValue) - : source(source), mapValue(mapValue), cancelled(false), released(false) { + : source(source), cancelled(false), released(false), mapValue(mapValue) { ThreadSingleAssignmentVar::addref(); int userParam; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 1c5124c12a..49f80dd6fb 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -113,7 +113,7 @@ ThreadFuture DLTransaction::getRange(const KeySelectorRef& begin, end.offset, limits.rows, limits.bytes, - FDBStreamingModes::EXACT, + FDB_STREAMING_MODE_EXACT, 0, snapshot, reverse); @@ -207,12 +207,12 @@ ThreadFuture>> DLTransaction::getRangeSplitPoints(c void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ)); + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); } void DLTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) { api->transactionAtomicOp( - tr, key.begin(), key.size(), value.begin(), value.size(), (FDBMutationTypes::Option)operationType); + tr, key.begin(), key.size(), value.begin(), value.size(), static_cast(operationType)); } void DLTransaction::set(const KeyRef& key, const ValueRef& value) { @@ -239,7 +239,7 @@ ThreadFuture DLTransaction::watch(const KeyRef& key) { void DLTransaction::addWriteConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::WRITE)); + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); } ThreadFuture DLTransaction::commit() { @@ -269,8 +269,10 @@ ThreadFuture DLTransaction::getApproximateSize() { } void DLTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { - throwIfError(api->transactionSetOption( - tr, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->transactionSetOption(tr, + static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } ThreadFuture DLTransaction::onError(Error const& e) { @@ -309,8 +311,10 @@ Reference DLDatabase::createTransaction() { } void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional value) { - throwIfError(api->databaseSetOption( - db, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->databaseSetOption(db, + static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } ThreadFuture DLDatabase::rebootWorker(const StringRef& address, bool check, int duration) { @@ -392,7 +396,7 @@ void loadClientFunction(T* fp, void* lib, std::string libPath, const char* funct } DLApi::DLApi(std::string fdbCPath, bool unlinkOnLoad) - : api(new FdbCApi()), fdbCPath(fdbCPath), unlinkOnLoad(unlinkOnLoad), networkSetup(false) {} + : fdbCPath(fdbCPath), api(new FdbCApi()), unlinkOnLoad(unlinkOnLoad), networkSetup(false) {} // Loads client API functions (definitions are in FdbCApi struct) void DLApi::init() { @@ -504,7 +508,7 @@ void DLApi::selectApiVersion(int apiVersion) { init(); throwIfError(api->selectApiVersion(apiVersion, headerVersion)); - throwIfError(api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT, nullptr, 0)); + throwIfError(api->setNetworkOption(static_cast(FDBNetworkOptions::EXTERNAL_CLIENT), nullptr, 0)); } const char* DLApi::getClientVersion() { @@ -516,8 +520,9 @@ const char* DLApi::getClientVersion() { } void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional value) { - throwIfError(api->setNetworkOption( - option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->setNetworkOption(static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } void DLApi::setupNetwork() { @@ -586,7 +591,7 @@ Reference DLApi::createDatabase609(const char* clusterFilePath) { Reference DLApi::createDatabase(const char* clusterFilePath) { if (headerVersion >= 610) { FdbCApi::FDBDatabase* db; - api->createDatabase(clusterFilePath, &db); + throwIfError(api->createDatabase(clusterFilePath, &db)); return Reference(new DLDatabase(api, db)); } else { return DLApi::createDatabase609(clusterFilePath); @@ -890,22 +895,43 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api, api->runOnExternalClients(threadIdx, [this](Reference client) { dbState->addClient(client); }); - if (!externalClientsInitialized.test_and_set()) { - api->runOnExternalClientsAllThreads([&clusterFilePath](Reference client) { - // This creates a database to initialize some client state on the external library - // We only do this on 6.2+ clients to avoid some bugs associated with older versions - // This deletes the new database immediately to discard its connections - if (client->protocolVersion.hasCloseUnusedConnection()) { + api->runOnExternalClientsAllThreads([&clusterFilePath](Reference client) { + // This creates a database to initialize some client state on the external library. + // We only do this on 6.2+ clients to avoid some bugs associated with older versions. + // This deletes the new database immediately to discard its connections. + // + // Simultaneous attempts to create a database could result in us running this initialization + // code in multiple threads simultaneously. It is necessary that each attempt have a chance + // to run this initialization in case the other fails, and it's safe to run them in parallel. + if (client->protocolVersion.hasCloseUnusedConnection() && !client->initialized) { + try { Reference newDb = client->api->createDatabase(clusterFilePath.c_str()); + client->initialized = true; + } catch (Error& e) { + // This connection is not initialized. It is still possible to connect with it, + // but we may not see trace logs from this client until a successful connection + // is established. + TraceEvent(SevWarnAlways, "FailedToInitializeExternalClient") + .detail("LibraryPath", client->libPath) + .detail("ClusterFilePath", clusterFilePath) + .error(e); } - }); - } + } + }); // For clients older than 6.2 we create and maintain our database connection api->runOnExternalClients(threadIdx, [this, &clusterFilePath](Reference client) { if (!client->protocolVersion.hasCloseUnusedConnection()) { - dbState->legacyDatabaseConnections[client->protocolVersion] = - client->api->createDatabase(clusterFilePath.c_str()); + try { + dbState->legacyDatabaseConnections[client->protocolVersion] = + client->api->createDatabase(clusterFilePath.c_str()); + } catch (Error& e) { + // This connection is discarded + TraceEvent(SevWarnAlways, "FailedToCreateLegacyDatabaseConnection") + .detail("LibraryPath", client->libPath) + .detail("ClusterFilePath", clusterFilePath) + .error(e); + } } }); @@ -988,8 +1014,8 @@ ThreadFuture MultiVersionDatabase::getServerProtocol(Optional

versionMonitorDb) - : clusterFilePath(clusterFilePath), versionMonitorDb(versionMonitorDb), - dbVar(new ThreadSafeAsyncVar>(Reference(nullptr))) {} + : dbVar(new ThreadSafeAsyncVar>(Reference(nullptr))), + clusterFilePath(clusterFilePath), versionMonitorDb(versionMonitorDb), closed(false) {} // Adds a client (local or externally loaded) that can be used to connect to the cluster void MultiVersionDatabase::DatabaseState::addClient(Reference client) { @@ -1053,6 +1079,10 @@ ThreadFuture MultiVersionDatabase::DatabaseState::monitorProtocolVersion() // Called when a change to the protocol version of the cluster has been detected. // Must be called from the main thread void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion protocolVersion) { + if (closed) { + return; + } + // If the protocol version changed but is still compatible, update our local version but keep the same connection if (dbProtocolVersion.present() && protocolVersion.normalizedVersion() == dbProtocolVersion.get().normalizedVersion()) { @@ -1079,7 +1109,20 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion .detail("Failed", client->failed) .detail("External", client->external); - Reference newDb = client->api->createDatabase(clusterFilePath.c_str()); + Reference newDb; + try { + newDb = client->api->createDatabase(clusterFilePath.c_str()); + } catch (Error& e) { + TraceEvent(SevWarnAlways, "MultiVersionClientFailedToCreateDatabase") + .detail("LibraryPath", client->libPath) + .detail("External", client->external) + .detail("ClusterFilePath", clusterFilePath) + .error(e); + + // Put the client in a disconnected state until the version changes again + updateDatabase(Reference(), Reference()); + return; + } if (client->external && !MultiVersionApi::apiVersionAtLeast(610)) { // Old API versions return a future when creating the database, so we need to wait for it @@ -1107,6 +1150,10 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion // Replaces the active database connection with a new one. Must be called from the main thread. void MultiVersionDatabase::DatabaseState::updateDatabase(Reference newDb, Reference client) { + if (closed) { + return; + } + if (newDb) { optionLock.enter(); for (auto option : options) { @@ -1138,12 +1185,28 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference ne versionMonitorDb = db; } else { // For older clients that don't have an API to get the protocol version, we have to monitor it locally - versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str()); + try { + versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str()); + } catch (Error& e) { + // We can't create a new database to monitor the cluster version. This means we will continue using the + // previous one, which should hopefully continue to work. + TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring") + .detail("ClusterFilePath", clusterFilePath) + .error(e); + } } } else { // We don't have a database connection, so use the local client to monitor the protocol version db = Reference(); - versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str()); + try { + versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str()); + } catch (Error& e) { + // We can't create a new database to monitor the cluster version. This means we will continue using the + // previous one, which should hopefully continue to work. + TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring") + .detail("ClusterFilePath", clusterFilePath) + .error(e); + } } dbVar->set(db); @@ -1173,6 +1236,7 @@ void MultiVersionDatabase::DatabaseState::close() { Reference self = Reference::addRef(this); onMainThreadVoid( [self]() { + self->closed = true; if (self->protocolVersionMonitor.isValid()) { self->protocolVersionMonitor.cancel(); } @@ -1250,8 +1314,6 @@ void MultiVersionDatabase::LegacyVersionMonitor::close() { } } -std::atomic_flag MultiVersionDatabase::externalClientsInitialized = ATOMIC_FLAG_INIT; - // MultiVersionApi bool MultiVersionApi::apiVersionAtLeast(int minVersion) { ASSERT_NE(MultiVersionApi::api->apiVersion, 0); @@ -1459,7 +1521,7 @@ std::vector> MultiVersionApi::copyExternalLibraryPe #else std::vector> MultiVersionApi::copyExternalLibraryPerThread(std::string path) { if (threadCount > 1) { - TraceEvent(SevError, "MultipleClientThreadsUnsupportedOnWindows"); + TraceEvent(SevError, "MultipleClientThreadsUnsupportedOnWindows").log(); throw unsupported_operation(); } std::vector> paths; @@ -1850,8 +1912,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() { } MultiVersionApi::MultiVersionApi() - : bypassMultiClientApi(false), networkStartSetup(false), networkSetup(false), callbackOnMainThread(true), - externalClient(false), localClientDisabled(false), apiVersion(0), envOptionsLoaded(false), threadCount(0) {} + : callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false), + bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), envOptionsLoaded(false) {} MultiVersionApi* MultiVersionApi::api = new MultiVersionApi(); @@ -1884,8 +1946,6 @@ bool ClientInfo::canReplace(Reference other) const { } // UNIT TESTS -extern bool noUnseed; - TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing") { auto vals = parseOptionValues("a"); ASSERT(vals.size() == 1 && vals[0] == "a"); diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index a98e16b440..274df7dd84 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -22,6 +22,7 @@ #define FDBCLIENT_MULTIVERSIONTRANSACTION_H #pragma once +#include "bindings/c/foundationdb/fdb_c_options.g.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/IClientApi.h" @@ -31,10 +32,10 @@ // FdbCApi is used as a wrapper around the FoundationDB C API that gets loaded from an external client library. // All of the required functions loaded from that external library are stored in function pointers in this struct. struct FdbCApi : public ThreadSafeReferenceCounted { - typedef struct future FDBFuture; - typedef struct cluster FDBCluster; - typedef struct database FDBDatabase; - typedef struct transaction FDBTransaction; + typedef struct FDB_future FDBFuture; + typedef struct FDB_cluster FDBCluster; + typedef struct FDB_database FDBDatabase; + typedef struct FDB_transaction FDBTransaction; #pragma pack(push, 4) typedef struct key { @@ -57,16 +58,16 @@ struct FdbCApi : public ThreadSafeReferenceCounted { // Network fdb_error_t (*selectApiVersion)(int runtimeVersion, int headerVersion); const char* (*getClientVersion)(); - fdb_error_t (*setNetworkOption)(FDBNetworkOptions::Option option, uint8_t const* value, int valueLength); + fdb_error_t (*setNetworkOption)(FDBNetworkOption option, uint8_t const* value, int valueLength); fdb_error_t (*setupNetwork)(); fdb_error_t (*runNetwork)(); fdb_error_t (*stopNetwork)(); - fdb_error_t* (*createDatabase)(const char* clusterFilePath, FDBDatabase** db); + fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db); // Database fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr); fdb_error_t (*databaseSetOption)(FDBDatabase* database, - FDBDatabaseOptions::Option option, + FDBDatabaseOption option, uint8_t const* value, int valueLength); void (*databaseDestroy)(FDBDatabase* database); @@ -86,7 +87,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { // Transaction fdb_error_t (*transactionSetOption)(FDBTransaction* tr, - FDBTransactionOptions::Option option, + FDBTransactionOption option, uint8_t const* value, int valueLength); void (*transactionDestroy)(FDBTransaction* tr); @@ -113,7 +114,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int endOffset, int limit, int targetBytes, - FDBStreamingModes::Option mode, + FDBStreamingMode mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse); @@ -135,7 +136,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int keyNameLength, uint8_t const* param, int paramLength, - FDBMutationTypes::Option operationType); + FDBMutationType operationType); FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, @@ -163,7 +164,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int beginKeyNameLength, uint8_t const* endKeyName, int endKeyNameLength, - FDBConflictRangeTypes::Option); + FDBConflictRangeType); // Future fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb); @@ -416,12 +417,15 @@ struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted { ProtocolVersion protocolVersion; IClientApi* api; bool failed; + std::atomic_bool initialized; std::vector> threadCompletionHooks; - ClientInfo() : ClientDesc(std::string(), false), protocolVersion(0), api(nullptr), failed(true) {} - ClientInfo(IClientApi* api) : ClientDesc("internal", false), protocolVersion(0), api(api), failed(false) {} + ClientInfo() + : ClientDesc(std::string(), false), protocolVersion(0), api(nullptr), failed(true), initialized(false) {} + ClientInfo(IClientApi* api) + : ClientDesc("internal", false), protocolVersion(0), api(api), failed(false), initialized(false) {} ClientInfo(IClientApi* api, std::string libPath) - : ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false) {} + : ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false), initialized(false) {} void loadProtocolVersion(); bool canReplace(Reference other) const; @@ -503,10 +507,9 @@ public: // this will be a specially created local db. Reference versionMonitorDb; + bool closed; + ThreadFuture changed; - - bool cancelled; - ThreadFuture dbReady; ThreadFuture protocolVersionMonitor; @@ -556,10 +559,6 @@ public: const Reference dbState; friend class MultiVersionTransaction; - - // Clients must create a database object in order to initialize some of their state. - // This needs to be done only once, and this flag tracks whether that has happened. - static std::atomic_flag externalClientsInitialized; }; // An implementation of IClientApi that can choose between multiple different client implementations either provided diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index b6b17d6133..b3a7421475 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -94,7 +94,8 @@ Future loadBalance( RequestStream Interface::*channel, const Request& request = Request(), TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint, - bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically + AtMostOnce atMostOnce = + AtMostOnce::False, // if true, throws request_maybe_delivered() instead of retrying automatically QueueModel* model = nullptr) { if (alternatives->hasCaches) { return loadBalance(alternatives->locations(), channel, request, taskID, atMostOnce, model); @@ -115,10 +116,10 @@ TLSConfig tlsConfig(TLSEndpointType::CLIENT); // The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h. NetworkOptions::NetworkOptions() - : localAddress(""), clusterFile(""), traceDirectory(Optional()), traceRollSize(TRACE_DEFAULT_ROLL_SIZE), - traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), traceFormat("xml"), - traceClockSource("now"), runLoopProfilingEnabled(false), - supportedVersions(new ReferencedObject>>()) {} + : traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), + traceFormat("xml"), traceClockSource("now"), + supportedVersions(new ReferencedObject>>()), runLoopProfilingEnabled(false) { +} static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/"); static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/"); @@ -154,6 +155,8 @@ void DatabaseContext::addTssMapping(StorageServerInterface const& ssi, StorageSe TSSEndpointData(tssi.id(), tssi.getKeyValues.getEndpoint(), metrics)); queueModel.updateTssEndpoint(ssi.watchValue.getEndpoint().token.first(), TSSEndpointData(tssi.id(), tssi.watchValue.getEndpoint(), metrics)); + queueModel.updateTssEndpoint(ssi.getKeyValuesStream.getEndpoint().token.first(), + TSSEndpointData(tssi.id(), tssi.getKeyValuesStream.getEndpoint(), metrics)); } } @@ -166,6 +169,7 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) { queueModel.removeTssEndpoint(ssi.getKey.getEndpoint().token.first()); queueModel.removeTssEndpoint(ssi.getKeyValues.getEndpoint().token.first()); queueModel.removeTssEndpoint(ssi.watchValue.getEndpoint().token.first()); + queueModel.removeTssEndpoint(ssi.getKeyValuesStream.getEndpoint().token.first()); } } @@ -309,7 +313,7 @@ std::string unprintable(std::string const& val) { return s; } -void DatabaseContext::validateVersion(Version version) { +void DatabaseContext::validateVersion(Version version) const { // Version could be 0 if the INITIALIZE_NEW_DATABASE option is set. In that case, it is illegal to perform any // reads. We throw client_invalid_operation because the caller didn't directly set the version, so the // version_invalid error might be confusing. @@ -324,11 +328,16 @@ void DatabaseContext::validateVersion(Version version) { ASSERT(version > 0 || version == latestVersion); } -void validateOptionValue(Optional value, bool shouldBePresent) { - if (shouldBePresent && !value.present()) +void validateOptionValuePresent(Optional value) { + if (!value.present()) { throw invalid_option_value(); - if (!shouldBePresent && value.present() && value.get().size() > 0) + } +} + +void validateOptionValueNotPresent(Optional value) { + if (value.present() && value.get().size() > 0) { throw invalid_option_value(); + } } void dumpMutations(const MutationListRef& mutations) { @@ -412,10 +421,11 @@ ACTOR Future databaseLogger(DatabaseContext* cx) { cx->bytesPerCommit.clear(); for (const auto& it : cx->tssMetrics) { - // TODO could skip this tss if request counter is zero? would potentially complicate elapsed calculation - // though + // TODO could skip this whole thing if tss if request counter is zero? + // That would potentially complicate elapsed calculation though if (it.second->mismatches.getIntervalDelta()) { - cx->tssMismatchStream.send(it.first); + cx->tssMismatchStream.send( + std::pair>(it.first, it.second->detailedMismatches)); } // do error histograms as separate event @@ -506,15 +516,15 @@ ACTOR static Future transactionInfoCommitActor(Transaction* tr, std::vecto ACTOR static Future delExcessClntTxnEntriesActor(Transaction* tr, int64_t clientTxInfoSizeLimit) { state const Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin); state const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin); - TraceEvent(SevInfo, "DelExcessClntTxnEntriesCalled"); + TraceEvent(SevInfo, "DelExcessClntTxnEntriesCalled").log(); loop { try { tr->reset(); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - Optional ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), true)); + Optional ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), Snapshot::True)); if (!ctrValue.present()) { - TraceEvent(SevInfo, "NumClntTxnEntriesNotFound"); + TraceEvent(SevInfo, "NumClntTxnEntriesNotFound").log(); return Void(); } state int64_t txInfoSize = 0; @@ -668,7 +678,7 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext* cx) { } } -ACTOR static Future monitorProxiesChange(Reference> clientDBInfo, +ACTOR static Future monitorProxiesChange(Reference const> clientDBInfo, AsyncTrigger* triggerVar) { state vector curCommitProxies; state vector curGrvProxies; @@ -854,13 +864,15 @@ ACTOR Future monitorCacheList(DatabaseContext* self) { ACTOR static Future handleTssMismatches(DatabaseContext* cx) { state Reference tr; state KeyBackedMap tssMapDB = KeyBackedMap(tssMappingKeys.begin); + state KeyBackedMap tssMismatchDB = KeyBackedMap(tssMismatchKeys.begin); loop { - state UID tssID = waitNext(cx->tssMismatchStream.getFuture()); + // + state std::pair> data = waitNext(cx->tssMismatchStream.getFuture()); // find ss pair id so we can remove it from the mapping state UID tssPairID; bool found = false; for (const auto& it : cx->tssMapping) { - if (it.second.id() == tssID) { + if (it.second.id() == data.first) { tssPairID = it.first; found = true; break; @@ -869,7 +881,7 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { if (found) { state bool quarantine = CLIENT_KNOBS->QUARANTINE_TSS_ON_MISMATCH; TraceEvent(SevWarnAlways, quarantine ? "TSS_QuarantineMismatch" : "TSS_KillMismatch") - .detail("TSSID", tssID.toString()); + .detail("TSSID", data.first.toString()); TEST(quarantine); // Quarantining TSS because it got mismatch TEST(!quarantine); // Killing TSS because it got mismatch @@ -879,14 +891,21 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { try { tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (quarantine) { - tr->set(tssQuarantineKeyFor(tssID), LiteralStringRef("")); + tr->set(tssQuarantineKeyFor(data.first), LiteralStringRef("")); } else { - tr->clear(serverTagKeyFor(tssID)); + tr->clear(serverTagKeyFor(data.first)); } tssMapDB.erase(tr, tssPairID); + for (const DetailedTSSMismatch& d : data.second) { + // -> mismatch data + tssMismatchDB.set( + tr, + Tuple().append(data.first.toString()).append(d.timestamp).append(d.mismatchId.toString()), + d.traceString); + } + wait(tr->commit()); break; @@ -896,7 +915,7 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { tries++; if (tries > 10) { // Give up, it'll get another mismatch or a human will investigate eventually - TraceEvent("TSS_MismatchGaveUp").detail("TSSID", tssID.toString()); + TraceEvent("TSS_MismatchGaveUp").detail("TSSID", data.first.toString()); break; } } @@ -1094,20 +1113,19 @@ Future HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction* DatabaseContext::DatabaseContext(Reference>> connectionFile, Reference> clientInfo, - Reference>> coordinator, + Reference> const> coordinator, Future clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, - bool enableLocalityLoadBalance, - bool lockAware, - bool internal, + EnableLocalityLoadBalance enableLocalityLoadBalance, + LockAware lockAware, + IsInternal internal, int apiVersion, - bool switchable) - : connectionFile(connectionFile), clientInfo(clientInfo), coordinator(coordinator), - clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality), - enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), apiVersion(apiVersion), - switchable(switchable), proxyProvisional(false), cc("TransactionMetrics"), - transactionReadVersions("ReadVersions", cc), transactionReadVersionsThrottled("ReadVersionsThrottled", cc), + IsSwitchable switchable) + : lockAware(lockAware), switchable(switchable), connectionFile(connectionFile), proxyProvisional(false), + clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), internal(internal), + cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc), + transactionReadVersionsThrottled("ReadVersionsThrottled", cc), transactionReadVersionsCompleted("ReadVersionsCompleted", cc), transactionReadVersionBatches("ReadVersionBatches", cc), transactionBatchReadVersions("BatchPriorityReadVersions", cc), @@ -1132,11 +1150,12 @@ DatabaseContext::DatabaseContext(ReferenceSHARD_STAT_SMOOTH_AMOUNT), - transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), + transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc), + transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000), + commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0), + transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), + coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0), + detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT), specialKeySpace(std::make_unique(specialKeys.begin, specialKeys.end, /* test */ false)) { dbId = deterministicRandom()->randomUniqueID(); connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) @@ -1349,8 +1368,8 @@ DatabaseContext::DatabaseContext(ReferenceSHARD_STAT_SMOOTH_AMOUNT), - transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(false), - transactionTracingEnabled(true) {} + transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc), + transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000), + commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), + transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT) {} // Static constructor used by server processes to create a DatabaseContext // For internal (fdbserver) use only Database DatabaseContext::create(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, - bool enableLocalityLoadBalance, + EnableLocalityLoadBalance enableLocalityLoadBalance, TaskPriority taskID, - bool lockAware, + LockAware lockAware, int apiVersion, - bool switchable) { + IsSwitchable switchable) { return Database(new DatabaseContext(Reference>>(), clientInfo, makeReference>>(), @@ -1399,7 +1417,7 @@ Database DatabaseContext::create(Reference> clientInfo, clientLocality, enableLocalityLoadBalance, lockAware, - true, + IsInternal::True, apiVersion, switchable)); } @@ -1415,7 +1433,7 @@ DatabaseContext::~DatabaseContext() { locationCache.insert(allKeys, Reference()); } -pair> DatabaseContext::getCachedLocation(const KeyRef& key, bool isBackward) { +pair> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) { if (isBackward) { auto range = locationCache.rangeContainingKeyBefore(key); return std::make_pair(range->range(), range->value()); @@ -1428,7 +1446,7 @@ pair> DatabaseContext::getCachedLocation(const bool DatabaseContext::getCachedLocations(const KeyRangeRef& range, vector>>& result, int limit, - bool reverse) { + Reverse reverse) { result.clear(); auto begin = locationCache.rangeContaining(range.begin); @@ -1476,7 +1494,7 @@ Reference DatabaseContext::setCachedLocation(const KeyRangeRef& ke return loc; } -void DatabaseContext::invalidateCache(const KeyRef& key, bool isBackward) { +void DatabaseContext::invalidateCache(const KeyRef& key, Reverse isBackward) { if (isBackward) { locationCache.rangeContainingKeyBefore(key)->value() = Reference(); } else { @@ -1491,7 +1509,7 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) { locationCache.insert(KeyRangeRef(begin, end), Reference()); } -Future DatabaseContext::onProxiesChanged() { +Future DatabaseContext::onProxiesChanged() const { return this->proxiesChangeTrigger.onTrigger(); } @@ -1509,7 +1527,7 @@ bool DatabaseContext::sampleOnCost(uint64_t cost) const { } int64_t extractIntOption(Optional value, int64_t minValue, int64_t maxValue) { - validateOptionValue(value, true); + validateOptionValuePresent(value); if (value.get().size() != 8) { throw invalid_option_value(); } @@ -1571,23 +1589,23 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional()); break; case FDBDatabaseOptions::SNAPSHOT_RYW_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); snapshotRywEnabled++; break; case FDBDatabaseOptions::SNAPSHOT_RYW_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); snapshotRywEnabled--; break; case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); transactionTracingEnabled++; break; case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); transactionTracingEnabled--; break; case FDBDatabaseOptions::USE_CONFIG_DATABASE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); useConfigDatabase = true; break; default: @@ -1636,7 +1654,7 @@ ACTOR static Future switchConnectionFileImpl(Reference connFile, int apiVersion, - bool internal, + IsInternal internal, LocalityData const& clientLocality, DatabaseContext* preallocatedDb) { if (!g_network) @@ -1708,7 +1726,8 @@ Database Database::createDatabase(Reference connFile, networkOptions.traceDirectory.get(), "trace", networkOptions.traceLogGroup, - networkOptions.traceFileIdentifier); + networkOptions.traceFileIdentifier, + networkOptions.tracePartialFileSuffix); TraceEvent("ClientStart") .detail("SourceVersion", getSourceVersion()) @@ -1748,11 +1767,11 @@ Database Database::createDatabase(Reference connFile, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, - true, - false, + EnableLocalityLoadBalance::True, + LockAware::False, internal, apiVersion, - /*switchable*/ true); + IsSwitchable::True); } else { db = new DatabaseContext(connectionFile, clientInfo, @@ -1760,21 +1779,22 @@ Database Database::createDatabase(Reference connFile, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, - true, - false, + EnableLocalityLoadBalance::True, + LockAware::False, internal, apiVersion, - /*switchable*/ true); + IsSwitchable::True); } auto database = Database(db); - GlobalConfig::create(database, clientInfo, std::addressof(clientInfo->get())); + GlobalConfig::create( + database, Reference const>(clientInfo), std::addressof(clientInfo->get())); return database; } Database Database::createDatabase(std::string connFileName, int apiVersion, - bool internal, + IsInternal internal, LocalityData const& clientLocality) { Reference rccf = Reference( new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFileName).first)); @@ -1821,15 +1841,15 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu networkOptions.traceDirectory = value.present() ? value.get().toString() : ""; break; case FDBNetworkOptions::TRACE_ROLL_SIZE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceRollSize = extractIntOption(value, 0, std::numeric_limits::max()); break; case FDBNetworkOptions::TRACE_MAX_LOGS_SIZE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceMaxLogsSize = extractIntOption(value, 0, std::numeric_limits::max()); break; case FDBNetworkOptions::TRACE_FORMAT: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceFormat = value.get().toString(); if (!validateTraceFormat(networkOptions.traceFormat)) { fprintf(stderr, "Unrecognized trace format: `%s'\n", networkOptions.traceFormat.c_str()); @@ -1837,7 +1857,7 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } break; case FDBNetworkOptions::TRACE_FILE_IDENTIFIER: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceFileIdentifier = value.get().toString(); if (networkOptions.traceFileIdentifier.length() > CLIENT_KNOBS->TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH) { fprintf(stderr, "Trace file identifier provided is too long.\n"); @@ -1858,15 +1878,19 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } break; case FDBNetworkOptions::TRACE_CLOCK_SOURCE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceClockSource = value.get().toString(); if (!validateTraceClockSource(networkOptions.traceClockSource)) { fprintf(stderr, "Unrecognized trace clock source: `%s'\n", networkOptions.traceClockSource.c_str()); throw invalid_option_value(); } break; + case FDBNetworkOptions::TRACE_PARTIAL_FILE_SUFFIX: + validateOptionValuePresent(value); + networkOptions.tracePartialFileSuffix = value.get().toString(); + break; case FDBNetworkOptions::KNOB: { - validateOptionValue(value, true); + validateOptionValuePresent(value); std::string optionValue = value.get().toString(); TraceEvent("SetKnob").detail("KnobString", optionValue); @@ -1890,42 +1914,42 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; } case FDBNetworkOptions::TLS_PLUGIN: - validateOptionValue(value, true); + validateOptionValuePresent(value); break; case FDBNetworkOptions::TLS_CERT_PATH: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCertificatePath(value.get().toString()); break; case FDBNetworkOptions::TLS_CERT_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCertificateBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_PATH: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCAPath(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCABytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_PASSWORD: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setPassword(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_PATH: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setKeyPath(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setKeyBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_VERIFY_PEERS: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.clearVerifyPeers(); tlsConfig.addVerifyPeers(value.get().toString()); break; @@ -1936,16 +1960,16 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu enableBuggify(false, BuggifyType::Client); break; case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_ACTIVATED_PROBABILITY: - validateOptionValue(value, true); + validateOptionValuePresent(value); clearBuggifySections(BuggifyType::Client); P_BUGGIFIED_SECTION_ACTIVATED[int(BuggifyType::Client)] = double(extractIntOption(value, 0, 100)) / 100.0; break; case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_FIRED_PROBABILITY: - validateOptionValue(value, true); + validateOptionValuePresent(value); P_BUGGIFIED_SECTION_FIRES[int(BuggifyType::Client)] = double(extractIntOption(value, 0, 100)) / 100.0; break; case FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); networkOptions.logClientInfo = false; break; case FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS: { @@ -1965,11 +1989,11 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; } case FDBNetworkOptions::ENABLE_RUN_LOOP_PROFILING: // Same as ENABLE_SLOW_TASK_PROFILING - validateOptionValue(value, false); + validateOptionValueNotPresent(value); networkOptions.runLoopProfilingEnabled = true; break; case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: { - validateOptionValue(value, true); + validateOptionValuePresent(value); std::string tracer = value.get().toString(); if (tracer == "none" || tracer == "disabled") { openTracer(TracerType::DISABLED); @@ -2012,7 +2036,7 @@ ACTOR Future monitorNetworkBusyness() { } // Setup g_network and start monitoring for network busyness -void setupNetwork(uint64_t transportId, bool useMetrics) { +void setupNetwork(uint64_t transportId, UseMetrics useMetrics) { if (g_network) throw network_already_setup(); @@ -2181,7 +2205,7 @@ Future getRange(Database const& cx, KeySelector const& begin, KeySelector const& end, GetRangeLimits const& limits, - bool const& reverse, + Reverse const& reverse, TransactionInfo const& info, TagSet const& tags); @@ -2263,7 +2287,7 @@ void updateTagMappings(Database cx, const GetKeyServerLocationsReply& reply) { ACTOR Future>> getKeyLocation_internal(Database cx, Key key, TransactionInfo info, - bool isBackward = false) { + Reverse isBackward = Reverse::False) { state Span span("NAPI:getKeyLocation"_loc, info.spanID); if (isBackward) { ASSERT(key != allKeys.begin && key <= allKeys.end); @@ -2303,7 +2327,7 @@ Future>> getKeyLocation(Database const& c Key const& key, F StorageServerInterface::*member, TransactionInfo const& info, - bool isBackward = false) { + Reverse isBackward = Reverse::False) { // we first check whether this range is cached auto ssi = cx->getCachedLocation(key, isBackward); if (!ssi.second) { @@ -2324,7 +2348,7 @@ Future>> getKeyLocation(Database const& c ACTOR Future>>> getKeyRangeLocations_internal(Database cx, KeyRange keys, int limit, - bool reverse, + Reverse reverse, TransactionInfo info) { state Span span("NAPI:getKeyRangeLocations"_loc, info.spanID); if (info.debugID.present()) @@ -2374,7 +2398,7 @@ template Future>>> getKeyRangeLocations(Database const& cx, KeyRange const& keys, int limit, - bool reverse, + Reverse reverse, F StorageServerInterface::*member, TransactionInfo const& info) { ASSERT(!keys.empty()); @@ -2411,8 +2435,8 @@ ACTOR Future warmRange_impl(Transaction* self, Database cx, KeyRange keys) state int totalRanges = 0; state int totalRequests = 0; loop { - vector>> locations = - wait(getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, false, self->info)); + vector>> locations = wait( + getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, Reverse::False, self->info)); totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT; totalRequests++; if (locations.size() == 0 || totalRanges >= cx->locationCacheSize || @@ -2501,7 +2525,7 @@ ACTOR Future> getValue(Future version, getValueID, ssLatestCommitVersions), TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::False, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { reply = _reply; } @@ -2588,7 +2612,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr Key locationKey(k.getKey(), k.arena()); state pair> ssi = - wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, k.isBackward())); + wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, Reverse{ k.isBackward() })); state VersionVector ssLatestCommitVersions; cx->getLatestCommitVersions(ssi.second, version.get(), ssLatestCommitVersions); @@ -2620,7 +2644,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr &StorageServerInterface::getKey, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::False, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { reply = _reply; } @@ -2643,7 +2667,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr if (info.debugID.present()) g_traceBatch.addEvent("GetKeyDebug", getKeyID.get().first(), "NativeAPI.getKey.Error"); if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) { - cx->invalidateCache(k.getKey(), k.isBackward()); + cx->invalidateCache(k.getKey(), Reverse{ k.isBackward() }); wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID)); } else { @@ -2936,7 +2960,7 @@ ACTOR Future watchValueMap(Future version, return Void(); } -void transformRangeLimits(GetRangeLimits limits, bool reverse, GetKeyValuesRequest& req) { +void transformRangeLimits(GetRangeLimits limits, Reverse reverse, GetKeyValuesRequest& req) { if (limits.bytes != 0) { if (!limits.hasRowLimit()) req.limit = CLIENT_KNOBS->REPLY_BYTE_LIMIT; // Can't get more than this many rows anyway @@ -2960,7 +2984,7 @@ ACTOR Future getExactRange(Database cx, Version version, KeyRange keys, GetRangeLimits limits, - bool reverse, + Reverse reverse, TransactionInfo info, TagSet tags) { state RangeResult output; @@ -3016,7 +3040,7 @@ ACTOR Future getExactRange(Database cx, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::False, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { rep = _rep; } @@ -3144,7 +3168,7 @@ ACTOR Future getRangeFallback(Database cx, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse, + Reverse reverse, TransactionInfo info, TagSet tags) { if (version == latestVersion) { @@ -3199,9 +3223,9 @@ void getRangeFinished(Database cx, double startTime, KeySelector begin, KeySelector end, - bool snapshot, + Snapshot snapshot, Promise> conflictRange, - bool reverse, + Reverse reverse, RangeResult result) { int64_t bytes = 0; for (const KeyValueRef& kv : result) { @@ -3255,8 +3279,8 @@ ACTOR Future getRange(Database cx, KeySelector end, GetRangeLimits limits, Promise> conflictRange, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags) { state GetRangeLimits originalLimits(limits); @@ -3291,7 +3315,7 @@ ACTOR Future getRange(Database cx, } Key locationKey = reverse ? Key(end.getKey(), end.arena()) : Key(begin.getKey(), begin.arena()); - bool locationBackward = reverse ? (end - 1).isBackward() : begin.isBackward(); + Reverse locationBackward{ reverse ? (end - 1).isBackward() : begin.isBackward() }; state pair> beginServer = wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKeyValues, info, locationBackward)); state KeyRange shard = beginServer.first; @@ -3369,7 +3393,7 @@ ACTOR Future getRange(Database cx, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::False, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr)); rep = _rep; ++cx->transactionPhysicalReadsCompleted; @@ -3503,7 +3527,7 @@ ACTOR Future getRange(Database cx, if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed || (e.code() == error_code_transaction_too_old && readVersion == latestVersion)) { cx->invalidateCache(reverse ? end.getKey() : begin.getKey(), - reverse ? (end - 1).isBackward() : begin.isBackward()); + Reverse{ reverse ? (end - 1).isBackward() : begin.isBackward() }); if (e.code() == error_code_wrong_shard_server) { RangeResult result = wait(getRangeFallback( @@ -3542,6 +3566,174 @@ ACTOR Future getRange(Database cx, } } +template +struct TSSDuplicateStreamData { + PromiseStream stream; + Promise tssComparisonDone; + + // empty constructor for optional? + TSSDuplicateStreamData() {} + + TSSDuplicateStreamData(PromiseStream stream) : stream(stream) {} + + bool done() { return tssComparisonDone.getFuture().isReady(); } + + void setDone() { + if (tssComparisonDone.canBeSet()) { + tssComparisonDone.send(Void()); + } + } + + ~TSSDuplicateStreamData() {} +}; + +// Error tracking here is weird, and latency doesn't really mean the same thing here as it does with normal tss +// comparisons, so this is pretty much just counting mismatches +ACTOR template +static Future tssStreamComparison(Request request, + TSSDuplicateStreamData streamData, + ReplyPromiseStream tssReplyStream, + TSSEndpointData tssData) { + state bool ssEndOfStream = false; + state bool tssEndOfStream = false; + state Optional ssReply = Optional(); + state Optional tssReply = Optional(); + + loop { + // reset replies + ssReply = Optional(); + tssReply = Optional(); + + state double startTime = now(); + // wait for ss response + try { + REPLYSTREAM_TYPE(Request) _ssReply = waitNext(streamData.stream.getFuture()); + ssReply = _ssReply; + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + streamData.setDone(); + throw; + } + if (e.code() == error_code_end_of_stream) { + // ss response will be set to empty, to compare to the SS response if it wasn't empty and cause a + // mismatch + ssEndOfStream = true; + } else { + tssData.metrics->ssError(e.code()); + } + TEST(e.code() != error_code_end_of_stream); // SS got error in TSS stream comparison + } + + state double sleepTime = std::max(startTime + FLOW_KNOBS->LOAD_BALANCE_TSS_TIMEOUT - now(), 0.0); + // wait for tss response + try { + choose { + when(REPLYSTREAM_TYPE(Request) _tssReply = waitNext(tssReplyStream.getFuture())) { + tssReply = _tssReply; + } + when(wait(delay(sleepTime))) { + ++tssData.metrics->tssTimeouts; + TEST(true); // Got TSS timeout in stream comparison + } + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + streamData.setDone(); + throw; + } + if (e.code() == error_code_end_of_stream) { + // tss response will be set to empty, to compare to the SS response if it wasn't empty and cause a + // mismatch + tssEndOfStream = true; + } else { + tssData.metrics->tssError(e.code()); + } + TEST(e.code() != error_code_end_of_stream); // TSS got error in TSS stream comparison + } + + if (!ssEndOfStream || !tssEndOfStream) { + ++tssData.metrics->streamComparisons; + } + + // if both are successful, compare + if (ssReply.present() && tssReply.present()) { + // compare results + // FIXME: this code is pretty much identical to LoadBalance.h + // TODO could add team check logic in if we added synchronous way to turn this into a fixed getRange request + // and send it to the whole team and compare? I think it's fine to skip that for streaming though + TEST(ssEndOfStream != tssEndOfStream); // SS or TSS stream finished early! + + // skip tss comparison if both are end of stream + if ((!ssEndOfStream || !tssEndOfStream) && !TSS_doCompare(ssReply.get(), tssReply.get())) { + TEST(true); // TSS mismatch in stream comparison + TraceEvent mismatchEvent( + (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(request)); + mismatchEvent.setMaxEventLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE); + mismatchEvent.detail("TSSID", tssData.tssId); + + if (tssData.metrics->shouldRecordDetailedMismatch()) { + TSS_traceMismatch(mismatchEvent, request, ssReply.get(), tssReply.get()); + + TEST(FLOW_KNOBS + ->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Full TSS Mismatch in stream comparison + TEST(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Partial TSS Mismatch in stream + // comparison and storing the rest in FDB + + if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) { + mismatchEvent.disable(); + UID mismatchUID = deterministicRandom()->randomUniqueID(); + tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString()); + + // record a summarized trace event instead + TraceEvent summaryEvent((g_network->isSimulated() && + g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(request)); + summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID); + } + } else { + // don't record trace event + mismatchEvent.disable(); + } + streamData.setDone(); + return Void(); + } + } + if (!ssReply.present() || !tssReply.present() || ssEndOfStream || tssEndOfStream) { + // if both streams don't still have more data, stop comparison + streamData.setDone(); + return Void(); + } + } +} + +// Currently only used for GetKeyValuesStream but could easily be plugged for other stream types +// User of the stream has to forward the SS's responses to the returned promise stream, if it is set +template +Optional> +maybeDuplicateTSSStreamFragment(Request& req, QueueModel* model, RequestStream const* ssStream) { + if (model) { + Optional tssData = model->getTssData(ssStream->getEndpoint().token.first()); + + if (tssData.present()) { + TEST(true); // duplicating stream to TSS + resetReply(req); + // FIXME: optimize to avoid creating new netNotifiedQueueWithAcknowledgements for each stream duplication + RequestStream tssRequestStream(tssData.get().endpoint); + ReplyPromiseStream tssReplyStream = tssRequestStream.getReplyStream(req); + PromiseStream ssDuplicateReplyStream; + TSSDuplicateStreamData streamData(ssDuplicateReplyStream); + model->addActor.send(tssStreamComparison(req, streamData, tssReplyStream, tssData.get())); + return Optional>(streamData); + } + } + return Optional>(); +} + // Streams all of the KV pairs in a target key range into a ParallelStream fragment ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* results, Database cx, @@ -3549,8 +3741,8 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* Version version, KeyRange keys, GetRangeLimits limits, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags, SpanID spanContext) { @@ -3562,6 +3754,7 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* loop { const KeyRange& range = locations[shard].first; + state Optional> tssDuplicateStream; state GetKeyValuesStreamRequest req; req.version = version; req.begin = firstGreaterOrEqual(range.begin); @@ -3571,6 +3764,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* req.limitBytes = std::numeric_limits::max(); cx->getLatestCommitVersions(locations[shard].second, version, req.ssLatestCommitVersions); + // keep shard's arena around in case of async tss comparison + req.arena.dependsOn(range.arena()); + ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse); // FIXME: buggify byte limits on internal functions that use them, instead of globally @@ -3634,6 +3830,12 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* locations[shard] .second->get(useIdx, &StorageServerInterface::getKeyValuesStream) .getReplyStream(req); + + tssDuplicateStream = maybeDuplicateTSSStreamFragment( + req, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr, + &locations[shard].second->get(useIdx, &StorageServerInterface::getKeyValuesStream)); + state bool breakAgain = false; loop { wait(results->onEmpty()); @@ -3641,6 +3843,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* choose { when(wait(cx->connectionFileChanged())) { results->sendError(transaction_too_old()); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(transaction_too_old()); + } return Void(); } @@ -3650,9 +3855,15 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* } catch (Error& e) { ++cx->transactionPhysicalReadsCompleted; if (e.code() == error_code_broken_promise) { + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(connection_failed()); + } throw connection_failed(); } if (e.code() != error_code_end_of_stream) { + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(e); + } throw; } rep = GetKeyValuesStreamReply(); @@ -3662,6 +3873,17 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* "TransactionDebug", info.debugID.get().first(), "NativeAPI.getExactRange.After"); RangeResult output(RangeResultRef(rep.data, rep.more), rep.arena); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + // shallow copy the reply with an arena depends, and send it to the duplicate stream for TSS + GetKeyValuesStreamReply replyCopy; + replyCopy.version = rep.version; + replyCopy.more = rep.more; + replyCopy.cached = rep.cached; + replyCopy.arena.dependsOn(rep.arena); + replyCopy.data.append(replyCopy.arena, rep.data.begin(), rep.data.size()); + tssDuplicateStream.get().stream.send(replyCopy); + } + int64_t bytes = 0; for (const KeyValueRef& kv : output) { bytes += kv.key.size() + kv.value.size(); @@ -3719,6 +3941,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* output.readThrough = reverse ? keys.begin : keys.end; results->send(std::move(output)); results->finish(); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(end_of_stream()); + } return Void(); } keys = KeyRangeRef(begin, end); @@ -3745,6 +3970,10 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* break; } } catch (Error& e) { + // send errors to tss duplicate stream, including actor_cancelled + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(e); + } if (e.code() == error_code_actor_cancelled) { throw; } @@ -3785,8 +4014,8 @@ ACTOR Future getRangeStream(PromiseStream _results, KeySelector end, GetRangeLimits limits, Promise> conflictRange, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags) { @@ -3866,7 +4095,7 @@ Future getRange(Database const& cx, KeySelector const& begin, KeySelector const& end, GetRangeLimits const& limits, - bool const& reverse, + Reverse const& reverse, TransactionInfo const& info, TagSet const& tags) { return getRange(cx, @@ -3876,7 +4105,7 @@ Future getRange(Database const& cx, end, limits, Promise>(), - true, + Snapshot::True, reverse, info, tags); @@ -3923,9 +4152,9 @@ Transaction::Transaction() : info(TaskPriority::DefaultEndpoint, generateSpanID(true)), span(info.spanID, "Transaction"_loc) {} Transaction::Transaction(Database const& cx) - : cx(cx), info(cx->taskID, generateSpanID(cx->transactionTracingEnabled)), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), - committedVersion(invalidVersion), versionstampPromise(Promise>()), options(cx), numErrors(0), - trLogInfo(createTrLogInfoProbabilistically(cx)), tr(info.spanID), span(info.spanID, "Transaction"_loc) { + : info(cx->taskID, generateSpanID(cx->transactionTracingEnabled)), numErrors(0), options(cx), + span(info.spanID, "Transaction"_loc), trLogInfo(createTrLogInfoProbabilistically(cx)), cx(cx), + backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), committedVersion(invalidVersion), tr(info.spanID) { if (DatabaseContext::debugUseTags) { debugAddTags(this); } @@ -3972,7 +4201,7 @@ void Transaction::setVersion(Version v) { readVersion = v; } -Future> Transaction::get(const Key& key, bool snapshot) { +Future> Transaction::get(const Key& key, Snapshot snapshot) { ++cx->transactionLogicalReads; ++cx->transactionGetValueRequests; // ASSERT (key < allKeys.end); @@ -4095,12 +4324,18 @@ ACTOR Future>> getAddressesForKeyActor(Key key lastLessOrEqual(serverTagKeys.begin), firstGreaterThan(serverTagKeys.end), GetRangeLimits(CLIENT_KNOBS->TOO_MANY), - false, + Reverse::False, info, options.readTags)); ASSERT(!serverTagResult.more && serverTagResult.size() < CLIENT_KNOBS->TOO_MANY); - Future futureServerUids = getRange( - cx, ver, lastLessOrEqual(ksKey), firstGreaterThan(ksKey), GetRangeLimits(1), false, info, options.readTags); + Future futureServerUids = getRange(cx, + ver, + lastLessOrEqual(ksKey), + firstGreaterThan(ksKey), + GetRangeLimits(1), + Reverse::False, + info, + options.readTags); RangeResult serverUids = wait(futureServerUids); ASSERT(serverUids.size()); // every shard needs to have a team @@ -4155,7 +4390,7 @@ ACTOR Future getKeyAndConflictRange(Database cx, } } -Future Transaction::getKey(const KeySelector& key, bool snapshot) { +Future Transaction::getKey(const KeySelector& key, Snapshot snapshot) { ++cx->transactionLogicalReads; ++cx->transactionGetKeyRequests; if (snapshot) @@ -4169,8 +4404,8 @@ Future Transaction::getKey(const KeySelector& key, bool snapshot) { Future Transaction::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { ++cx->transactionLogicalReads; ++cx->transactionGetRangeRequests; @@ -4211,8 +4446,8 @@ Future Transaction::getRange(const KeySelector& begin, Future Transaction::getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -4222,8 +4457,8 @@ Future Transaction::getRangeStream(const PromiseStream& resul const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { ++cx->transactionLogicalReads; ++cx->transactionGetRangeStreamRequests; @@ -4272,8 +4507,8 @@ Future Transaction::getRangeStream(const PromiseStream& resul const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRangeStream(results, begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -4316,7 +4551,7 @@ void Transaction::makeSelfConflicting() { tr.transaction.write_conflict_ranges.push_back(tr.arena, r); } -void Transaction::set(const KeyRef& key, const ValueRef& value, bool addConflictRange) { +void Transaction::set(const KeyRef& key, const ValueRef& value, AddConflictRange addConflictRange) { ++cx->transactionSetMutations; if (key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) @@ -4338,7 +4573,7 @@ void Transaction::set(const KeyRef& key, const ValueRef& value, bool addConflict void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationRef::Type operationType, - bool addConflictRange) { + AddConflictRange addConflictRange) { ++cx->transactionAtomicMutations; if (key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) @@ -4366,7 +4601,7 @@ void Transaction::atomicOp(const KeyRef& key, TEST(true); // NativeAPI atomic operation } -void Transaction::clear(const KeyRangeRef& range, bool addConflictRange) { +void Transaction::clear(const KeyRangeRef& range, AddConflictRange addConflictRange) { ++cx->transactionClearMutations; auto& req = tr; auto& t = req.transaction; @@ -4398,7 +4633,7 @@ void Transaction::clear(const KeyRangeRef& range, bool addConflictRange) { if (addConflictRange) t.write_conflict_ranges.push_back(req.arena, r); } -void Transaction::clear(const KeyRef& key, bool addConflictRange) { +void Transaction::clear(const KeyRef& key, AddConflictRange addConflictRange) { ++cx->transactionClearMutations; // There aren't any keys in the database with size larger than KEY_SIZE_LIMIT if (key.size() > @@ -4742,11 +4977,12 @@ ACTOR Future> estimateCommitCosts(Transac wait(getKeyRangeLocations(self->getDatabase(), keyRange, CLIENT_KNOBS->TOO_MANY, - false, + Reverse::False, &StorageServerInterface::getShardState, self->info)); - if (locations.empty()) + if (locations.empty()) { continue; + } uint64_t bytes = 0; if (locations.size() == 1) { @@ -4842,7 +5078,7 @@ ACTOR static Future tryCommit(Database cx, &CommitProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, - true); + AtMostOnce::True); } choose { @@ -5022,7 +5258,7 @@ Future Transaction::commitMutations() { if (options.debugDump) { UID u = nondeterministicRandom()->randomUniqueID(); - TraceEvent("TransactionDump", u); + TraceEvent("TransactionDump", u).log(); for (auto i = tr.transaction.mutations.begin(); i != tr.transaction.mutations.end(); ++i) TraceEvent("TransactionMutation", u) .detail("T", i->type) @@ -5067,7 +5303,10 @@ ACTOR Future commitAndWatch(Transaction* self) { self->setupWatches(); } - self->reset(); + if (!self->apiVersionAtLeast(700)) { + self->reset(); + } + return Void(); } catch (Error& e) { if (e.code() != error_code_actor_cancelled) { @@ -5076,7 +5315,10 @@ ACTOR Future commitAndWatch(Transaction* self) { } self->versionstampPromise.sendError(transaction_invalid_version()); - self->reset(); + + if (!self->apiVersionAtLeast(700)) { + self->reset(); + } } throw; @@ -5092,7 +5334,7 @@ Future Transaction::commit() { void Transaction::setOption(FDBTransactionOptions::Option option, Optional value) { switch (option) { case FDBTransactionOptions::INITIALIZE_NEW_DATABASE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (readVersion.isValid()) throw read_version_already_set(); readVersion = Version(0); @@ -5100,37 +5342,37 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional 100 || value.get().size() == 0) { throw invalid_option_value(); @@ -5167,7 +5409,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optionalidentifier.empty()) { trLogInfo->logTo(TransactionLogInfo::TRACE_LOG); } else { @@ -5178,7 +5420,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional::max()); if (maxFieldLength == 0) { @@ -5192,7 +5434,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, OptionalrandomUniqueID()); if (trLogInfo && !trLogInfo->identifier.empty()) { TraceEvent(SevInfo, "TransactionBeingTraced") @@ -5202,23 +5444,23 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional::max()) / 1000.0; break; case FDBTransactionOptions::SIZE_LIMIT: - validateOptionValue(value, true); + validateOptionValuePresent(value); options.sizeLimit = extractIntOption(value, 32, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT); break; case FDBTransactionOptions::LOCK_AWARE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.lockAware = true; options.readOnly = false; break; case FDBTransactionOptions::READ_LOCK_AWARE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (!options.lockAware) { options.lockAware = true; options.readOnly = true; @@ -5226,34 +5468,34 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional extractReadVersion(Location location, TransactionPriority priority, Reference trLogInfo, Future f, - bool lockAware, + LockAware lockAware, double startTime, Promise> metadataVersion, TagSet tags) { @@ -5552,7 +5794,7 @@ Future Transaction::getReadVersion(uint32_t flags) { options.priority, trLogInfo, req.reply.getFuture(), - options.lockAware, + LockAware{ options.lockAware }, startTime, metadataVersion, options.tags); @@ -5591,7 +5833,7 @@ ACTOR Future> getCoordinatorProtocolFromConnectPacket( NetworkAddress coordinatorAddress, Optional expectedVersion) { - state Reference>> protocolVersion = + state Reference> const> protocolVersion = FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress); loop { @@ -5616,7 +5858,7 @@ ACTOR Future> getCoordinatorProtocolFromConnectPacket( // Returns the protocol version reported by the given coordinator // If an expected version is given, the future won't return until the protocol version is different than expected ACTOR Future getClusterProtocolImpl( - Reference>> coordinator, + Reference> const> coordinator, Optional expectedVersion) { state bool needToConnect = true; @@ -5742,7 +5984,7 @@ ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRang wait(getKeyRangeLocations(cx, keys, std::numeric_limits::max(), - false, + Reverse::False, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); state int nLocs = locations.size(); @@ -5841,7 +6083,7 @@ ACTOR Future>> getReadHotRanges(Da wait(getKeyRangeLocations(cx, keys, shardLimit, - false, + Reverse::False, &StorageServerInterface::getReadHotRanges, TransactionInfo(TaskPriority::DataDistribution, span.context))); try { @@ -5909,7 +6151,7 @@ ACTOR Future, int>> waitStorageMetrics(Databa wait(getKeyRangeLocations(cx, keys, shardLimit, - false, + Reverse::False, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); if (expectedShardCount >= 0 && locations.size() != expectedShardCount) { @@ -6001,7 +6243,7 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->TOO_MANY, - false, + Reverse::False, &StorageServerInterface::getRangeSplitPoints, TransactionInfo(TaskPriority::DataDistribution, span.context))); try { @@ -6062,7 +6304,7 @@ ACTOR Future>> splitStorageMetrics(Database cx, wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT, - false, + Reverse::False, &StorageServerInterface::splitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); state StorageMetrics used; @@ -6156,7 +6398,7 @@ void Transaction::setToken(uint64_t token) { void enableClientInfoLogging() { ASSERT(networkOptions.logClientInfo.present() == false); networkOptions.logClientInfo = true; - TraceEvent(SevInfo, "ClientInfoLoggingEnabled"); + TraceEvent(SevInfo, "ClientInfoLoggingEnabled").log(); } ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID snapUID) { @@ -6169,7 +6411,7 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn &CommitProxyInterface::proxySnapReq, ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, - true /*atmostOnce*/))) { + AtMostOnce::True))) { TraceEvent("SnapCreateExit").detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID); return Void(); } @@ -6210,7 +6452,7 @@ ACTOR Future checkSafeExclusions(Database cx, vector exc } throw; } - TraceEvent("ExclusionSafetyCheckCoordinators"); + TraceEvent("ExclusionSafetyCheckCoordinators").log(); state ClientCoordinators coordinatorList(cx->getConnectionFile()); state vector>> leaderServers; leaderServers.reserve(coordinatorList.clientLeaderServers.size()); @@ -6223,7 +6465,7 @@ ACTOR Future checkSafeExclusions(Database cx, vector exc choose { when(wait(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.0))) {} when(wait(delay(3.0))) { - TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum"); + TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum").log(); return false; } } @@ -6332,16 +6574,16 @@ Future DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command); } -ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, bool lock_aware) { +ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) { state ReadYourWritesTransaction tr(cx); loop { try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) { + if (lockAware) { tr.setOption(FDBTransactionOptions::LOCK_AWARE); } - tr.set(perpetualStorageWiggleKey, enable ? LiteralStringRef("1") : LiteralStringRef("0")); + tr.set(perpetualStorageWiggleKey, enable ? "1"_sr : "0"_sr); wait(tr.commit()); break; } catch (Error& e) { diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 043bcaf4f2..3636b30c46 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -27,10 +27,12 @@ #elif !defined(FDBCLIENT_NATIVEAPI_ACTOR_H) #define FDBCLIENT_NATIVEAPI_ACTOR_H +#include "flow/BooleanParam.h" #include "flow/flow.h" #include "flow/TDMetric.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/CommitProxyInterface.h" +#include "fdbclient/ClientBooleanParams.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" @@ -51,7 +53,8 @@ void addref(DatabaseContext* ptr); template <> void delref(DatabaseContext* ptr); -void validateOptionValue(Optional value, bool shouldBePresent); +void validateOptionValuePresent(Optional value); +void validateOptionValueNotPresent(Optional value); void enableClientInfoLogging(); @@ -65,6 +68,7 @@ struct NetworkOptions { std::string traceFormat; std::string traceClockSource; std::string traceFileIdentifier; + std::string tracePartialFileSuffix; Optional logClientInfo; Reference>>> supportedVersions; bool runLoopProfilingEnabled; @@ -81,13 +85,13 @@ public: // on another thread static Database createDatabase(Reference connFile, int apiVersion, - bool internal = true, + IsInternal internal = IsInternal::True, LocalityData const& clientLocality = LocalityData(), DatabaseContext* preallocatedDb = nullptr); static Database createDatabase(std::string connFileName, int apiVersion, - bool internal = true, + IsInternal internal = IsInternal::True, LocalityData const& clientLocality = LocalityData()); Database() {} // an uninitialized database can be destructed or reassigned safely; that's it @@ -112,7 +116,7 @@ private: void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()); // Configures the global networking machinery -void setupNetwork(uint64_t transportId = 0, bool useMetrics = false); +void setupNetwork(uint64_t transportId = 0, UseMetrics = UseMetrics::False); // This call blocks while the network is running. To use the API in a single-threaded // environment, the calling program must have ACTORs already launched that are waiting @@ -186,7 +190,7 @@ struct TransactionLogInfo : public ReferenceCounted, NonCopy TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {} TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {} TransactionLogInfo(std::string id, LoggingLocation location) - : logLocation(location), identifier(id), maxFieldLength(0) {} + : logLocation(location), maxFieldLength(0), identifier(id) {} void setIdentifier(std::string id) { identifier = id; } void logTo(LoggingLocation loc) { logLocation = logLocation | loc; } @@ -228,10 +232,10 @@ struct Watch : public ReferenceCounted, NonCopyable { Promise onSetWatchTrigger; Future watchFuture; - Watch() : watchFuture(Never()), valuePresent(false), setPresent(false) {} - Watch(Key key) : key(key), watchFuture(Never()), valuePresent(false), setPresent(false) {} + Watch() : valuePresent(false), setPresent(false), watchFuture(Never()) {} + Watch(Key key) : key(key), valuePresent(false), setPresent(false), watchFuture(Never()) {} Watch(Key key, Optional val) - : key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) {} + : key(key), value(val), valuePresent(true), setPresent(false), watchFuture(Never()) {} void setWatch(Future watchFuture); }; @@ -241,31 +245,29 @@ public: explicit Transaction(Database const& cx); ~Transaction(); - void preinitializeOnForeignThread() { committedVersion = invalidVersion; } - void setVersion(Version v); Future getReadVersion() { return getReadVersion(0); } Future getRawReadVersion(); Optional getCachedReadVersion() const; - [[nodiscard]] Future> get(const Key& key, bool snapshot = false); + [[nodiscard]] Future> get(const Key& key, Snapshot = Snapshot::False); [[nodiscard]] Future watch(Reference watch); - [[nodiscard]] Future getKey(const KeySelector& key, bool snapshot = false); + [[nodiscard]] Future getKey(const KeySelector& key, Snapshot = Snapshot::False); // Future< Optional > get( const KeySelectorRef& key ); [[nodiscard]] Future getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::False, + Reverse = Reverse::False); [[nodiscard]] Future getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::False, + Reverse = Reverse::False); [[nodiscard]] Future getRange(const KeyRange& keys, int limit, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, @@ -274,8 +276,8 @@ public: } [[nodiscard]] Future getRange(const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, @@ -289,19 +291,19 @@ public: const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::False, + Reverse = Reverse::False); [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::False, + Reverse = Reverse::False); [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeyRange& keys, int limit, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRangeStream(results, KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), @@ -312,8 +314,8 @@ public: [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRangeStream(results, KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), @@ -348,13 +350,13 @@ public: // The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end] Future>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize); // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key - void set(const KeyRef& key, const ValueRef& value, bool addConflictRange = true); + void set(const KeyRef& key, const ValueRef& value, AddConflictRange = AddConflictRange::True); void atomicOp(const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, - bool addConflictRange = true); - void clear(const KeyRangeRef& range, bool addConflictRange = true); - void clear(const KeyRef& key, bool addConflictRange = true); + AddConflictRange = AddConflictRange::True); + void clear(const KeyRangeRef& range, AddConflictRange = AddConflictRange::True); + void clear(const KeyRef& key, AddConflictRange = AddConflictRange::True); [[nodiscard]] Future commit(); // Throws not_committed or commit_unknown_result errors in normal operation void setOption(FDBTransactionOptions::Option option, Optional value = Optional()); @@ -418,7 +420,7 @@ private: Database cx; double backoff; - Version committedVersion; + Version committedVersion{ invalidVersion }; CommitTransactionRequest tr; Future readVersion; Promise> metadataVersion; @@ -451,7 +453,7 @@ inline uint64_t getWriteOperationCost(uint64_t bytes) { // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value // will be 1. Otherwise, the value will be 0. -ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, bool lock_aware = false); +ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::False); #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp index 8b7ef9f06d..d47cf26ae3 100644 --- a/fdbclient/PaxosConfigTransaction.actor.cpp +++ b/fdbclient/PaxosConfigTransaction.actor.cpp @@ -18,113 +18,269 @@ * limitations under the License. */ +#include "fdbclient/DatabaseContext.h" #include "fdbclient/PaxosConfigTransaction.h" #include "flow/actorcompiler.h" // must be last include -class PaxosConfigTransactionImpl {}; +class PaxosConfigTransactionImpl { + ConfigTransactionCommitRequest toCommit; + Future getGenerationFuture; + std::vector ctis; + int numRetries{ 0 }; + bool committed{ false }; + Optional dID; + Database cx; + + ACTOR static Future getGeneration(PaxosConfigTransactionImpl* self) { + state std::vector> getGenerationFutures; + getGenerationFutures.reserve(self->ctis.size()); + for (auto const& cti : self->ctis) { + getGenerationFutures.push_back(cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{})); + } + // FIXME: Must tolerate failures and disagreement + wait(waitForAll(getGenerationFutures)); + return getGenerationFutures[0].get().generation; + } + + ACTOR static Future> get(PaxosConfigTransactionImpl* self, Key key) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); + } + state ConfigKey configKey = ConfigKey::decodeKey(key); + ConfigGeneration generation = wait(self->getGenerationFuture); + // TODO: Load balance + ConfigTransactionGetReply reply = + wait(self->ctis[0].get.getReply(ConfigTransactionGetRequest{ generation, configKey })); + if (reply.value.present()) { + return reply.value.get().toValue(); + } else { + return Optional{}; + } + } + + ACTOR static Future getConfigClasses(PaxosConfigTransactionImpl* self) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); + } + ConfigGeneration generation = wait(self->getGenerationFuture); + // TODO: Load balance + ConfigTransactionGetConfigClassesReply reply = + wait(self->ctis[0].getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ generation })); + RangeResult result; + result.reserve(result.arena(), reply.configClasses.size()); + for (const auto& configClass : reply.configClasses) { + result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr)); + } + return result; + } + + ACTOR static Future getKnobs(PaxosConfigTransactionImpl* self, Optional configClass) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); + } + ConfigGeneration generation = wait(self->getGenerationFuture); + // TODO: Load balance + ConfigTransactionGetKnobsReply reply = + wait(self->ctis[0].getKnobs.getReply(ConfigTransactionGetKnobsRequest{ generation, configClass })); + RangeResult result; + result.reserve(result.arena(), reply.knobNames.size()); + for (const auto& knobName : reply.knobNames) { + result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr)); + } + return result; + } + + ACTOR static Future commit(PaxosConfigTransactionImpl* self) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); + } + wait(store(self->toCommit.generation, self->getGenerationFuture)); + self->toCommit.annotation.timestamp = now(); + std::vector> commitFutures; + commitFutures.reserve(self->ctis.size()); + for (const auto& cti : self->ctis) { + commitFutures.push_back(cti.commit.getReply(self->toCommit)); + } + // FIXME: Must tolerate failures and disagreement + wait(quorum(commitFutures, commitFutures.size() / 2 + 1)); + self->committed = true; + return Void(); + } + +public: + Future getReadVersion() { + if (!getGenerationFuture.isValid()) { + getGenerationFuture = getGeneration(this); + } + return map(getGenerationFuture, [](auto const& gen) { return gen.committedVersion; }); + } + + Optional getCachedReadVersion() const { + if (getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError()) { + return getGenerationFuture.get().committedVersion; + } else { + return {}; + } + } + + Version getCommittedVersion() const { return committed ? getGenerationFuture.get().liveVersion : ::invalidVersion; } + + int64_t getApproximateSize() const { return toCommit.expectedSize(); } + + void set(KeyRef key, ValueRef value) { toCommit.set(key, value); } + + void clear(KeyRef key) { toCommit.clear(key); } + + Future> get(Key const& key) { return get(this, key); } + + Future getRange(KeyRangeRef keys) { + if (keys == configClassKeys) { + return getConfigClasses(this); + } else if (keys == globalConfigKnobKeys) { + return getKnobs(this, {}); + } else if (configKnobKeys.contains(keys) && keys.singleKeyRange()) { + const auto configClass = keys.begin.removePrefix(configKnobKeys.begin); + return getKnobs(this, configClass); + } else { + throw invalid_config_db_range_read(); + } + } + + Future onError(Error const& e) { + // TODO: Improve this: + if (e.code() == error_code_transaction_too_old) { + reset(); + return delay((1 << numRetries++) * 0.01 * deterministicRandom()->random01()); + } + throw e; + } + + void debugTransaction(UID dID) { this->dID = dID; } + + void reset() { + getGenerationFuture = Future{}; + toCommit = {}; + committed = false; + } + + void fullReset() { + numRetries = 0; + dID = {}; + reset(); + } + + void checkDeferredError(Error const& deferredError) const { + if (deferredError.code() != invalid_error_code) { + throw deferredError; + } + if (cx.getPtr()) { + cx->checkDeferredError(); + } + } + + Future commit() { return commit(this); } + + PaxosConfigTransactionImpl(Database const& cx) : cx(cx) { + auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators(); + ctis.reserve(coordinators.size()); + for (const auto& coordinator : coordinators) { + ctis.emplace_back(coordinator); + } + } + + PaxosConfigTransactionImpl(std::vector const& ctis) : ctis(ctis) {} +}; Future PaxosConfigTransaction::getReadVersion() { - // TODO: Implement - return ::invalidVersion; + return impl().getReadVersion(); } Optional PaxosConfigTransaction::getCachedReadVersion() const { - // TODO: Implement - return ::invalidVersion; + return impl().getCachedReadVersion(); } -Future> PaxosConfigTransaction::get(Key const& key, bool snapshot) { - // TODO: Implement - return Optional{}; +Future> PaxosConfigTransaction::get(Key const& key, Snapshot) { + return impl().get(key); } -Future> PaxosConfigTransaction::getRange(KeySelector const& begin, - KeySelector const& end, - int limit, - bool snapshot, - bool reverse) { - // TODO: Implement - ASSERT(false); - return Standalone{}; +Future PaxosConfigTransaction::getRange(KeySelector const& begin, + KeySelector const& end, + int limit, + Snapshot snapshot, + Reverse reverse) { + if (reverse) { + throw client_invalid_operation(); + } + return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } -Future> PaxosConfigTransaction::getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot, - bool reverse) { - // TODO: Implememnt - ASSERT(false); - return Standalone{}; +Future PaxosConfigTransaction::getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot snapshot, + Reverse reverse) { + if (reverse) { + throw client_invalid_operation(); + } + return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } void PaxosConfigTransaction::set(KeyRef const& key, ValueRef const& value) { - // TODO: Implememnt - ASSERT(false); + return impl().set(key, value); } void PaxosConfigTransaction::clear(KeyRef const& key) { - // TODO: Implememnt - ASSERT(false); + return impl().clear(key); } Future PaxosConfigTransaction::commit() { - // TODO: Implememnt - ASSERT(false); - return Void(); + return impl().commit(); } Version PaxosConfigTransaction::getCommittedVersion() const { - // TODO: Implement - ASSERT(false); - return ::invalidVersion; + return impl().getCommittedVersion(); } int64_t PaxosConfigTransaction::getApproximateSize() const { - // TODO: Implement - ASSERT(false); - return 0; + return impl().getApproximateSize(); } void PaxosConfigTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { - // TODO: Implement - ASSERT(false); + // TODO: Support using this option to determine atomicity } Future PaxosConfigTransaction::onError(Error const& e) { - // TODO: Implement - ASSERT(false); - return Void(); + return impl().onError(e); } void PaxosConfigTransaction::cancel() { - // TODO: Implement - ASSERT(false); + // TODO: Implement someday + throw client_invalid_operation(); } void PaxosConfigTransaction::reset() { - // TODO: Implement - ASSERT(false); + impl().reset(); } void PaxosConfigTransaction::fullReset() { - // TODO: Implement - ASSERT(false); + impl().fullReset(); } void PaxosConfigTransaction::debugTransaction(UID dID) { - // TODO: Implement - ASSERT(false); + impl().debugTransaction(dID); } void PaxosConfigTransaction::checkDeferredError() const { - // TODO: Implement - ASSERT(false); + impl().checkDeferredError(deferredError); } -PaxosConfigTransaction::PaxosConfigTransaction(Database const& cx) { - // TODO: Implement - ASSERT(false); -} +PaxosConfigTransaction::PaxosConfigTransaction(std::vector const& ctis) + : _impl(std::make_unique(ctis)) {} + +PaxosConfigTransaction::PaxosConfigTransaction() = default; PaxosConfigTransaction::~PaxosConfigTransaction() = default; + +void PaxosConfigTransaction::setDatabase(Database const& cx) { + _impl = std::make_unique(cx); +} diff --git a/fdbclient/PaxosConfigTransaction.h b/fdbclient/PaxosConfigTransaction.h index 884afdb2d1..758507b7ec 100644 --- a/fdbclient/PaxosConfigTransaction.h +++ b/fdbclient/PaxosConfigTransaction.h @@ -33,22 +33,24 @@ class PaxosConfigTransaction final : public IConfigTransaction, public FastAlloc PaxosConfigTransactionImpl& impl() { return *_impl; } public: - PaxosConfigTransaction(Database const&); + PaxosConfigTransaction(std::vector const&); + PaxosConfigTransaction(); ~PaxosConfigTransaction(); + void setDatabase(Database const&) override; Future getReadVersion() override; Optional getCachedReadVersion() const override; - Future> get(Key const& key, bool snapshot = false) override; - Future> getRange(KeySelector const& begin, - KeySelector const& end, - int limit, - bool snapshot = false, - bool reverse = false) override; - Future> getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; + Future> get(Key const& key, Snapshot = Snapshot::False) override; + Future getRange(KeySelector const& begin, + KeySelector const& end, + int limit, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; + Future getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; void set(KeyRef const& key, ValueRef const& value) override; void clear(KeyRangeRef const&) override { throw client_invalid_operation(); } void clear(KeyRef const&) override; diff --git a/fdbclient/RYWIterator.h b/fdbclient/RYWIterator.h index 8bc9091fe2..90ab1884e0 100644 --- a/fdbclient/RYWIterator.h +++ b/fdbclient/RYWIterator.h @@ -28,7 +28,7 @@ class RYWIterator { public: RYWIterator(SnapshotCache* snapshotCache, WriteMap* writeMap) - : cache(snapshotCache), writes(writeMap), begin_key_cmp(0), end_key_cmp(0), bypassUnreadable(false) {} + : begin_key_cmp(0), end_key_cmp(0), cache(snapshotCache), writes(writeMap), bypassUnreadable(false) {} enum SEGMENT_TYPE { UNKNOWN_RANGE, EMPTY_RANGE, KV }; static const SEGMENT_TYPE typeMap[12]; diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 4db07f527b..77dcf2ef05 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -65,7 +65,7 @@ public: typedef Key Result; }; - template + template struct GetRangeReq { GetRangeReq(KeySelector begin, KeySelector end, GetRangeLimits limits) : begin(begin), end(end), limits(limits) {} @@ -99,7 +99,7 @@ public: } else if (it->is_empty_range()) { return Optional(); } else { - Optional res = wait(ryw->tr.get(read.key, true)); + Optional res = wait(ryw->tr.get(read.key, Snapshot::True)); KeyRef k(ryw->arena, read.key); if (res.present()) { @@ -162,20 +162,22 @@ public: // transaction. Responsible for clipping results to the non-system keyspace when appropriate, since NativeAPI // doesn't do that. - static Future> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, bool snapshot) { + static Future> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, Snapshot snapshot) { return ryw->tr.get(read.key, snapshot); } - ACTOR static Future readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, bool snapshot) { + ACTOR static Future readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, Snapshot snapshot) { Key key = wait(ryw->tr.getKey(read.key, snapshot)); if (ryw->getMaxReadKey() < key) return ryw->getMaxReadKey(); // Filter out results in the system keys if they are not accessible return key; } - ACTOR template - static Future readThrough(ReadYourWritesTransaction* ryw, GetRangeReq read, bool snapshot) { - if (Reverse && read.end.offset > 1) { + ACTOR template + static Future readThrough(ReadYourWritesTransaction* ryw, + GetRangeReq read, + Snapshot snapshot) { + if (backwards && read.end.offset > 1) { // FIXME: Optimistically assume that this will not run into the system keys, and only reissue if the result // actually does. Key key = wait(ryw->tr.getKey(read.end, snapshot)); @@ -185,10 +187,11 @@ public: read.end = KeySelector(firstGreaterOrEqual(key), key.arena()); } - RangeResult v = wait(ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, Reverse)); + RangeResult v = wait( + ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, backwards ? Reverse::True : Reverse::False)); KeyRef maxKey = ryw->getMaxReadKey(); if (v.size() > 0) { - if (!Reverse && v[v.size() - 1].key >= maxKey) { + if (!backwards && v[v.size() - 1].key >= maxKey) { state RangeResult _v = v; int i = _v.size() - 2; for (; i >= 0 && _v[i].key >= maxKey; --i) { @@ -299,7 +302,7 @@ public: ACTOR template static Future readWithConflictRangeThrough(ReadYourWritesTransaction* ryw, Req req, - bool snapshot) { + Snapshot snapshot) { choose { when(typename Req::Result result = wait(readThrough(ryw, req, snapshot))) { return result; } when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); } @@ -316,7 +319,7 @@ public: ACTOR template static Future readWithConflictRangeRYW(ReadYourWritesTransaction* ryw, Req req, - bool snapshot) { + Snapshot snapshot) { state RYWIterator it(&ryw->cache, &ryw->writes); choose { when(typename Req::Result result = wait(read(ryw, req, &it))) { @@ -332,7 +335,7 @@ public: template static inline Future readWithConflictRange(ReadYourWritesTransaction* ryw, Req const& req, - bool snapshot) { + Snapshot snapshot) { if (ryw->options.readYourWritesDisabled) { return readWithConflictRangeThrough(ryw, req, snapshot); } else if (snapshot && ryw->options.snapshotRywEnabled <= 0) { @@ -690,7 +693,8 @@ public: //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows); additionalRows = 0; - RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, false)); + RangeResult snapshot_read = + wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::False)); KeyRangeRef range = getKnownKeyRange(snapshot_read, read_begin, read_end, ryw->arena); //TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough); @@ -993,7 +997,8 @@ public: //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows); additionalRows = 0; - RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, true)); + RangeResult snapshot_read = + wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::True)); KeyRangeRef range = getKnownKeyRangeBack(snapshot_read, read_begin, read_end, ryw->arena); //TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough); @@ -1110,7 +1115,7 @@ public: if (!ryw->options.readYourWritesDisabled) { ryw->watchMap[key].push_back(watch); - val = readWithConflictRange(ryw, GetValueReq(key), false); + val = readWithConflictRange(ryw, GetValueReq(key), Snapshot::False); } else { ryw->approximateSize += 2 * key.expectedSize() + 1; val = ryw->tr.get(key); @@ -1159,7 +1164,7 @@ public: if (!ryw->resetPromise.isSet()) ryw->resetPromise.sendError(transaction_timed_out()); wait(delay(deterministicRandom()->random01() * 5)); - TraceEvent("ClientBuggifyInFlightCommit"); + TraceEvent("ClientBuggifyInFlightCommit").log(); wait(ryw->tr.commit()); } @@ -1280,14 +1285,18 @@ public: }; ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx) - : ISingleThreadTransaction(cx->deferredError), cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0), - creationTime(now()), commitStarted(false), options(tr), versionStampFuture(tr.getVersionstamp()), - specialKeySpaceWriteMap(std::make_pair(false, Optional()), specialKeys.end) { + : ISingleThreadTransaction(cx->deferredError), tr(cx), cache(&arena), writes(&arena), retries(0), approximateSize(0), + creationTime(now()), commitStarted(false), versionStampFuture(tr.getVersionstamp()), + specialKeySpaceWriteMap(std::make_pair(false, Optional()), specialKeys.end), options(tr) { std::copy( cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(), std::back_inserter(persistentOptions)); applyPersistentOptions(); } +void ReadYourWritesTransaction::setDatabase(Database const& cx) { + *this = ReadYourWritesTransaction(cx); +} + ACTOR Future timebomb(double endTime, Promise resetPromise) { while (now() < endTime) { wait(delayUntil(std::min(endTime + 0.0001, now() + CLIENT_KNOBS->TRANSACTION_TIMEOUT_DELAY_INTERVAL))); @@ -1352,7 +1361,7 @@ ACTOR Future getWorkerInterfaces(Reference c } } -Future> ReadYourWritesTransaction::get(const Key& key, bool snapshot) { +Future> ReadYourWritesTransaction::get(const Key& key, Snapshot snapshot) { TEST(true); // ReadYourWritesTransaction::get if (getDatabase()->apiVersionAtLeast(630)) { @@ -1416,7 +1425,7 @@ Future> ReadYourWritesTransaction::get(const Key& key, bool snap return result; } -Future ReadYourWritesTransaction::getKey(const KeySelector& key, bool snapshot) { +Future ReadYourWritesTransaction::getKey(const KeySelector& key, Snapshot snapshot) { if (checkUsedDuringCommit()) { return used_during_commit(); } @@ -1435,8 +1444,8 @@ Future ReadYourWritesTransaction::getKey(const KeySelector& key, bool snaps Future ReadYourWritesTransaction::getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { if (getDatabase()->apiVersionAtLeast(630)) { if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() && end.getKey() <= specialKeys.end) { @@ -1495,8 +1504,8 @@ Future ReadYourWritesTransaction::getRange(KeySelector begin, Future ReadYourWritesTransaction::getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -1627,13 +1636,14 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& clearBegin = std::max(ExtStringRef(keys.begin), it.beginKey()); inClearRange = true; } else if (!it.is_cleared_range() && inClearRange) { - tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), false); + tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), + AddConflictRange::False); inClearRange = false; } } if (inClearRange) { - tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), false); + tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), AddConflictRange::False); } it.skip(keys.begin); @@ -1657,9 +1667,9 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& switch (op[i].type) { case MutationRef::SetValue: if (op[i].value.present()) { - tr.set(it.beginKey().assertRef(), op[i].value.get(), false); + tr.set(it.beginKey().assertRef(), op[i].value.get(), AddConflictRange::False); } else { - tr.clear(it.beginKey().assertRef(), false); + tr.clear(it.beginKey().assertRef(), AddConflictRange::False); } break; case MutationRef::AddValue: @@ -1676,7 +1686,7 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& case MutationRef::MinV2: case MutationRef::AndV2: case MutationRef::CompareAndClear: - tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, false); + tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, AddConflictRange::False); break; default: break; @@ -1729,10 +1739,6 @@ void ReadYourWritesTransaction::getWriteConflicts(KeyRangeMap* result) { } } -void ReadYourWritesTransaction::preinitializeOnForeignThread() { - tr.preinitializeOnForeignThread(); -} - void ReadYourWritesTransaction::setTransactionID(uint64_t id) { tr.setTransactionID(id); } @@ -1845,7 +1851,7 @@ RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRang } void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& operand, uint32_t operationType) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -1893,7 +1899,7 @@ void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& oper // this does validation of the key and needs to be performed before the readYourWritesDisabled path KeyRangeRef range = getVersionstampKeyRange(arena, k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey()); versionStampKeys.push_back(arena, k); - addWriteConflict = false; + addWriteConflict = AddConflictRange::False; if (!options.readYourWritesDisabled) { writeRangeToNativeTransaction(range); writes.addUnmodifiedAndUnreadableRange(range); @@ -1953,7 +1959,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) { } } - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -1983,7 +1989,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) { } void ReadYourWritesTransaction::clear(const KeyRangeRef& range) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -2036,7 +2042,7 @@ void ReadYourWritesTransaction::clear(const KeyRangeRef& range) { } void ReadYourWritesTransaction::clear(const KeyRef& key) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -2165,7 +2171,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option, void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional value) { switch (option) { case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (!reading.isReady() || !cache.empty() || !writes.empty()) throw client_invalid_operation(); @@ -2174,26 +2180,26 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti break; case FDBTransactionOptions::READ_AHEAD_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readAheadDisabled = true; break; case FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.nextWriteDisableConflictRange = true; break; case FDBTransactionOptions::ACCESS_SYSTEM_KEYS: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readSystemKeys = true; options.writeSystemKeys = true; break; case FDBTransactionOptions::READ_SYSTEM_KEYS: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readSystemKeys = true; break; @@ -2217,30 +2223,30 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti transactionDebugInfo->transactionName = value.present() ? value.get().toString() : ""; break; case FDBTransactionOptions::SNAPSHOT_RYW_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.snapshotRywEnabled++; break; case FDBTransactionOptions::SNAPSHOT_RYW_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.snapshotRywEnabled--; break; case FDBTransactionOptions::USED_DURING_COMMIT_PROTECTION_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.disableUsedDuringCommitProtection = true; break; case FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.specialKeySpaceRelaxed = true; break; case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.specialKeySpaceChangeConfiguration = true; break; case FDBTransactionOptions::BYPASS_UNREADABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.bypassUnreadable = true; break; default: @@ -2278,10 +2284,11 @@ void ReadYourWritesTransaction::operator=(ReadYourWritesTransaction&& r) noexcep } ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept - : ISingleThreadTransaction(std::move(r.deferredError)), cache(std::move(r.cache)), writes(std::move(r.writes)), - arena(std::move(r.arena)), reading(std::move(r.reading)), retries(r.retries), approximateSize(r.approximateSize), - creationTime(r.creationTime), timeoutActor(std::move(r.timeoutActor)), resetPromise(std::move(r.resetPromise)), - commitStarted(r.commitStarted), options(r.options), transactionDebugInfo(r.transactionDebugInfo) { + : ISingleThreadTransaction(std::move(r.deferredError)), arena(std::move(r.arena)), cache(std::move(r.cache)), + writes(std::move(r.writes)), resetPromise(std::move(r.resetPromise)), reading(std::move(r.reading)), + retries(r.retries), approximateSize(r.approximateSize), timeoutActor(std::move(r.timeoutActor)), + creationTime(r.creationTime), commitStarted(r.commitStarted), transactionDebugInfo(r.transactionDebugInfo), + options(r.options) { cache.arena = &arena; writes.arena = &arena; tr = std::move(r.tr); diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index 65bb972da9..53431e00ed 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -68,25 +68,26 @@ public: explicit ReadYourWritesTransaction(Database const& cx); ~ReadYourWritesTransaction(); + void setDatabase(Database const&) override; void setVersion(Version v) override { tr.setVersion(v); } Future getReadVersion() override; Optional getCachedReadVersion() const override { return tr.getCachedReadVersion(); } - Future> get(const Key& key, bool snapshot = false) override; - Future getKey(const KeySelector& key, bool snapshot = false) override; - Future> getRange(const KeySelector& begin, - const KeySelector& end, - int limit, - bool snapshot = false, - bool reverse = false) override; - Future> getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; - Future> getRange(const KeyRange& keys, - int limit, - bool snapshot = false, - bool reverse = false) { + Future> get(const Key& key, Snapshot = Snapshot::False) override; + Future getKey(const KeySelector& key, Snapshot = Snapshot::False) override; + Future getRange(const KeySelector& begin, + const KeySelector& end, + int limit, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; + Future getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; + Future getRange(const KeyRange& keys, + int limit, + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, @@ -95,8 +96,8 @@ public: } Future getRange(const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, @@ -153,8 +154,6 @@ public: void getWriteConflicts(KeyRangeMap* result) override; - void preinitializeOnForeignThread(); - Database getDatabase() const { return tr.getDatabase(); } const TransactionInfo& getTransactionInfo() const { return tr.info; } diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 96f8473947..5e2c20120f 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -26,9 +26,7 @@ ServerKnobs::ServerKnobs(Randomize randomize, ClientKnobs* clientKnobs, IsSimula initialize(randomize, clientKnobs, isSimulated); } -void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsSimulated _isSimulated) { - bool const randomize = _randomize == Randomize::YES; - bool const isSimulated = _isSimulated == IsSimulated::YES; +void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) { // clang-format off // Versions init( VERSIONS_PER_SECOND, 1e6 ); @@ -103,6 +101,8 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( PUSH_STATS_SLOW_AMOUNT, 2 ); init( PUSH_STATS_SLOW_RATIO, 0.5 ); init( TLOG_POP_BATCH_SIZE, 1000 ); if ( randomize && BUGGIFY ) TLOG_POP_BATCH_SIZE = 10; + init( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE, 250e6 ); + init( ENABLE_DETAILED_TLOG_POP_TRACE, true ); // disk snapshot max timeout, to be put in TLog, storage and coordinator nodes init( MAX_FORKED_PROCESS_OUTPUT, 1024 ); @@ -256,6 +256,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( DD_TEAMS_INFO_PRINT_YIELD_COUNT, 100 ); if( randomize && BUGGIFY ) DD_TEAMS_INFO_PRINT_YIELD_COUNT = deterministicRandom()->random01() * 1000 + 1; init( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY, 120 ); if( randomize && BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5; init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD, 1 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 10; + init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 50 ); // TeamRemover init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER, false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true @@ -335,11 +336,14 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS // KeyValueStoreRocksDB init( ROCKSDB_BACKGROUND_PARALLELISM, 0 ); init( ROCKSDB_READ_PARALLELISM, 4 ); - init( ROCKSDB_MEMTABLE_BYTES, 512 * 1024 * 1024 ); + // Use a smaller memtable in simulation to avoid OOMs. + int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024; + init( ROCKSDB_MEMTABLE_BYTES, memtableBytes ); init( ROCKSDB_UNSAFE_AUTO_FSYNC, false ); init( ROCKSDB_PERIODIC_COMPACTION_SECONDS, 0 ); init( ROCKSDB_PREFIX_LEN, 0 ); init( ROCKSDB_BLOCK_CACHE_SIZE, 0 ); + init( ROCKSDB_METRICS_DELAY, 60.0 ); // Leader election bool longLeaderElection = randomize && BUGGIFY; @@ -364,6 +368,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET, 10.0 ); init( START_TRANSACTION_MAX_QUEUE_SIZE, 1e6 ); init( KEY_LOCATION_MAX_QUEUE_SIZE, 1e6 ); + init( COMMIT_PROXY_LIVENESS_TIMEOUT, 20.0 ); init( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, 0.0005 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005; init( COMMIT_TRANSACTION_BATCH_INTERVAL_MIN, 0.001 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_MIN = 0.1; @@ -463,7 +468,15 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( REPLACE_INTERFACE_CHECK_DELAY, 5.0 ); init( COORDINATOR_REGISTER_INTERVAL, 5.0 ); init( CLIENT_REGISTER_INTERVAL, 600.0 ); - init( CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR, false ); + init( CC_ENABLE_WORKER_HEALTH_MONITOR, false ); + init( CC_WORKER_HEALTH_CHECKING_INTERVAL, 60.0 ); + init( CC_DEGRADED_LINK_EXPIRATION_INTERVAL, 300.0 ); + init( CC_MIN_DEGRADATION_INTERVAL, 120.0 ); + init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, 3 ); + init( CC_MAX_EXCLUSION_DUE_TO_HEALTH, 2 ); + init( CC_HEALTH_TRIGGER_RECOVERY, false ); + init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL, 3600.0 ); + init( CC_MAX_HEALTH_RECOVERY_COUNT, 2 ); init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0; init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit ); @@ -634,6 +647,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( DBINFO_FAILED_DELAY, 1.0 ); init( ENABLE_WORKER_HEALTH_MONITOR, false ); init( WORKER_HEALTH_MONITOR_INTERVAL, 60.0 ); + init( PEER_LATENCY_CHECK_MIN_POPULATION, 30 ); init( PEER_LATENCY_DEGRADATION_PERCENTILE, 0.90 ); init( PEER_LATENCY_DEGRADATION_THRESHOLD, 0.05 ); init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD, 0.1 ); @@ -643,7 +657,9 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS // Coordination init( COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL, 1.0 ); if( randomize && BUGGIFY ) COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL = 10.0; + init( FORWARD_REQUEST_TOO_OLD, 4*24*60*60 ); if( randomize && BUGGIFY ) FORWARD_REQUEST_TOO_OLD = 60.0; init( ENABLE_CROSS_CLUSTER_SUPPORT, true ); if( randomize && BUGGIFY ) ENABLE_CROSS_CLUSTER_SUPPORT = false; + init( COORDINATOR_LEADER_CONNECTION_TIMEOUT, 20.0 ); // Buggification init( BUGGIFIED_EVENTUAL_CONSISTENCY, 1.0 ); @@ -721,6 +737,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( REDWOOD_DEFAULT_EXTENT_READ_SIZE, 1024 * 1024 ); init( REDWOOD_EXTENT_CONCURRENT_READS, 4 ); init( REDWOOD_KVSTORE_CONCURRENT_READS, 64 ); + init( REDWOOD_KVSTORE_RANGE_PREFETCH, true ); init( REDWOOD_PAGE_REBUILD_MAX_SLACK, 0.33 ); init( REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES, 10 ); init( REDWOOD_LAZY_CLEAR_MIN_PAGES, 0 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 5e67f465ab..5205b881b2 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -20,6 +20,7 @@ #pragma once +#include "flow/BooleanParam.h" #include "flow/Knobs.h" #include "fdbrpc/fdbrpc.h" #include "fdbrpc/Locality.h" @@ -65,6 +66,8 @@ public: // message (measured in 1/1024ths, e.g. a value of 2048 yields a // factor of 2). int64_t VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD; + int64_t TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE; + bool ENABLE_DETAILED_TLOG_POP_TRACE; double TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR; int64_t TLOG_MESSAGE_BLOCK_BYTES; int64_t MAX_MESSAGE_SIZE; @@ -206,6 +209,7 @@ public: int DD_TEAMS_INFO_PRINT_YIELD_COUNT; int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY; int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle + int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle // TeamRemover to remove redundant teams bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor @@ -223,10 +227,6 @@ public: double DD_FAILURE_TIME; double DD_ZERO_HEALTHY_TEAM_DELAY; - // Redwood Storage Engine - int PREFIX_TREE_IMMEDIATE_KEY_SIZE_LIMIT; - int PREFIX_TREE_IMMEDIATE_KEY_SIZE_MIN; - // KeyValueStore SQLITE int CLEAR_BUFFER_SIZE; double READ_VALUE_TIME_ESTIMATE; @@ -275,6 +275,7 @@ public: int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS; int ROCKSDB_PREFIX_LEN; int64_t ROCKSDB_BLOCK_CACHE_SIZE; + double ROCKSDB_METRICS_DELAY; // Leader election int MAX_NOTIFICATIONS; @@ -298,6 +299,7 @@ public: double START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET; int START_TRANSACTION_MAX_QUEUE_SIZE; int KEY_LOCATION_MAX_QUEUE_SIZE; + double COMMIT_PROXY_LIVENESS_TIMEOUT; double COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE; double COMMIT_TRANSACTION_BATCH_INTERVAL_MIN; @@ -390,7 +392,23 @@ public: double REPLACE_INTERFACE_CHECK_DELAY; double COORDINATOR_REGISTER_INTERVAL; double CLIENT_REGISTER_INTERVAL; - bool CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR; + bool CC_ENABLE_WORKER_HEALTH_MONITOR; + double CC_WORKER_HEALTH_CHECKING_INTERVAL; // The interval of refreshing the degraded server list. + double CC_DEGRADED_LINK_EXPIRATION_INTERVAL; // The time period from the last degradation report after which a + // degraded server is considered healthy. + double CC_MIN_DEGRADATION_INTERVAL; // The minimum interval that a server is reported as degraded to be considered + // as degraded by Cluster Controller. + int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE; // The maximum number of degraded peers when excluding a server. When the + // number of degraded peers is more than this value, we will not exclude + // this server since it may because of server overload. + int CC_MAX_EXCLUSION_DUE_TO_HEALTH; // The max number of degraded servers to exclude by Cluster Controller due to + // degraded health. + bool CC_HEALTH_TRIGGER_RECOVERY; // If true, cluster controller will kill the master to trigger recovery when + // detecting degraded servers. If false, cluster controller only prints a warning. + double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed + // CC_MAX_HEALTH_RECOVERY_COUNT within + // CC_TRACKING_HEALTH_RECOVERY_INTERVAL. + int CC_MAX_HEALTH_RECOVERY_COUNT; // Knobs used to select the best policy (via monte carlo) int POLICY_RATING_TESTS; // number of tests per policy (in order to compare) @@ -578,6 +596,8 @@ public: double COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL; bool ENABLE_CROSS_CLUSTER_SUPPORT; // Allow a coordinator to serve requests whose connection string does not match // the local descriptor + double FORWARD_REQUEST_TOO_OLD; // Do not forward requests older than this setting + double COORDINATOR_LEADER_CONNECTION_TIMEOUT; // Buggification double BUGGIFIED_EVENTUAL_CONSISTENCY; @@ -660,7 +680,7 @@ public: int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files int REDWOOD_EXTENT_CONCURRENT_READS; // Max number of simultaneous extent disk reads in progress. int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress. - int REDWOOD_COMMIT_CONCURRENT_READS; // Max number of concurrent reads done to support commit operations + bool REDWOOD_KVSTORE_RANGE_PREFETCH; // Whether to use range read prefetching double REDWOOD_PAGE_REBUILD_MAX_SLACK; // When rebuilding pages, max slack to allow in page int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at // once diff --git a/fdbclient/SimpleConfigTransaction.actor.cpp b/fdbclient/SimpleConfigTransaction.actor.cpp index 8b03fb0d91..1e0068e288 100644 --- a/fdbclient/SimpleConfigTransaction.actor.cpp +++ b/fdbclient/SimpleConfigTransaction.actor.cpp @@ -30,39 +30,40 @@ class SimpleConfigTransactionImpl { ConfigTransactionCommitRequest toCommit; - Future getVersionFuture; + Future getGenerationFuture; ConfigTransactionInterface cti; int numRetries{ 0 }; bool committed{ false }; Optional dID; Database cx; - ACTOR static Future getReadVersion(SimpleConfigTransactionImpl* self) { + ACTOR static Future getGeneration(SimpleConfigTransactionImpl* self) { if (self->dID.present()) { TraceEvent("SimpleConfigTransactionGettingReadVersion", self->dID.get()); } - ConfigTransactionGetVersionRequest req; - ConfigTransactionGetVersionReply reply = - wait(self->cti.getVersion.getReply(ConfigTransactionGetVersionRequest{})); + ConfigTransactionGetGenerationRequest req; + ConfigTransactionGetGenerationReply reply = + wait(self->cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{})); if (self->dID.present()) { - TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get()).detail("Version", reply.version); + TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get()) + .detail("Version", reply.generation.liveVersion); } - return reply.version; + return reply.generation; } ACTOR static Future> get(SimpleConfigTransactionImpl* self, KeyRef key) { - if (!self->getVersionFuture.isValid()) { - self->getVersionFuture = getReadVersion(self); + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); } state ConfigKey configKey = ConfigKey::decodeKey(key); - Version version = wait(self->getVersionFuture); + ConfigGeneration generation = wait(self->getGenerationFuture); if (self->dID.present()) { TraceEvent("SimpleConfigTransactionGettingValue", self->dID.get()) .detail("ConfigClass", configKey.configClass) .detail("KnobName", configKey.knobName); } ConfigTransactionGetReply reply = - wait(self->cti.get.getReply(ConfigTransactionGetRequest{ version, configKey })); + wait(self->cti.get.getReply(ConfigTransactionGetRequest{ generation, configKey })); if (self->dID.present()) { TraceEvent("SimpleConfigTransactionGotValue", self->dID.get()) .detail("Value", reply.value.get().toString()); @@ -70,33 +71,32 @@ class SimpleConfigTransactionImpl { if (reply.value.present()) { return reply.value.get().toValue(); } else { - return {}; + return Optional{}; } } - ACTOR static Future> getConfigClasses(SimpleConfigTransactionImpl* self) { - if (!self->getVersionFuture.isValid()) { - self->getVersionFuture = getReadVersion(self); + ACTOR static Future getConfigClasses(SimpleConfigTransactionImpl* self) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); } - Version version = wait(self->getVersionFuture); + ConfigGeneration generation = wait(self->getGenerationFuture); ConfigTransactionGetConfigClassesReply reply = - wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ version })); - Standalone result; + wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ generation })); + RangeResult result; for (const auto& configClass : reply.configClasses) { result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr)); } return result; } - ACTOR static Future> getKnobs(SimpleConfigTransactionImpl* self, - Optional configClass) { - if (!self->getVersionFuture.isValid()) { - self->getVersionFuture = getReadVersion(self); + ACTOR static Future getKnobs(SimpleConfigTransactionImpl* self, Optional configClass) { + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); } - Version version = wait(self->getVersionFuture); + ConfigGeneration generation = wait(self->getGenerationFuture); ConfigTransactionGetKnobsReply reply = - wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ version, configClass })); - Standalone result; + wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ generation, configClass })); + RangeResult result; for (const auto& knobName : reply.knobNames) { result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr)); } @@ -104,10 +104,10 @@ class SimpleConfigTransactionImpl { } ACTOR static Future commit(SimpleConfigTransactionImpl* self) { - if (!self->getVersionFuture.isValid()) { - self->getVersionFuture = getReadVersion(self); + if (!self->getGenerationFuture.isValid()) { + self->getGenerationFuture = getGeneration(self); } - wait(store(self->toCommit.version, self->getVersionFuture)); + wait(store(self->toCommit.generation, self->getGenerationFuture)); self->toCommit.annotation.timestamp = now(); wait(self->cti.commit.getReply(self->toCommit)); self->committed = true; @@ -123,29 +123,13 @@ public: SimpleConfigTransactionImpl(ConfigTransactionInterface const& cti) : cti(cti) {} - void set(KeyRef key, ValueRef value) { - if (key == configTransactionDescriptionKey) { - toCommit.annotation.description = KeyRef(toCommit.arena, value); - } else { - ConfigKey configKey = ConfigKeyRef::decodeKey(key); - auto knobValue = IKnobCollection::parseKnobValue( - configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST); - toCommit.mutations.emplace_back_deep(toCommit.arena, configKey, knobValue.contents()); - } - } + void set(KeyRef key, ValueRef value) { toCommit.set(key, value); } - void clear(KeyRef key) { - if (key == configTransactionDescriptionKey) { - toCommit.annotation.description = ""_sr; - } else { - toCommit.mutations.emplace_back_deep( - toCommit.arena, ConfigKeyRef::decodeKey(key), Optional{}); - } - } + void clear(KeyRef key) { toCommit.clear(key); } Future> get(KeyRef key) { return get(this, key); } - Future> getRange(KeyRangeRef keys) { + Future getRange(KeyRangeRef keys) { if (keys == configClassKeys) { return getConfigClasses(this); } else if (keys == globalConfigKnobKeys) { @@ -170,23 +154,23 @@ public: } Future getReadVersion() { - if (!getVersionFuture.isValid()) - getVersionFuture = getReadVersion(this); - return getVersionFuture; + if (!getGenerationFuture.isValid()) + getGenerationFuture = getGeneration(this); + return map(getGenerationFuture, [](auto const& gen) { return gen.committedVersion; }); } Optional getCachedReadVersion() const { - if (getVersionFuture.isValid() && getVersionFuture.isReady() && !getVersionFuture.isError()) { - return getVersionFuture.get(); + if (getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError()) { + return getGenerationFuture.get().committedVersion; } else { return {}; } } - Version getCommittedVersion() const { return committed ? getVersionFuture.get() : ::invalidVersion; } + Version getCommittedVersion() const { return committed ? getGenerationFuture.get().liveVersion : ::invalidVersion; } void reset() { - getVersionFuture = Future{}; + getGenerationFuture = Future{}; toCommit = {}; committed = false; } @@ -221,23 +205,29 @@ Optional SimpleConfigTransaction::getCachedReadVersion() const { return impl().getCachedReadVersion(); } -Future> SimpleConfigTransaction::get(Key const& key, bool snapshot) { +Future> SimpleConfigTransaction::get(Key const& key, Snapshot snapshot) { return impl().get(key); } -Future> SimpleConfigTransaction::getRange(KeySelector const& begin, - KeySelector const& end, - int limit, - bool snapshot, - bool reverse) { +Future SimpleConfigTransaction::getRange(KeySelector const& begin, + KeySelector const& end, + int limit, + Snapshot snapshot, + Reverse reverse) { + if (reverse) { + throw client_invalid_operation(); + } return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } -Future> SimpleConfigTransaction::getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot, - bool reverse) { +Future SimpleConfigTransaction::getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot snapshot, + Reverse reverse) { + if (reverse) { + throw client_invalid_operation(); + } return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } @@ -290,10 +280,13 @@ void SimpleConfigTransaction::checkDeferredError() const { impl().checkDeferredError(deferredError); } -SimpleConfigTransaction::SimpleConfigTransaction(Database const& cx) - : _impl(std::make_unique(cx)) {} +void SimpleConfigTransaction::setDatabase(Database const& cx) { + _impl = std::make_unique(cx); +} SimpleConfigTransaction::SimpleConfigTransaction(ConfigTransactionInterface const& cti) : _impl(std::make_unique(cti)) {} +SimpleConfigTransaction::SimpleConfigTransaction() = default; + SimpleConfigTransaction::~SimpleConfigTransaction() = default; diff --git a/fdbclient/SimpleConfigTransaction.h b/fdbclient/SimpleConfigTransaction.h index dd779922bd..8190123271 100644 --- a/fdbclient/SimpleConfigTransaction.h +++ b/fdbclient/SimpleConfigTransaction.h @@ -43,21 +43,23 @@ class SimpleConfigTransaction final : public IConfigTransaction, public FastAllo public: SimpleConfigTransaction(ConfigTransactionInterface const&); SimpleConfigTransaction(Database const&); + SimpleConfigTransaction(); + void setDatabase(Database const&) override; ~SimpleConfigTransaction(); Future getReadVersion() override; Optional getCachedReadVersion() const override; - Future> get(Key const& key, bool snapshot = false) override; - Future> getRange(KeySelector const& begin, - KeySelector const& end, - int limit, - bool snapshot = false, - bool reverse = false) override; - Future> getRange(KeySelector begin, - KeySelector end, - GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; + Future> get(Key const& key, Snapshot = Snapshot::False) override; + Future getRange(KeySelector const& begin, + KeySelector const& end, + int limit, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; + Future getRange(KeySelector begin, + KeySelector end, + GetRangeLimits limits, + Snapshot = Snapshot::False, + Reverse = Reverse::False) override; Future commit() override; Version getCommittedVersion() const override; void setOption(FDBTransactionOptions::Option option, Optional value = Optional()) override; diff --git a/fdbclient/SnapshotCache.h b/fdbclient/SnapshotCache.h index eabd289aee..f4e110edc4 100644 --- a/fdbclient/SnapshotCache.h +++ b/fdbclient/SnapshotCache.h @@ -311,7 +311,7 @@ public: entries.insert(Entry(allKeys.end, afterAllKeys, VectorRef()), NoMetric(), true); } // Visual Studio refuses to generate these, apparently despite the standard - SnapshotCache(SnapshotCache&& r) noexcept : entries(std::move(r.entries)), arena(r.arena) {} + SnapshotCache(SnapshotCache&& r) noexcept : arena(r.arena), entries(std::move(r.entries)) {} SnapshotCache& operator=(SnapshotCache&& r) noexcept { entries = std::move(r.entries); arena = r.arena; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index fe1a2d5409..441699df2d 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -248,8 +248,9 @@ ACTOR Future normalizeKeySelectorActor(SpecialKeySpace* sks, } SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool testOnly) - : range(KeyRangeRef(spaceStartKey, spaceEndKey)), readImpls(nullptr, spaceEndKey), writeImpls(nullptr, spaceEndKey), - modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey) { + : readImpls(nullptr, spaceEndKey), + modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey), + writeImpls(nullptr, spaceEndKey), range(KeyRangeRef(spaceStartKey, spaceEndKey)) { // Default begin of KeyRangeMap is Key(), insert the range to update start key readImpls.insert(range, nullptr); writeImpls.insert(range, nullptr); @@ -277,7 +278,7 @@ ACTOR Future SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { ASSERT(ryw); choose { when(RangeResult result = @@ -293,7 +294,7 @@ ACTOR Future SpecialKeySpace::getRangeAggregationActor(SpecialKeySp KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { // This function handles ranges which cover more than one keyrange and aggregates all results // KeySelector, GetRangeLimits and reverse are all handled here state RangeResult result; @@ -413,7 +414,7 @@ Future SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { // validate limits here if (!limits.isValid()) return range_limits_invalid(); @@ -441,7 +442,7 @@ ACTOR Future> SpecialKeySpace::getActor(SpecialKeySpace* sks, KeySelector(firstGreaterOrEqual(key)), KeySelector(firstGreaterOrEqual(keyAfter(key))), GetRangeLimits(CLIENT_KNOBS->TOO_MANY), - false)); + Reverse::False)); ASSERT(result.size() <= 1); if (result.size()) { return Optional(result[0].value); diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 8076c320b9..ed7e6da46a 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -168,7 +168,7 @@ public: KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse = false); + Reverse = Reverse::False); void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value); @@ -209,13 +209,13 @@ private: KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse); + Reverse reverse); ACTOR static Future getRangeAggregationActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse); + Reverse reverse); KeyRangeMap readImpls; KeyRangeMap modules; diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp index 79f2e2bc4b..d379a0fa69 100644 --- a/fdbclient/StorageServerInterface.cpp +++ b/fdbclient/StorageServerInterface.cpp @@ -30,32 +30,31 @@ std::string traceChecksumValue(ValueRef s) { return s.size() > 12 ? format("(%d)%08x", s.size(), crc32c_append(0, s.begin(), s.size())) : s.toString(); } +// point reads template <> -bool TSS_doCompare(const GetValueRequest& req, - const GetValueReply& src, - const GetValueReply& tss, - Severity traceSeverity, - UID tssId) { - if (src.value.present() != tss.value.present() || (src.value.present() && src.value.get() != tss.value.get())) { - TraceEvent(traceSeverity, "TSSMismatchGetValue") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail("Key", req.key.printable()) - .detail("Version", req.version) - .detail("SSReply", src.value.present() ? traceChecksumValue(src.value.get()) : "missing") - .detail("TSSReply", tss.value.present() ? traceChecksumValue(tss.value.get()) : "missing"); - - return false; - } - return true; +bool TSS_doCompare(const GetValueReply& src, const GetValueReply& tss) { + return src.value.present() == tss.value.present() && (!src.value.present() || src.value.get() == tss.value.get()); } template <> -bool TSS_doCompare(const GetKeyRequest& req, - const GetKeyReply& src, - const GetKeyReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const GetValueRequest& req) { + return "TSSMismatchGetValue"; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetValueRequest& req, + const GetValueReply& src, + const GetValueReply& tss) { + event.detail("Key", req.key.printable()) + .detail("Version", req.version) + .detail("SSReply", src.value.present() ? traceChecksumValue(src.value.get()) : "missing") + .detail("TSSReply", tss.value.present() ? traceChecksumValue(tss.value.get()) : "missing"); +} + +// key selector reads +template <> +bool TSS_doCompare(const GetKeyReply& src, const GetKeyReply& tss) { // This process is a bit complicated. Since the tss and ss can return different results if neighboring shards to // req.sel.key are currently being moved, We validate that the results are the same IF the returned key selectors // are final. Otherwise, we only mark the request as a mismatch if the difference between the two returned key @@ -92,107 +91,211 @@ bool TSS_doCompare(const GetKeyRequest& req, bool tssOffsetLarger = (src.sel.offset == tss.sel.offset) ? tss.sel.orEqual : src.sel.offset < tss.sel.offset; matches = tssKeyLarger != tssOffsetLarger; } - if (!matches) { - TraceEvent(traceSeverity, "TSSMismatchGetKey") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail("KeySelector", - format("%s%s:%d", req.sel.orEqual ? "=" : "", req.sel.getKey().printable().c_str(), req.sel.offset)) - .detail("Version", req.version) - .detail("SSReply", - format("%s%s:%d", src.sel.orEqual ? "=" : "", src.sel.getKey().printable().c_str(), src.sel.offset)) - .detail( - "TSSReply", - format("%s%s:%d", tss.sel.orEqual ? "=" : "", tss.sel.getKey().printable().c_str(), tss.sel.offset)); - } return matches; } template <> -bool TSS_doCompare(const GetKeyValuesRequest& req, - const GetKeyValuesReply& src, - const GetKeyValuesReply& tss, - Severity traceSeverity, - UID tssId) { - if (src.more != tss.more || src.data != tss.data) { +const char* TSS_mismatchTraceName(const GetKeyRequest& req) { + return "TSSMismatchGetKey"; +} - std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); - for (auto& it : src.data) { - ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); - } +template <> +void TSS_traceMismatch(TraceEvent& event, const GetKeyRequest& req, const GetKeyReply& src, const GetKeyReply& tss) { + event + .detail("KeySelector", + format("%s%s:%d", req.sel.orEqual ? "=" : "", req.sel.getKey().printable().c_str(), req.sel.offset)) + .detail("Version", req.version) + .detail("SSReply", + format("%s%s:%d", src.sel.orEqual ? "=" : "", src.sel.getKey().printable().c_str(), src.sel.offset)) + .detail("TSSReply", + format("%s%s:%d", tss.sel.orEqual ? "=" : "", tss.sel.getKey().printable().c_str(), tss.sel.offset)); +} - std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); - for (auto& it : tss.data) { - tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); - } +// range reads +template <> +bool TSS_doCompare(const GetKeyValuesReply& src, const GetKeyValuesReply& tss) { + return src.more == tss.more && src.data == tss.data; +} - TraceEvent(traceSeverity, "TSSMismatchGetKeyValues") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail( - "Begin", - format( - "%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) - .detail("End", - format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) - .detail("Version", req.version) - .detail("Limit", req.limit) - .detail("LimitBytes", req.limitBytes) - .detail("SSReply", ssResultsString) - .detail("TSSReply", tssResultsString); +template <> +const char* TSS_mismatchTraceName(const GetKeyValuesRequest& req) { + return "TSSMismatchGetKeyValues"; +} - return false; +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetKeyValuesRequest& req, + const GetKeyValuesReply& src, + const GetKeyValuesReply& tss) { + std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); + for (auto& it : src.data) { + ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); } + + std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); + for (auto& it : tss.data) { + tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + event + .detail( + "Begin", + format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) + .detail("End", + format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) + .detail("Version", req.version) + .detail("Limit", req.limit) + .detail("LimitBytes", req.limitBytes) + .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10) + .detail("SSReply", ssResultsString) + .detail("TSSReply", tssResultsString); +} + +// streaming range reads +template <> +bool TSS_doCompare(const GetKeyValuesStreamReply& src, const GetKeyValuesStreamReply& tss) { + return src.more == tss.more && src.data == tss.data; +} + +template <> +const char* TSS_mismatchTraceName(const GetKeyValuesStreamRequest& req) { + return "TSSMismatchGetKeyValuesStream"; +} + +// TODO this is all duplicated from above, simplify? +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetKeyValuesStreamRequest& req, + const GetKeyValuesStreamReply& src, + const GetKeyValuesStreamReply& tss) { + std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); + for (auto& it : src.data) { + ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + + std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); + for (auto& it : tss.data) { + tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + event + .detail( + "Begin", + format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) + .detail("End", + format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) + .detail("Version", req.version) + .detail("Limit", req.limit) + .detail("LimitBytes", req.limitBytes) + .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10) + .detail("SSReply", ssResultsString) + .detail("TSSReply", tssResultsString); +} + +template <> +bool TSS_doCompare(const WatchValueReply& src, const WatchValueReply& tss) { + // We duplicate watches just for load, no need to validate replies. return true; } template <> -bool TSS_doCompare(const WatchValueRequest& req, - const WatchValueReply& src, - const WatchValueReply& tss, - Severity traceSeverity, - UID tssId) { - // We duplicate watches just for load, no need to validte replies. - return true; +const char* TSS_mismatchTraceName(const WatchValueRequest& req) { + ASSERT(false); + return ""; } -// no-op template specializations for metrics replies template <> -bool TSS_doCompare(const WaitMetricsRequest& req, - const StorageMetrics& src, - const StorageMetrics& tss, - Severity traceSeverity, - UID tssId) { +void TSS_traceMismatch(TraceEvent& event, + const WatchValueRequest& req, + const WatchValueReply& src, + const WatchValueReply& tss) { + ASSERT(false); +} + +// template specializations for metrics replies that should never be called because these requests aren't duplicated + +// storage metrics +template <> +bool TSS_doCompare(const StorageMetrics& src, const StorageMetrics& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const SplitMetricsRequest& req, - const SplitMetricsReply& src, - const SplitMetricsReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const WaitMetricsRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const WaitMetricsRequest& req, + const StorageMetrics& src, + const StorageMetrics& tss) { + ASSERT(false); +} + +// split metrics +template <> +bool TSS_doCompare(const SplitMetricsReply& src, const SplitMetricsReply& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const ReadHotSubRangeRequest& req, - const ReadHotSubRangeReply& src, - const ReadHotSubRangeReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const SplitMetricsRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const SplitMetricsRequest& req, + const SplitMetricsReply& src, + const SplitMetricsReply& tss) { + ASSERT(false); +} + +// read hot sub range +template <> +bool TSS_doCompare(const ReadHotSubRangeReply& src, const ReadHotSubRangeReply& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const SplitRangeRequest& req, - const SplitRangeReply& src, - const SplitRangeReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const ReadHotSubRangeRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const ReadHotSubRangeRequest& req, + const ReadHotSubRangeReply& src, + const ReadHotSubRangeReply& tss) { + ASSERT(false); +} + +// split range +template <> +bool TSS_doCompare(const SplitRangeReply& src, const SplitRangeReply& tss) { + ASSERT(false); return true; } +template <> +const char* TSS_mismatchTraceName(const SplitRangeRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const SplitRangeRequest& req, + const SplitRangeReply& src, + const SplitRangeReply& tss) { + ASSERT(false); +} + // only record metrics for data reads template <> @@ -228,6 +331,9 @@ void TSSMetrics::recordLatency(const ReadHotSubRangeRequest& req, double ssLaten template <> void TSSMetrics::recordLatency(const SplitRangeRequest& req, double ssLatency, double tssLatency) {} +template <> +void TSSMetrics::recordLatency(const GetKeyValuesStreamRequest& req, double ssLatency, double tssLatency) {} + // ------------------- TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { @@ -240,32 +346,20 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { std::string s_d = "d"; std::string s_e = "e"; - // test getValue - GetValueRequest gvReq; - gvReq.key = StringRef(s_a); - gvReq.version = 5; - UID tssId; GetValueReply gvReplyMissing; GetValueReply gvReplyA(Optional(StringRef(s_a)), false); GetValueReply gvReplyB(Optional(StringRef(s_b)), false); - ASSERT(TSS_doCompare(gvReq, gvReplyMissing, gvReplyMissing, SevInfo, tssId)); - ASSERT(TSS_doCompare(gvReq, gvReplyA, gvReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gvReq, gvReplyB, gvReplyB, SevInfo, tssId)); + ASSERT(TSS_doCompare(gvReplyMissing, gvReplyMissing)); + ASSERT(TSS_doCompare(gvReplyA, gvReplyA)); + ASSERT(TSS_doCompare(gvReplyB, gvReplyB)); - ASSERT(!TSS_doCompare(gvReq, gvReplyMissing, gvReplyA, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gvReq, gvReplyA, gvReplyB, SevInfo, tssId)); + ASSERT(!TSS_doCompare(gvReplyMissing, gvReplyA)); + ASSERT(!TSS_doCompare(gvReplyA, gvReplyB)); // test GetKeyValues - Arena a; // for all of the refs. ASAN complains if this isn't done. Could also make them all standalone i guess - GetKeyValuesRequest gkvReq; - gkvReq.begin = firstGreaterOrEqual(StringRef(a, s_a)); - gkvReq.end = firstGreaterOrEqual(StringRef(a, s_b)); - gkvReq.version = 5; - gkvReq.limit = 100; - gkvReq.limitBytes = 1000; - + Arena a; GetKeyValuesReply gkvReplyEmpty; GetKeyValuesReply gkvReplyOne; KeyValueRef v; @@ -276,16 +370,11 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { gkvReplyOneMore.data.push_back_deep(gkvReplyOneMore.arena, v); gkvReplyOneMore.more = true; - ASSERT(TSS_doCompare(gkvReq, gkvReplyEmpty, gkvReplyEmpty, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkvReq, gkvReplyOne, gkvReplyOne, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkvReq, gkvReplyOneMore, gkvReplyOneMore, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gkvReq, gkvReplyEmpty, gkvReplyOne, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gkvReq, gkvReplyOne, gkvReplyOneMore, SevInfo, tssId)); - - // test GetKey - GetKeyRequest gkReq; - gkReq.sel = KeySelectorRef(StringRef(a, s_a), false, 1); - gkReq.version = 5; + ASSERT(TSS_doCompare(gkvReplyEmpty, gkvReplyEmpty)); + ASSERT(TSS_doCompare(gkvReplyOne, gkvReplyOne)); + ASSERT(TSS_doCompare(gkvReplyOneMore, gkvReplyOneMore)); + ASSERT(!TSS_doCompare(gkvReplyEmpty, gkvReplyOne)); + ASSERT(!TSS_doCompare(gkvReplyOne, gkvReplyOneMore)); GetKeyReply gkReplyA(KeySelectorRef(StringRef(a, s_a), false, 20), false); GetKeyReply gkReplyB(KeySelectorRef(StringRef(a, s_b), false, 10), false); @@ -294,85 +383,58 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { GetKeyReply gkReplyE(KeySelectorRef(StringRef(a, s_e), false, -20), false); // identical cases - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyB, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyD, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyE, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyB)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyD)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyE)); // relative offset cases - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyB, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyB, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyB)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyB)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyD, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyE, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyE, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyD, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyD)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyE)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyE)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyD)); // test same offset/orEqual wrong key - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); // this could be from different shard boundaries, so don't say it's a mismatch - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 10), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 10), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false))); // test offsets and key difference don't match - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, -10), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false), - SevInfo, - tssId)); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, -10), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false))); // test key is next over in one shard, one found it and other didn't // positive // one that didn't find is +1 - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 1), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 1), false))); // negative will have zero offset but not equal set - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); // test shard boundary key returned by incomplete query is the same as the key found by the other (only possible in // positive direction) - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false))); // explictly test checksum function std::string s12 = "ABCDEFGHIJKL"; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index a7a6396039..c3b7e85343 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -129,7 +129,7 @@ void decodeKeyServersValue(RangeResult result, std::sort(src.begin(), src.end()); std::sort(dest.begin(), dest.end()); if (missingIsError && (src.size() != srcTag.size() || dest.size() != destTag.size())) { - TraceEvent(SevError, "AttemptedToDecodeMissingTag"); + TraceEvent(SevError, "AttemptedToDecodeMissingTag").log(); for (const KeyValueRef& kv : result) { Tag tag = decodeServerTagValue(kv.value); UID serverID = decodeServerTagKey(kv.key); @@ -364,6 +364,8 @@ UID decodeTssQuarantineKey(KeyRef const& key) { return serverID; } +const KeyRangeRef tssMismatchKeys(LiteralStringRef("\xff/tssMismatch/"), LiteralStringRef("\xff/tssMismatch0")); + const KeyRangeRef serverTagKeys(LiteralStringRef("\xff/serverTag/"), LiteralStringRef("\xff/serverTag0")); const KeyRef serverTagPrefix = serverTagKeys.begin; diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 5bdf88419c..a60998edf4 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -124,6 +124,10 @@ extern const KeyRangeRef tssQuarantineKeys; const Key tssQuarantineKeyFor(UID serverID); UID decodeTssQuarantineKey(KeyRef const&); +// \xff/tssMismatch/[[Tuple]] := [[TraceEventString]] +// For recording tss mismatch details in the system keyspace +extern const KeyRangeRef tssMismatchKeys; + // "\xff/serverTag/[[serverID]]" = "[[Tag]]" // Provides the Tag for the given serverID. Used to access a // storage server's corresponding TLog in order to apply mutations. diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.actor.cpp index 4e17a1c9f7..a1f6526fd3 100644 --- a/fdbclient/TaskBucket.actor.cpp +++ b/fdbclient/TaskBucket.actor.cpp @@ -22,6 +22,11 @@ #include "fdbclient/ReadYourWrites.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DEFINE_BOOLEAN_PARAM(AccessSystemKeys); +FDB_DEFINE_BOOLEAN_PARAM(PriorityBatch); +FDB_DEFINE_BOOLEAN_PARAM(VerifyTask); +FDB_DEFINE_BOOLEAN_PARAM(UpdateParams); + Reference Task::getDoneFuture(Reference fb) { return fb->unpack(params[reservedTaskParamKeyDone]); } @@ -168,14 +173,14 @@ public: { // Get a task key that is <= a random UID task key, if successful then return it - Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), true)); + Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), Snapshot::True)); if (space.contains(k)) return Optional(k); } { // Get a task key that is <= the maximum possible UID, if successful return it. - Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), true)); + Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), Snapshot::True)); if (space.contains(k)) return Optional(k); } @@ -328,7 +333,7 @@ public: Reference futureBucket, Reference task, Reference taskFunc, - bool verifyTask) { + VerifyTask verifyTask) { bool isFinished = wait(taskBucket->isFinished(tr, task)); if (isFinished) { return Void(); @@ -390,7 +395,7 @@ public: taskBucket->setOptions(tr); // Attempt to extend the task's timeout - state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, false)); + state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::False)); wait(tr->commit()); task->timeoutVersion = newTimeout; versionNow = tr->getCommittedVersion(); @@ -406,15 +411,16 @@ public: Reference taskBucket, Reference futureBucket, Reference task) { + state Reference taskFunc; + state VerifyTask verifyTask = false; + if (!task || !TaskFuncBase::isValidTask(task)) return false; - state Reference taskFunc; - try { taskFunc = TaskFuncBase::create(task->params[Task::reservedTaskParamKeyType]); if (taskFunc) { - state bool verifyTask = (task->params.find(Task::reservedTaskParamValidKey) != task->params.end()); + verifyTask.set(task->params.find(Task::reservedTaskParamValidKey) != task->params.end()); if (verifyTask) { loop { @@ -472,7 +478,7 @@ public: ACTOR static Future dispatch(Database cx, Reference taskBucket, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { state std::vector> tasks(maxConcurrentTasks); for (auto& f : tasks) @@ -569,7 +575,7 @@ public: ACTOR static Future run(Database cx, Reference taskBucket, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { state Reference> paused = makeReference>(true); state Future watchPausedFuture = watchPaused(cx, taskBucket, paused); @@ -812,7 +818,7 @@ public: ACTOR static Future extendTimeout(Reference tr, Reference taskBucket, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion) { taskBucket->setOptions(tr); @@ -863,14 +869,17 @@ public: } }; -TaskBucket::TaskBucket(const Subspace& subspace, bool sysAccess, bool priorityBatch, bool lockAware) - : prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), available(prefix.get(LiteralStringRef("av"))), - available_prioritized(prefix.get(LiteralStringRef("avp"))), timeouts(prefix.get(LiteralStringRef("to"))), - pauseKey(prefix.pack(LiteralStringRef("pause"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS), - system_access(sysAccess), priority_batch(priorityBatch), lock_aware(lockAware), cc("TaskBucket"), - dbgid(deterministicRandom()->randomUniqueID()), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc), - dispatchErrors("DispatchErrors", cc), dispatchDoTasks("DispatchDoTasks", cc), - dispatchEmptyTasks("DispatchEmptyTasks", cc), dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc) {} +TaskBucket::TaskBucket(const Subspace& subspace, + AccessSystemKeys sysAccess, + PriorityBatch priorityBatch, + LockAware lockAware) + : cc("TaskBucket"), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc), dispatchErrors("DispatchErrors", cc), + dispatchDoTasks("DispatchDoTasks", cc), dispatchEmptyTasks("DispatchEmptyTasks", cc), + dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc), dbgid(deterministicRandom()->randomUniqueID()), + prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), pauseKey(prefix.pack(LiteralStringRef("pause"))), + available(prefix.get(LiteralStringRef("av"))), available_prioritized(prefix.get(LiteralStringRef("avp"))), + timeouts(prefix.get(LiteralStringRef("to"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS), + system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware) {} TaskBucket::~TaskBucket() {} @@ -971,7 +980,7 @@ Future TaskBucket::doTask(Database cx, Reference futureBucke Future TaskBucket::run(Database cx, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { return TaskBucketImpl::run(cx, Reference::addRef(this), futureBucket, pollDelay, maxConcurrentTasks); } @@ -1001,7 +1010,7 @@ Future TaskBucket::finish(Reference tr, Referen Future TaskBucket::extendTimeout(Reference tr, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion) { return TaskBucketImpl::extendTimeout( tr, Reference::addRef(this), task, updateParams, newTimeoutVersion); @@ -1041,8 +1050,8 @@ public: } }; -FutureBucket::FutureBucket(const Subspace& subspace, bool sysAccess, bool lockAware) - : prefix(subspace), system_access(sysAccess), lock_aware(lockAware) {} +FutureBucket::FutureBucket(const Subspace& subspace, AccessSystemKeys sysAccess, LockAware lockAware) + : prefix(subspace), system_access(sysAccess), lockAware(lockAware) {} FutureBucket::~FutureBucket() {} diff --git a/fdbclient/TaskBucket.h b/fdbclient/TaskBucket.h index dcdf0dad0c..e492f26226 100644 --- a/fdbclient/TaskBucket.h +++ b/fdbclient/TaskBucket.h @@ -35,6 +35,11 @@ class FutureBucket; class TaskFuture; +FDB_DECLARE_BOOLEAN_PARAM(AccessSystemKeys); +FDB_DECLARE_BOOLEAN_PARAM(PriorityBatch); +FDB_DECLARE_BOOLEAN_PARAM(VerifyTask); +FDB_DECLARE_BOOLEAN_PARAM(UpdateParams); + // A Task is a set of key=value parameters that constitute a unit of work for a TaskFunc to perform. // The parameter keys are specific to the TaskFunc that the Task is for, except for a set of reserved // parameter keys which are used by TaskBucket to determine which TaskFunc to run and provide @@ -134,13 +139,16 @@ class FutureBucket; // instance may declare the Task a failure and move it back to the available subspace. class TaskBucket : public ReferenceCounted { public: - TaskBucket(const Subspace& subspace, bool sysAccess = false, bool priorityBatch = false, bool lockAware = false); + TaskBucket(const Subspace& subspace, + AccessSystemKeys = AccessSystemKeys::False, + PriorityBatch = PriorityBatch::False, + LockAware = LockAware::False); virtual ~TaskBucket(); void setOptions(Reference tr) { if (system_access) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) + if (lockAware) tr->setOption(FDBTransactionOptions::LOCK_AWARE); } @@ -191,7 +199,10 @@ public: Future doOne(Database cx, Reference futureBucket); - Future run(Database cx, Reference futureBucket, double* pollDelay, int maxConcurrentTasks); + Future run(Database cx, + Reference futureBucket, + std::shared_ptr pollDelay, + int maxConcurrentTasks); Future watchPaused(Database cx, Reference> paused); Future isEmpty(Reference tr); @@ -207,11 +218,11 @@ public: // Extend the task's timeout as if it just started and also save any parameter changes made to the task Future extendTimeout(Reference tr, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion = invalidVersion); Future extendTimeout(Database cx, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion = invalidVersion) { return map(runRYWTransaction(cx, [=](Reference tr) { @@ -250,7 +261,7 @@ public: bool getSystemAccess() const { return system_access; } - bool getLockAware() const { return lock_aware; } + bool getLockAware() const { return lockAware; } Key getPauseKey() const { return pauseKey; } @@ -293,20 +304,20 @@ private: uint32_t timeout; bool system_access; bool priority_batch; - bool lock_aware; + bool lockAware; }; class TaskFuture; class FutureBucket : public ReferenceCounted { public: - FutureBucket(const Subspace& subspace, bool sysAccess = false, bool lockAware = false); + FutureBucket(const Subspace& subspace, AccessSystemKeys = AccessSystemKeys::False, LockAware = LockAware::False); virtual ~FutureBucket(); void setOptions(Reference tr) { if (system_access) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) + if (lockAware) tr->setOption(FDBTransactionOptions::LOCK_AWARE); } @@ -324,7 +335,7 @@ public: Reference unpack(Key key); bool isSystemAccess() const { return system_access; }; - bool isLockAware() const { return lock_aware; }; + bool isLockAware() const { return lockAware; }; private: friend class TaskFuture; @@ -333,7 +344,7 @@ private: Subspace prefix; bool system_access; - bool lock_aware; + bool lockAware; }; class TaskFuture : public ReferenceCounted { diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 5e01474712..47649747a9 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -122,7 +122,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) [db, connFile, apiVersion]() { try { Database::createDatabase( - Reference(connFile), apiVersion, false, LocalityData(), db) + Reference(connFile), apiVersion, IsInternal::False, LocalityData(), db) .extractPtr(); } catch (Error& e) { new (db) DatabaseContext(e); @@ -149,9 +149,9 @@ ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx, ISingleThreadT auto tr = this->tr = ISingleThreadTransaction::allocateOnForeignThread(type); // No deferred error -- if the construction of the RYW transaction fails, we have no where to put it onMainThreadVoid( - [tr, type, cx]() { + [tr, cx]() { cx->addref(); - ISingleThreadTransaction::create(tr, type, Database(cx)); + tr->setDatabase(Database(cx)); }, nullptr); } @@ -192,7 +192,7 @@ ThreadFuture> ThreadSafeTransaction::get(const KeyRef& key, bool ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, k, snapshot]() -> Future> { tr->checkDeferredError(); - return tr->get(k, snapshot); + return tr->get(k, Snapshot{ snapshot }); }); } @@ -202,7 +202,7 @@ ThreadFuture ThreadSafeTransaction::getKey(const KeySelectorRef& key, bool ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, k, snapshot]() -> Future { tr->checkDeferredError(); - return tr->getKey(k, snapshot); + return tr->getKey(k, Snapshot{ snapshot }); }); } @@ -238,7 +238,7 @@ ThreadFuture ThreadSafeTransaction::getRange(const KeySelectorRef& ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, b, e, limit, snapshot, reverse]() -> Future { tr->checkDeferredError(); - return tr->getRange(b, e, limit, snapshot, reverse); + return tr->getRange(b, e, limit, Snapshot{ snapshot }, Reverse{ reverse }); }); } @@ -253,7 +253,7 @@ ThreadFuture ThreadSafeTransaction::getRange(const KeySelectorRef& ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, b, e, limits, snapshot, reverse]() -> Future { tr->checkDeferredError(); - return tr->getRange(b, e, limits, snapshot, reverse); + return tr->getRange(b, e, limits, Snapshot{ snapshot }, Reverse{ reverse }); }); } diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h index b9da8621a0..32371689a2 100644 --- a/fdbclient/VersionedMap.h +++ b/fdbclient/VersionedMap.h @@ -58,11 +58,11 @@ struct PTree : public ReferenceCounted>, FastAllocated>, NonCo Reference left(Version at) const { return child(false, at); } Reference right(Version at) const { return child(true, at); } - PTree(const T& data, Version ver) : data(data), lastUpdateVersion(ver), updated(false) { + PTree(const T& data, Version ver) : lastUpdateVersion(ver), updated(false), data(data) { priority = deterministicRandom()->randomUInt32(); } PTree(uint32_t pri, T const& data, Reference const& left, Reference const& right, Version ver) - : priority(pri), data(data), lastUpdateVersion(ver), updated(false) { + : priority(pri), lastUpdateVersion(ver), updated(false), data(data) { pointer[0] = left; pointer[1] = right; } diff --git a/fdbclient/WriteMap.h b/fdbclient/WriteMap.h index 0471c16270..129509b1b4 100644 --- a/fdbclient/WriteMap.h +++ b/fdbclient/WriteMap.h @@ -168,7 +168,7 @@ private: typedef Reference Tree; public: - explicit WriteMap(Arena* arena) : arena(arena), ver(-1), scratch_iterator(this), writeMapEmpty(true) { + explicit WriteMap(Arena* arena) : arena(arena), writeMapEmpty(true), ver(-1), scratch_iterator(this) { PTreeImpl::insert( writes, ver, WriteMapEntry(allKeys.begin, OperationStack(), false, false, false, false, false)); PTreeImpl::insert(writes, ver, WriteMapEntry(allKeys.end, OperationStack(), false, false, false, false, false)); @@ -177,8 +177,8 @@ public: } WriteMap(WriteMap&& r) noexcept - : writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver), - scratch_iterator(std::move(r.scratch_iterator)), arena(r.arena) {} + : arena(r.arena), writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver), + scratch_iterator(std::move(r.scratch_iterator)) {} WriteMap& operator=(WriteMap&& r) noexcept { writeMapEmpty = r.writeMapEmpty; writes = std::move(r.writes); diff --git a/fdbclient/rapidjson/internal/stack.h b/fdbclient/rapidjson/internal/stack.h index 7ab15d42a0..fa43aa0171 100644 --- a/fdbclient/rapidjson/internal/stack.h +++ b/fdbclient/rapidjson/internal/stack.h @@ -17,6 +17,7 @@ #include "../allocators.h" #include "swap.h" +#include #if defined(__clang__) RAPIDJSON_DIAG_PUSH @@ -106,7 +107,7 @@ public: template RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + if (RAPIDJSON_UNLIKELY(static_cast(sizeof(T) * count) > (stackEnd_ - stackTop_))) Expand(count); } @@ -118,7 +119,7 @@ public: template RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + RAPIDJSON_ASSERT(static_cast(sizeof(T) * count) <= (stackEnd_ - stackTop_)); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; diff --git a/fdbclient/vexillographer/c.cs b/fdbclient/vexillographer/c.cs index 2ea6675dff..dab01ff5cf 100644 --- a/fdbclient/vexillographer/c.cs +++ b/fdbclient/vexillographer/c.cs @@ -52,7 +52,7 @@ namespace vexillographer { string parameterComment = ""; if (o.scope.ToString().EndsWith("Option")) - parameterComment = String.Format("{0}/* {1} */\n", indent, "Parameter: " + o.getParameterComment()); + parameterComment = String.Format("{0}/* {1} {2}*/\n", indent, "Parameter: " + o.getParameterComment(), o.hidden ? "This is a hidden parameter and should not be used directly by applications." : ""); return String.Format("{0}/* {2} */\n{5}{0}{1}{3}={4}", indent, prefix, o.comment, o.name.ToUpper(), o.code, parameterComment); } @@ -64,7 +64,7 @@ namespace vexillographer options = new Option[] { new Option{ scope = scope, comment = "This option is only a placeholder for C compatibility and should not be used", code = -1, name = "DUMMY_DO_NOT_USE", paramDesc = null } }; - outFile.WriteLine(string.Join(",\n\n", options.Where(f => !f.hidden).Select(f => getCLine(f, " ", prefix)).ToArray())); + outFile.WriteLine(string.Join(",\n\n", options.Select(f => getCLine(f, " ", prefix)).ToArray())); outFile.WriteLine("}} FDB{0};", scope.ToString()); outFile.WriteLine(); } diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index 15ba1250ca..6eede67882 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -57,6 +57,9 @@ description is not currently required but encouraged.