wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature-range-feed' into blob_full

This commit is contained in:
Josh Slocum 2021-08-10 11:25:48 -05:00
parent 247602753d a1b0053b57
commit 921a2cfca1
274 changed files with 5554 additions and 2938 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -8,7 +8,7 @@ bindings/java/foundationdb-client*.jar
bindings/java/foundationdb-tests*.jar
bindings/java/fdb-java-*-sources.jar
packaging/msi/FDBInstaller.msi
builds/
build/
cmake-build-debug/
# Generated source, build, and packaging files
*.g.cpp

2
CMakeLists.txt
View File

@ -171,7 +171,7 @@ add_subdirectory(fdbbackup)
add_subdirectory(contrib)
add_subdirectory(tests)
add_subdirectory(flowbench EXCLUDE_FROM_ALL)
if(WITH_PYTHON)
if(WITH_PYTHON AND WITH_C_BINDING)
add_subdirectory(bindings)
endif()
if(WITH_DOCUMENTATION)

42
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -42,7 +42,7 @@ FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin)
key_data_set(false), verify_peers_set(false) {
if ((tls_cfg = tls_config_new()) == nullptr) {
TraceEvent(SevError, "FDBLibTLSConfigError");
TraceEvent(SevError, "FDBLibTLSConfigError").log();
throw std::runtime_error("FDBLibTLSConfigError");
}
@ -67,14 +67,14 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
// servername, since this will be ignored - the servername should be
// matched by the verify criteria instead.
if (verify_peers_set && servername != nullptr) {
TraceEvent(SevError, "FDBLibTLSVerifyPeersWithServerName");
TraceEvent(SevError, "FDBLibTLSVerifyPeersWithServerName").log();
return nullptr;
}
// If verify peers has not been set, then require a server name to
// avoid an accidental lack of name validation.
if (!verify_peers_set && servername == nullptr) {
TraceEvent(SevError, "FDBLibTLSNoServerName");
TraceEvent(SevError, "FDBLibTLSNoServerName").log();
return nullptr;
}
}
@ -123,18 +123,18 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
if (cert_pem_len > INT_MAX)
goto err;
if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == nullptr) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
if ((certs = sk_X509_new_null()) == nullptr) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
ERR_clear_error();
while ((cert = PEM_read_bio_X509(bio, nullptr, password_cb, nullptr)) != nullptr) {
if (!sk_X509_push(certs, cert)) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
}
@ -150,7 +150,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
}
if (sk_X509_num(certs) < 1) {
TraceEvent(SevError, "FDBLibTLSNoCerts");
TraceEvent(SevError, "FDBLibTLSNoCerts").log();
goto err;
}
@ -168,11 +168,11 @@ err:
bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
if (ca_data_set) {
TraceEvent(SevError, "FDBLibTLSCAAlreadySet");
TraceEvent(SevError, "FDBLibTLSCAAlreadySet").log();
return false;
}
if (session_created) {
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive");
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log();
return false;
}
@ -194,11 +194,11 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) {
if (cert_data_set) {
TraceEvent(SevError, "FDBLibTLSCertAlreadySet");
TraceEvent(SevError, "FDBLibTLSCertAlreadySet").log();
return false;
}
if (session_created) {
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive");
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log();
return false;
}
@ -218,11 +218,11 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
bool rc = false;
if (key_data_set) {
TraceEvent(SevError, "FDBLibTLSKeyAlreadySet");
TraceEvent(SevError, "FDBLibTLSKeyAlreadySet").log();
goto err;
}
if (session_created) {
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive");
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log();
goto err;
}
@ -231,7 +231,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
long len;
if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
ERR_clear_error();
@ -241,7 +241,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) ||
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
TraceEvent(SevError, "FDBLibTLSIncorrectPassword");
TraceEvent(SevError, "FDBLibTLSIncorrectPassword").log();
} else {
ERR_error_string_n(errnum, errbuf, sizeof(errbuf));
TraceEvent(SevError, "FDBLibTLSPrivateKeyError").detail("LibcryptoErrorMessage", errbuf);
@ -250,15 +250,15 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
}
BIO_free(bio);
if ((bio = BIO_new(BIO_s_mem())) == nullptr) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
if (!PEM_write_bio_PrivateKey(bio, key, nullptr, nullptr, 0, nullptr, nullptr)) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
if ((len = BIO_get_mem_data(bio, &data)) <= 0) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
TraceEvent(SevError, "FDBLibTLSOutOfMemory").log();
goto err;
}
if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) {
@ -283,16 +283,16 @@ err:
bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[], int verify_peers_len[]) {
if (verify_peers_set) {
TraceEvent(SevError, "FDBLibTLSVerifyPeersAlreadySet");
TraceEvent(SevError, "FDBLibTLSVerifyPeersAlreadySet").log();
return false;
}
if (session_created) {
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive");
TraceEvent(SevError, "FDBLibTLSPolicyAlreadyActive").log();
return false;
}
if (count < 1) {
TraceEvent(SevError, "FDBLibTLSNoVerifyPeers");
TraceEvent(SevError, "FDBLibTLSNoVerifyPeers").log();
return false;
}

16
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -73,7 +73,7 @@ FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
if (is_client) {
if ((tls_ctx = tls_client()) == nullptr) {
TraceEvent(SevError, "FDBLibTLSClientError", uid);
TraceEvent(SevError, "FDBLibTLSClientError", uid).log();
throw std::runtime_error("FDBLibTLSClientError");
}
if (tls_configure(tls_ctx, policy->tls_cfg) == -1) {
@ -88,7 +88,7 @@ FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
}
} else {
if ((tls_sctx = tls_server()) == nullptr) {
TraceEvent(SevError, "FDBLibTLSServerError", uid);
TraceEvent(SevError, "FDBLibTLSServerError", uid).log();
throw std::runtime_error("FDBLibTLSServerError");
}
if (tls_configure(tls_sctx, policy->tls_cfg) == -1) {
@ -250,7 +250,7 @@ std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLS
// Verify the certificate.
if ((store_ctx = X509_STORE_CTX_new()) == nullptr) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory", uid);
TraceEvent(SevError, "FDBLibTLSOutOfMemory", uid).log();
reason = "Out of memory";
goto err;
}
@ -333,7 +333,7 @@ bool FDBLibTLSSession::verify_peer() {
return true;
if ((cert_pem = tls_peer_cert_chain_pem(tls_ctx, &cert_pem_len)) == nullptr) {
TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
TraceEvent(SevError, "FDBLibTLSNoCertError", uid).log();
goto err;
}
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr)
@ -388,14 +388,14 @@ int FDBLibTLSSession::handshake() {
int FDBLibTLSSession::read(uint8_t* data, int length) {
if (!handshake_completed) {
TraceEvent(SevError, "FDBLibTLSReadHandshakeError");
TraceEvent(SevError, "FDBLibTLSReadHandshakeError").log();
return FAILED;
}
ssize_t n = tls_read(tls_ctx, data, length);
if (n > 0) {
if (n > INT_MAX) {
TraceEvent(SevError, "FDBLibTLSReadOverflow");
TraceEvent(SevError, "FDBLibTLSReadOverflow").log();
return FAILED;
}
return (int)n;
@ -415,14 +415,14 @@ int FDBLibTLSSession::read(uint8_t* data, int length) {
int FDBLibTLSSession::write(const uint8_t* data, int length) {
if (!handshake_completed) {
TraceEvent(SevError, "FDBLibTLSWriteHandshakeError", uid);
TraceEvent(SevError, "FDBLibTLSWriteHandshakeError", uid).log();
return FAILED;
}
ssize_t n = tls_write(tls_ctx, data, length);
if (n > 0) {
if (n > INT_MAX) {
TraceEvent(SevError, "FDBLibTLSWriteOverflow", uid);
TraceEvent(SevError, "FDBLibTLSWriteOverflow", uid).log();
return FAILED;
}
return (int)n;

10
bindings/CMakeLists.txt
View File

@ -3,14 +3,16 @@ if(NOT OPEN_FOR_IDE)
add_subdirectory(c)
add_subdirectory(flow)
endif()
add_subdirectory(python)
if(WITH_JAVA)
if(WITH_PYTHON_BINDING)
add_subdirectory(python)
endif()
if(WITH_JAVA_BINDING)
add_subdirectory(java)
endif()
if(WITH_GO AND NOT OPEN_FOR_IDE)
if(WITH_GO_BINDING AND NOT OPEN_FOR_IDE)
add_subdirectory(go)
endif()
if(WITH_RUBY)
if(WITH_RUBY_BINDING)
add_subdirectory(ruby)
endif()
if(NOT WIN32 AND NOT OPEN_FOR_IDE)

10
bindings/c/CMakeLists.txt
View File

@ -79,6 +79,7 @@ if(NOT WIN32)
test/unit/fdb_api.hpp)
set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
if(OPEN_FOR_IDE)
add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
@ -88,6 +89,7 @@ if(NOT WIN32)
add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
else()
add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
@ -96,6 +98,7 @@ if(NOT WIN32)
add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
strip_debug_symbols(fdb_c_performance_test)
strip_debug_symbols(fdb_c_ryw_benchmark)
strip_debug_symbols(fdb_c_txn_size_test)
@ -106,12 +109,14 @@ if(NOT WIN32)
add_dependencies(fdb_c_setup_tests doctest)
add_dependencies(fdb_c_unit_tests doctest)
add_dependencies(fdb_c_unit_tests_version_510 doctest)
target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads)
# do not set RPATH for mako
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
@ -146,6 +151,11 @@ if(NOT WIN32)
COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>
@CLUSTER_FILE@
fdb)
add_fdbclient_test(
NAME trace_partial_file_suffix_test
COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>
@CLUSTER_FILE@
fdb)
add_fdbclient_test(
NAME fdb_c_external_client_unit_tests
COMMAND $<TARGET_FILE:fdb_c_unit_tests>

2
bindings/c/fdb_c.cpp
View File

@ -162,7 +162,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) {
return TSAVB(f)->isReady();
}
class CAPICallback : public ThreadCallback {
class CAPICallback final : public ThreadCallback {
public:
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata)
: callbackf(callbackf), f(f), userdata(userdata) {}

1
bindings/c/foundationdb/ClientWorkload.h
View File

@ -66,6 +66,7 @@ public:
};
struct FDBPromise {
virtual ~FDBPromise() = default;
virtual void send(void*) = 0;
};

111
bindings/c/test/unit/trace_partial_file_suffix_test.cpp Normal file
View File

@ -0,0 +1,111 @@
/*
* trace_partial_file_suffix_test.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <thread>
#include "flow/Platform.h"
#define FDB_API_VERSION 710
#include "foundationdb/fdb_c.h"
#undef NDEBUG
#include <cassert>
void fdb_check(fdb_error_t e) {
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
}
void set_net_opt(FDBNetworkOption option, const std::string& value) {
fdb_check(fdb_network_set_option(option, reinterpret_cast<const uint8_t*>(value.c_str()), value.size()));
}
bool file_exists(const char* path) {
FILE* f = fopen(path, "r");
if (f) {
fclose(f);
return true;
}
return false;
}
int main(int argc, char** argv) {
fdb_check(fdb_select_api_version(710));
std::string file_identifier = "trace_partial_file_suffix_test" + std::to_string(std::random_device{}());
std::string trace_partial_file_suffix = ".tmp";
std::string simulated_stray_partial_file =
"trace.127.0.0.1." + file_identifier + ".simulated.xml" + trace_partial_file_suffix;
// Simulate this process crashing previously by creating a ".tmp" file
{ std::ofstream file{ simulated_stray_partial_file }; }
set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE, "");
set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_FILE_IDENTIFIER, file_identifier);
set_net_opt(FDBNetworkOption::FDB_NET_OPTION_TRACE_PARTIAL_FILE_SUFFIX, trace_partial_file_suffix);
fdb_check(fdb_setup_network());
std::thread network_thread{ &fdb_run_network };
// Apparently you need to open a database to initialize logging
FDBDatabase* out;
fdb_check(fdb_create_database(nullptr, &out));
fdb_database_destroy(out);
// Eventually there's a new trace file for this test ending in .tmp
std::string name;
for (;;) {
for (const auto& path : platform::listFiles(".")) {
if (path.find(file_identifier) != std::string::npos && path.find(".simulated.") == std::string::npos) {
assert(path.substr(path.size() - trace_partial_file_suffix.size()) == trace_partial_file_suffix);
name = path;
break;
}
}
if (!name.empty()) {
break;
}
}
fdb_check(fdb_stop_network());
network_thread.join();
// After shutting down, the suffix is removed for both the simulated stray file and our new file
if (!trace_partial_file_suffix.empty()) {
assert(!file_exists(name.c_str()));
assert(!file_exists(simulated_stray_partial_file.c_str()));
}
auto new_name = name.substr(0, name.size() - trace_partial_file_suffix.size());
auto new_stray_name =
simulated_stray_partial_file.substr(0, simulated_stray_partial_file.size() - trace_partial_file_suffix.size());
assert(file_exists(new_name.c_str()));
assert(file_exists(new_stray_name.c_str()));
remove(new_name.c_str());
remove(new_stray_name.c_str());
assert(!file_exists(new_name.c_str()));
assert(!file_exists(new_stray_name.c_str()));
}

75
bindings/c/test/unit/unit_tests.cpp
View File

@ -2177,6 +2177,81 @@ TEST_CASE("monitor_network_busyness") {
CHECK(containsGreaterZero);
}
// Commit a transaction and confirm it has not been reset
TEST_CASE("commit_does_not_reset") {
fdb::Transaction tr(db);
fdb::Transaction tr2(db);
// Commit two transactions, one that will fail with conflict and the other
// that will succeed. Ensure both transactions are not reset at the end.
while (1) {
fdb::Int64Future tr1GrvFuture = tr.get_read_version();
fdb_error_t err = wait_future(tr1GrvFuture);
if (err) {
fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err);
fdb_check(wait_future(tr1OnErrorFuture));
continue;
}
int64_t tr1StartVersion;
CHECK(!tr1GrvFuture.get(&tr1StartVersion));
fdb::Int64Future tr2GrvFuture = tr2.get_read_version();
err = wait_future(tr2GrvFuture);
if (err) {
fdb::EmptyFuture tr2OnErrorFuture = tr2.on_error(err);
fdb_check(wait_future(tr2OnErrorFuture));
continue;
}
int64_t tr2StartVersion;
CHECK(!tr2GrvFuture.get(&tr2StartVersion));
tr.set(key("foo"), "bar");
fdb::EmptyFuture tr1CommitFuture = tr.commit();
err = wait_future(tr1CommitFuture);
if (err) {
fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err);
fdb_check(wait_future(tr1OnErrorFuture));
continue;
}
fdb_check(tr2.add_conflict_range(key("foo"), strinc(key("foo")), FDB_CONFLICT_RANGE_TYPE_READ));
tr2.set(key("foo"), "bar");
fdb::EmptyFuture tr2CommitFuture = tr2.commit();
err = wait_future(tr2CommitFuture);
CHECK(err == 1020); // not_committed
fdb::Int64Future tr1GrvFuture2 = tr.get_read_version();
err = wait_future(tr1GrvFuture2);
if (err) {
fdb::EmptyFuture tr1OnErrorFuture = tr.on_error(err);
fdb_check(wait_future(tr1OnErrorFuture));
continue;
}
int64_t tr1EndVersion;
CHECK(!tr1GrvFuture2.get(&tr1EndVersion));
fdb::Int64Future tr2GrvFuture2 = tr2.get_read_version();
err = wait_future(tr2GrvFuture2);
if (err) {
fdb::EmptyFuture tr2OnErrorFuture = tr2.on_error(err);
fdb_check(wait_future(tr2OnErrorFuture));
continue;
}
int64_t tr2EndVersion;
CHECK(!tr2GrvFuture2.get(&tr2EndVersion));
// If we reset the transaction, then the read version will change
CHECK(tr1StartVersion == tr1EndVersion);
CHECK(tr2StartVersion == tr2EndVersion);
break;
}
}
int main(int argc, char** argv) {
if (argc < 3) {
std::cout << "Unit tests for the FoundationDB C API.\n"

4
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -36,8 +36,8 @@ const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
: nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
: rootNode(nodeSubspace.get(nodeSubspace.key())), nodeSubspace(nodeSubspace), contentSubspace(contentSubspace),
allocator(rootNode.get(HIGH_CONTENTION_KEY)), allowManualPrefixes(allowManualPrefixes) {}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);

4
bindings/flow/FDBLoanerTypes.h
View File

@ -167,9 +167,9 @@ struct RangeResultRef : VectorRef<KeyValueRef> {
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
: VectorRef<KeyValueRef>(p, toCopy), more(toCopy.more),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}

2
bindings/java/JavaWorkload.cpp
View File

@ -513,7 +513,7 @@ struct JVM {
}
};
struct JavaWorkload : FDBWorkload {
struct JavaWorkload final : FDBWorkload {
std::shared_ptr<JVM> jvm;
FDBLogger& log;
FDBWorkloadContext* context = nullptr;

188
bindings/java/src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java Normal file
View File

@ -0,0 +1,188 @@
/*
* RepeatableReadMultiThreadClientTest
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import com.apple.foundationdb.tuple.Tuple;
import org.junit.jupiter.api.Assertions;
/**
* This test verify transcations have repeatable read.
* 1 First set initialValue to key.
* 2 Have transactions to read the key and verify the initialValue in a loop, if it does not
* see the initialValue as the value, it set the flag to false.
*
* 3 Then have new transactions set the value and then read to verify the new value is set,
* if it does not read the new value, set the flag to false.
*
* 4 Verify that old transactions have not finished when new transactions have finished,
* then verify old transactions does not have false flag -- it means that old transactions
* are still seeting the initialValue even after new transactions set them to a new value.
*/
public class RepeatableReadMultiThreadClientTest {
public static final MultiClientHelper clientHelper = new MultiClientHelper();
private static final int oldValueReadCount = 30;
private static final int threadPerDB = 5;
private static final String key = "foo";
private static final String initialValue = "bar";
private static final String newValue = "cool";
private static final Map<Thread, OldValueReader> threadToOldValueReaders = new HashMap<>();
public static void main(String[] args) throws Exception {
FDB fdb = FDB.selectAPIVersion(710);
setupThreads(fdb);
Collection<Database> dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us
System.out.println("Starting tests");
setup(dbs);
System.out.println("Start processing and validating");
readOldValue(dbs);
setNewValueAndRead(dbs);
System.out.println("Test finished");
}
private static synchronized void setupThreads(FDB fdb) {
int clientThreadsPerVersion = clientHelper.readClusterFromEnv().length;
fdb.options().setClientThreadsPerVersion(clientThreadsPerVersion);
System.out.printf("thread per version is %d\n", clientThreadsPerVersion);
fdb.options().setExternalClientDirectory("/var/dynamic-conf/lib");
fdb.options().setTraceEnable("/tmp");
fdb.options().setKnob("min_trace_severity=5");
}
private static void setup(Collection<Database> dbs) {
// 0 -> 1 -> 2 -> 3 -> 0
for (Database db : dbs) {
db.run(tr -> {
tr.set(Tuple.from(key).pack(), Tuple.from(initialValue).pack());
return null;
});
}
}
private static void readOldValue(Collection<Database> dbs) throws InterruptedException {
for (Database db : dbs) {
for (int i = 0; i < threadPerDB; i++) {
final OldValueReader oldValueReader = new OldValueReader(db);
final Thread thread = new Thread(OldValueReader.create(db));
thread.start();
threadToOldValueReaders.put(thread, oldValueReader);
}
}
}
private static void setNewValueAndRead(Collection<Database> dbs) throws InterruptedException {
// threads running NewValueReader need to wait for threads to start first who run OldValueReader
Thread.sleep(1000);
final Map<Thread, NewValueReader> threads = new HashMap<>();
for (Database db : dbs) {
for (int i = 0; i < threadPerDB; i++) {
final NewValueReader newValueReader = new NewValueReader(db);
final Thread thread = new Thread(NewValueReader.create(db));
thread.start();
threads.put(thread, newValueReader);
}
}
for (Map.Entry<Thread, NewValueReader> entry : threads.entrySet()) {
entry.getKey().join();
Assertions.assertTrue(entry.getValue().succeed, "new value reader failed to read the correct value");
}
for (Map.Entry<Thread, OldValueReader> entry : threadToOldValueReaders.entrySet()) {
Assertions.assertTrue(entry.getKey().isAlive(), "Old value reader finished too soon, cannot verify repeatable read, succeed is " + entry.getValue().succeed);
}
for (Map.Entry<Thread, OldValueReader> entry : threadToOldValueReaders.entrySet()) {
entry.getKey().join();
Assertions.assertTrue(entry.getValue().succeed, "old value reader failed to read the correct value");
}
}
public static class OldValueReader implements Runnable {
private final Database db;
private boolean succeed;
private OldValueReader(Database db) {
this.db = db;
this.succeed = true;
}
public static OldValueReader create(Database db) {
return new OldValueReader(db);
}
@Override
public void run() {
db.run(tr -> {
try {
for (int i = 0; i < oldValueReadCount; i++) {
byte[] result = tr.get(Tuple.from(key).pack()).join();
String value = Tuple.fromBytes(result).getString(0);
if (!initialValue.equals(value)) {
succeed = false;
break;
}
Thread.sleep(100);
}
}
catch (Exception e) {
succeed = false;
}
return null;
});
}
}
public static class NewValueReader implements Runnable {
private final Database db;
private boolean succeed;
public NewValueReader(Database db) {
this.db = db;
this.succeed = true;
}
public static NewValueReader create(Database db) {
return new NewValueReader(db);
}
@Override
public void run() {
db.run(tr -> {
tr.set(Tuple.from(key).pack(), Tuple.from(newValue).pack());
return null;
});
String value = db.run(tr -> {
byte[] result = tr.get(Tuple.from(key).pack()).join();
return Tuple.fromBytes(result).getString(0);
});
if (!newValue.equals(value)) {
succeed = false;
}
}
}
}

8
bindings/java/src/main/com/apple/foundationdb/KeySelector.java
View File

@ -165,9 +165,13 @@ public class KeySelector {
}
/**
* Returns the {@code or-equal} parameter of this {@code KeySelector}. For internal use.
* Returns the orEqual parameter for this {@code KeySelector}. See the
* {@link #KeySelector(byte[], boolean, int)} KeySelector constructor}
* for more details.
*
* @return the {@code or-equal} parameter of this {@code KeySelector}.
*/
boolean orEqual() {
public boolean orEqual() {
return orEqual;
}

1
bindings/java/src/tests.cmake
View File

@ -51,6 +51,7 @@ set(JAVA_INTEGRATION_TESTS
src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java
src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java
src/integration/com/apple/foundationdb/SidebandMultiThreadClientTest.java
src/integration/com/apple/foundationdb/RepeatableReadMultiThreadClientTest.java
)
# Resources that are used in integration testing, but are not explicitly test files (JUnit rules,

64
bindings/python/tests/fdbcli_tests.py
View File

@ -332,9 +332,10 @@ def transaction(logger):
output7 = run_fdbcli_command('get', 'key')
assert output7 == "`key': not found"
def get_fdb_process_addresses():
def get_fdb_process_addresses(logger):
# get all processes' network addresses
output = run_fdbcli_command('kill')
logger.debug(output)
# except the first line, each line is one process
addresses = output.split('\n')[1:]
assert len(addresses) == process_number
@ -354,7 +355,7 @@ def coordinators(logger):
assert coordinator_list[0]['address'] == coordinators
# verify the cluster description
assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description))
addresses = get_fdb_process_addresses()
addresses = get_fdb_process_addresses(logger)
# set all 5 processes as coordinators and update the cluster description
new_cluster_description = 'a_simple_description'
run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description))
@ -369,7 +370,7 @@ def coordinators(logger):
@enable_logging()
def exclude(logger):
# get all processes' network addresses
addresses = get_fdb_process_addresses()
addresses = get_fdb_process_addresses(logger)
logger.debug("Cluster processes: {}".format(' '.join(addresses)))
# There should be no excluded process for now
no_excluded_process_output = 'There are currently no servers or localities excluded from the database.'
@ -377,16 +378,35 @@ def exclude(logger):
assert no_excluded_process_output in output1
# randomly pick one and exclude the process
excluded_address = random.choice(addresses)
# If we see "not enough space" error, use FORCE option to proceed
# this should be a safe operation as we do not need any storage space for the test
force = False
# sometimes we need to retry the exclude
while True:
logger.debug("Excluding process: {}".format(excluded_address))
error_message = run_fdbcli_command_and_get_error('exclude', excluded_address)
if not error_message:
if force:
error_message = run_fdbcli_command_and_get_error('exclude', 'FORCE', excluded_address)
else:
error_message = run_fdbcli_command_and_get_error('exclude', excluded_address)
if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address):
# exclude coordinator will print the warning, verify the randomly selected process is the coordinator
coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators')
assert len(coordinator_list) == 1
assert coordinator_list[0]['address'] == excluded_address
break
elif 'ERROR: This exclude may cause the total free space in the cluster to drop below 10%.' in error_message:
# exclude the process may cause the free space not enough
# use FORCE option to ignore it and proceed
assert not force
force = True
logger.debug("Use FORCE option to exclude the process")
elif not error_message:
break
else:
logger.debug("Error message: {}\n".format(error_message))
logger.debug("Retry exclude after 1 second")
time.sleep(1)
output2 = run_fdbcli_command('exclude')
# logger.debug(output3)
assert 'There are currently 1 servers or localities being excluded from the database' in output2
assert excluded_address in output2
run_fdbcli_command('include', excluded_address)
@ -394,6 +414,32 @@ def exclude(logger):
output4 = run_fdbcli_command('exclude')
assert no_excluded_process_output in output4
# read the system key 'k', need to enable the option first
def read_system_key(k):
output = run_fdbcli_command('option', 'on', 'READ_SYSTEM_KEYS;', 'get', k)
if 'is' not in output:
# key not present
return None
_, value = output.split(' is ')
return value
@enable_logging()
def throttle(logger):
# no throttled tags at the beginning
no_throttle_tags_output = 'There are no throttled tags'
assert run_fdbcli_command('throttle', 'list') == no_throttle_tags_output
# test 'throttle enable auto'
run_fdbcli_command('throttle', 'enable', 'auto')
# verify the change is applied by reading the system key
# not an elegant way, may change later
enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled')
assert enable_flag == "`1'"
run_fdbcli_command('throttle', 'disable', 'auto')
enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled')
# verify disabled
assert enable_flag == "`0'"
# TODO : test manual throttling, not easy to do now
if __name__ == '__main__':
# fdbcli_tests.py <path_to_fdbcli_binary> <path_to_fdb_cluster_file> <process_number>
assert len(sys.argv) == 4, "Please pass arguments: <path_to_fdbcli_binary> <path_to_fdb_cluster_file> <process_number>"
@ -403,7 +449,8 @@ if __name__ == '__main__':
# assertions will fail if fdbcli does not work as expected
process_number = int(sys.argv[3])
if process_number == 1:
advanceversion()
# TODO: disable for now, the change can cause the database unavailable
#advanceversion()
cache_range()
consistencycheck()
datadistribution()
@ -413,9 +460,10 @@ if __name__ == '__main__':
setclass()
suspend()
transaction()
throttle()
else:
assert process_number > 1, "Process number should be positive"
coordinators()
# exclude()
exclude()

59
cmake/AddFdbTest.cmake
View File

@ -39,6 +39,9 @@ function(configure_testing)
endfunction()
function(verify_testing)
if(NOT ENABLE_SIMULATION_TESTS)
return()
endif()
foreach(test_file IN LISTS fdb_test_files)
message(SEND_ERROR "${test_file} found but it is not associated with a test")
endforeach()
@ -119,28 +122,30 @@ function(add_fdb_test)
set(VALGRIND_OPTION "--use-valgrind")
endif()
list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
add_test(NAME ${test_name}
COMMAND $<TARGET_FILE:Python::Interpreter> ${TestRunner}
-n ${test_name}
-b ${PROJECT_BINARY_DIR}
-t ${test_type}
-O ${OLD_FDBSERVER_BINARY}
--crash
--aggregate-traces ${TEST_AGGREGATE_TRACES}
--log-format ${TEST_LOG_FORMAT}
--keep-logs ${TEST_KEEP_LOGS}
--keep-simdirs ${TEST_KEEP_SIMDIR}
--seed ${SEED}
--test-number ${assigned_id}
${BUGGIFY_OPTION}
${VALGRIND_OPTION}
${ADD_FDB_TEST_TEST_FILES}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
get_filename_component(test_dir_full ${first_file} DIRECTORY)
if(NOT ${test_dir_full} STREQUAL "")
get_filename_component(test_dir ${test_dir_full} NAME)
set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}")
if (ENABLE_SIMULATION_TESTS)
add_test(NAME ${test_name}
COMMAND $<TARGET_FILE:Python::Interpreter> ${TestRunner}
-n ${test_name}
-b ${PROJECT_BINARY_DIR}
-t ${test_type}
-O ${OLD_FDBSERVER_BINARY}
--crash
--aggregate-traces ${TEST_AGGREGATE_TRACES}
--log-format ${TEST_LOG_FORMAT}
--keep-logs ${TEST_KEEP_LOGS}
--keep-simdirs ${TEST_KEEP_SIMDIR}
--seed ${SEED}
--test-number ${assigned_id}
${BUGGIFY_OPTION}
${VALGRIND_OPTION}
${ADD_FDB_TEST_TEST_FILES}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
get_filename_component(test_dir_full ${first_file} DIRECTORY)
if(NOT ${test_dir_full} STREQUAL "")
get_filename_component(test_dir ${test_dir_full} NAME)
set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}")
endif()
endif()
# set variables used for generating test packages
set(TEST_NAMES ${TEST_NAMES} ${test_name} PARENT_SCOPE)
@ -347,7 +352,7 @@ function(package_bindingtester)
COMMENT "Copy Flow tester for bindingtester")
set(generated_binding_files python/fdb/fdboptions.py)
if(WITH_JAVA)
if(WITH_JAVA_BINDING)
if(NOT FDB_RELEASE)
set(prerelease_string "-PRERELEASE")
else()
@ -364,7 +369,7 @@ function(package_bindingtester)
set(generated_binding_files ${generated_binding_files} java/foundationdb-tests.jar)
endif()
if(WITH_GO AND NOT OPEN_FOR_IDE)
if(WITH_GO_BINDING AND NOT OPEN_FOR_IDE)
add_dependencies(copy_binding_output_files fdb_go_tester fdb_go)
add_custom_command(
TARGET copy_binding_output_files
@ -416,14 +421,14 @@ function(add_fdbclient_test)
message(STATUS "Adding Client test ${T_NAME}")
if (T_PROCESS_NUMBER)
add_test(NAME "${T_NAME}"
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py
--build-dir ${CMAKE_BINARY_DIR}
--process-number ${T_PROCESS_NUMBER}
--
${T_COMMAND})
else()
add_test(NAME "${T_NAME}"
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py
--build-dir ${CMAKE_BINARY_DIR}
--
${T_COMMAND})
@ -459,7 +464,7 @@ function(add_multi_fdbclient_test)
endif()
message(STATUS "Adding Client test ${T_NAME}")
add_test(NAME "${T_NAME}"
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
--build-dir ${CMAKE_BINARY_DIR}
--clusters 3
--

94
cmake/CompileBoost.cmake
View File

@ -1,61 +1,73 @@
function(compile_boost)
# Initialize function incoming parameters
set(options)
set(oneValueArgs TARGET)
set(multiValueArgs BUILD_ARGS CXXFLAGS LDFLAGS)
cmake_parse_arguments(MY "${options}" "${oneValueArgs}"
cmake_parse_arguments(COMPILE_BOOST "${options}" "${oneValueArgs}"
"${multiValueArgs}" ${ARGN} )
# Configure the boost toolset to use
set(BOOTSTRAP_ARGS "--with-libraries=context")
set(B2_COMMAND "./b2")
set(BOOST_COMPILER_FLAGS -fvisibility=hidden -fPIC -std=c++14 -w)
# Configure bootstrap command
set(BOOTSTRAP_COMMAND "./bootstrap.sh")
set(BOOTSTRAP_LIBRARIES "context")
set(BOOST_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
if(APPLE)
set(BOOST_TOOLSET "clang-darwin")
# this is to fix a weird macOS issue -- by default
# cmake would otherwise pass a compiler that can't
# compile boost
set(BOOST_CXX_COMPILER "/usr/bin/clang++")
elseif(CLANG)
if(CLANG)
set(BOOST_TOOLSET "clang")
list(APPEND BOOTSTRAP_ARGS "${BOOTSTRAP_COMMAND} --with-toolset=clang")
if(APPLE)
# this is to fix a weird macOS issue -- by default
# cmake would otherwise pass a compiler that can't
# compile boost
set(BOOST_CXX_COMPILER "/usr/bin/clang++")
endif()
else()
set(BOOST_TOOLSET "gcc")
endif()
if(APPLE OR USE_LIBCXX)
list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++)
endif()
set(BOOST_ADDITIONAL_COMPILE_OPTIOINS "")
foreach(flag IN LISTS BOOST_COMPILER_FLAGS MY_CXXFLAGS)
string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIOINS "<cxxflags>${flag} ")
endforeach()
foreach(flag IN LISTS MY_LDFLAGS)
string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIOINS "<linkflags>${flag} ")
endforeach()
configure_file(${CMAKE_SOURCE_DIR}/cmake/user-config.jam.cmake ${CMAKE_BINARY_DIR}/user-config.jam)
message(STATUS "Use ${BOOST_TOOLSET} to build boost")
# Configure b2 command
set(B2_COMMAND "./b2")
set(BOOST_COMPILER_FLAGS -fvisibility=hidden -fPIC -std=c++17 -w)
set(BOOST_LINK_FLAGS "")
if(APPLE OR CLANG OR USE_LIBCXX)
list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
list(APPEND BOOST_LINK_FLAGS -static-libgcc -lc++ -lc++abi)
endif()
# Update the user-config.jam
set(BOOST_ADDITIONAL_COMPILE_OPTIOINS "")
foreach(flag IN LISTS BOOST_COMPILER_FLAGS COMPILE_BOOST_CXXFLAGS)
string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<cxxflags>${flag} ")
endforeach()
#foreach(flag IN LISTS BOOST_LINK_FLAGS COMPILE_BOOST_LDFLAGS)
# string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<linkflags>${flag} ")
#endforeach()
configure_file(${CMAKE_SOURCE_DIR}/cmake/user-config.jam.cmake ${CMAKE_BINARY_DIR}/user-config.jam)
set(USER_CONFIG_FLAG --user-config=${CMAKE_BINARY_DIR}/user-config.jam)
# Build boost
include(ExternalProject)
set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install")
ExternalProject_add("${MY_TARGET}Project"
ExternalProject_add("${COMPILE_BOOST_TARGET}Project"
URL "https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2"
URL_HASH SHA256=59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722
CONFIGURE_COMMAND ./bootstrap.sh ${BOOTSTRAP_ARGS}
BUILD_COMMAND ${B2_COMMAND} link=static ${MY_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install
CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET}
BUILD_COMMAND ${B2_COMMAND} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install
BUILD_IN_SOURCE ON
INSTALL_COMMAND ""
UPDATE_COMMAND ""
BUILD_BYPRODUCTS "${BOOST_INSTALL_DIR}/boost/config.hpp"
"${BOOST_INSTALL_DIR}/lib/libboost_context.a")
add_library(${MY_TARGET}_context STATIC IMPORTED)
add_dependencies(${MY_TARGET}_context ${MY_TARGET}Project)
set_target_properties(${MY_TARGET}_context PROPERTIES IMPORTED_LOCATION "${BOOST_INSTALL_DIR}/lib/libboost_context.a")
add_library(${COMPILE_BOOST_TARGET}_context STATIC IMPORTED)
add_dependencies(${COMPILE_BOOST_TARGET}_context ${COMPILE_BOOST_TARGET}Project)
set_target_properties(${COMPILE_BOOST_TARGET}_context PROPERTIES IMPORTED_LOCATION "${BOOST_INSTALL_DIR}/lib/libboost_context.a")
add_library(${MY_TARGET} INTERFACE)
target_include_directories(${MY_TARGET} SYSTEM INTERFACE ${BOOST_INSTALL_DIR}/include)
target_link_libraries(${MY_TARGET} INTERFACE ${MY_TARGET}_context)
endfunction()
add_library(${COMPILE_BOOST_TARGET} INTERFACE)
target_include_directories(${COMPILE_BOOST_TARGET} SYSTEM INTERFACE ${BOOST_INSTALL_DIR}/include)
target_link_libraries(${COMPILE_BOOST_TARGET} INTERFACE ${COMPILE_BOOST_TARGET}_context)
endfunction(compile_boost)
if(USE_SANITIZER)
if(WIN32)
@ -72,10 +84,20 @@ if(USE_SANITIZER)
return()
endif()
list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0)
# since boost 1.72 boost installs cmake configs. We will enforce config mode
set(Boost_USE_STATIC_LIBS ON)
set(BOOST_HINT_PATHS /opt/boost_1_72_0)
# Clang and Gcc will have different name mangling to std::call_once, etc.
if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$")
list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0_clang)
set(BOOST_HINT_PATHS /opt/boost_1_72_0_clang)
message(STATUS "Using Clang version of boost::context")
else ()
list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0)
set(BOOST_HINT_PATHS /opt/boost_1_72_0)
message(STATUS "Using g++ version of boost::context")
endif ()
if(BOOST_ROOT)
list(APPEND BOOST_HINT_PATHS ${BOOST_ROOT})
endif()

4
cmake/CompileRocksDB.cmake
View File

@ -36,8 +36,8 @@ if (RocksDB_FOUND)
${BINARY_DIR}/librocksdb.a)
else()
ExternalProject_Add(rocksdb
URL https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz
URL_HASH SHA256=d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee
URL https://github.com/facebook/rocksdb/archive/v6.22.1.tar.gz
URL_HASH SHA256=2df8f34a44eda182e22cf84dee7a14f17f55d305ff79c06fb3cd1e5f8831e00d
CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

11
cmake/ConfigureCompiler.cmake
View File

@ -261,10 +261,6 @@ else()
if (CLANG)
add_compile_options()
# Clang has link errors unless `atomic` is specifically requested.
if(NOT APPLE)
#add_link_options(-latomic)
endif()
if (APPLE OR USE_LIBCXX)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
if (NOT APPLE)
@ -285,25 +281,20 @@ else()
-Wpessimizing-move
-Woverloaded-virtual
-Wshift-sign-overflow
# Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 10
# Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 11
-Wno-comment
-Wno-dangling-else
-Wno-delete-non-virtual-dtor
-Wno-format
-Wno-mismatched-tags
-Wno-missing-field-initializers
-Wno-reorder
-Wno-reorder-ctor
-Wno-sign-compare
-Wno-tautological-pointer-compare
-Wno-undefined-var-template
-Wno-tautological-pointer-compare
-Wno-unknown-pragmas
-Wno-unknown-warning-option
-Wno-unused-function
-Wno-unused-local-typedef
-Wno-unused-parameter
-Wno-self-assign
)
if (USE_CCACHE)
add_compile_options(

117
cmake/FDBComponents.cmake
View File

@ -47,22 +47,6 @@ else()
endif()
endif()
################################################################################
# Java Bindings
################################################################################
set(WITH_JAVA OFF)
find_package(JNI 1.8)
find_package(Java 1.8 COMPONENTS Development)
# leave FreeBSD JVM compat for later
if(JNI_FOUND AND Java_FOUND AND Java_Development_FOUND AND NOT (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD"))
set(WITH_JAVA ON)
include(UseJava)
enable_language(Java)
else()
set(WITH_JAVA OFF)
endif()
################################################################################
# Python Bindings
################################################################################
@ -75,12 +59,57 @@ else()
set(WITH_PYTHON OFF)
endif()
option(BUILD_PYTHON_BINDING "build python binding" ON)
if(NOT BUILD_PYTHON_BINDING OR NOT WITH_PYTHON)
set(WITH_PYTHON_BINDING OFF)
else()
if(WITH_PYTHON)
set(WITH_PYTHON_BINDING ON)
else()
#message(FATAL_ERROR "Could not found a suitable python interpreter")
set(WITH_PYTHON_BINDING OFF)
endif()
endif()
################################################################################
# C Bindings
################################################################################
option(BUILD_C_BINDING "build C binding" ON)
if(BUILD_C_BINDING AND WITH_PYTHON)
set(WITH_C_BINDING ON)
else()
set(WITH_C_BINDING OFF)
endif()
################################################################################
# Java Bindings
################################################################################
option(BUILD_JAVA_BINDING "build java binding" ON)
if(NOT BUILD_JAVA_BINDING OR NOT WITH_C_BINDING)
set(WITH_JAVA_BINDING OFF)
else()
set(WITH_JAVA_BINDING OFF)
find_package(JNI 1.8)
find_package(Java 1.8 COMPONENTS Development)
# leave FreeBSD JVM compat for later
if(JNI_FOUND AND Java_FOUND AND Java_Development_FOUND AND NOT (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") AND WITH_C_BINDING)
set(WITH_JAVA_BINDING ON)
include(UseJava)
enable_language(Java)
else()
set(WITH_JAVA_BINDING OFF)
endif()
endif()
################################################################################
# Pip
################################################################################
option(BUILD_DOCUMENTATION "build documentation" ON)
find_package(Python3 COMPONENTS Interpreter)
if (Python3_Interpreter_FOUND)
if (WITH_PYTHON AND Python3_Interpreter_FOUND AND BUILD_DOCUMENTATION)
set(WITH_DOCUMENTATION ON)
else()
set(WITH_DOCUMENTATION OFF)
@ -90,27 +119,37 @@ endif()
# GO
################################################################################
find_program(GO_EXECUTABLE go)
# building the go binaries is currently not supported on Windows
if(GO_EXECUTABLE AND NOT WIN32)
set(WITH_GO ON)
option(BUILD_GO_BINDING "build go binding" ON)
if(NOT BUILD_GO_BINDING OR NOT BUILD_C_BINDING)
set(WITH_GO_BINDING OFF)
else()
set(WITH_GO OFF)
endif()
if (USE_SANITIZER)
# Disable building go for sanitizers, since _stacktester doesn't link properly
set(WITH_GO OFF)
find_program(GO_EXECUTABLE go)
# building the go binaries is currently not supported on Windows
if(GO_EXECUTABLE AND NOT WIN32 AND WITH_C_BINDING)
set(WITH_GO_BINDING ON)
else()
set(WITH_GO_BINDING OFF)
endif()
if (USE_SANITIZER)
# Disable building go for sanitizers, since _stacktester doesn't link properly
set(WITH_GO_BINDING OFF)
endif()
endif()
################################################################################
# Ruby
################################################################################
find_program(GEM_EXECUTABLE gem)
set(WITH_RUBY OFF)
if(GEM_EXECUTABLE)
set(GEM_COMMAND ${RUBY_EXECUTABLE} ${GEM_EXECUTABLE})
set(WITH_RUBY ON)
option(BUILD_RUBY_BINDING "build ruby binding" ON)
if(NOT BUILD_RUBY_BINDING OR NOT BUILD_C_BINDING)
set(WITH_RUBY_BINDING OFF)
else()
find_program(GEM_EXECUTABLE gem)
set(WITH_RUBY_BINDING OFF)
if(GEM_EXECUTABLE AND WITH_C_BINDING)
set(GEM_COMMAND ${RUBY_EXECUTABLE} ${GEM_EXECUTABLE})
set(WITH_RUBY_BINDING ON)
endif()
endif()
################################################################################
@ -160,20 +199,22 @@ function(print_components)
message(STATUS "=========================================")
message(STATUS " Components Build Overview ")
message(STATUS "=========================================")
message(STATUS "Build Java Bindings: ${WITH_JAVA}")
message(STATUS "Build with TLS support: ${WITH_TLS}")
message(STATUS "Build Go bindings: ${WITH_GO}")
message(STATUS "Build Ruby bindings: ${WITH_RUBY}")
message(STATUS "Build Python sdist (make package): ${WITH_PYTHON}")
message(STATUS "Build Documentation (make html): ${WITH_DOCUMENTATION}")
message(STATUS "Build Bindings (depends on Python): ${WITH_PYTHON}")
message(STATUS "Build C Bindings: ${WITH_C_BINDING}")
message(STATUS "Build Python Bindings: ${WITH_PYTHON_BINDING}")
message(STATUS "Build Java Bindings: ${WITH_JAVA_BINDING}")
message(STATUS "Build Go bindings: ${WITH_GO_BINDING}")
message(STATUS "Build Ruby bindings: ${WITH_RUBY_BINDING}")
message(STATUS "Build with TLS support: ${WITH_TLS}")
message(STATUS "Build Documentation (make html): ${WITH_DOCUMENTATION}")
message(STATUS "Build Python sdist (make package): ${WITH_PYTHON_BINDING}")
message(STATUS "Configure CTest (depends on Python): ${WITH_PYTHON}")
message(STATUS "Build with RocksDB: ${WITH_ROCKSDB_EXPERIMENTAL}")
message(STATUS "=========================================")
endfunction()
if(FORCE_ALL_COMPONENTS)
if(NOT WITH_JAVA OR NOT WITH_TLS OR NOT WITH_GO OR NOT WITH_RUBY OR NOT WITH_PYTHON OR NOT WITH_DOCUMENTATION)
if(NOT WITH_C_BINDING OR NOT WITH_JAVA_BINDING OR NOT WITH_TLS OR NOT WITH_GO_BINDING OR NOT WITH_RUBY_BINDING OR NOT WITH_PYTHON_BINDING OR NOT WITH_DOCUMENTATION)
print_components()
message(FATAL_ERROR "FORCE_ALL_COMPONENTS is set but not all dependencies could be found")
endif()

2
cmake/user-config.jam.cmake
View File

@ -1 +1 @@
using @BOOST_TOOLSET@ : : @BOOST_CXX_COMPILER@ : @BOOST_ADDITIONAL_COMPILE_OPTIOINS@ ;
using @BOOST_TOOLSET@ : : @BOOST_CXX_COMPILER@ : @BOOST_ADDITIONAL_COMPILE_OPTIONS@ ;

57
contrib/TestHarness/Program.cs.cmake
View File

@ -144,7 +144,9 @@ namespace SummarizeTest
string oldBinaryFolder = (args.Length > 1) ? args[1] : Path.Combine("/opt", "joshua", "global_data", "oldBinaries");
bool useValgrind = args.Length > 2 && args[2].ToLower() == "true";
int maxTries = (args.Length > 3) ? int.Parse(args[3]) : 3;
return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN));
bool buggifyEnabled = (args.Length > 4) ? bool.Parse(args[4]) : true;
bool faultInjectionEnabled = (args.Length > 5) ? bool.Parse(args[5]) : true;
return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN), buggifyEnabled, faultInjectionEnabled);
}
catch(Exception e)
{
@ -240,10 +242,10 @@ namespace SummarizeTest
}
}
static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "")
static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "", bool buggifyEnabled = true, bool faultInjectionEnabled = true)
{
int seed = random.Next(1000000000);
bool buggify = random.NextDouble() < buggifyOnRatio;
bool buggify = buggifyEnabled ? (random.NextDouble() < buggifyOnRatio) : false;
string testFile = null;
string testDir = "";
string oldServerName = "";
@ -353,11 +355,11 @@ namespace SummarizeTest
bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
bool useToml = File.Exists(testFile + "-1.toml");
string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt";
result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim);
result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
if (result == 0)
{
string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt";
result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim);
result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
}
}
else
@ -365,13 +367,13 @@ namespace SummarizeTest
int expectedUnseed = -1;
if (!useValgrind && unseedCheck)
{
result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout, noSim);
result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout, noSim, faultInjectionEnabled);
}
if (!retryableError)
{
int unseed;
result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout, noSim);
result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout, noSim, faultInjectionEnabled);
}
}
@ -386,7 +388,7 @@ namespace SummarizeTest
private static int RunTest(string fdbserverName, string tlsPluginFile, string summaryFileName, string errorFileName, int seed,
bool buggify, string testFile, string runDir, string uid, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, bool useValgrind, bool restarting = false,
bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false, bool noSim = false)
bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false, bool noSim = false, bool faultInjectionEnabled = true)
{
unseed = -1;
@ -407,7 +409,7 @@ namespace SummarizeTest
Directory.CreateDirectory(tempPath);
Directory.SetCurrentDirectory(tempPath);
if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, oldBinaryName);
if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, faultInjectionEnabled, oldBinaryName);
string valgrindOutputFile = null;
using (var process = new System.Diagnostics.Process())
@ -422,15 +424,16 @@ namespace SummarizeTest
process.StartInfo.RedirectStandardOutput = true;
string role = (noSim) ? "test" : "simulation";
var args = "";
string faultInjectionArg = string.IsNullOrEmpty(oldBinaryName) ? string.Format("-fi {0}", faultInjectionEnabled ? "on" : "off") : "";
if (willRestart && oldBinaryName.EndsWith("alpha6"))
{
args = string.Format("-Rs 1000000000 -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} --crash",
role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg);
args = string.Format("-Rs 1000000000 -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} {6} --crash",
role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg);
}
else
{
args = string.Format("-Rs 1GB -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} --crash",
role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg);
args = string.Format("-Rs 1GB -r {0} {1} -s {2} -f \"{3}\" -b {4} {5} {6} --crash",
role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg);
}
if (restarting) args = args + " --restarting";
if (useValgrind && !willRestart)
@ -524,7 +527,7 @@ namespace SummarizeTest
var xout = new XElement("UnableToKillProcess",
new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways));
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName);
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled);
return 104;
}
}
@ -549,7 +552,7 @@ namespace SummarizeTest
new XAttribute("Plugin", tlsPluginFile),
new XAttribute("MachineName", System.Environment.MachineName));
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName);
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled);
ok = useValgrind ? 0 : 103;
}
else
@ -588,7 +591,7 @@ namespace SummarizeTest
new XAttribute("Severity", (int)Magnesium.Severity.SevError),
new XAttribute("ErrorMessage", e.Message));
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName);
AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName, faultInjectionEnabled);
return 101;
}
finally
@ -638,6 +641,15 @@ namespace SummarizeTest
{
if(!String.IsNullOrEmpty(errLine.Data))
{
if (errLine.Data.EndsWith("WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!")) {
// When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives.
return;
}
if (errLine.Data.EndsWith("Warning: unimplemented fcntl command: 1036")) {
// Valgrind produces this warning when F_SET_RW_HINT is used
return;
}
hasError = true;
if(Errors.Count < maxErrors) {
if(errLine.Data.Length > maxErrorLength) {
@ -695,13 +707,14 @@ namespace SummarizeTest
}
}
static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, string oldBinary="")
static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, bool faultInjectionEnabled, string oldBinary="")
{
var xout = new XElement("TestPlan",
new XAttribute("TestUID", uid),
new XAttribute("RandomSeed", randomSeed),
new XAttribute("TestFile", testFileName),
new XAttribute("BuggifyEnabled", buggify ? "1" : "0"),
new XAttribute("FaultInjectionEnabled", faultInjectionEnabled ? "1" : "0"),
new XAttribute("DeterminismCheck", testDeterminism ? "1" : "0"),
new XAttribute("OldBinary", Path.GetFileName(oldBinary)));
AppendToSummary(summaryFileName, xout);
@ -791,6 +804,8 @@ namespace SummarizeTest
new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"),
new XAttribute("OldBinary", Path.GetFileName(oldBinaryName)));
testBeginFound = true;
if (ev.DDetails.ContainsKey("FaultInjectionEnabled"))
xout.Add(new XAttribute("FaultInjectionEnabled", ev.Details.FaultInjectionEnabled));
}
if (ev.Type == "Simulation")
{
@ -962,10 +977,6 @@ namespace SummarizeTest
int stderrBytes = 0;
foreach (string err in outputErrors)
{
if (err.EndsWith("WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!")) {
// When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives.
continue;
}
if (stderrSeverity == (int)Magnesium.Severity.SevError)
{
error = true;
@ -1230,7 +1241,7 @@ namespace SummarizeTest
}
private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null,
int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null)
int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null, bool? faultInjectionEnabled = null)
{
var test = new XElement("Test", xout);
if(testFile != null)
@ -1239,6 +1250,8 @@ namespace SummarizeTest
test.Add(new XAttribute("RandomSeed", seed));
if(buggify != null)
test.Add(new XAttribute("BuggifyEnabled", buggify.Value ? "1" : "0"));
if(faultInjectionEnabled != null)
test.Add(new XAttribute("FaultInjectionEnabled", faultInjectionEnabled.Value ? "1" : "0"));
if(determinismCheck != null)
test.Add(new XAttribute("DeterminismCheck", determinismCheck.Value ? "1" : "0"));
if(oldBinaryName != null)

2
documentation/sphinx/source/api-version-upgrade-guide.rst
View File

@ -25,6 +25,8 @@ API version 700
General
-------
* Committing a transaction will no longer partially reset it. In particular, getting the read version from a transaction that has committed or failed to commit with an error will return the original read version.
Python bindings
---------------

8
documentation/sphinx/source/ha-write-path.rst
View File

@ -43,7 +43,7 @@ Tag is an overloaded term in FDB. In the early history of FDB, a tag is a number
* As FDB scales and we work to reduce the recovery time, a special tag for transaction state store (txnStateStore) is introduced;
* and so on.
* FDB also have transaction tags which are used for transaction throttling, not for the tag-partitioned log system mentioned in this article. See :ref:`transaction-tagging`
To distinguish the types of tags used for different purposes at different locations (primary DC or remote DC), we introduce Tag structure, which has two fields:
@ -64,7 +64,7 @@ To simplify the description, we ignore the batching mechanisms happening in each
Figure 1 illustrates how a mutation is routed inside FDB. The solid lines are asynchronous pull operations, while the dotted lines are synchronous push operations.
.. image:: /images/FDB_ha_write_path.png
.. image:: images/FDB_ha_write_path.png
At Client
---------
@ -86,7 +86,7 @@ At Proxy
* 1 tag for log router. Assume it is (-2, 3), where -2 is the locality value for all log router tags. The tag id is randomly chosen by proxy as well.
* No tag for satellite tLog. The "satellite TLog locality" -5 in the code is used when recruiting a satellite TLog to tell it that it is a satellite TLog. This causes the TLog to only index log router tags (-2) and not bother indexing any of the >0 tags.
* No tag for satellite tLog. The "satellite TLog locality" -5 in the code is used when recruiting a satellite TLog to tell it that it is a satellite TLog. This causes the satellite TLog to only index log router tags (-2) and not bother indexing any of the >0 tags.
Why do we need log routers? Why cannot we let remote tLog directly pull data from primary tLogs?
@ -98,7 +98,7 @@ Another alternative is to use remote SSes’ tags to decide which satellite tLog
Proxy groups mutations with the same tag as messages. Proxy then synchronously pushes these mutation messages to tLogs based on the tags. Proxy cannot acknowledge that the transaction is committed until the message has been durable on all primary and satellite tLogs.
**Commit empty messages to tLogs.** When a proxy commits a tagged mutation message at version V1 to tLogs, it also has to commit an empty message at the same version V1 to the rest of tLogs. This makes sure every tLog has the same versions of messages, even though some messages are empty. This is a trick used in FDB to let all tLogs march at the same versions. The reason why FDB does the trick is because the master hands out segments of versions as 'from v1 to v2', and the TLogs need to be able to piece all of them back together into one consistent timeline. It may or may not be a good design decision, because a slow tLog can delay other tLogs of the same kind. We may want to revisit the design later.
**Commit empty messages to tLogs.** When a proxy commits a tagged mutation message at version V1 to tLogs, it also has to commit an empty message at the same version V1 to the rest of tLogs. This makes sure every tLog has the same versions of messages, even though some messages are empty. This is a trick used in FDB to let all tLogs march at the same versions. The reason why FDB does the trick is that the master hands out segments of versions as 'from v1 to v2', and the TLogs need to be able to piece all of them back together into one consistent timeline. It may or may not be a good design decision, because a slow tLog can delay other tLogs of the same kind. We may want to revisit the design later.
At primary tLogs and satellite tLogs

22
documentation/sphinx/source/release-notes/release-notes-630.rst
View File

@ -2,6 +2,28 @@
Release Notes
#############
6.3.19
======
* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) <https://github.com/apple/foundationdb/pull/5330>`_
6.3.18
======
* The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5221) <https://github.com/apple/foundationdb/pull/5221>`_
* Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5221) <https://github.com/apple/foundationdb/pull/5221>`_
* Added Mako latency measurements. `(PR #5255) <https://github.com/apple/foundationdb/pull/5255>`_
* Fixed a bug introduced when porting restoring an inconsistent snapshot feature from 7.0 branch to 6.3 branch. The parameter that controls whether to perform an inconsistent snapshot restore may instead be used to lock the database during restore. `(PR #5228) <https://github.com/apple/foundationdb/pull/5228>`_
* Added SidebandMultiThreadClientTest, which validates causal consistency for multi-threaded client. `(PR #5173) <https://github.com/apple/foundationdb/pull/5173>`_
6.3.17
======
* Made readValuePrefix consistent regarding error messages. `(PR #5160) <https://github.com/apple/foundationdb/pull/5160>`_
* Added ``TLogPopDetails`` trace event to tLog pop. `(PR #5134) <https://github.com/apple/foundationdb/pull/5134>`_
* Added ``CommitBatchingEmptyMessageRatio`` metric to track the ratio of empty messages to tlogs. `(PR #5087) <https://github.com/apple/foundationdb/pull/5087>`_
* Observability improvements in ProxyStats. `(PR #5046) <https://github.com/apple/foundationdb/pull/5046>`_
* Added ``RecoveryInternal`` and ``ProxyReplies`` trace events to recovery_transaction step in recovery. `(PR #5038) <https://github.com/apple/foundationdb/pull/5038>`_
* Multi-threaded client documentation improvements. `(PR #5033) <https://github.com/apple/foundationdb/pull/5033>`_
* Added ``ClusterControllerWorkerFailed`` trace event when a worker is removed from cluster controller. `(PR #5035) <https://github.com/apple/foundationdb/pull/5035>`_
* Added histograms for storage server write path components. `(PR #5019) <https://github.com/apple/foundationdb/pull/5019>`_
6.3.15
======

6
documentation/sphinx/source/release-notes/release-notes-700.rst
View File

@ -28,6 +28,8 @@ Features
* Added the Testing Storage Server (TSS), which allows FoundationDB to run an "untrusted" storage engine with identical workload to the current storage engine, with zero impact on durability or correctness, and minimal impact on performance. `(Documentation) <https://github.com/apple/foundationdb/blob/master/documentation/sphinx/source/tss.rst>`_ `(PR #4556) <https://github.com/apple/foundationdb/pull/4556>`_
* Added perpetual storage wiggle that supports less impactful B-trees recreation and data migration. These will also be used for deploying the Testing Storage Server which compares 2 storage engines' results. See :ref:`Documentation <perpetual-storage-wiggle>` for details. `(PR #4838) <https://github.com/apple/foundationdb/pull/4838>`_
* Improved the efficiency with which storage servers replicate data between themselves. `(PR #5017) <https://github.com/apple/foundationdb/pull/5017>`_
* Added support to ``exclude command`` to exclude based on locality match. `(PR #5113) <https://github.com/apple/foundationdb/pull/5113>`_
* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5328) <https://github.com/apple/foundationdb/pull/5328>`_
Performance
-----------
@ -60,6 +62,8 @@ Fixes
* Added a new pre-backup action when creating a backup. Backups can now either verify the range data is being saved to is empty before the backup begins (current behavior) or clear the range where data is being saved to. Fixes a ``restore_destination_not_empty`` failure after a backup retry due to ``commit_unknown_failure``. `(PR #4595) <https://github.com/apple/foundationdb/pull/4595>`_
* When configured with ``usable_regions=2``, a cluster would not fail over to a region which contained only storage class processes. `(PR #4599) <https://github.com/apple/foundationdb/pull/4599>`_
* If a restore is done using a prefix to remove and specific key ranges to restore, the key range boundaries must begin with the prefix to remove. `(PR #4684) <https://github.com/apple/foundationdb/pull/4684>`_
* The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5220) <https://github.com/apple/foundationdb/pull/5220>`_
* Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5220) <https://github.com/apple/foundationdb/pull/5220>`_
Status
------
@ -89,6 +93,8 @@ Other Changes
* The ``foundationdb`` service installed by the RPM packages will now automatically restart ``fdbmonitor`` after 60 seconds when it fails. `(PR #3841) <https://github.com/apple/foundationdb/pull/3841>`_
* Capture output of forked snapshot processes in trace events. `(PR #4254) <https://github.com/apple/foundationdb/pull/4254/files>`_
* Add ErrorKind field to Severity 40 trace events. `(PR #4741) <https://github.com/apple/foundationdb/pull/4741/files>`_
* Added histograms for the storage server write path components. `(PR #5021) <https://github.com/apple/foundationdb/pull/5021/files>`_
* Committing a transaction will no longer partially reset it as of API version 700. `(PR #5271) <https://github.com/apple/foundationdb/pull/5271/files>`_
Earlier release notes
---------------------

10
fdbbackup/FileConverter.h
View File

@ -41,6 +41,11 @@ enum {
OPT_TRACE_LOG_GROUP,
OPT_INPUT_FILE,
OPT_BUILD_FLAGS,
OPT_LIST_ONLY,
OPT_KEY_PREFIX,
OPT_HEX_KEY_PREFIX,
OPT_BEGIN_VERSION_FILTER,
OPT_END_VERSION_FILTER,
OPT_HELP
};
@ -62,6 +67,11 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
TLS_OPTION_FLAGS
#endif
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_LIST_ONLY, "--list_only", SO_NONE },
{ OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
{ OPT_HEX_KEY_PREFIX, "--hex_prefix", SO_REQ_SEP },
{ OPT_BEGIN_VERSION_FILTER, "--begin_version_filter", SO_REQ_SEP },
{ OPT_END_VERSION_FILTER, "--end_version_filter", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },

452
fdbbackup/FileDecoder.actor.cpp
View File

@ -19,14 +19,20 @@
*/
#include <algorithm>
#include <cstdlib>
#include <iostream>
#include <limits>
#include <string>
#include <vector>
#include "fdbbackup/BackupTLSConfig.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbbackup/FileConverter.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/MutationList.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/flow.h"
#include "flow/serialize.h"
@ -65,6 +71,14 @@ void printDecodeUsage() {
TLS_HELP
#endif
" --build_flags Print build information and exit.\n"
" --list_only Print file list and exit.\n"
" -k KEY_PREFIX Use the prefix for filtering mutations\n"
" --hex_prefix HEX_PREFIX\n"
" The prefix specified in HEX format, e.g., \\x05\\x01.\n"
" --begin_version_filter BEGIN_VERSION\n"
" The version range's begin version (inclusive) for filtering.\n"
" --end_version_filter END_VERSION\n"
" The version range's end version (exclusive) for filtering.\n"
"\n";
return;
}
@ -76,9 +90,19 @@ void printBuildInformation() {
struct DecodeParams {
std::string container_url;
std::string fileFilter; // only files match the filter will be decoded
bool log_enabled = false;
bool log_enabled = true;
std::string log_dir, trace_format, trace_log_group;
BackupTLSConfig tlsConfig;
bool list_only = false;
std::string prefix; // Key prefix for filtering
Version beginVersionFilter = 0;
Version endVersionFilter = std::numeric_limits<Version>::max();
// Returns if [begin, end) overlap with the filter range
bool overlap(Version begin, Version end) const {
// Filter [100, 200), [50,75) [200, 300)
return !(begin >= endVersionFilter || end <= beginVersionFilter);
}
std::string toString() {
std::string s;
@ -97,12 +121,69 @@ struct DecodeParams {
s.append(" LogGroup:").append(trace_log_group);
}
}
s.append(", list_only: ").append(list_only ? "true" : "false");
if (beginVersionFilter != 0) {
s.append(", beginVersionFilter: ").append(std::to_string(beginVersionFilter));
}
if (endVersionFilter < std::numeric_limits<Version>::max()) {
s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter));
}
if (!prefix.empty()) {
s.append(", KeyPrefix: ").append(printable(KeyRef(prefix)));
}
return s;
}
};
// Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
// into the binary string.
std::string decode_hex_string(std::string line) {
size_t i = 0;
std::string ret;
while (i <= line.length()) {
switch (line[i]) {
case '\\':
if (i + 2 > line.length()) {
std::cerr << "Invalid hex string at: " << i << "\n";
return ret;
}
switch (line[i + 1]) {
char ent, save;
case '"':
case '\\':
case ' ':
case ';':
line.erase(i, 1);
break;
case 'x':
if (i + 4 > line.length()) {
std::cerr << "Invalid hex string at: " << i << "\n";
return ret;
}
char* pEnd;
save = line[i + 4];
line[i + 4] = 0;
ent = char(strtoul(line.data() + i + 2, &pEnd, 16));
if (*pEnd) {
std::cerr << "Invalid hex string at: " << i << "\n";
return ret;
}
line[i + 4] = save;
line.replace(i, 4, 1, ent);
break;
default:
std::cerr << "Invalid hex string at: " << i << "\n";
return ret;
}
default:
i++;
}
}
return line.substr(0, i);
}
int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
while (args->Next()) {
auto lastError = args->LastError();
@ -124,6 +205,26 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
param->container_url = args->OptionArg();
break;
case OPT_LIST_ONLY:
param->list_only = true;
break;
case OPT_KEY_PREFIX:
param->prefix = args->OptionArg();
break;
case OPT_HEX_KEY_PREFIX:
param->prefix = decode_hex_string(args->OptionArg());
break;
case OPT_BEGIN_VERSION_FILTER:
param->beginVersionFilter = std::atoll(args->OptionArg());
break;
case OPT_END_VERSION_FILTER:
param->endVersionFilter = std::atoll(args->OptionArg());
break;
case OPT_CRASHONERROR:
g_crashOnError = true;
break;
@ -141,7 +242,7 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
break;
case OPT_TRACE_FORMAT:
if (!validateTraceFormat(args->OptionArg())) {
if (!selectTraceFormatter(args->OptionArg())) {
std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n";
return FDB_EXIT_ERROR;
}
@ -202,78 +303,18 @@ void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, const DecodeParams& params) {
std::vector<LogFile> filtered;
for (const auto& file : files) {
if (file.fileName.find(params.fileFilter) != std::string::npos) {
if (file.fileName.find(params.fileFilter) != std::string::npos &&
params.overlap(file.beginVersion, file.endVersion + 1)) {
filtered.push_back(file);
}
}
return filtered;
}
std::pair<Version, int32_t> decode_key(const StringRef& key) {
ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t));
uint8_t hash;
Version version;
int32_t part;
BinaryReader rd(key, Unversioned());
rd >> hash >> version >> part;
version = bigEndian64(version);
part = bigEndian32(part);
int32_t v = version / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
ASSERT(((uint8_t)hashlittle(&v, sizeof(v), 0)) == hash);
return std::make_pair(version, part);
}
// Decodes an encoded list of mutations in the format of:
// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k],
// where a mutation is encoded as:
// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][key][value]
std::vector<MutationRef> decode_value(const StringRef& value) {
StringRefReader reader(value, restore_corrupted_data());
reader.consume<uint64_t>(); // Consume the includeVersion
uint32_t val_length = reader.consume<uint32_t>();
if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) {
TraceEvent(SevError, "ValueError")
.detail("ValueLen", val_length)
.detail("ValueSize", value.size())
.detail("Value", printable(value));
}
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
type = reader.consume<uint32_t>();
p1len = reader.consume<uint32_t>();
p2len = reader.consume<uint32_t>();
const uint8_t* key = reader.consume(p1len);
const uint8_t* val = reader.consume(p2len);
mutations.emplace_back((MutationRef::Type)type, StringRef(key, p1len), StringRef(val, p2len));
}
return mutations;
}
struct VersionedMutations {
Version version;
std::vector<MutationRef> mutations;
Arena arena; // The arena that contains the mutations.
};
struct VersionedKVPart {
Arena arena;
Version version;
int32_t part;
StringRef kv;
VersionedKVPart(Arena arena, Version version, int32_t part, StringRef kv)
: arena(arena), version(version), part(part), kv(kv) {}
std::string serializedMutations; // buffer that contains mutations
};
/*
@ -293,174 +334,66 @@ struct VersionedKVPart {
* at any time this object might have two blocks of data in memory.
*/
class DecodeProgress {
std::vector<VersionedKVPart> keyValues;
std::vector<Standalone<VectorRef<KeyValueRef>>> blocks;
std::unordered_map<Version, fileBackup::AccumulatedMutations> mutationBlocksByVersion;
public:
DecodeProgress() = default;
template <class U>
DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
DecodeProgress(const LogFile& file) : file(file) {}
// If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true,
// which means we should look for data in the next file. The caller
// should call getUnfinishedBuffer() to get these left data.
bool finished() const { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); }
std::vector<VersionedKVPart>&& getUnfinishedBuffer() && { return std::move(keyValues); }
// Returns all mutations of the next version in a batch.
Future<VersionedMutations> getNextBatch() { return getNextBatchImpl(this); }
bool finished() const { return done; }
// Open and loads file into memory
Future<Void> openFile(Reference<IBackupContainer> container) { return openFileImpl(this, container); }
// The following are private APIs:
// Returns true if value contains complete data.
static bool isValueComplete(StringRef value) {
StringRefReader reader(value, restore_corrupted_data());
reader.consume<uint64_t>(); // Consume the includeVersion
uint32_t val_length = reader.consume<uint32_t>();
return val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t);
}
// PRECONDITION: finished() must return false before calling this function.
// Returns the next batch of mutations along with the arena backing it.
// Note the returned batch can be empty when the file has unfinished
// version batch data that are in the next file.
ACTOR static Future<VersionedMutations> getNextBatchImpl(DecodeProgress* self) {
ASSERT(!self->finished());
VersionedMutations getNextBatch() {
ASSERT(!finished());
loop {
if (self->keyValues.size() <= 1) {
// Try to decode another block when less than one left
wait(readAndDecodeFile(self));
}
const auto& kv = self->keyValues[0];
ASSERT(kv.part == 0);
// decode next versions, check if they are continuous parts
int idx = 1; // next kv pair in "keyValues"
int bufSize = kv.kv.size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size())
break;
const auto& nextKV = self->keyValues[idx];
if (kv.version != nextKV.version) {
break;
}
if (lastPart + 1 != nextKV.part) {
TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", nextKV.part);
throw restore_corrupted_data();
}
bufSize += nextKV.kv.size();
}
VersionedMutations m;
m.version = kv.version;
TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size());
StringRef value = kv.kv;
if (idx > 1) {
// Stitch parts into one and then decode one by one
Standalone<StringRef> buf = self->combineValues(idx, bufSize);
value = buf;
m.arena = buf.arena();
}
if (isValueComplete(value)) {
m.mutations = decode_value(value);
if (m.arena.getSize() == 0) {
m.arena = kv.arena;
}
self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx);
return m;
} else if (!self->eof) {
// Read one more block, hopefully the missing part of the value can be found.
wait(readAndDecodeFile(self));
} else {
TraceEvent(SevWarn, "MissingValue").detail("Version", m.version);
self->leftover = true;
return m; // Empty mutations
VersionedMutations vms;
for (auto& [version, m] : mutationBlocksByVersion) {
if (m.isComplete()) {
vms.version = version;
std::vector<MutationRef> mutations = fileBackup::decodeMutationLogValue(m.serializedMutations);
TraceEvent("Decode").detail("Version", vms.version).detail("N", mutations.size());
vms.mutations.insert(vms.mutations.end(), mutations.begin(), mutations.end());
vms.serializedMutations = m.serializedMutations;
mutationBlocksByVersion.erase(version);
return vms;
}
}
}
// Returns a buffer which stitches first "idx" values into one.
// "len" MUST equal the summation of these values.
Standalone<StringRef> combineValues(const int idx, const int len) {
ASSERT(idx <= keyValues.size() && idx > 1);
Standalone<StringRef> buf = makeString(len);
int n = 0;
for (int i = 0; i < idx; i++) {
const auto& value = keyValues[i].kv;
memcpy(mutateString(buf) + n, value.begin(), value.size());
n += value.size();
}
ASSERT(n == len);
return buf;
}
// Decodes a block into KeyValueRef stored in "keyValues".
void decode_block(const Standalone<StringRef>& buf, int len) {
StringRef block(buf.begin(), len);
StringRefReader reader(block, restore_corrupted_data());
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
const uint8_t* k = reader.consume(kLen);
std::pair<Version, int32_t> version_part = decode_key(StringRef(k, kLen));
uint32_t vLen = reader.consumeNetworkUInt32();
const uint8_t* v = reader.consume(vLen);
TraceEvent(SevDecodeInfo, "Block")
.detail("KeySize", kLen)
.detail("valueSize", vLen)
.detail("Offset", reader.rptr - buf.begin())
.detail("Version", version_part.first)
.detail("Part", version_part.second);
keyValues.emplace_back(buf.arena(), version_part.first, version_part.second, StringRef(v, vLen));
}
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) {
if (b != 0xFF)
throw restore_corrupted_data_padding();
}
// The (version, part) in a block can be out of order, i.e., (3, 0)
// can be followed by (4, 0), and then (3, 1). So we need to sort them
// first by version, and then by part number.
std::sort(keyValues.begin(), keyValues.end(), [](const VersionedKVPart& a, const VersionedKVPart& b) {
return a.version == b.version ? a.part < b.part : a.version < b.version;
});
return;
} catch (Error& e) {
TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin());
throw;
// No complete versions
if (!mutationBlocksByVersion.empty()) {
TraceEvent(SevWarn, "UnfishedBlocks").detail("NumberOfVersions", mutationBlocksByVersion.size());
}
done = true;
return vms;
}
ACTOR static Future<Void> openFileImpl(DecodeProgress* self, Reference<IBackupContainer> container) {
Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
self->fd = fd;
wait(readAndDecodeFile(self));
while (!self->eof) {
wait(readAndDecodeFile(self));
}
return Void();
}
// Add chunks to mutationBlocksByVersion
void addBlockKVPairs(VectorRef<KeyValueRef> chunks) {
for (auto& kv : chunks) {
auto versionAndChunkNumber = fileBackup::decodeMutationLogKey(kv.key);
mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv);
}
}
// Reads a file block, decodes it into key/value pairs, and stores these pairs.
ACTOR static Future<Void> readAndDecodeFile(DecodeProgress* self) {
try {
@ -470,17 +403,18 @@ public:
return Void();
}
state Standalone<StringRef> buf = makeString(len);
state int rLen = wait(self->fd->read(mutateString(buf), len, self->offset));
// Decode a file block into log_key and log_value chunks
Standalone<VectorRef<KeyValueRef>> chunks =
wait(fileBackup::decodeMutationLogFileBlock(self->fd, self->offset, len));
self->blocks.push_back(chunks);
TraceEvent("ReadFile")
.detail("Name", self->file.fileName)
.detail("Len", rLen)
.detail("Len", len)
.detail("Offset", self->offset);
if (rLen != len) {
throw restore_corrupted_data();
}
self->decode_block(buf, rLen);
self->offset += rLen;
self->addBlockKVPairs(chunks);
self->offset += len;
return Void();
} catch (Error& e) {
TraceEvent(SevWarn, "CorruptLogFileBlock")
@ -496,12 +430,55 @@ public:
Reference<IAsyncFile> fd;
int64_t offset = 0;
bool eof = false;
bool leftover = false; // Done but has unfinished version batch data left
bool done = false;
};
ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile file, UID uid, DecodeParams params) {
if (file.fileSize == 0) {
TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
return Void();
}
state DecodeProgress progress(file);
wait(progress.openFile(container));
while (!progress.finished()) {
VersionedMutations vms = progress.getNextBatch();
if (vms.version < params.beginVersionFilter || vms.version >= params.endVersionFilter) {
TraceEvent("SkipVersion").detail("Version", vms.version);
continue;
}
int sub = 0;
for (const auto& m : vms.mutations) {
sub++; // sub sequence number starts at 1
bool print = params.prefix.empty(); // no filtering
if (!print) {
if (isSingleKeyMutation((MutationRef::Type)m.type)) {
print = m.param1.startsWith(StringRef(params.prefix));
} else if (m.type == MutationRef::ClearRange) {
KeyRange range(KeyRangeRef(m.param1, m.param2));
print = range.contains(StringRef(params.prefix));
} else {
ASSERT(false);
}
}
if (print) {
TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
.detail("Version", vms.version)
.setMaxFieldLength(10000)
.detail("M", m.toString());
std::cout << vms.version << " " << m.toString() << "\n";
}
}
}
TraceEvent("ProcessFileDone", uid).detail("File", file.fileName);
return Void();
}
ACTOR Future<Void> decode_logs(DecodeParams params) {
state Reference<IBackupContainer> container = IBackupContainer::openContainer(params.container_url);
state UID uid = deterministicRandom()->randomUniqueID();
state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(),
@ -512,7 +489,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
}),
listing.logs.end());
std::sort(listing.logs.begin(), listing.logs.end());
TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size());
TraceEvent("Container", uid).detail("URL", params.container_url).detail("Logs", listing.logs.size());
TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params.toString());
BackupDescription desc = wait(container->describeBackup());
std::cout << "\n" << desc.toString() << "\n";
@ -520,26 +498,15 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params);
printLogFiles("Relevant files are: ", logs);
state int i = 0;
// Previous file's unfinished version data
state std::vector<VersionedKVPart> left;
for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0)
continue;
if (params.list_only) return Void();
state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container));
while (!progress.finished()) {
VersionedMutations vms = wait(progress.getNextBatch());
for (const auto& m : vms.mutations) {
std::cout << vms.version << " " << m.toString() << "\n";
}
}
left = std::move(progress).getUnfinishedBuffer();
if (!left.empty()) {
TraceEvent("UnfinishedFile").detail("File", logs[i].fileName).detail("Q", left.size());
}
state int idx = 0;
while (idx < logs.size()) {
TraceEvent("ProcessFile").detail("Name", logs[idx].fileName).detail("I", idx);
wait(process_file(container, logs[idx], uid, params));
idx++;
}
TraceEvent("DecodeDone", uid);
return Void();
}
@ -564,6 +531,8 @@ int main(int argc, char** argv) {
}
if (!param.trace_format.empty()) {
setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format));
} else {
setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr);
}
if (!param.trace_log_group.empty()) {
setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group));
@ -571,7 +540,7 @@ int main(int argc, char** argv) {
}
if (!param.tlsConfig.setupTLS()) {
TraceEvent(SevError, "TLSError");
TraceEvent(SevError, "TLSError").log();
throw tls_error();
}
@ -582,12 +551,17 @@ int main(int argc, char** argv) {
setupNetwork(0, UseMetrics::True);
TraceEvent::setNetworkThread();
openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "decode", param.trace_log_group);
openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group);
param.tlsConfig.setupBlobCredentials();
auto f = stopAfter(decode_logs(param));
runNetwork();
flushTraceFileVoid();
fflush(stdout);
closeTraceFile();
return status;
} catch (Error& e) {
std::cerr << "ERROR: " << e.what() << "\n";

2
fdbbackup/backup.actor.cpp
View File

@ -1667,7 +1667,7 @@ ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
readMore = true;
} catch (Error& e) {
// If doc can't be parsed or isn't alive, delete it.
TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key);
TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key).error(e, true);
tr->clear(docs[i].key);
// If limit is 1 then read more.
if (limit == 1)

2
fdbcli/CMakeLists.txt
View File

@ -7,7 +7,9 @@ set(FDBCLI_SRCS
FlowLineNoise.h
ForceRecoveryWithDataLossCommand.actor.cpp
MaintenanceCommand.actor.cpp
SetClassCommand.actor.cpp
SnapshotCommand.actor.cpp
ThrottleCommand.actor.cpp
Util.cpp
linenoise/linenoise.h)

123
fdbcli/SetClassCommand.actor.cpp Normal file
View File

@ -0,0 +1,123 @@
/*
* SetClassCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace {
ACTOR Future<Void> printProcessClass(Reference<IDatabase> db) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
try {
// Hold the reference to the memory
state ThreadFuture<RangeResult> classTypeFuture =
tr->getRange(fdb_cli::processClassTypeSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
state ThreadFuture<RangeResult> classSourceFuture =
tr->getRange(fdb_cli::processClassSourceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
wait(success(safeThreadFutureToFuture(classSourceFuture)) &&
success(safeThreadFutureToFuture(classTypeFuture)));
RangeResult processTypeList = classTypeFuture.get();
RangeResult processSourceList = classSourceFuture.get();
ASSERT(processSourceList.size() == processTypeList.size());
if (!processTypeList.size())
printf("No processes are registered in the database.\n");
printf("There are currently %zu processes in the database:\n", processTypeList.size());
for (int index = 0; index < processTypeList.size(); index++) {
std::string address =
processTypeList[index].key.removePrefix(fdb_cli::processClassTypeSpecialKeyRange.begin).toString();
// check the addresses are the same in each list
std::string addressFromSourceList =
processSourceList[index]
.key.removePrefix(fdb_cli::processClassSourceSpecialKeyRange.begin)
.toString();
ASSERT(address == addressFromSourceList);
printf(" %s: %s (%s)\n",
address.c_str(),
processTypeList[index].value.toString().c_str(),
processSourceList[index].value.toString().c_str());
}
return Void();
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
};
ACTOR Future<bool> setProcessClass(Reference<IDatabase> db, KeyRef network_address, KeyRef class_type) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
try {
tr->set(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin), class_type);
wait(safeThreadFutureToFuture(tr->commit()));
return true;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
} // namespace
namespace fdb_cli {
const KeyRangeRef processClassSourceSpecialKeyRange =
KeyRangeRef(LiteralStringRef("\xff\xff/configuration/process/class_source/"),
LiteralStringRef("\xff\xff/configuration/process/class_source0"));
const KeyRangeRef processClassTypeSpecialKeyRange =
KeyRangeRef(LiteralStringRef("\xff\xff/configuration/process/class_type/"),
LiteralStringRef("\xff\xff/configuration/process/class_type0"));
ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
if (tokens.size() != 3 && tokens.size() != 1) {
printUsage(tokens[0]);
return false;
} else if (tokens.size() == 1) {
wait(printProcessClass(db));
} else {
bool successful = wait(setProcessClass(db, tokens[1], tokens[2]));
return successful;
}
return true;
}
CommandFactory setClassFactory(
"setclass",
CommandHelp("setclass [<ADDRESS> <CLASS>]",
"change the class of a process",
"If no address and class are specified, lists the classes of all servers.\n\nSetting the class to "
"`default' resets the process class to the class specified on the command line. The available "
"classes are `unset', `storage', `transaction', `resolution', `commit_proxy', `grv_proxy', "
"`master', `test', "
"`stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', "
"`coordinator', `ratekeeper', `storage_cache', `backup', and `default'."));
} // namespace fdb_cli

645
fdbcli/ThrottleCommand.actor.cpp Normal file
View File

@ -0,0 +1,645 @@
/*
* ThrottleCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/TagThrottle.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/CommitTransaction.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/genericactors.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace {
// Helper functions copied from TagThrottle.actor.cpp
// The only difference is transactions are changed to go through MultiversionTransaction,
// instead of the native Transaction(i.e., RYWTransaction)
ACTOR Future<bool> getValidAutoEnabled(Reference<ITransaction> tr) {
state bool result;
loop {
Optional<Value> value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey)));
if (!value.present()) {
tr->reset();
wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
continue;
} else if (value.get() == LiteralStringRef("1")) {
result = true;
} else if (value.get() == LiteralStringRef("0")) {
result = false;
} else {
TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue").detail("Value", value.get());
tr->reset();
wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
continue;
}
return result;
};
}
ACTOR Future<std::vector<TagThrottleInfo>> getThrottledTags(Reference<IDatabase> db,
int limit,
bool containsRecommend = false) {
state Reference<ITransaction> tr = db->createTransaction();
state bool reportAuto = containsRecommend;
loop {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
try {
if (!containsRecommend) {
wait(store(reportAuto, getValidAutoEnabled(tr)));
}
state ThreadFuture<RangeResult> f = tr->getRange(
reportAuto ? tagThrottleKeys : KeyRangeRef(tagThrottleKeysPrefix, tagThrottleAutoKeysPrefix), limit);
RangeResult throttles = wait(safeThreadFutureToFuture(f));
std::vector<TagThrottleInfo> results;
for (auto throttle : throttles) {
results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key),
TagThrottleValue::fromValue(throttle.value)));
}
return results;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<std::vector<TagThrottleInfo>> getRecommendedTags(Reference<IDatabase> db, int limit) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
try {
bool enableAuto = wait(getValidAutoEnabled(tr));
if (enableAuto) {
return std::vector<TagThrottleInfo>();
}
state ThreadFuture<RangeResult> f =
tr->getRange(KeyRangeRef(tagThrottleAutoKeysPrefix, tagThrottleKeys.end), limit);
RangeResult throttles = wait(safeThreadFutureToFuture(f));
std::vector<TagThrottleInfo> results;
for (auto throttle : throttles) {
results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key),
TagThrottleValue::fromValue(throttle.value)));
}
return results;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<Void> updateThrottleCount(Reference<ITransaction> tr, int64_t delta) {
state ThreadFuture<Optional<Value>> countVal = tr->get(tagThrottleCountKey);
state ThreadFuture<Optional<Value>> limitVal = tr->get(tagThrottleLimitKey);
wait(success(safeThreadFutureToFuture(countVal)) && success(safeThreadFutureToFuture(limitVal)));
int64_t count = 0;
int64_t limit = 0;
if (countVal.get().present()) {
BinaryReader reader(countVal.get().get(), Unversioned());
reader >> count;
}
if (limitVal.get().present()) {
BinaryReader reader(limitVal.get().get(), Unversioned());
reader >> limit;
}
count += delta;
if (count > limit) {
throw too_many_tag_throttles();
}
BinaryWriter writer(Unversioned());
writer << count;
tr->set(tagThrottleCountKey, writer.toValue());
return Void();
}
void signalThrottleChange(Reference<ITransaction> tr) {
tr->atomicOp(
tagThrottleSignalKey, LiteralStringRef("XXXXXXXXXX\x00\x00\x00\x00"), MutationRef::SetVersionstampedValue);
}
ACTOR Future<Void> throttleTags(Reference<IDatabase> db,
TagSet tags,
double tpsRate,
double initialDuration,
TagThrottleType throttleType,
TransactionPriority priority,
Optional<double> expirationTime = Optional<double>(),
Optional<TagThrottledReason> reason = Optional<TagThrottledReason>()) {
state Reference<ITransaction> tr = db->createTransaction();
state Key key = TagThrottleKey(tags, throttleType, priority).toKey();
ASSERT(initialDuration > 0);
if (throttleType == TagThrottleType::MANUAL) {
reason = TagThrottledReason::MANUAL;
}
TagThrottleValue throttle(tpsRate,
expirationTime.present() ? expirationTime.get() : 0,
initialDuration,
reason.present() ? reason.get() : TagThrottledReason::UNSET);
BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
wr << throttle;
state Value value = wr.toValue();
loop {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
if (throttleType == TagThrottleType::MANUAL) {
Optional<Value> oldThrottle = wait(safeThreadFutureToFuture(tr->get(key)));
if (!oldThrottle.present()) {
wait(updateThrottleCount(tr, 1));
}
}
tr->set(key, value);
if (throttleType == TagThrottleType::MANUAL) {
signalThrottleChange(tr);
}
wait(safeThreadFutureToFuture(tr->commit()));
return Void();
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<bool> unthrottleTags(Reference<IDatabase> db,
TagSet tags,
Optional<TagThrottleType> throttleType,
Optional<TransactionPriority> priority) {
state Reference<ITransaction> tr = db->createTransaction();
state std::vector<Key> keys;
for (auto p : allTransactionPriorities) {
if (!priority.present() || priority.get() == p) {
if (!throttleType.present() || throttleType.get() == TagThrottleType::AUTO) {
keys.push_back(TagThrottleKey(tags, TagThrottleType::AUTO, p).toKey());
}
if (!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) {
keys.push_back(TagThrottleKey(tags, TagThrottleType::MANUAL, p).toKey());
}
}
}
state bool removed = false;
loop {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
state std::vector<Future<Optional<Value>>> values;
values.reserve(keys.size());
for (auto key : keys) {
values.push_back(safeThreadFutureToFuture(tr->get(key)));
}
wait(waitForAll(values));
int delta = 0;
for (int i = 0; i < values.size(); ++i) {
if (values[i].get().present()) {
if (TagThrottleKey::fromKey(keys[i]).throttleType == TagThrottleType::MANUAL) {
delta -= 1;
}
tr->clear(keys[i]);
// Report that we are removing this tag if we ever see it present.
// This protects us from getting confused if the transaction is maybe committed.
// It's ok if someone else actually ends up removing this tag at the same time
// and we aren't the ones to actually do it.
removed = true;
}
}
if (delta != 0) {
wait(updateThrottleCount(tr, delta));
}
if (removed) {
signalThrottleChange(tr);
wait(safeThreadFutureToFuture(tr->commit()));
}
return removed;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<Void> enableAuto(Reference<IDatabase> db, bool enabled) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
Optional<Value> value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey)));
if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) ||
(!enabled && value.get() != LiteralStringRef("0"))) {
tr->set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0"));
signalThrottleChange(tr);
wait(safeThreadFutureToFuture(tr->commit()));
}
return Void();
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<bool> unthrottleMatchingThrottles(Reference<IDatabase> db,
KeyRef beginKey,
KeyRef endKey,
Optional<TransactionPriority> priority,
bool onlyExpiredThrottles) {
state Reference<ITransaction> tr = db->createTransaction();
state KeySelector begin = firstGreaterOrEqual(beginKey);
state KeySelector end = firstGreaterOrEqual(endKey);
state bool removed = false;
loop {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
// holds memory of the RangeResult
state ThreadFuture<RangeResult> f = tr->getRange(begin, end, 1000);
state RangeResult tags = wait(safeThreadFutureToFuture(f));
state uint64_t unthrottledTags = 0;
uint64_t manualUnthrottledTags = 0;
for (auto tag : tags) {
if (onlyExpiredThrottles) {
double expirationTime = TagThrottleValue::fromValue(tag.value).expirationTime;
if (expirationTime == 0 || expirationTime > now()) {
continue;
}
}
TagThrottleKey key = TagThrottleKey::fromKey(tag.key);
if (priority.present() && key.priority != priority.get()) {
continue;
}
if (key.throttleType == TagThrottleType::MANUAL) {
++manualUnthrottledTags;
}
removed = true;
tr->clear(tag.key);
unthrottledTags++;
}
if (manualUnthrottledTags > 0) {
wait(updateThrottleCount(tr, -manualUnthrottledTags));
}
if (unthrottledTags > 0) {
signalThrottleChange(tr);
}
wait(safeThreadFutureToFuture(tr->commit()));
if (!tags.more) {
return removed;
}
ASSERT(tags.size() > 0);
begin = KeySelector(firstGreaterThan(tags[tags.size() - 1].key), tags.arena());
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
Future<bool> unthrottleAll(Reference<IDatabase> db,
Optional<TagThrottleType> tagThrottleType,
Optional<TransactionPriority> priority) {
KeyRef begin = tagThrottleKeys.begin;
KeyRef end = tagThrottleKeys.end;
if (tagThrottleType.present() && tagThrottleType == TagThrottleType::AUTO) {
begin = tagThrottleAutoKeysPrefix;
} else if (tagThrottleType.present() && tagThrottleType == TagThrottleType::MANUAL) {
end = tagThrottleAutoKeysPrefix;
}
return unthrottleMatchingThrottles(db, begin, end, priority, false);
}
} // namespace
namespace fdb_cli {
ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
if (tokens.size() == 1) {
printUsage(tokens[0]);
return false;
} else if (tokencmp(tokens[1], "list")) {
if (tokens.size() > 4) {
printf("Usage: throttle list [throttled|recommended|all] [LIMIT]\n");
printf("\n");
printf("Lists tags that are currently throttled.\n");
printf("The default LIMIT is 100 tags.\n");
return false;
}
state bool reportThrottled = true;
state bool reportRecommended = false;
if (tokens.size() >= 3) {
if (tokencmp(tokens[2], "recommended")) {
reportThrottled = false;
reportRecommended = true;
} else if (tokencmp(tokens[2], "all")) {
reportThrottled = true;
reportRecommended = true;
} else if (!tokencmp(tokens[2], "throttled")) {
printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str());
return false;
}
}
state int throttleListLimit = 100;
if (tokens.size() >= 4) {
char* end;
throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10);
if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) {
fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str());
return false;
}
}
state std::vector<TagThrottleInfo> tags;
if (reportThrottled && reportRecommended) {
wait(store(tags, getThrottledTags(db, throttleListLimit, true)));
} else if (reportThrottled) {
wait(store(tags, getThrottledTags(db, throttleListLimit)));
} else if (reportRecommended) {
wait(store(tags, getRecommendedTags(db, throttleListLimit)));
}
bool anyLogged = false;
for (auto itr = tags.begin(); itr != tags.end(); ++itr) {
if (itr->expirationTime > now()) {
if (!anyLogged) {
printf("Throttled tags:\n\n");
printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n");
printf(" --------------+----------------+-----------+--------+------------+------\n");
anyLogged = true;
}
std::string reasonStr = "unset";
if (itr->reason == TagThrottledReason::MANUAL) {
reasonStr = "manual";
} else if (itr->reason == TagThrottledReason::BUSY_WRITE) {
reasonStr = "busy write";
} else if (itr->reason == TagThrottledReason::BUSY_READ) {
reasonStr = "busy read";
}
printf(" %12d | %13ds | %9s | %6s | %10s |%s\n",
(int)(itr->tpsRate),
std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)),
transactionPriorityToString(itr->priority, false),
itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual",
reasonStr.c_str(),
itr->tag.toString().c_str());
}
}
if (tags.size() == throttleListLimit) {
printf("\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n",
throttleListLimit);
printf("Usage: throttle list [LIMIT]\n");
}
if (!anyLogged) {
printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended");
}
} else if (tokencmp(tokens[1], "on")) {
if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) {
printf("Usage: throttle on tag <TAG> [RATE] [DURATION] [PRIORITY]\n");
printf("\n");
printf("Enables throttling for transactions with the specified tag.\n");
printf("An optional transactions per second rate can be specified (default 0).\n");
printf("An optional duration can be specified, which must include a time suffix (s, m, h, "
"d) (default 1h).\n");
printf("An optional priority can be specified. Choices are `default', `immediate', and "
"`batch' (default `default').\n");
return false;
}
double tpsRate = 0.0;
uint64_t duration = 3600;
TransactionPriority priority = TransactionPriority::DEFAULT;
if (tokens.size() >= 5) {
char* end;
tpsRate = std::strtod((const char*)tokens[4].begin(), &end);
if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) {
fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str());
return false;
}
if (tpsRate < 0) {
fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate);
return false;
}
}
if (tokens.size() == 6) {
Optional<uint64_t> parsedDuration = parseDuration(tokens[5].toString());
if (!parsedDuration.present()) {
fprintf(stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str());
return false;
}
duration = parsedDuration.get();
if (duration == 0) {
fprintf(stderr, "ERROR: throttle duration cannot be 0\n");
return false;
}
}
if (tokens.size() == 7) {
if (tokens[6] == LiteralStringRef("default")) {
priority = TransactionPriority::DEFAULT;
} else if (tokens[6] == LiteralStringRef("immediate")) {
priority = TransactionPriority::IMMEDIATE;
} else if (tokens[6] == LiteralStringRef("batch")) {
priority = TransactionPriority::BATCH;
} else {
fprintf(stderr,
"ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', "
"or `batch'.\n",
tokens[6].toString().c_str());
return false;
}
}
TagSet tags;
tags.addTag(tokens[3]);
wait(throttleTags(db, tags, tpsRate, duration, TagThrottleType::MANUAL, priority));
printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str());
} else if (tokencmp(tokens[1], "off")) {
int nextIndex = 2;
TagSet tags;
bool throttleTypeSpecified = false;
bool is_error = false;
Optional<TagThrottleType> throttleType = TagThrottleType::MANUAL;
Optional<TransactionPriority> priority;
if (tokens.size() == 2) {
is_error = true;
}
while (nextIndex < tokens.size() && !is_error) {
if (tokencmp(tokens[nextIndex], "all")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = Optional<TagThrottleType>();
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "auto")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = TagThrottleType::AUTO;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "manual")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = TagThrottleType::MANUAL;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "default")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::DEFAULT;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "immediate")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::IMMEDIATE;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "batch")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::BATCH;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "tag")) {
if (tags.size() > 0 || nextIndex == tokens.size() - 1) {
is_error = true;
continue;
}
tags.addTag(tokens[nextIndex + 1]);
nextIndex += 2;
}
}
if (!is_error) {
state const char* throttleTypeString =
!throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually ");
state std::string priorityString =
priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : "";
if (tags.size() > 0) {
bool success = wait(unthrottleTags(db, tags, throttleType, priority));
if (success) {
printf("Unthrottled tag `%s'%s\n", tokens[3].toString().c_str(), priorityString.c_str());
} else {
printf("Tag `%s' was not %sthrottled%s\n",
tokens[3].toString().c_str(),
throttleTypeString,
priorityString.c_str());
}
} else {
bool unthrottled = wait(unthrottleAll(db, throttleType, priority));
if (unthrottled) {
printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str());
} else {
printf("There were no tags being %sthrottled%s\n", throttleTypeString, priorityString.c_str());
}
}
} else {
printf("Usage: throttle off [all|auto|manual] [tag <TAG>] [PRIORITY]\n");
printf("\n");
printf("Disables throttling for throttles matching the specified filters. At least one "
"filter must be used.\n\n");
printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type "
"of throttle\n");
printf("affected. `all' targets all throttles, `auto' targets those created by the "
"cluster, and\n");
printf("`manual' targets those created manually (default `manual').\n\n");
printf("The `tag' filter can be use to turn off only a specific tag.\n\n");
printf("The priority filter can be used to turn off only throttles at specific priorities. "
"Choices are\n");
printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n");
}
} else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) {
if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) {
printf("Usage: throttle <enable|disable> auto\n");
printf("\n");
printf("Enables or disable automatic tag throttling.\n");
return false;
}
state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable");
wait(enableAuto(db, autoTagThrottlingEnabled));
printf("Automatic tag throttling has been %s\n", autoTagThrottlingEnabled ? "enabled" : "disabled");
} else {
printUsage(tokens[0]);
return false;
}
return true;
}
CommandFactory throttleFactory(
"throttle",
CommandHelp("throttle <on|off|enable auto|disable auto|list> [ARGS]",
"view and control throttled tags",
"Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' "
"to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n"));
} // namespace fdb_cli

429
fdbcli/fdbcli.actor.cpp
View File

@ -566,15 +566,6 @@ void initHelp() {
"pair in <ADDRESS...> or any LocalityData (like dcid, zoneid, machineid, processid), removes any "
"matching exclusions from the excluded servers and localities list. "
"(A specified IP will match all IP:* exclusion entries)");
helpMap["setclass"] =
CommandHelp("setclass [<ADDRESS> <CLASS>]",
"change the class of a process",
"If no address and class are specified, lists the classes of all servers.\n\nSetting the class to "
"`default' resets the process class to the class specified on the command line. The available "
"classes are `unset', `storage', `transaction', `resolution', `commit_proxy', `grv_proxy', "
"`master', `test', "
"`stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', "
"`coordinator', `ratekeeper', `storage_cache', `backup', and `default'.");
helpMap["status"] =
CommandHelp("status [minimal|details|json]",
"get the status of a FoundationDB cluster",
@ -648,11 +639,6 @@ void initHelp() {
"namespace for all the profiling-related commands.",
"Different types support different actions. Run `profile` to get a list of "
"types, and iteratively explore the help.\n");
helpMap["throttle"] =
CommandHelp("throttle <on|off|enable auto|disable auto|list> [ARGS]",
"view and control throttled tags",
"Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' "
"to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n");
helpMap["cache_range"] = CommandHelp(
"cache_range <set|clear> <BEGINKEY> <ENDKEY>",
"Mark a key range to add to or remove from storage caches.",
@ -671,6 +657,7 @@ void initHelp() {
CommandHelp("triggerddteaminfolog",
"trigger the data distributor teams logging",
"Trigger the data distributor to log detailed information about its teams.");
helpMap["rangefeed"] = CommandHelp("rangefeed <register|get|pop> <RANGEID> <BEGINKEY> <ENDKEY>", "", "");
helpMap["tssq"] =
CommandHelp("tssq start|stop <StorageUID>",
"start/stop tss quarantine",
@ -2771,45 +2758,6 @@ ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
return false;
}
ACTOR Future<bool> setClass(Database db, std::vector<StringRef> tokens) {
if (tokens.size() == 1) {
vector<ProcessData> _workers = wait(makeInterruptable(getWorkers(db)));
auto workers = _workers; // strip const
if (!workers.size()) {
printf("No processes are registered in the database.\n");
return false;
}
std::sort(workers.begin(), workers.end(), ProcessData::sort_by_address());
printf("There are currently %zu processes in the database:\n", workers.size());
for (const auto& w : workers)
printf(" %s: %s (%s)\n",
w.address.toString().c_str(),
w.processClass.toString().c_str(),
w.processClass.sourceString().c_str());
return false;
}
AddressExclusion addr = AddressExclusion::parse(tokens[1]);
if (!addr.isValid()) {
fprintf(stderr, "ERROR: '%s' is not a valid network endpoint address\n", tokens[1].toString().c_str());
if (tokens[1].toString().find(":tls") != std::string::npos)
printf(" Do not include the `:tls' suffix when naming a process\n");
return true;
}
ProcessClass processClass(tokens[2].toString(), ProcessClass::DBSource);
if (processClass.classType() == ProcessClass::InvalidClass && tokens[2] != LiteralStringRef("default")) {
fprintf(stderr, "ERROR: '%s' is not a valid process class\n", tokens[2].toString().c_str());
return true;
}
wait(makeInterruptable(setClass(db, addr, processClass)));
return false;
};
Reference<ReadYourWritesTransaction> getTransaction(Database db,
Reference<ReadYourWritesTransaction>& tr,
FdbOptions* options,
@ -3580,6 +3528,77 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
continue;
}
if (tokencmp(tokens[0], "rangefeed")) {
if (tokens.size() == 1) {
printUsage(tokens[0]);
is_error = true;
continue;
}
if (tokencmp(tokens[1], "register")) {
if (tokens.size() != 5) {
printUsage(tokens[0]);
is_error = true;
continue;
}
state Transaction trx(db);
loop {
try {
wait(trx.registerRangeFeed(tokens[2], KeyRangeRef(tokens[3], tokens[4])));
wait(trx.commit());
break;
} catch (Error& e) {
wait(trx.onError(e));
}
}
} else if (tokencmp(tokens[1], "get")) {
if (tokens.size() < 3 || tokens.size() > 5) {
printUsage(tokens[0]);
is_error = true;
continue;
}
Version begin = 0;
Version end = std::numeric_limits<Version>::max();
if (tokens.size() > 3) {
int n = 0;
if (sscanf(tokens[3].toString().c_str(), "%ld%n", &begin, &n) != 1 ||
n != tokens[3].size()) {
printUsage(tokens[0]);
is_error = true;
continue;
}
}
if (tokens.size() > 4) {
int n = 0;
if (sscanf(tokens[4].toString().c_str(), "%ld%n", &end, &n) != 1 || n != tokens[4].size()) {
printUsage(tokens[0]);
is_error = true;
continue;
}
}
Standalone<VectorRef<MutationsAndVersionRef>> res =
wait(db->getRangeFeedMutations(tokens[2], begin, end));
for (auto& it : res) {
for (auto& it2 : it.mutations) {
printf("%lld %s\n", it.version, it2.toString().c_str());
}
}
} else if (tokencmp(tokens[1], "pop")) {
if (tokens.size() != 4) {
printUsage(tokens[0]);
is_error = true;
continue;
}
Version v;
int n = 0;
if (sscanf(tokens[3].toString().c_str(), "%ld%n", &v, &n) != 1 || n != tokens[3].size()) {
printUsage(tokens[0]);
is_error = true;
} else {
wait(db->popRangeFeedMutations(tokens[2], v));
}
}
continue;
}
if (tokencmp(tokens[0], "tssq")) {
if (tokens.size() == 2) {
if (tokens[1] != LiteralStringRef("list")) {
@ -3718,14 +3737,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if (tokencmp(tokens[0], "setclass")) {
if (tokens.size() != 3 && tokens.size() != 1) {
printUsage(tokens[0]);
bool _result = wait(makeInterruptable(setClassCommandActor(db2, tokens)));
if (!_result)
is_error = true;
} else {
bool err = wait(setClass(db, tokens));
if (err)
is_error = true;
}
continue;
}
@ -3984,6 +3998,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
is_error = true;
continue;
}
wait(makeInterruptable(GlobalConfig::globalConfig().onInitialized()));
if (tokencmp(tokens[2], "get")) {
if (tokens.size() != 3) {
fprintf(stderr, "ERROR: Addtional arguments to `get` are not supported.\n");
@ -4551,300 +4566,12 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if (tokencmp(tokens[0], "throttle")) {
if (tokens.size() == 1) {
printUsage(tokens[0]);
bool _result = wait(throttleCommandActor(db2, tokens));
if (!_result)
is_error = true;
continue;
} else if (tokencmp(tokens[1], "list")) {
if (tokens.size() > 4) {
printf("Usage: throttle list [throttled|recommended|all] [LIMIT]\n");
printf("\n");
printf("Lists tags that are currently throttled.\n");
printf("The default LIMIT is 100 tags.\n");
is_error = true;
continue;
}
state bool reportThrottled = true;
state bool reportRecommended = false;
if (tokens.size() >= 3) {
if (tokencmp(tokens[2], "recommended")) {
reportThrottled = false;
reportRecommended = true;
} else if (tokencmp(tokens[2], "all")) {
reportThrottled = true;
reportRecommended = true;
} else if (!tokencmp(tokens[2], "throttled")) {
printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str());
is_error = true;
continue;
}
}
state int throttleListLimit = 100;
if (tokens.size() >= 4) {
char* end;
throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10);
if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) {
fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str());
is_error = true;
continue;
}
}
state std::vector<TagThrottleInfo> tags;
if (reportThrottled && reportRecommended) {
wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit, true)));
} else if (reportThrottled) {
wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit)));
} else if (reportRecommended) {
wait(store(tags, ThrottleApi::getRecommendedTags(db, throttleListLimit)));
}
bool anyLogged = false;
for (auto itr = tags.begin(); itr != tags.end(); ++itr) {
if (itr->expirationTime > now()) {
if (!anyLogged) {
printf("Throttled tags:\n\n");
printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n");
printf(
" --------------+----------------+-----------+--------+------------+------\n");
anyLogged = true;
}
std::string reasonStr = "unset";
if (itr->reason == TagThrottledReason::MANUAL) {
reasonStr = "manual";
} else if (itr->reason == TagThrottledReason::BUSY_WRITE) {
reasonStr = "busy write";
} else if (itr->reason == TagThrottledReason::BUSY_READ) {
reasonStr = "busy read";
}
printf(" %12d | %13ds | %9s | %6s | %10s |%s\n",
(int)(itr->tpsRate),
std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)),
transactionPriorityToString(itr->priority, false),
itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual",
reasonStr.c_str(),
itr->tag.toString().c_str());
}
}
if (tags.size() == throttleListLimit) {
printf(
"\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n",
throttleListLimit);
printf("Usage: throttle list [LIMIT]\n");
}
if (!anyLogged) {
printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended");
}
} else if (tokencmp(tokens[1], "on")) {
if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) {
printf("Usage: throttle on tag <TAG> [RATE] [DURATION] [PRIORITY]\n");
printf("\n");
printf("Enables throttling for transactions with the specified tag.\n");
printf("An optional transactions per second rate can be specified (default 0).\n");
printf("An optional duration can be specified, which must include a time suffix (s, m, h, "
"d) (default 1h).\n");
printf("An optional priority can be specified. Choices are `default', `immediate', and "
"`batch' (default `default').\n");
is_error = true;
continue;
}
double tpsRate = 0.0;
uint64_t duration = 3600;
TransactionPriority priority = TransactionPriority::DEFAULT;
if (tokens.size() >= 5) {
char* end;
tpsRate = std::strtod((const char*)tokens[4].begin(), &end);
if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) {
fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str());
is_error = true;
continue;
}
if (tpsRate < 0) {
fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate);
is_error = true;
continue;
}
}
if (tokens.size() == 6) {
Optional<uint64_t> parsedDuration = parseDuration(tokens[5].toString());
if (!parsedDuration.present()) {
fprintf(
stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str());
is_error = true;
continue;
}
duration = parsedDuration.get();
if (duration == 0) {
fprintf(stderr, "ERROR: throttle duration cannot be 0\n");
is_error = true;
continue;
}
}
if (tokens.size() == 7) {
if (tokens[6] == LiteralStringRef("default")) {
priority = TransactionPriority::DEFAULT;
} else if (tokens[6] == LiteralStringRef("immediate")) {
priority = TransactionPriority::IMMEDIATE;
} else if (tokens[6] == LiteralStringRef("batch")) {
priority = TransactionPriority::BATCH;
} else {
fprintf(stderr,
"ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', "
"or `batch'.\n",
tokens[6].toString().c_str());
is_error = true;
continue;
}
}
TagSet tags;
tags.addTag(tokens[3]);
wait(ThrottleApi::throttleTags(db, tags, tpsRate, duration, TagThrottleType::MANUAL, priority));
printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str());
} else if (tokencmp(tokens[1], "off")) {
int nextIndex = 2;
TagSet tags;
bool throttleTypeSpecified = false;
Optional<TagThrottleType> throttleType = TagThrottleType::MANUAL;
Optional<TransactionPriority> priority;
if (tokens.size() == 2) {
is_error = true;
}
while (nextIndex < tokens.size() && !is_error) {
if (tokencmp(tokens[nextIndex], "all")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = Optional<TagThrottleType>();
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "auto")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = TagThrottleType::AUTO;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "manual")) {
if (throttleTypeSpecified) {
is_error = true;
continue;
}
throttleTypeSpecified = true;
throttleType = TagThrottleType::MANUAL;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "default")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::DEFAULT;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "immediate")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::IMMEDIATE;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "batch")) {
if (priority.present()) {
is_error = true;
continue;
}
priority = TransactionPriority::BATCH;
++nextIndex;
} else if (tokencmp(tokens[nextIndex], "tag")) {
if (tags.size() > 0 || nextIndex == tokens.size() - 1) {
is_error = true;
continue;
}
tags.addTag(tokens[nextIndex + 1]);
nextIndex += 2;
}
}
if (!is_error) {
state const char* throttleTypeString =
!throttleType.present()
? ""
: (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually ");
state std::string priorityString =
priority.present()
? format(" at %s priority", transactionPriorityToString(priority.get(), false))
: "";
if (tags.size() > 0) {
bool success = wait(ThrottleApi::unthrottleTags(db, tags, throttleType, priority));
if (success) {
printf("Unthrottled tag `%s'%s\n",
tokens[3].toString().c_str(),
priorityString.c_str());
} else {
printf("Tag `%s' was not %sthrottled%s\n",
tokens[3].toString().c_str(),
throttleTypeString,
priorityString.c_str());
}
} else {
bool unthrottled = wait(ThrottleApi::unthrottleAll(db, throttleType, priority));
if (unthrottled) {
printf("Unthrottled all %sthrottled tags%s\n",
throttleTypeString,
priorityString.c_str());
} else {
printf("There were no tags being %sthrottled%s\n",
throttleTypeString,
priorityString.c_str());
}
}
} else {
printf("Usage: throttle off [all|auto|manual] [tag <TAG>] [PRIORITY]\n");
printf("\n");
printf("Disables throttling for throttles matching the specified filters. At least one "
"filter must be used.\n\n");
printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type "
"of throttle\n");
printf("affected. `all' targets all throttles, `auto' targets those created by the "
"cluster, and\n");
printf("`manual' targets those created manually (default `manual').\n\n");
printf("The `tag' filter can be use to turn off only a specific tag.\n\n");
printf("The priority filter can be used to turn off only throttles at specific priorities. "
"Choices are\n");
printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n");
}
} else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) {
if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) {
printf("Usage: throttle <enable|disable> auto\n");
printf("\n");
printf("Enables or disable automatic tag throttling.\n");
is_error = true;
continue;
}
state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable");
wait(ThrottleApi::enableAuto(db, autoTagThrottlingEnabled));
printf("Automatic tag throttling has been %s\n",
autoTagThrottlingEnabled ? "enabled" : "disabled");
} else {
printUsage(tokens[0]);
is_error = true;
}
continue;
}
if (tokencmp(tokens[0], "cache_range")) {
if (tokens.size() != 4) {
printUsage(tokens[0]);

8
fdbcli/fdbcli.actor.h
View File

@ -64,7 +64,9 @@ extern const KeyRef consistencyCheckSpecialKey;
// maintenance
extern const KeyRangeRef maintenanceSpecialKeyRange;
extern const KeyRef ignoreSSFailureSpecialKey;
// setclass
extern const KeyRangeRef processClassSourceSpecialKeyRange;
extern const KeyRangeRef processClassTypeSpecialKeyRange;
// help functions (Copied from fdbcli.actor.cpp)
// compare StringRef with the given c string
@ -81,8 +83,12 @@ ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr, std:
ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// maintenance command
ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// setclass command
ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// snapshot command
ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// throttle command
ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
} // namespace fdb_cli

45
fdbclient/AsyncTaskThread.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include <atomic>
#include "fdbclient/AsyncTaskThread.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -30,13 +32,22 @@ public:
bool isTerminate() const override { return true; }
};
ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, int* sum, int count) {
ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, std::atomic<int> *sum, int count, int clientId, double meanSleep) {
state int i = 0;
state double randomSleep = 0.0;
for (; i < count; ++i) {
randomSleep = deterministicRandom()->random01() * 2 * meanSleep;
wait(delay(randomSleep));
wait(asyncTaskThread->execAsync([sum = sum] {
++(*sum);
sum->fetch_add(1);
return Void();
}));
TraceEvent("AsyncTaskThreadIncrementedSum")
.detail("Index", i)
.detail("Sum", sum->load())
.detail("ClientId", clientId)
.detail("RandomSleep", randomSleep)
.detail("MeanSleep", meanSleep);
}
return Void();
}
@ -51,7 +62,7 @@ AsyncTaskThread::~AsyncTaskThread() {
bool wakeUp = false;
{
std::lock_guard<std::mutex> g(m);
wakeUp = queue.push(std::make_shared<TerminateTask>());
wakeUp = queue.push(std::make_unique<TerminateTask>());
}
if (wakeUp) {
cv.notify_one();
@ -61,7 +72,7 @@ AsyncTaskThread::~AsyncTaskThread() {
void AsyncTaskThread::run(AsyncTaskThread* self) {
while (true) {
std::shared_ptr<IAsyncTask> task;
std::unique_ptr<IAsyncTask> task;
{
std::unique_lock<std::mutex> lk(self->m);
self->cv.wait(lk, [self] { return !self->queue.canSleep(); });
@ -75,14 +86,30 @@ void AsyncTaskThread::run(AsyncTaskThread* self) {
}
TEST_CASE("/asynctaskthread/add") {
state int sum = 0;
state std::atomic<int> sum = 0;
state AsyncTaskThread asyncTaskThread;
state int numClients = 10;
state int incrementsPerClient = 100;
std::vector<Future<Void>> clients;
clients.reserve(10);
for (int i = 0; i < 10; ++i) {
clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100));
clients.reserve(numClients);
for (int clientId = 0; clientId < numClients; ++clientId) {
clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, incrementsPerClient, clientId, deterministicRandom()->random01() * 0.01));
}
wait(waitForAll(clients));
ASSERT_EQ(sum, 1000);
ASSERT_EQ(sum.load(), numClients * incrementsPerClient);
return Void();
}
TEST_CASE("/asynctaskthread/error") {
state AsyncTaskThread asyncTaskThread;
try {
wait(asyncTaskThread.execAsync([]{
throw operation_failed();
return Void();
}));
ASSERT(false);
} catch (Error &e) {
ASSERT_EQ(e.code(), error_code_operation_failed);
}
return Void();
}

5
fdbclient/AsyncTaskThread.h
View File

@ -48,7 +48,7 @@ public:
};
class AsyncTaskThread {
ThreadSafeQueue<std::shared_ptr<IAsyncTask>> queue;
ThreadSafeQueue<std::unique_ptr<IAsyncTask>> queue;
std::condition_variable cv;
std::mutex m;
std::thread thread;
@ -60,7 +60,7 @@ class AsyncTaskThread {
bool wakeUp = false;
{
std::lock_guard<std::mutex> g(m);
wakeUp = queue.push(std::make_shared<AsyncTask<F>>(func));
wakeUp = queue.push(std::make_unique<AsyncTask<F>>(func));
}
if (wakeUp) {
cv.notify_one();
@ -88,6 +88,7 @@ public:
auto funcResult = func();
onMainThreadVoid([promise, funcResult] { promise.send(funcResult); }, nullptr, priority);
} catch (Error& e) {
TraceEvent("ErrorExecutingAsyncTask").error(e);
onMainThreadVoid([promise, e] { promise.sendError(e); }, nullptr, priority);
}
});

24
fdbclient/BackupAgent.actor.h
View File

@ -368,8 +368,8 @@ public:
DatabaseBackupAgent(DatabaseBackupAgent&& r) noexcept
: subspace(std::move(r.subspace)), states(std::move(r.states)), config(std::move(r.config)),
errors(std::move(r.errors)), ranges(std::move(r.ranges)), tagNames(std::move(r.tagNames)),
taskBucket(std::move(r.taskBucket)), futureBucket(std::move(r.futureBucket)),
sourceStates(std::move(r.sourceStates)), sourceTagNames(std::move(r.sourceTagNames)) {}
sourceStates(std::move(r.sourceStates)), sourceTagNames(std::move(r.sourceTagNames)),
taskBucket(std::move(r.taskBucket)), futureBucket(std::move(r.futureBucket)) {}
void operator=(DatabaseBackupAgent&& r) noexcept {
subspace = std::move(r.subspace);
@ -717,11 +717,22 @@ protected:
template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL()));
Tuple tuple;
tuple.append(StringRef(bc->getURL()));
if (bc->getEncryptionKeyFileName().present()) {
tuple.append(bc->getEncryptionKeyFileName().get());
}
return tuple;
}
template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString());
ASSERT(val.size() == 1 || val.size() == 2);
auto url = val.getString(0).toString();
Optional<std::string> encryptionKeyFileName;
if (val.size() == 2) {
encryptionKeyFileName = val.getString(1).toString();
}
return IBackupContainer::openContainer(url, encryptionKeyFileName);
}
class BackupConfig : public KeyBackedConfig {
@ -970,6 +981,11 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
int64_t offset,
int len);
// Reads a mutation log block from file and parses into batch mutation blocks for further parsing.
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size);
} // namespace fileBackup

7
fdbclient/BackupContainer.actor.cpp
View File

@ -284,11 +284,11 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
#ifdef BUILD_AZURE_BACKUP
else if (u.startsWith("azure://"_sr)) {
u.eat("azure://"_sr);
auto address = NetworkAddress::parse(u.eat("/"_sr).toString());
auto accountName = u.eat("@"_sr).toString();
auto endpoint = u.eat("/"_sr).toString();
auto containerName = u.eat("/"_sr).toString();
auto accountName = u.eat("/"_sr).toString();
r = makeReference<BackupContainerAzureBlobStore>(
address, containerName, accountName, encryptionKeyFileName);
endpoint, accountName, containerName, encryptionKeyFileName);
}
#endif
else {
@ -296,6 +296,7 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
throw backup_invalid_url();
}
r->encryptionKeyFileName = encryptionKeyFileName;
r->URL = url;
return r;
} catch (Error& e) {

43
fdbclient/BackupContainer.h
View File

@ -298,12 +298,53 @@ public:
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string getURL() const { return URL; }
std::string const &getURL() const { return URL; }
Optional<std::string> const &getEncryptionKeyFileName() const { return encryptionKeyFileName; }
static std::string lastOpenError;
private:
std::string URL;
Optional<std::string> encryptionKeyFileName;
};
namespace fileBackup {
// Accumulates mutation log value chunks, as both a vector of chunks and as a combined chunk,
// in chunk order, and can check the chunk set for completion or intersection with a set
// of ranges.
struct AccumulatedMutations {
AccumulatedMutations() : lastChunkNumber(-1) {}
// Add a KV pair for this mutation chunk set
// It will be accumulated onto serializedMutations if the chunk number is
// the next expected value.
void addChunk(int chunkNumber, const KeyValueRef& kv);
// Returns true if both
// - 1 or more chunks were added to this set
// - The header of the first chunk contains a valid protocol version and a length
// that matches the bytes after the header in the combined value in serializedMutations
bool isComplete() const;
// Returns true if a complete chunk contains any MutationRefs which intersect with any
// range in ranges.
// It is undefined behavior to run this if isComplete() does not return true.
bool matchesAnyRange(const std::vector<KeyRange>& ranges) const;
std::vector<KeyValueRef> kvs;
std::string serializedMutations;
int lastChunkNumber;
};
// Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and
// returns (commitVersion, chunkNumber)
std::pair<Version, int32_t> decodeMutationLogKey(const StringRef& key);
// Decodes an encoded list of mutations in the format of:
// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k],
// where a mutation is encoded as:
// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][param1][param2]
std::vector<MutationRef> decodeMutationLogValue(const StringRef& value);
} // namespace fileBackup
#endif

259
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -19,40 +19,70 @@
*/
#include "fdbclient/BackupContainerAzureBlobStore.h"
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
#include "fdbrpc/AsyncFileEncrypted.h"
#emdif
#include <future>
#include "flow/actorcompiler.h" // This must be the last #include.
namespace {
std::string const notFoundErrorCode = "404";
void printAzureError(std::string const& operationName, azure::storage_lite::storage_error const& err) {
printf("(%s) : Error from Azure SDK : %s (%s) : %s",
operationName.c_str(),
err.code_name.c_str(),
err.code.c_str(),
err.message.c_str());
}
template <class T>
T waitAzureFuture(std::future<azure::storage_lite::storage_outcome<T>>&& f, std::string const& operationName) {
auto outcome = f.get();
if (outcome.success()) {
return outcome.response();
} else {
printAzureError(operationName, outcome.error());
throw backup_error();
}
}
} // namespace
class BackupContainerAzureBlobStoreImpl {
public:
using AzureClient = azure::storage_lite::blob_client;
class ReadFile final : public IAsyncFile, ReferenceCounted<ReadFile> {
AsyncTaskThread& asyncTaskThread;
AsyncTaskThread* asyncTaskThread;
std::string containerName;
std::string blobName;
AzureClient* client;
std::shared_ptr<AzureClient> client;
public:
ReadFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
std::shared_ptr<AzureClient> const& client)
: asyncTaskThread(&asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<ReadFile>::addref(); }
void delref() override { ReferenceCounted<ReadFile>::delref(); }
Future<int> read(void* data, int length, int64_t offset) {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
Future<int> read(void* data, int length, int64_t offset) override {
TraceEvent(SevDebug, "BCAzureBlobStoreRead")
.detail("Length", length)
.detail("Offset", offset)
.detail("ContainerName", containerName)
.detail("BlobName", blobName);
return asyncTaskThread->execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
std::ostringstream oss(std::ios::out | std::ios::binary);
client->download_blob_to_stream(containerName, blobName, offset, length, oss);
waitAzureFuture(client->download_blob_to_stream(containerName, blobName, offset, length, oss),
"download_blob_to_stream");
auto str = std::move(oss).str();
memcpy(data, str.c_str(), str.size());
return static_cast<int>(str.size());
@ -63,19 +93,23 @@ public:
Future<Void> truncate(int64_t size) override { throw file_not_writable(); }
Future<Void> sync() override { throw file_not_writable(); }
Future<int64_t> size() const override {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName] {
return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size);
});
TraceEvent(SevDebug, "BCAzureBlobStoreReadFileSize")
.detail("ContainerName", containerName)
.detail("BlobName", blobName);
return asyncTaskThread->execAsync(
[client = this->client, containerName = this->containerName, blobName = this->blobName] {
auto resp =
waitAzureFuture(client->get_blob_properties(containerName, blobName), "get_blob_properties");
return static_cast<int64_t>(resp.size);
});
}
std::string getFilename() const override { return blobName; }
int64_t debugFD() const override { return 0; }
};
class WriteFile final : public IAsyncFile, ReferenceCounted<WriteFile> {
AsyncTaskThread& asyncTaskThread;
AzureClient* client;
AsyncTaskThread* asyncTaskThread;
std::shared_ptr<AzureClient> client;
std::string containerName;
std::string blobName;
int64_t m_cursor{ 0 };
@ -90,8 +124,8 @@ public:
WriteFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
std::shared_ptr<AzureClient> const& client)
: asyncTaskThread(&asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<WriteFile>::addref(); }
void delref() override { ReferenceCounted<WriteFile>::delref(); }
@ -116,22 +150,33 @@ public:
return Void();
}
Future<Void> sync() override {
TraceEvent(SevDebug, "BCAzureBlobStoreSync")
.detail("Length", buffer.size())
.detail("ContainerName", containerName)
.detail("BlobName", blobName);
auto movedBuffer = std::move(buffer);
buffer.clear();
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer));
auto resp = client->append_block_from_stream(containerName, blobName, iss).get();
return Void();
});
buffer = {};
if (!movedBuffer.empty()) {
return asyncTaskThread->execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer));
waitAzureFuture(client->append_block_from_stream(containerName, blobName, iss),
"append_block_from_stream");
return Void();
});
}
return Void();
}
Future<int64_t> size() const override {
return asyncTaskThread.execAsync(
TraceEvent(SevDebug, "BCAzureBlobStoreSize")
.detail("ContainerName", containerName)
.detail("BlobName", blobName);
return asyncTaskThread->execAsync(
[client = this->client, containerName = this->containerName, blobName = this->blobName] {
auto resp = client->get_blob_properties(containerName, blobName).get().response();
ASSERT(resp.valid()); // TODO: Should instead throw here
auto resp =
waitAzureFuture(client->get_blob_properties(containerName, blobName), "get_blob_properties");
return static_cast<int64_t>(resp.size);
});
}
@ -165,44 +210,53 @@ public:
static bool isDirectory(const std::string& blobName) { return blobName.size() && blobName.back() == '/'; }
// Hack to get around the fact that macros don't work inside actor functions
static Reference<IAsyncFile> encryptFile(Reference<IAsyncFile> const& f, AsyncFileEncrypted::Mode mode) {
Reference<IAsyncFile> result = f;
#if ENCRYPTION_ENABLED
result = makeReference<AsyncFileEncrypted>(result, mode);
#endif
return result;
}
ACTOR static Future<Reference<IAsyncFile>> readFile(BackupContainerAzureBlobStore* self, std::string fileName) {
bool exists = wait(self->blobExists(fileName));
if (!exists) {
throw file_not_found();
}
Reference<IAsyncFile> f =
makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
#if ENCRYPTION_ENABLED
makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client);
if (self->usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, false);
f = encryptFile(f, AsyncFileEncrypted::Mode::READ_ONLY);
}
#endif
return f;
}
ACTOR static Future<Reference<IBackupFile>> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) {
TraceEvent(SevDebug, "BCAzureBlobStoreCreateWriteFile")
.detail("ContainerName", self->containerName)
.detail("FileName", fileName);
wait(self->asyncTaskThread.execAsync(
[client = self->client.get(), containerName = self->containerName, fileName = fileName] {
auto outcome = client->create_append_blob(containerName, fileName).get();
[client = self->client, containerName = self->containerName, fileName = fileName] {
waitAzureFuture(client->create_append_blob(containerName, fileName), "create_append_blob");
return Void();
}));
auto f = makeReference<WriteFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
#if ENCRYPTION_ENABLED
Reference<IAsyncFile> f =
makeReference<WriteFile>(self->asyncTaskThread, self->containerName, fileName, self->client);
if (self->usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, true);
f = encryptFile(f, AsyncFileEncrypted::Mode::APPEND_ONLY);
}
#endif
return makeReference<BackupFile>(fileName, f);
}
static void listFiles(AzureClient* client,
static void listFiles(std::shared_ptr<AzureClient> const& client,
const std::string& containerName,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter,
BackupContainerFileSystem::FilesAndSizesT& result) {
auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response();
auto resp = waitAzureFuture(client->list_blobs_segmented(containerName, "/", "", path), "list_blobs_segmented");
for (const auto& blob : resp.blobs) {
if (isDirectory(blob.name) && folderPathFilter(blob.name)) {
if (isDirectory(blob.name) && (!folderPathFilter || folderPathFilter(blob.name))) {
listFiles(client, containerName, blob.name, folderPathFilter, result);
} else {
result.emplace_back(blob.name, blob.content_length);
@ -216,8 +270,12 @@ public:
BackupContainerFileSystem::FilesAndSizesT files = wait(self->listFiles());
filesToDelete = files.size();
}
wait(self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] {
client->delete_container(containerName).wait();
TraceEvent(SevDebug, "BCAzureBlobStoreDeleteContainer")
.detail("FilesToDelete", filesToDelete)
.detail("ContainerName", self->containerName)
.detail("TrackNumDeleted", pNumDeleted != nullptr);
wait(self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client] {
waitAzureFuture(client->delete_container(containerName), "delete_container");
return Void();
}));
if (pNumDeleted) {
@ -226,38 +284,44 @@ public:
return Void();
}
ACTOR static Future<Void> create(BackupContainerAzureBlobStore* self) {
state Future<Void> f1 =
self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] {
client->create_container(containerName).wait();
return Void();
});
state Future<Void> f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void();
return f1 && f2;
}
};
Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileName) {
return asyncTaskThread.execAsync(
[client = this->client.get(), containerName = this->containerName, fileName = fileName] {
auto resp = client->get_blob_properties(containerName, fileName).get().response();
return resp.valid();
});
TraceEvent(SevDebug, "BCAzureBlobStoreCheckExists")
.detail("FileName", fileName)
.detail("ContainerName", containerName);
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, fileName = fileName] {
auto outcome = client->get_blob_properties(containerName, fileName).get();
if (outcome.success()) {
return true;
} else {
auto const& err = outcome.error();
if (err.code == notFoundErrorCode) {
return false;
} else {
printAzureError("get_blob_properties", err);
throw backup_error();
}
}
});
}
BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address,
BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const std::string& endpoint,
const std::string& accountName,
const std::string& containerName,
const Optional<std::string>& encryptionKeyFileName)
: containerName(containerName) {
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
setEncryptionKey(encryptionKeyFileName);
#endif
std::string accountKey = std::getenv("AZURE_KEY");
const char* _accountKey = std::getenv("AZURE_KEY");
if (!_accountKey) {
TraceEvent(SevError, "EnvironmentVariableNotFound").detail("EnvVariable", "AZURE_KEY");
// TODO: More descriptive error?
throw backup_error();
}
std::string accountKey = _accountKey;
auto credential = std::make_shared<azure::storage_lite::shared_key_credential>(accountName, accountKey);
auto storageAccount = std::make_shared<azure::storage_lite::storage_account>(
accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str()));
accountName, credential, true, format("https://%s", endpoint.c_str()));
client = std::make_unique<AzureClient>(storageAccount, 1);
}
@ -269,12 +333,30 @@ void BackupContainerAzureBlobStore::delref() {
}
Future<Void> BackupContainerAzureBlobStore::create() {
return BackupContainerAzureBlobStoreImpl::create(this);
TraceEvent(SevDebug, "BCAzureBlobStoreCreateContainer").detail("ContainerName", containerName);
Future<Void> createContainerFuture =
asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] {
waitAzureFuture(client->create_container(containerName), "create_container");
return Void();
});
Future<Void> encryptionSetupFuture = usesEncryption() ? encryptionSetupComplete() : Void();
return createContainerFuture && encryptionSetupFuture;
}
Future<bool> BackupContainerAzureBlobStore::exists() {
return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {
auto resp = client->get_container_properties(containerName).get().response();
return resp.valid();
TraceEvent(SevDebug, "BCAzureBlobStoreCheckContainerExists").detail("ContainerName", containerName);
return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client] {
auto outcome = client->get_container_properties(containerName).get();
if (outcome.success()) {
return true;
} else {
auto const& err = outcome.error();
if (err.code == notFoundErrorCode) {
return false;
} else {
printAzureError("got_container_properties", err);
throw backup_error();
}
}
});
}
@ -289,22 +371,23 @@ Future<Reference<IBackupFile>> BackupContainerAzureBlobStore::writeFile(const st
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles(
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(),
containerName = this->containerName,
path = path,
folderPathFilter = folderPathFilter] {
FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);
return result;
});
TraceEvent(SevDebug, "BCAzureBlobStoreListFiles").detail("ContainerName", containerName).detail("Path", path);
return asyncTaskThread.execAsync(
[client = this->client, containerName = this->containerName, path = path, folderPathFilter = folderPathFilter] {
FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);
return result;
});
}
Future<Void> BackupContainerAzureBlobStore::deleteFile(const std::string& fileName) {
return asyncTaskThread.execAsync(
[containerName = this->containerName, fileName = fileName, client = client.get()]() {
client->delete_blob(containerName, fileName).wait();
return Void();
});
TraceEvent(SevDebug, "BCAzureBlobStoreDeleteFile")
.detail("ContainerName", containerName)
.detail("FileName", fileName);
return asyncTaskThread.execAsync([containerName = this->containerName, fileName = fileName, client = client]() {
client->delete_blob(containerName, fileName).wait();
return Void();
});
}
Future<Void> BackupContainerAzureBlobStore::deleteContainer(int* pNumDeleted) {
@ -317,5 +400,5 @@ Future<std::vector<std::string>> BackupContainerAzureBlobStore::listURLs(const s
}
std::string BackupContainerAzureBlobStore::getURLFormat() {
return "azure://<ip>:<port>/<accountname>/<container>/<path_to_file>";
return "azure://<accountname>@<endpoint>/<container>/";
}

4
fdbclient/BackupContainerAzureBlobStore.h
View File

@ -33,7 +33,7 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
ReferenceCounted<BackupContainerAzureBlobStore> {
using AzureClient = azure::storage_lite::blob_client;
std::unique_ptr<AzureClient> client;
std::shared_ptr<AzureClient> client;
std::string containerName;
AsyncTaskThread asyncTaskThread;
@ -42,7 +42,7 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
friend class BackupContainerAzureBlobStoreImpl;
public:
BackupContainerAzureBlobStore(const NetworkAddress& address,
BackupContainerAzureBlobStore(const std::string& endpoint,
const std::string& accountName,
const std::string& containerName,
const Optional<std::string>& encryptionKeyFileName);

26
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -23,9 +23,7 @@
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/BackupContainerLocalDirectory.h"
#include "fdbclient/JsonBuilder.h"
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
#include "flow/StreamCipher.h"
#endif
#include "flow/UnitTest.h"
#include <algorithm>
@ -165,7 +163,6 @@ public:
state Version maxVer = 0;
state RangeFile rf;
state json_spirit::mArray fileArray;
state int i;
// Validate each filename, update version range
for (const auto& f : fileNames) {
@ -1481,7 +1478,6 @@ Future<Void> BackupContainerFileSystem::encryptionSetupComplete() const {
return encryptionSetupFuture;
}
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& encryptionKeyFileName) {
if (encryptionKeyFileName.present()) {
#if ENCRYPTION_ENABLED
@ -1491,18 +1487,13 @@ void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& en
#endif
}
}
Future<Void> BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const &filename) {
Future<Void> BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const& filename) {
#if ENCRYPTION_ENABLED
return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename);
#else
return Void();
#endif
}
#else
Future<Void> BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const& filename) {
return Void();
}
#endif
namespace backup_test {
@ -1515,13 +1506,16 @@ int chooseFileSize(std::vector<int>& sizes) {
return deterministicRandom()->randomInt(0, 2e6);
}
ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c, Reference<IBackupFile> f, int size, FlowLock* lock) {
ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c,
Reference<IBackupFile> f,
int size,
FlowLock* lock) {
state Standalone<VectorRef<uint8_t>> content;
wait(lock->take(TaskPriority::DefaultYield, size));
state FlowLock::Releaser releaser(*lock, size);
state FlowLock::Releaser releaser(*lock, size);
printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str());
printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str());
content.resize(content.arena(), size);
for (int i = 0; i < content.size(); ++i) {
content[i] = (uint8_t)deterministicRandom()->randomInt(0, 256);
@ -1609,9 +1603,9 @@ ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> en
// List of sizes to use to test edge cases on underlying file implementations
state std::vector<int> fileSizes = { 0 };
if (StringRef(url).startsWith(LiteralStringRef("blob"))) {
fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE);
fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10);
}
fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE);
fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10);
}
loop {
state Version logStart = v;

2
fdbclient/BackupContainerFileSystem.h
View File

@ -156,9 +156,7 @@ public:
protected:
bool usesEncryption() const;
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
#endif
Future<Void> encryptionSetupComplete() const;
private:

7
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -31,7 +31,8 @@ namespace {
class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
public:
BackupFile(const std::string& fileName, Reference<IAsyncFile> file, const std::string& finalFullPath)
: IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0), m_blockSize(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) {
: IBackupFile(fileName), m_file(file), m_writeOffset(0), m_finalFullPath(finalFullPath),
m_blockSize(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) {
if (BUGGIFY) {
m_blockSize = deterministicRandom()->randomInt(100, 20000);
}
@ -133,9 +134,7 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
const Optional<std::string>& encryptionKeyFileName) {
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
setEncryptionKey(encryptionKeyFileName);
#endif
std::string path;
if (url.find("file://") != 0) {
@ -229,7 +228,7 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
}
if (g_simulator.getCurrentProcess()->uid == UID()) {
TraceEvent(SevError, "BackupContainerReadFileOnUnsetProcessID");
TraceEvent(SevError, "BackupContainerReadFileOnUnsetProcessID").log();
}
std::string uniquePath = fullPath + "." + g_simulator.getCurrentProcess()->uid.toString() + ".lnk";
unlink(uniquePath.c_str());

2
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -147,9 +147,7 @@ BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndp
const S3BlobStoreEndpoint::ParametersT& params,
const Optional<std::string>& encryptionKeyFileName)
: m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
#if (!defined(TLS_DISABLED) && !defined(_WIN32))
setEncryptionKey(encryptionKeyFileName);
#endif
// Currently only one parameter is supported, "bucket"
for (const auto& [name, value] : params) {
if (name == "bucket") {

4
fdbclient/CommitProxyInterface.h
View File

@ -244,7 +244,7 @@ struct GetReadVersionRequest : TimedRequest {
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>())
: spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags),
: spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags),
debugID(debugID) {
flags = flags & ~FLAG_PRIORITY_MASK;
switch (priority) {
@ -313,7 +313,7 @@ struct GetKeyServerLocationsRequest {
int limit,
bool reverse,
Arena const& arena)
: spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
: arena(arena), spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse) {}
template <class Ar>
void serialize(Ar& ar) {

34
fdbclient/ConfigTransactionInterface.cpp
View File

@ -20,12 +20,13 @@
#include "fdbclient/ConfigTransactionInterface.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/SystemData.h"
#include "flow/IRandom.h"
ConfigTransactionInterface::ConfigTransactionInterface() : _id(deterministicRandom()->randomUniqueID()) {}
void ConfigTransactionInterface::setupWellKnownEndpoints() {
getVersion.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETVERSION, TaskPriority::Coordination);
getGeneration.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETGENERATION, TaskPriority::Coordination);
get.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GET, TaskPriority::Coordination);
getClasses.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETCLASSES, TaskPriority::Coordination);
getKnobs.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETKNOBS, TaskPriority::Coordination);
@ -33,8 +34,8 @@ void ConfigTransactionInterface::setupWellKnownEndpoints() {
}
ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote)
: getVersion(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETVERSION)), get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)),
getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
: getGeneration(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)),
get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)), getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
getKnobs(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), commit(Endpoint({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {
}
@ -45,3 +46,30 @@ bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rh
bool ConfigTransactionInterface::operator!=(ConfigTransactionInterface const& rhs) const {
return !(*this == rhs);
}
bool ConfigGeneration::operator==(ConfigGeneration const& rhs) const {
return liveVersion == rhs.liveVersion && committedVersion == rhs.committedVersion;
}
bool ConfigGeneration::operator!=(ConfigGeneration const& rhs) const {
return !(*this == rhs);
}
void ConfigTransactionCommitRequest::set(KeyRef key, ValueRef value) {
if (key == configTransactionDescriptionKey) {
annotation.description = KeyRef(arena, value);
} else {
ConfigKey configKey = ConfigKeyRef::decodeKey(key);
auto knobValue = IKnobCollection::parseKnobValue(
configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST);
mutations.emplace_back_deep(arena, configKey, knobValue.contents());
}
}
void ConfigTransactionCommitRequest::clear(KeyRef key) {
if (key == configTransactionDescriptionKey) {
annotation.description = ""_sr;
} else {
mutations.emplace_back_deep(arena, ConfigKeyRef::decodeKey(key), Optional<KnobValueRef>{});
}
}

77
fdbclient/ConfigTransactionInterface.h
View File

@ -27,22 +27,38 @@
#include "fdbrpc/fdbrpc.h"
#include "flow/flow.h"
struct ConfigTransactionGetVersionReply {
static constexpr FileIdentifier file_identifier = 2934851;
ConfigTransactionGetVersionReply() = default;
explicit ConfigTransactionGetVersionReply(Version version) : version(version) {}
Version version;
struct ConfigGeneration {
// The live version of each node is monotonically increasing
Version liveVersion{ 0 };
// The committedVersion of each node is the version of the last commit made durable.
// Each committedVersion was previously given to clients as a liveVersion, prior to commit.
Version committedVersion{ 0 };
bool operator==(ConfigGeneration const&) const;
bool operator!=(ConfigGeneration const&) const;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, version);
serializer(ar, liveVersion, committedVersion);
}
};
struct ConfigTransactionGetVersionRequest {
struct ConfigTransactionGetGenerationReply {
static constexpr FileIdentifier file_identifier = 2934851;
ConfigTransactionGetGenerationReply() = default;
explicit ConfigTransactionGetGenerationReply(ConfigGeneration generation) : generation(generation) {}
ConfigGeneration generation;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, generation);
}
};
struct ConfigTransactionGetGenerationRequest {
static constexpr FileIdentifier file_identifier = 138941;
ReplyPromise<ConfigTransactionGetVersionReply> reply;
ConfigTransactionGetVersionRequest() = default;
ReplyPromise<ConfigTransactionGetGenerationReply> reply;
ConfigTransactionGetGenerationRequest() = default;
template <class Ar>
void serialize(Ar& ar) {
@ -64,45 +80,36 @@ struct ConfigTransactionGetReply {
struct ConfigTransactionGetRequest {
static constexpr FileIdentifier file_identifier = 923040;
Version version;
ConfigGeneration generation;
ConfigKey key;
ReplyPromise<ConfigTransactionGetReply> reply;
ConfigTransactionGetRequest() = default;
explicit ConfigTransactionGetRequest(Version version, ConfigKey key) : version(version), key(key) {}
explicit ConfigTransactionGetRequest(ConfigGeneration generation, ConfigKey key)
: generation(generation), key(key) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, version, key, reply);
serializer(ar, generation, key, reply);
}
};
struct ConfigTransactionCommitRequest {
static constexpr FileIdentifier file_identifier = 103841;
Arena arena;
Version version{ ::invalidVersion };
ConfigGeneration generation{ ::invalidVersion, ::invalidVersion };
VectorRef<ConfigMutationRef> mutations;
ConfigCommitAnnotationRef annotation;
ReplyPromise<Void> reply;
size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, arena, version, mutations, annotation, reply);
}
};
struct ConfigTransactionGetRangeReply {
static constexpr FileIdentifier file_identifier = 430263;
Standalone<RangeResultRef> range;
ConfigTransactionGetRangeReply() = default;
explicit ConfigTransactionGetRangeReply(Standalone<RangeResultRef> range) : range(range) {}
void set(KeyRef key, ValueRef value);
void clear(KeyRef key);
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, range);
serializer(ar, arena, generation, mutations, annotation, reply);
}
};
@ -122,15 +129,15 @@ struct ConfigTransactionGetConfigClassesReply {
struct ConfigTransactionGetConfigClassesRequest {
static constexpr FileIdentifier file_identifier = 7163400;
Version version;
ConfigGeneration generation;
ReplyPromise<ConfigTransactionGetConfigClassesReply> reply;
ConfigTransactionGetConfigClassesRequest() = default;
explicit ConfigTransactionGetConfigClassesRequest(Version version) : version(version) {}
explicit ConfigTransactionGetConfigClassesRequest(ConfigGeneration generation) : generation(generation) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, version);
serializer(ar, generation);
}
};
@ -149,17 +156,17 @@ struct ConfigTransactionGetKnobsReply {
struct ConfigTransactionGetKnobsRequest {
static constexpr FileIdentifier file_identifier = 987410;
Version version;
ConfigGeneration generation;
Optional<Key> configClass;
ReplyPromise<ConfigTransactionGetKnobsReply> reply;
ConfigTransactionGetKnobsRequest() = default;
explicit ConfigTransactionGetKnobsRequest(Version version, Optional<Key> configClass)
: version(version), configClass(configClass) {}
explicit ConfigTransactionGetKnobsRequest(ConfigGeneration generation, Optional<Key> configClass)
: generation(generation), configClass(configClass) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, version, configClass, reply);
serializer(ar, generation, configClass, reply);
}
};
@ -172,7 +179,7 @@ struct ConfigTransactionInterface {
public:
static constexpr FileIdentifier file_identifier = 982485;
struct RequestStream<ConfigTransactionGetVersionRequest> getVersion;
struct RequestStream<ConfigTransactionGetGenerationRequest> getGeneration;
struct RequestStream<ConfigTransactionGetRequest> get;
struct RequestStream<ConfigTransactionGetConfigClassesRequest> getClasses;
struct RequestStream<ConfigTransactionGetKnobsRequest> getKnobs;
@ -188,6 +195,6 @@ public:
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, getVersion, get, getClasses, getKnobs, commit);
serializer(ar, getGeneration, get, getClasses, getKnobs, commit);
}
};

2
fdbclient/CoordinationInterface.h
View File

@ -38,7 +38,7 @@ constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10);
constexpr UID WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE(-1, 11);
constexpr UID WLTOKEN_CONFIGTXN_GETVERSION(-1, 12);
constexpr UID WLTOKEN_CONFIGTXN_GETGENERATION(-1, 12);
constexpr UID WLTOKEN_CONFIGTXN_GET(-1, 13);
constexpr UID WLTOKEN_CONFIGTXN_GETCLASSES(-1, 14);
constexpr UID WLTOKEN_CONFIGTXN_GETKNOBS(-1, 15);

47
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -44,28 +44,29 @@ const Key DatabaseBackupAgent::keyDatabasesInSync = LiteralStringRef("databases_
const int DatabaseBackupAgent::LATEST_DR_VERSION = 1;
DatabaseBackupAgent::DatabaseBackupAgent()
: subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)),
errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)),
: subspace(Subspace(databaseBackupPrefixRange.begin)), states(subspace.get(BackupAgentBase::keyStates)),
config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)),
ranges(subspace.get(BackupAgentBase::keyRanges)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
sourceTagNames(subspace.get(BackupAgentBase::keyTagName)),
taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
AccessSystemKeys::True,
PriorityBatch::False,
LockAware::True)),
futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)),
sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {}
futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) {
}
DatabaseBackupAgent::DatabaseBackupAgent(Database src)
: subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)),
errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)),
: subspace(Subspace(databaseBackupPrefixRange.begin)), states(subspace.get(BackupAgentBase::keyStates)),
config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)),
ranges(subspace.get(BackupAgentBase::keyRanges)), tagNames(subspace.get(BackupAgentBase::keyTagName)),
sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
sourceTagNames(subspace.get(BackupAgentBase::keyTagName)),
taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks),
AccessSystemKeys::True,
PriorityBatch::False,
LockAware::True)),
futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)),
sourceStates(subspace.get(BackupAgentBase::keySourceStates)),
sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {
futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::True, LockAware::True)) {
taskBucket->src = src;
}
@ -364,7 +365,7 @@ struct BackupRangeTaskFunc : TaskFuncBase {
TEST(true); // range insert delayed because too versionMap is too large
if (rangeCount > CLIENT_KNOBS->BACKUP_MAP_KEY_UPPER_LIMIT)
TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge");
TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge").log();
wait(delay(1));
task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin;
@ -1882,7 +1883,7 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
if (task->params[BackupAgentBase::destUid].size() == 0) {
TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade");
TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade").log();
wait(success(AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone))));
} else {
Version beginVersion =
@ -2377,11 +2378,11 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade
try {
// Check if src is unlocked and dest is locked
if (getLockedStatus(srcStatus) != false) {
TraceEvent(SevWarn, "DBA_AtomicSwitchOverSrcLocked");
TraceEvent(SevWarn, "DBA_AtomicSwitchOverSrcLocked").log();
throw backup_error();
}
if (getLockedStatus(destStatus) != true) {
TraceEvent(SevWarn, "DBA_AtomicSwitchOverDestUnlocked");
TraceEvent(SevWarn, "DBA_AtomicSwitchOverDestUnlocked").log();
throw backup_error();
}
// Check if mutation-stream-id matches
@ -2402,7 +2403,7 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade
destDRAgents.end(),
std::inserter(intersectingAgents, intersectingAgents.begin()));
if (intersectingAgents.empty()) {
TraceEvent(SevWarn, "DBA_SwitchOverPossibleDRAgentsIncorrectSetup");
TraceEvent(SevWarn, "DBA_SwitchOverPossibleDRAgentsIncorrectSetup").log();
throw backup_error();
}
} catch (std::runtime_error& e) {
@ -2757,7 +2758,7 @@ public:
}
}
TraceEvent("DBA_SwitchoverReady");
TraceEvent("DBA_SwitchoverReady").log();
try {
wait(backupAgent->discontinueBackup(dest, tagName));
@ -2768,7 +2769,7 @@ public:
wait(success(backupAgent->waitBackup(dest, tagName, StopWhenDone::True)));
TraceEvent("DBA_SwitchoverStopped");
TraceEvent("DBA_SwitchoverStopped").log();
state ReadYourWritesTransaction tr3(dest);
loop {
@ -2789,7 +2790,7 @@ public:
}
}
TraceEvent("DBA_SwitchoverVersionUpgraded");
TraceEvent("DBA_SwitchoverVersionUpgraded").log();
try {
wait(drAgent.submitBackup(backupAgent->taskBucket->src,
@ -2805,15 +2806,15 @@ public:
throw;
}
TraceEvent("DBA_SwitchoverSubmitted");
TraceEvent("DBA_SwitchoverSubmitted").log();
wait(success(drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName)));
TraceEvent("DBA_SwitchoverStarted");
TraceEvent("DBA_SwitchoverStarted").log();
wait(backupAgent->unlockBackup(dest, tagName));
TraceEvent("DBA_SwitchoverUnlocked");
TraceEvent("DBA_SwitchoverUnlocked").log();
return Void();
}

22
fdbclient/DatabaseContext.h
View File

@ -51,7 +51,7 @@ public:
private:
DatabaseContext* cx;
StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
: ReferencedInterface<StorageServerInterface>(interf, locality), cx(cx) {}
};
struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> {
@ -196,7 +196,7 @@ public:
Reference<CommitProxyInfo> getCommitProxies(bool useProvisionalProxies);
Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(bool useProvisionalProxies);
Reference<GrvProxyInfo> getGrvProxies(bool useProvisionalProxies);
Future<Void> onProxiesChanged();
Future<Void> onProxiesChanged() const;
Future<HealthMetrics> getHealthMetrics(bool detailed);
// Returns the protocol version reported by the coordinator this client is connected to
@ -252,10 +252,18 @@ public:
// Management API, create snapshot
Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
Future<Standalone<VectorRef<MutationsAndVersionRef>>> getRangeFeedMutations(
StringRef rangeID,
Version begin = 0,
Version end = std::numeric_limits<Version>::max(),
KeyRange range = allKeys);
Future<std::vector<std::pair<Key, KeyRange>>> getOverlappingRangeFeeds(KeyRangeRef ranges, Version minVersion);
Future<Void> popRangeFeedMutations(StringRef rangeID, Version version);
// private:
explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
@ -307,7 +315,7 @@ public:
// trust that the read version (possibly set manually by the application) is actually from the correct cluster.
// Updated everytime we get a GRV response
Version minAcceptableReadVersion = std::numeric_limits<Version>::max();
void validateVersion(Version);
void validateVersion(Version) const;
// Client status updater
struct ClientStatusUpdater {
@ -399,7 +407,7 @@ public:
Future<Void> connected;
// An AsyncVar that reports the coordinator this DatabaseContext is interacting with
Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator;
Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator;
Reference<AsyncVar<Optional<ClusterInterface>>> statusClusterInterface;
Future<Void> statusLeaderMon;
@ -428,7 +436,6 @@ public:
static bool debugUseTags;
static const std::vector<std::string> debugTransactionTagChoices;
std::unordered_map<KeyRef, Reference<WatchMetadata>> watchMap;
// Adds or updates the specified (SS, TSS) pair in the TSS mapping (if not already present).
// Requests to the storage server will be duplicated to the TSS.
@ -437,6 +444,9 @@ public:
// Removes the storage server and its TSS pair from the TSS mapping (if present).
// Requests to the storage server will no longer be duplicated to its pair TSS.
void removeTssMapping(StorageServerInterface const& ssi);
private:
std::unordered_map<KeyRef, Reference<WatchMetadata>> watchMap;
};
#endif

4
fdbclient/FDBTypes.h
View File

@ -655,9 +655,9 @@ struct RangeResultRef : VectorRef<KeyValueRef> {
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
: VectorRef<KeyValueRef>(p, toCopy), more(toCopy.more),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}

114
fdbclient/FileBackupAgent.actor.cpp
View File

@ -691,9 +691,9 @@ private:
int64_t blockEnd;
};
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len) {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len) {
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(file->read(mutateString(buf), len, offset));
if (rLen != len)
@ -3244,7 +3244,7 @@ REGISTER_TASKFUNC(RestoreRangeTaskFunc);
// Decodes a mutation log key, which contains (hash, commitVersion, chunkNumber) and
// returns (commitVersion, chunkNumber)
std::pair<Version, int32_t> decodeLogKey(const StringRef& key) {
std::pair<Version, int32_t> decodeMutationLogKey(const StringRef& key) {
ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t));
uint8_t hash;
@ -3265,7 +3265,7 @@ std::pair<Version, int32_t> decodeLogKey(const StringRef& key) {
// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k],
// where a mutation is encoded as:
// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][param1][param2]
std::vector<MutationRef> decodeLogValue(const StringRef& value) {
std::vector<MutationRef> decodeMutationLogValue(const StringRef& value) {
StringRefReader reader(value, restore_corrupted_data());
Version protocolVersion = reader.consume<uint64_t>();
@ -3300,72 +3300,54 @@ std::vector<MutationRef> decodeLogValue(const StringRef& value) {
return mutations;
}
// Accumulates mutation log value chunks, as both a vector of chunks and as a combined chunk,
// in chunk order, and can check the chunk set for completion or intersection with a set
// of ranges.
struct AccumulatedMutations {
AccumulatedMutations() : lastChunkNumber(-1) {}
// Add a KV pair for this mutation chunk set
// It will be accumulated onto serializedMutations if the chunk number is
// the next expected value.
void addChunk(int chunkNumber, const KeyValueRef& kv) {
if (chunkNumber == lastChunkNumber + 1) {
lastChunkNumber = chunkNumber;
serializedMutations += kv.value.toString();
} else {
lastChunkNumber = -2;
serializedMutations.clear();
}
kvs.push_back(kv);
void AccumulatedMutations::addChunk(int chunkNumber, const KeyValueRef& kv) {
if (chunkNumber == lastChunkNumber + 1) {
lastChunkNumber = chunkNumber;
serializedMutations += kv.value.toString();
} else {
lastChunkNumber = -2;
serializedMutations.clear();
}
kvs.push_back(kv);
}
// Returns true if both
// - 1 or more chunks were added to this set
// - The header of the first chunk contains a valid protocol version and a length
// that matches the bytes after the header in the combined value in serializedMutations
bool isComplete() const {
if (lastChunkNumber >= 0) {
StringRefReader reader(serializedMutations, restore_corrupted_data());
bool AccumulatedMutations::isComplete() const {
if (lastChunkNumber >= 0) {
StringRefReader reader(serializedMutations, restore_corrupted_data());
Version protocolVersion = reader.consume<uint64_t>();
if (protocolVersion <= 0x0FDB00A200090001) {
throw incompatible_protocol_version();
}
uint32_t vLen = reader.consume<uint32_t>();
return vLen == reader.remainder().size();
Version protocolVersion = reader.consume<uint64_t>();
if (protocolVersion <= 0x0FDB00A200090001) {
throw incompatible_protocol_version();
}
return false;
uint32_t vLen = reader.consume<uint32_t>();
return vLen == reader.remainder().size();
}
// Returns true if a complete chunk contains any MutationRefs which intersect with any
// range in ranges.
// It is undefined behavior to run this if isComplete() does not return true.
bool matchesAnyRange(const std::vector<KeyRange>& ranges) const {
std::vector<MutationRef> mutations = decodeLogValue(serializedMutations);
for (auto& m : mutations) {
for (auto& r : ranges) {
if (m.type == MutationRef::ClearRange) {
if (r.intersects(KeyRangeRef(m.param1, m.param2))) {
return true;
}
} else {
if (r.contains(m.param1)) {
return true;
}
return false;
}
// Returns true if a complete chunk contains any MutationRefs which intersect with any
// range in ranges.
// It is undefined behavior to run this if isComplete() does not return true.
bool AccumulatedMutations::matchesAnyRange(const std::vector<KeyRange>& ranges) const {
std::vector<MutationRef> mutations = decodeMutationLogValue(serializedMutations);
for (auto& m : mutations) {
for (auto& r : ranges) {
if (m.type == MutationRef::ClearRange) {
if (r.intersects(KeyRangeRef(m.param1, m.param2))) {
return true;
}
} else {
if (r.contains(m.param1)) {
return true;
}
}
}
return false;
}
std::vector<KeyValueRef> kvs;
std::string serializedMutations;
int lastChunkNumber;
};
return false;
}
// Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
// and have data relevant to one of the KV ranges in ranges
@ -3373,7 +3355,7 @@ std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, c
std::unordered_map<Version, AccumulatedMutations> mutationBlocksByVersion;
for (auto& kv : data) {
auto versionAndChunkNumber = decodeLogKey(kv.key);
auto versionAndChunkNumber = decodeMutationLogKey(kv.key);
mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv);
}
@ -3444,7 +3426,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
state Key mutationLogPrefix = restore.mutationLogPrefix();
state Reference<IAsyncFile> inFile = wait(bc->readFile(logFile.fileName));
state Standalone<VectorRef<KeyValueRef>> dataOriginal = wait(decodeLogFileBlock(inFile, readOffset, readLen));
state Standalone<VectorRef<KeyValueRef>> dataOriginal = wait(decodeMutationLogFileBlock(inFile, readOffset, readLen));
// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this
// restore based on the restore range set.
@ -5478,7 +5460,7 @@ public:
try {
wait(discontinueBackup(backupAgent, ryw_tr, tagName));
wait(ryw_tr->commit());
TraceEvent("AS_DiscontinuedBackup");
TraceEvent("AS_DiscontinuedBackup").log();
break;
} catch (Error& e) {
if (e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate) {
@ -5489,7 +5471,7 @@ public:
}
wait(success(waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::True)));
TraceEvent("AS_BackupStopped");
TraceEvent("AS_BackupStopped").log();
ryw_tr->reset();
loop {
@ -5502,7 +5484,7 @@ public:
ryw_tr->clear(range);
}
wait(ryw_tr->commit());
TraceEvent("AS_ClearedRange");
TraceEvent("AS_ClearedRange").log();
break;
} catch (Error& e) {
wait(ryw_tr->onError(e));
@ -5512,7 +5494,7 @@ public:
Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
if (fastRestore) {
TraceEvent("AtomicParallelRestoreStartRestore");
TraceEvent("AtomicParallelRestoreStartRestore").log();
Version targetVersion = ::invalidVersion;
wait(submitParallelRestore(cx,
tagName,
@ -5533,7 +5515,7 @@ public:
}
return -1;
} else {
TraceEvent("AS_StartRestore");
TraceEvent("AS_StartRestore").log();
Version ver = wait(restore(backupAgent,
cx,
cx,

5
fdbclient/GlobalConfig.actor.cpp
View File

@ -77,6 +77,7 @@ void GlobalConfig::trigger(KeyRef key, std::function<void(std::optional<std::any
}
void GlobalConfig::insert(KeyRef key, ValueRef value) {
// TraceEvent(SevInfo, "GlobalConfig_Insert").detail("Key", key).detail("Value", value);
data.erase(key);
Arena arena(key.expectedSize() + value.expectedSize());
@ -112,6 +113,7 @@ void GlobalConfig::erase(Key key) {
}
void GlobalConfig::erase(KeyRangeRef range) {
// TraceEvent(SevInfo, "GlobalConfig_Erase").detail("Range", range);
auto it = data.begin();
while (it != data.end()) {
if (range.contains(it->first)) {
@ -165,7 +167,7 @@ ACTOR Future<Void> GlobalConfig::migrate(GlobalConfig* self) {
// attempt this migration at the same time, sometimes resulting in
// aborts due to conflicts. Purposefully avoid retrying, making this
// migration best-effort.
TraceEvent(SevInfo, "GlobalConfigMigrationError").detail("What", e.what());
TraceEvent(SevInfo, "GlobalConfig_MigrationError").detail("What", e.what());
}
return Void();
@ -174,6 +176,7 @@ ACTOR Future<Void> GlobalConfig::migrate(GlobalConfig* self) {
// Updates local copy of global configuration by reading the entire key-range
// from storage.
ACTOR Future<Void> GlobalConfig::refresh(GlobalConfig* self) {
// TraceEvent trace(SevInfo, "GlobalConfig_Refresh");
self->erase(KeyRangeRef(""_sr, "\xff"_sr));
Transaction tr(self->cx);

2
fdbclient/GlobalConfig.actor.h
View File

@ -72,7 +72,7 @@ public:
// to allow global configuration to run transactions on the latest
// database.
template <class T>
static void create(Database& cx, Reference<AsyncVar<T>> db, const ClientDBInfo* dbInfo) {
static void create(Database& cx, Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
if (g_network->global(INetwork::enGlobalConfig) == nullptr) {
auto config = new GlobalConfig{ cx };
g_network->setGlobal(INetwork::enGlobalConfig, config);

6
fdbclient/IConfigTransaction.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include <vector>
#include "fdbclient/IConfigTransaction.h"
#include "fdbclient/SimpleConfigTransaction.h"
#include "fdbclient/PaxosConfigTransaction.h"
@ -25,3 +27,7 @@
Reference<IConfigTransaction> IConfigTransaction::createTestSimple(ConfigTransactionInterface const& cti) {
return makeReference<SimpleConfigTransaction>(cti);
}
Reference<IConfigTransaction> IConfigTransaction::createTestPaxos(std::vector<ConfigTransactionInterface> const& ctis) {
return makeReference<PaxosConfigTransaction>(ctis);
}

1
fdbclient/IConfigTransaction.h
View File

@ -40,6 +40,7 @@ public:
virtual ~IConfigTransaction() = default;
static Reference<IConfigTransaction> createTestSimple(ConfigTransactionInterface const&);
static Reference<IConfigTransaction> createTestPaxos(std::vector<ConfigTransactionInterface> const&);
// Not implemented:
void setVersion(Version) override { throw client_invalid_operation(); }

20
fdbclient/ISingleThreadTransaction.h
View File

@ -52,16 +52,16 @@ public:
virtual Optional<Version> getCachedReadVersion() const = 0;
virtual Future<Optional<Value>> get(const Key& key, Snapshot = Snapshot::False) = 0;
virtual Future<Key> getKey(const KeySelector& key, Snapshot = Snapshot::False) = 0;
virtual Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) = 0;
virtual Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) = 0;
virtual Future<RangeResult> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) = 0;
virtual Future<RangeResult> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) = 0;
virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;
virtual Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0;
virtual Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) = 0;

41
fdbclient/MonitorLeader.actor.cpp
View File

@ -26,6 +26,28 @@
#include "flow/Platform.h"
#include "flow/actorcompiler.h" // has to be last include
namespace {
std::string trim(std::string const& connectionString) {
// Strip out whitespace
// Strip out characters between a # and a newline
std::string trimmed;
auto end = connectionString.end();
for (auto c = connectionString.begin(); c != end; ++c) {
if (*c == '#') {
++c;
while (c != end && *c != '\n' && *c != '\r')
++c;
if (c == end)
break;
} else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
trimmed += *c;
}
return trimmed;
}
} // namespace
std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
if (filename.length())
return std::make_pair(filename, false);
@ -154,24 +176,6 @@ std::string ClusterConnectionString::getErrorString(std::string const& source, E
}
}
std::string trim(std::string const& connectionString) {
// Strip out whitespace
// Strip out characters between a # and a newline
std::string trimmed;
auto end = connectionString.end();
for (auto c = connectionString.begin(); c != end; ++c) {
if (*c == '#') {
++c;
while (c != end && *c != '\n' && *c != '\r')
++c;
if (c == end)
break;
} else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
trimmed += *c;
}
return trimmed;
}
ClusterConnectionString::ClusterConnectionString(std::string const& connectionString) {
auto trimmed = trim(connectionString);
@ -838,6 +842,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
clientInfo->set(ni);
successIdx = idx;
} else {
TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cannot talk to cluster controller
idx = (idx + 1) % addrs.size();
if (idx == successIdx) {
wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));

4
fdbclient/MonitorLeader.h
View File

@ -49,7 +49,7 @@ struct ClientData {
OpenDatabaseRequest getRequest();
ClientData() : clientInfo(new AsyncVar<CachedSerialization<ClientDBInfo>>(CachedSerialization<ClientDBInfo>())) {}
ClientData() : clientInfo(makeReference<AsyncVar<CachedSerialization<ClientDBInfo>>>()) {}
};
struct MonitorLeaderInfo {
@ -58,7 +58,7 @@ struct MonitorLeaderInfo {
MonitorLeaderInfo() : hasConnected(false) {}
explicit MonitorLeaderInfo(Reference<ClusterConnectionFile> intermediateConnFile)
: intermediateConnFile(intermediateConnFile), hasConnected(false) {}
: hasConnected(false), intermediateConnFile(intermediateConnFile) {}
};
// Monitors the given coordination group's leader election process and provides a best current guess

2
fdbclient/MultiVersionAssignmentVars.h
View File

@ -281,7 +281,7 @@ template <class S, class T>
class FlatMapSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public:
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue), cancelled(false), released(false) {
: source(source), cancelled(false), released(false), mapValue(mapValue) {
ThreadSingleAssignmentVar<T>::addref();
int userParam;

101
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -396,7 +396,7 @@ void loadClientFunction(T* fp, void* lib, std::string libPath, const char* funct
}
DLApi::DLApi(std::string fdbCPath, bool unlinkOnLoad)
: api(new FdbCApi()), fdbCPath(fdbCPath), unlinkOnLoad(unlinkOnLoad), networkSetup(false) {}
: fdbCPath(fdbCPath), api(new FdbCApi()), unlinkOnLoad(unlinkOnLoad), networkSetup(false) {}
// Loads client API functions (definitions are in FdbCApi struct)
void DLApi::init() {
@ -591,7 +591,7 @@ Reference<IDatabase> DLApi::createDatabase609(const char* clusterFilePath) {
Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath) {
if (headerVersion >= 610) {
FdbCApi::FDBDatabase* db;
api->createDatabase(clusterFilePath, &db);
throwIfError(api->createDatabase(clusterFilePath, &db));
return Reference<IDatabase>(new DLDatabase(api, db));
} else {
return DLApi::createDatabase609(clusterFilePath);
@ -895,22 +895,43 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
api->runOnExternalClients(threadIdx, [this](Reference<ClientInfo> client) { dbState->addClient(client); });
if (!externalClientsInitialized.test_and_set()) {
api->runOnExternalClientsAllThreads([&clusterFilePath](Reference<ClientInfo> client) {
// This creates a database to initialize some client state on the external library
// We only do this on 6.2+ clients to avoid some bugs associated with older versions
// This deletes the new database immediately to discard its connections
if (client->protocolVersion.hasCloseUnusedConnection()) {
api->runOnExternalClientsAllThreads([&clusterFilePath](Reference<ClientInfo> client) {
// This creates a database to initialize some client state on the external library.
// We only do this on 6.2+ clients to avoid some bugs associated with older versions.
// This deletes the new database immediately to discard its connections.
//
// Simultaneous attempts to create a database could result in us running this initialization
// code in multiple threads simultaneously. It is necessary that each attempt have a chance
// to run this initialization in case the other fails, and it's safe to run them in parallel.
if (client->protocolVersion.hasCloseUnusedConnection() && !client->initialized) {
try {
Reference<IDatabase> newDb = client->api->createDatabase(clusterFilePath.c_str());
client->initialized = true;
} catch (Error& e) {
// This connection is not initialized. It is still possible to connect with it,
// but we may not see trace logs from this client until a successful connection
// is established.
TraceEvent(SevWarnAlways, "FailedToInitializeExternalClient")
.detail("LibraryPath", client->libPath)
.detail("ClusterFilePath", clusterFilePath)
.error(e);
}
});
}
}
});
// For clients older than 6.2 we create and maintain our database connection
api->runOnExternalClients(threadIdx, [this, &clusterFilePath](Reference<ClientInfo> client) {
if (!client->protocolVersion.hasCloseUnusedConnection()) {
dbState->legacyDatabaseConnections[client->protocolVersion] =
client->api->createDatabase(clusterFilePath.c_str());
try {
dbState->legacyDatabaseConnections[client->protocolVersion] =
client->api->createDatabase(clusterFilePath.c_str());
} catch (Error& e) {
// This connection is discarded
TraceEvent(SevWarnAlways, "FailedToCreateLegacyDatabaseConnection")
.detail("LibraryPath", client->libPath)
.detail("ClusterFilePath", clusterFilePath)
.error(e);
}
}
});
@ -993,8 +1014,8 @@ ThreadFuture<ProtocolVersion> MultiVersionDatabase::getServerProtocol(Optional<P
}
MultiVersionDatabase::DatabaseState::DatabaseState(std::string clusterFilePath, Reference<IDatabase> versionMonitorDb)
: clusterFilePath(clusterFilePath), versionMonitorDb(versionMonitorDb),
dbVar(new ThreadSafeAsyncVar<Reference<IDatabase>>(Reference<IDatabase>(nullptr))) {}
: dbVar(new ThreadSafeAsyncVar<Reference<IDatabase>>(Reference<IDatabase>(nullptr))),
clusterFilePath(clusterFilePath), versionMonitorDb(versionMonitorDb), closed(false) {}
// Adds a client (local or externally loaded) that can be used to connect to the cluster
void MultiVersionDatabase::DatabaseState::addClient(Reference<ClientInfo> client) {
@ -1058,6 +1079,10 @@ ThreadFuture<Void> MultiVersionDatabase::DatabaseState::monitorProtocolVersion()
// Called when a change to the protocol version of the cluster has been detected.
// Must be called from the main thread
void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion protocolVersion) {
if (closed) {
return;
}
// If the protocol version changed but is still compatible, update our local version but keep the same connection
if (dbProtocolVersion.present() &&
protocolVersion.normalizedVersion() == dbProtocolVersion.get().normalizedVersion()) {
@ -1084,7 +1109,20 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
.detail("Failed", client->failed)
.detail("External", client->external);
Reference<IDatabase> newDb = client->api->createDatabase(clusterFilePath.c_str());
Reference<IDatabase> newDb;
try {
newDb = client->api->createDatabase(clusterFilePath.c_str());
} catch (Error& e) {
TraceEvent(SevWarnAlways, "MultiVersionClientFailedToCreateDatabase")
.detail("LibraryPath", client->libPath)
.detail("External", client->external)
.detail("ClusterFilePath", clusterFilePath)
.error(e);
// Put the client in a disconnected state until the version changes again
updateDatabase(Reference<IDatabase>(), Reference<ClientInfo>());
return;
}
if (client->external && !MultiVersionApi::apiVersionAtLeast(610)) {
// Old API versions return a future when creating the database, so we need to wait for it
@ -1112,6 +1150,10 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
// Replaces the active database connection with a new one. Must be called from the main thread.
void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> newDb, Reference<ClientInfo> client) {
if (closed) {
return;
}
if (newDb) {
optionLock.enter();
for (auto option : options) {
@ -1143,12 +1185,28 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
versionMonitorDb = db;
} else {
// For older clients that don't have an API to get the protocol version, we have to monitor it locally
versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str());
try {
versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str());
} catch (Error& e) {
// We can't create a new database to monitor the cluster version. This means we will continue using the
// previous one, which should hopefully continue to work.
TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring")
.detail("ClusterFilePath", clusterFilePath)
.error(e);
}
}
} else {
// We don't have a database connection, so use the local client to monitor the protocol version
db = Reference<IDatabase>();
versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str());
try {
versionMonitorDb = MultiVersionApi::api->getLocalClient()->api->createDatabase(clusterFilePath.c_str());
} catch (Error& e) {
// We can't create a new database to monitor the cluster version. This means we will continue using the
// previous one, which should hopefully continue to work.
TraceEvent(SevWarnAlways, "FailedToCreateDatabaseForVersionMonitoring")
.detail("ClusterFilePath", clusterFilePath)
.error(e);
}
}
dbVar->set(db);
@ -1178,6 +1236,7 @@ void MultiVersionDatabase::DatabaseState::close() {
Reference<DatabaseState> self = Reference<DatabaseState>::addRef(this);
onMainThreadVoid(
[self]() {
self->closed = true;
if (self->protocolVersionMonitor.isValid()) {
self->protocolVersionMonitor.cancel();
}
@ -1255,8 +1314,6 @@ void MultiVersionDatabase::LegacyVersionMonitor::close() {
}
}
std::atomic_flag MultiVersionDatabase::externalClientsInitialized = ATOMIC_FLAG_INIT;
// MultiVersionApi
bool MultiVersionApi::apiVersionAtLeast(int minVersion) {
ASSERT_NE(MultiVersionApi::api->apiVersion, 0);
@ -1464,7 +1521,7 @@ std::vector<std::pair<std::string, bool>> MultiVersionApi::copyExternalLibraryPe
#else
std::vector<std::pair<std::string, bool>> MultiVersionApi::copyExternalLibraryPerThread(std::string path) {
if (threadCount > 1) {
TraceEvent(SevError, "MultipleClientThreadsUnsupportedOnWindows");
TraceEvent(SevError, "MultipleClientThreadsUnsupportedOnWindows").log();
throw unsupported_operation();
}
std::vector<std::pair<std::string, bool>> paths;
@ -1855,8 +1912,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
}
MultiVersionApi::MultiVersionApi()
: bypassMultiClientApi(false), networkStartSetup(false), networkSetup(false), callbackOnMainThread(true),
externalClient(false), localClientDisabled(false), apiVersion(0), envOptionsLoaded(false), threadCount(0) {}
: callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false),
bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), envOptionsLoaded(false) {}
MultiVersionApi* MultiVersionApi::api = new MultiVersionApi();

18
fdbclient/MultiVersionTransaction.h
View File

@ -417,12 +417,15 @@ struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted<ClientInfo> {
ProtocolVersion protocolVersion;
IClientApi* api;
bool failed;
std::atomic_bool initialized;
std::vector<std::pair<void (*)(void*), void*>> threadCompletionHooks;
ClientInfo() : ClientDesc(std::string(), false), protocolVersion(0), api(nullptr), failed(true) {}
ClientInfo(IClientApi* api) : ClientDesc("internal", false), protocolVersion(0), api(api), failed(false) {}
ClientInfo()
: ClientDesc(std::string(), false), protocolVersion(0), api(nullptr), failed(true), initialized(false) {}
ClientInfo(IClientApi* api)
: ClientDesc("internal", false), protocolVersion(0), api(api), failed(false), initialized(false) {}
ClientInfo(IClientApi* api, std::string libPath)
: ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false) {}
: ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false), initialized(false) {}
void loadProtocolVersion();
bool canReplace(Reference<ClientInfo> other) const;
@ -504,10 +507,9 @@ public:
// this will be a specially created local db.
Reference<IDatabase> versionMonitorDb;
bool closed;
ThreadFuture<Void> changed;
bool cancelled;
ThreadFuture<Void> dbReady;
ThreadFuture<Void> protocolVersionMonitor;
@ -557,10 +559,6 @@ public:
const Reference<DatabaseState> dbState;
friend class MultiVersionTransaction;
// Clients must create a database object in order to initialize some of their state.
// This needs to be done only once, and this flag tracks whether that has happened.
static std::atomic_flag externalClientsInitialized;
};
// An implementation of IClientApi that can choose between multiple different client implementations either provided

226
fdbclient/NativeAPI.actor.cpp
View File

@ -116,10 +116,10 @@ TLSConfig tlsConfig(TLSEndpointType::CLIENT);
// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h.
NetworkOptions::NetworkOptions()
: localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()), traceRollSize(TRACE_DEFAULT_ROLL_SIZE),
traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), traceFormat("xml"),
traceClockSource("now"), runLoopProfilingEnabled(false),
supportedVersions(new ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>()) {}
: traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
traceFormat("xml"), traceClockSource("now"),
supportedVersions(new ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>()), runLoopProfilingEnabled(false) {
}
static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/");
static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/");
@ -285,7 +285,7 @@ std::string unprintable(std::string const& val) {
return s;
}
void DatabaseContext::validateVersion(Version version) {
void DatabaseContext::validateVersion(Version version) const {
// Version could be 0 if the INITIALIZE_NEW_DATABASE option is set. In that case, it is illegal to perform any
// reads. We throw client_invalid_operation because the caller didn't directly set the version, so the
// version_invalid error might be confusing.
@ -488,7 +488,7 @@ ACTOR static Future<Void> transactionInfoCommitActor(Transaction* tr, std::vecto
ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t clientTxInfoSizeLimit) {
state const Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
state const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
TraceEvent(SevInfo, "DelExcessClntTxnEntriesCalled");
TraceEvent(SevInfo, "DelExcessClntTxnEntriesCalled").log();
loop {
try {
tr->reset();
@ -496,7 +496,7 @@ ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), Snapshot::True));
if (!ctrValue.present()) {
TraceEvent(SevInfo, "NumClntTxnEntriesNotFound");
TraceEvent(SevInfo, "NumClntTxnEntriesNotFound").log();
return Void();
}
state int64_t txInfoSize = 0;
@ -650,7 +650,7 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
}
}
ACTOR static Future<Void> monitorProxiesChange(Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
ACTOR static Future<Void> monitorProxiesChange(Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
AsyncTrigger* triggerVar) {
state vector<CommitProxyInterface> curCommitProxies;
state vector<GrvProxyInterface> curGrvProxies;
@ -1085,7 +1085,7 @@ Future<RangeResult> HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction*
DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
@ -1094,11 +1094,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
IsInternal internal,
int apiVersion,
IsSwitchable switchable)
: connectionFile(connectionFile), clientInfo(clientInfo), coordinator(coordinator),
clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality),
enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), apiVersion(apiVersion),
switchable(switchable), proxyProvisional(false), cc("TransactionMetrics"),
transactionReadVersions("ReadVersions", cc), transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
: lockAware(lockAware), switchable(switchable), connectionFile(connectionFile), proxyProvisional(false),
clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), internal(internal),
cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc),
transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc),
transactionReadVersionBatches("ReadVersionBatches", cc),
transactionBatchReadVersions("BatchPriorityReadVersions", cc),
@ -1123,11 +1122,12 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
transactionStatusRequests("StatusRequests", cc), transactionsTooOld("TooOld", cc),
transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), outstandingWatches(0),
latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
bytesPerCommit(1000), mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
internal(internal), transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0),
transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
dbId = deterministicRandom()->randomUniqueID();
connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size())
@ -1340,8 +1340,8 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
}
DatabaseContext::DatabaseContext(const Error& err)
: deferredError(err), cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc),
transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
: deferredError(err), internal(IsInternal::False), cc("TransactionMetrics"),
transactionReadVersions("ReadVersions", cc), transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc),
transactionReadVersionBatches("ReadVersionBatches", cc),
transactionBatchReadVersions("BatchPriorityReadVersions", cc),
@ -1366,11 +1366,10 @@ DatabaseContext::DatabaseContext(const Error& err)
transactionStatusRequests("StatusRequests", cc), transactionsTooOld("TooOld", cc),
transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), latencies(1000),
readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(IsInternal::False),
transactionTracingEnabled(true) {}
transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT) {}
// Static constructor used by server processes to create a DatabaseContext
// For internal (fdbserver) use only
@ -1482,7 +1481,7 @@ void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
locationCache.insert(KeyRangeRef(begin, end), Reference<LocationInfo>());
}
Future<Void> DatabaseContext::onProxiesChanged() {
Future<Void> DatabaseContext::onProxiesChanged() const {
return this->proxiesChangeTrigger.onTrigger();
}
@ -1627,7 +1626,7 @@ ACTOR static Future<Void> switchConnectionFileImpl(Reference<ClusterConnectionFi
loop {
tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
try {
TraceEvent("SwitchConnectionFileAttemptingGRV");
TraceEvent("SwitchConnectionFileAttemptingGRV").log();
Version v = wait(tr.getReadVersion());
TraceEvent("SwitchConnectionFileGotRV")
.detail("ReadVersion", v)
@ -1699,7 +1698,8 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
networkOptions.traceDirectory.get(),
"trace",
networkOptions.traceLogGroup,
networkOptions.traceFileIdentifier);
networkOptions.traceFileIdentifier,
networkOptions.tracePartialFileSuffix);
TraceEvent("ClientStart")
.detail("SourceVersion", getSourceVersion())
@ -1759,7 +1759,8 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
}
auto database = Database(db);
GlobalConfig::create(database, clientInfo, std::addressof(clientInfo->get()));
GlobalConfig::create(
database, Reference<AsyncVar<ClientDBInfo> const>(clientInfo), std::addressof(clientInfo->get()));
return database;
}
@ -1856,6 +1857,10 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
throw invalid_option_value();
}
break;
case FDBNetworkOptions::TRACE_PARTIAL_FILE_SUFFIX:
validateOptionValuePresent(value);
networkOptions.tracePartialFileSuffix = value.get().toString();
break;
case FDBNetworkOptions::KNOB: {
validateOptionValuePresent(value);
@ -4092,9 +4097,9 @@ Transaction::Transaction()
: info(TaskPriority::DefaultEndpoint, generateSpanID(true)), span(info.spanID, "Transaction"_loc) {}
Transaction::Transaction(Database const& cx)
: cx(cx), info(cx->taskID, generateSpanID(cx->transactionTracingEnabled)), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF),
committedVersion(invalidVersion), versionstampPromise(Promise<Standalone<StringRef>>()), options(cx), numErrors(0),
trLogInfo(createTrLogInfoProbabilistically(cx)), tr(info.spanID), span(info.spanID, "Transaction"_loc) {
: info(cx->taskID, generateSpanID(cx->transactionTracingEnabled)), numErrors(0), options(cx),
span(info.spanID, "Transaction"_loc), trLogInfo(createTrLogInfoProbabilistically(cx)), cx(cx),
backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), committedVersion(invalidVersion), tr(info.spanID) {
if (DatabaseContext::debugUseTags) {
debugAddTags(this);
}
@ -4310,6 +4315,21 @@ Future<Standalone<VectorRef<const char*>>> Transaction::getAddressesForKey(const
return getAddressesForKeyActor(key, ver, cx, info, options);
}
ACTOR Future<Void> registerRangeFeedActor(Transaction* tr, Key rangeID, KeyRange range) {
state Key rangeIDKey = rangeID.withPrefix(rangeFeedPrefix);
Optional<Value> val = wait(tr->get(rangeIDKey));
if (!val.present()) {
tr->set(rangeIDKey, rangeFeedValue(range));
} else if (decodeRangeFeedValue(val.get()) != range) {
throw unsupported_operation();
}
return Void();
}
Future<Void> Transaction::registerRangeFeed(const Key& rangeID, const KeyRange& range) {
return registerRangeFeedActor(this, rangeID, range);
}
ACTOR Future<Key> getKeyAndConflictRange(Database cx,
KeySelector k,
Future<Version> version,
@ -5198,7 +5218,7 @@ Future<Void> Transaction::commitMutations() {
if (options.debugDump) {
UID u = nondeterministicRandom()->randomUniqueID();
TraceEvent("TransactionDump", u);
TraceEvent("TransactionDump", u).log();
for (auto i = tr.transaction.mutations.begin(); i != tr.transaction.mutations.end(); ++i)
TraceEvent("TransactionMutation", u)
.detail("T", i->type)
@ -5243,7 +5263,10 @@ ACTOR Future<Void> commitAndWatch(Transaction* self) {
self->setupWatches();
}
self->reset();
if (!self->apiVersionAtLeast(700)) {
self->reset();
}
return Void();
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled) {
@ -5252,7 +5275,10 @@ ACTOR Future<Void> commitAndWatch(Transaction* self) {
}
self->versionstampPromise.sendError(transaction_invalid_version());
self->reset();
if (!self->apiVersionAtLeast(700)) {
self->reset();
}
}
throw;
@ -5760,7 +5786,7 @@ ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
NetworkAddress coordinatorAddress,
Optional<ProtocolVersion> expectedVersion) {
state Reference<AsyncVar<Optional<ProtocolVersion>>> protocolVersion =
state Reference<AsyncVar<Optional<ProtocolVersion>> const> protocolVersion =
FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress);
loop {
@ -5785,7 +5811,7 @@ ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
// Returns the protocol version reported by the given coordinator
// If an expected version is given, the future won't return until the protocol version is different than expected
ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
Optional<ProtocolVersion> expectedVersion) {
state bool needToConnect = true;
@ -6325,7 +6351,7 @@ void Transaction::setToken(uint64_t token) {
void enableClientInfoLogging() {
ASSERT(networkOptions.logClientInfo.present() == false);
networkOptions.logClientInfo = true;
TraceEvent(SevInfo, "ClientInfoLoggingEnabled");
TraceEvent(SevInfo, "ClientInfoLoggingEnabled").log();
}
ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
@ -6379,7 +6405,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exc
}
throw;
}
TraceEvent("ExclusionSafetyCheckCoordinators");
TraceEvent("ExclusionSafetyCheckCoordinators").log();
state ClientCoordinators coordinatorList(cx->getConnectionFile());
state vector<Future<Optional<LeaderInfo>>> leaderServers;
leaderServers.reserve(coordinatorList.clientLeaderServers.size());
@ -6392,7 +6418,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exc
choose {
when(wait(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.0))) {}
when(wait(delay(3.0))) {
TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum");
TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum").log();
return false;
}
}
@ -6501,6 +6527,124 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
}
ACTOR Future<Standalone<VectorRef<MutationsAndVersionRef>>> getRangeFeedMutationsActor(Reference<DatabaseContext> db,
StringRef rangeID,
Version begin,
Version end,
KeyRange range) {
state Database cx(db);
state Transaction tr(cx);
state Key rangeIDKey = rangeID.withPrefix(rangeFeedPrefix);
state Span span("NAPI:GetRangeFeedMutations"_loc);
Optional<Value> val = wait(tr.get(rangeIDKey));
if (!val.present()) {
throw unsupported_operation();
}
KeyRange keys = decodeRangeFeedValue(val.get());
state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
wait(getKeyRangeLocations(cx,
keys,
100,
Reverse::False,
&StorageServerInterface::rangeFeed,
TransactionInfo(TaskPriority::DefaultEndpoint, span.context)));
if (locations.size() > 1) {
throw unsupported_operation();
}
state RangeFeedRequest req;
req.rangeID = rangeID;
req.begin = begin;
req.end = end;
RangeFeedReply rep = wait(loadBalance(cx.getPtr(),
locations[0].second,
&StorageServerInterface::rangeFeed,
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr));
return Standalone<VectorRef<MutationsAndVersionRef>>(rep.mutations, rep.arena);
}
Future<Standalone<VectorRef<MutationsAndVersionRef>>> DatabaseContext::getRangeFeedMutations(StringRef rangeID,
Version begin,
Version end,
KeyRange range) {
return getRangeFeedMutationsActor(Reference<DatabaseContext>::addRef(this), rangeID, begin, end, range);
}
ACTOR Future<std::vector<std::pair<Key, KeyRange>>> getOverlappingRangeFeedsActor(Reference<DatabaseContext> db,
KeyRangeRef range,
Version minVersion) {
state Database cx(db);
state Transaction tr(cx);
state Span span("NAPI:GetOverlappingRangeFeeds"_loc);
state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
wait(getKeyRangeLocations(cx,
range,
100,
Reverse::False,
&StorageServerInterface::rangeFeed,
TransactionInfo(TaskPriority::DefaultEndpoint, span.context)));
if (locations.size() > 1) {
throw unsupported_operation();
}
state OverlappingRangeFeedsRequest req;
req.range = range;
OverlappingRangeFeedsReply rep = wait(loadBalance(cx.getPtr(),
locations[0].second,
&StorageServerInterface::overlappingRangeFeeds,
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr));
return rep.rangeIds;
}
Future<std::vector<std::pair<Key, KeyRange>>> DatabaseContext::getOverlappingRangeFeeds(KeyRangeRef range,
Version minVersion) {
return getOverlappingRangeFeedsActor(Reference<DatabaseContext>::addRef(this), range, minVersion);
}
ACTOR Future<Void> popRangeFeedMutationsActor(Reference<DatabaseContext> db, StringRef rangeID, Version version) {
state Database cx(db);
state Transaction tr(cx);
state Key rangeIDKey = rangeID.withPrefix(rangeFeedPrefix);
state Span span("NAPI:PopRangeFeedMutations"_loc);
Optional<Value> val = wait(tr.get(rangeIDKey));
if (!val.present()) {
throw unsupported_operation();
}
KeyRange keys = decodeRangeFeedValue(val.get());
state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
wait(getKeyRangeLocations(cx,
keys,
100,
Reverse::False,
&StorageServerInterface::rangeFeed,
TransactionInfo(TaskPriority::DefaultEndpoint, span.context)));
if (locations.size() > 1) {
throw unsupported_operation();
}
state std::vector<Future<Void>> popRequests;
for (int i = 0; i < locations[0].second->size(); i++) {
popRequests.push_back(
locations[0].second->getInterface(i).rangeFeedPop.getReply(RangeFeedPopRequest(rangeID, version)));
}
wait(waitForAll(popRequests));
return Void();
}
Future<Void> DatabaseContext::popRangeFeedMutations(StringRef rangeID, Version version) {
return popRangeFeedMutationsActor(Reference<DatabaseContext>::addRef(this), rangeID, version);
}
ACTOR Future<Void> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) {
state ReadYourWritesTransaction tr(cx);
loop {

11
fdbclient/NativeAPI.actor.h
View File

@ -68,6 +68,7 @@ struct NetworkOptions {
std::string traceFormat;
std::string traceClockSource;
std::string traceFileIdentifier;
std::string tracePartialFileSuffix;
Optional<bool> logClientInfo;
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions;
bool runLoopProfilingEnabled;
@ -189,7 +190,7 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {}
TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {}
TransactionLogInfo(std::string id, LoggingLocation location)
: logLocation(location), identifier(id), maxFieldLength(0) {}
: logLocation(location), maxFieldLength(0), identifier(id) {}
void setIdentifier(std::string id) { identifier = id; }
void logTo(LoggingLocation loc) { logLocation = logLocation | loc; }
@ -231,10 +232,10 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
Promise<Void> onSetWatchTrigger;
Future<Void> watchFuture;
Watch() : watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch(Key key) : key(key), watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch() : valuePresent(false), setPresent(false), watchFuture(Never()) {}
Watch(Key key) : key(key), valuePresent(false), setPresent(false), watchFuture(Never()) {}
Watch(Key key, Optional<Value> val)
: key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) {}
: key(key), value(val), valuePresent(true), setPresent(false), watchFuture(Never()) {}
void setWatch(Future<Void> watchFuture);
};
@ -325,6 +326,8 @@ public:
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
Future<Void> registerRangeFeed(const Key& rangeID, const KeyRange& range);
void enableCheckWrites();
void addReadConflictRange(KeyRangeRef const& keys);
void addWriteConflictRange(KeyRangeRef const& keys);

267
fdbclient/PaxosConfigTransaction.actor.cpp
View File

@ -18,118 +18,269 @@
* limitations under the License.
*/
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/PaxosConfigTransaction.h"
#include "flow/actorcompiler.h" // must be last include
class PaxosConfigTransactionImpl {};
class PaxosConfigTransactionImpl {
ConfigTransactionCommitRequest toCommit;
Future<ConfigGeneration> getGenerationFuture;
std::vector<ConfigTransactionInterface> ctis;
int numRetries{ 0 };
bool committed{ false };
Optional<UID> dID;
Database cx;
ACTOR static Future<ConfigGeneration> getGeneration(PaxosConfigTransactionImpl* self) {
state std::vector<Future<ConfigTransactionGetGenerationReply>> getGenerationFutures;
getGenerationFutures.reserve(self->ctis.size());
for (auto const& cti : self->ctis) {
getGenerationFutures.push_back(cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{}));
}
// FIXME: Must tolerate failures and disagreement
wait(waitForAll(getGenerationFutures));
return getGenerationFutures[0].get().generation;
}
ACTOR static Future<Optional<Value>> get(PaxosConfigTransactionImpl* self, Key key) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
state ConfigKey configKey = ConfigKey::decodeKey(key);
ConfigGeneration generation = wait(self->getGenerationFuture);
// TODO: Load balance
ConfigTransactionGetReply reply =
wait(self->ctis[0].get.getReply(ConfigTransactionGetRequest{ generation, configKey }));
if (reply.value.present()) {
return reply.value.get().toValue();
} else {
return Optional<Value>{};
}
}
ACTOR static Future<RangeResult> getConfigClasses(PaxosConfigTransactionImpl* self) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
ConfigGeneration generation = wait(self->getGenerationFuture);
// TODO: Load balance
ConfigTransactionGetConfigClassesReply reply =
wait(self->ctis[0].getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ generation }));
RangeResult result;
result.reserve(result.arena(), reply.configClasses.size());
for (const auto& configClass : reply.configClasses) {
result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
}
return result;
}
ACTOR static Future<RangeResult> getKnobs(PaxosConfigTransactionImpl* self, Optional<Key> configClass) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
ConfigGeneration generation = wait(self->getGenerationFuture);
// TODO: Load balance
ConfigTransactionGetKnobsReply reply =
wait(self->ctis[0].getKnobs.getReply(ConfigTransactionGetKnobsRequest{ generation, configClass }));
RangeResult result;
result.reserve(result.arena(), reply.knobNames.size());
for (const auto& knobName : reply.knobNames) {
result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
}
return result;
}
ACTOR static Future<Void> commit(PaxosConfigTransactionImpl* self) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
wait(store(self->toCommit.generation, self->getGenerationFuture));
self->toCommit.annotation.timestamp = now();
std::vector<Future<Void>> commitFutures;
commitFutures.reserve(self->ctis.size());
for (const auto& cti : self->ctis) {
commitFutures.push_back(cti.commit.getReply(self->toCommit));
}
// FIXME: Must tolerate failures and disagreement
wait(quorum(commitFutures, commitFutures.size() / 2 + 1));
self->committed = true;
return Void();
}
public:
Future<Version> getReadVersion() {
if (!getGenerationFuture.isValid()) {
getGenerationFuture = getGeneration(this);
}
return map(getGenerationFuture, [](auto const& gen) { return gen.committedVersion; });
}
Optional<Version> getCachedReadVersion() const {
if (getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError()) {
return getGenerationFuture.get().committedVersion;
} else {
return {};
}
}
Version getCommittedVersion() const { return committed ? getGenerationFuture.get().liveVersion : ::invalidVersion; }
int64_t getApproximateSize() const { return toCommit.expectedSize(); }
void set(KeyRef key, ValueRef value) { toCommit.set(key, value); }
void clear(KeyRef key) { toCommit.clear(key); }
Future<Optional<Value>> get(Key const& key) { return get(this, key); }
Future<RangeResult> getRange(KeyRangeRef keys) {
if (keys == configClassKeys) {
return getConfigClasses(this);
} else if (keys == globalConfigKnobKeys) {
return getKnobs(this, {});
} else if (configKnobKeys.contains(keys) && keys.singleKeyRange()) {
const auto configClass = keys.begin.removePrefix(configKnobKeys.begin);
return getKnobs(this, configClass);
} else {
throw invalid_config_db_range_read();
}
}
Future<Void> onError(Error const& e) {
// TODO: Improve this:
if (e.code() == error_code_transaction_too_old) {
reset();
return delay((1 << numRetries++) * 0.01 * deterministicRandom()->random01());
}
throw e;
}
void debugTransaction(UID dID) { this->dID = dID; }
void reset() {
getGenerationFuture = Future<ConfigGeneration>{};
toCommit = {};
committed = false;
}
void fullReset() {
numRetries = 0;
dID = {};
reset();
}
void checkDeferredError(Error const& deferredError) const {
if (deferredError.code() != invalid_error_code) {
throw deferredError;
}
if (cx.getPtr()) {
cx->checkDeferredError();
}
}
Future<Void> commit() { return commit(this); }
PaxosConfigTransactionImpl(Database const& cx) : cx(cx) {
auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators();
ctis.reserve(coordinators.size());
for (const auto& coordinator : coordinators) {
ctis.emplace_back(coordinator);
}
}
PaxosConfigTransactionImpl(std::vector<ConfigTransactionInterface> const& ctis) : ctis(ctis) {}
};
Future<Version> PaxosConfigTransaction::getReadVersion() {
// TODO: Implement
return ::invalidVersion;
return impl().getReadVersion();
}
Optional<Version> PaxosConfigTransaction::getCachedReadVersion() const {
// TODO: Implement
return ::invalidVersion;
return impl().getCachedReadVersion();
}
Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, Snapshot snapshot) {
// TODO: Implement
return Optional<Value>{};
Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, Snapshot) {
return impl().get(key);
}
Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
// TODO: Implement
ASSERT(false);
return Standalone<RangeResultRef>{};
Future<RangeResult> PaxosConfigTransaction::getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
if (reverse) {
throw client_invalid_operation();
}
return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
}
Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
// TODO: Implement
ASSERT(false);
return Standalone<RangeResultRef>{};
Future<RangeResult> PaxosConfigTransaction::getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
if (reverse) {
throw client_invalid_operation();
}
return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
}
void PaxosConfigTransaction::set(KeyRef const& key, ValueRef const& value) {
// TODO: Implememnt
ASSERT(false);
return impl().set(key, value);
}
void PaxosConfigTransaction::clear(KeyRef const& key) {
// TODO: Implememnt
ASSERT(false);
return impl().clear(key);
}
Future<Void> PaxosConfigTransaction::commit() {
// TODO: Implememnt
ASSERT(false);
return Void();
return impl().commit();
}
Version PaxosConfigTransaction::getCommittedVersion() const {
// TODO: Implement
ASSERT(false);
return ::invalidVersion;
return impl().getCommittedVersion();
}
int64_t PaxosConfigTransaction::getApproximateSize() const {
// TODO: Implement
ASSERT(false);
return 0;
return impl().getApproximateSize();
}
void PaxosConfigTransaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
// TODO: Implement
ASSERT(false);
// TODO: Support using this option to determine atomicity
}
Future<Void> PaxosConfigTransaction::onError(Error const& e) {
// TODO: Implement
ASSERT(false);
return Void();
return impl().onError(e);
}
void PaxosConfigTransaction::cancel() {
// TODO: Implement
ASSERT(false);
// TODO: Implement someday
throw client_invalid_operation();
}
void PaxosConfigTransaction::reset() {
// TODO: Implement
ASSERT(false);
impl().reset();
}
void PaxosConfigTransaction::fullReset() {
// TODO: Implement
ASSERT(false);
impl().fullReset();
}
void PaxosConfigTransaction::debugTransaction(UID dID) {
// TODO: Implement
ASSERT(false);
impl().debugTransaction(dID);
}
void PaxosConfigTransaction::checkDeferredError() const {
// TODO: Implement
ASSERT(false);
impl().checkDeferredError(deferredError);
}
PaxosConfigTransaction::PaxosConfigTransaction() {
// TODO: Implement
ASSERT(false);
}
PaxosConfigTransaction::PaxosConfigTransaction(std::vector<ConfigTransactionInterface> const& ctis)
: _impl(std::make_unique<PaxosConfigTransactionImpl>(ctis)) {}
PaxosConfigTransaction::PaxosConfigTransaction() = default;
PaxosConfigTransaction::~PaxosConfigTransaction() = default;
void PaxosConfigTransaction::setDatabase(Database const& cx) {
// TODO: Implement
ASSERT(false);
_impl = std::make_unique<PaxosConfigTransactionImpl>(cx);
}

21
fdbclient/PaxosConfigTransaction.h
View File

@ -33,6 +33,7 @@ class PaxosConfigTransaction final : public IConfigTransaction, public FastAlloc
PaxosConfigTransactionImpl& impl() { return *_impl; }
public:
PaxosConfigTransaction(std::vector<ConfigTransactionInterface> const&);
PaxosConfigTransaction();
~PaxosConfigTransaction();
void setDatabase(Database const&) override;
@ -40,16 +41,16 @@ public:
Optional<Version> getCachedReadVersion() const override;
Future<Optional<Value>> get(Key const& key, Snapshot = Snapshot::False) override;
Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
void set(KeyRef const& key, ValueRef const& value) override;
void clear(KeyRangeRef const&) override { throw client_invalid_operation(); }
void clear(KeyRef const&) override;

2
fdbclient/RYWIterator.h
View File

@ -28,7 +28,7 @@
class RYWIterator {
public:
RYWIterator(SnapshotCache* snapshotCache, WriteMap* writeMap)
: cache(snapshotCache), writes(writeMap), begin_key_cmp(0), end_key_cmp(0), bypassUnreadable(false) {}
: begin_key_cmp(0), end_key_cmp(0), cache(snapshotCache), writes(writeMap), bypassUnreadable(false) {}
enum SEGMENT_TYPE { UNKNOWN_RANGE, EMPTY_RANGE, KV };
static const SEGMENT_TYPE typeMap[12];

17
fdbclient/ReadYourWrites.actor.cpp
View File

@ -1164,7 +1164,7 @@ public:
if (!ryw->resetPromise.isSet())
ryw->resetPromise.sendError(transaction_timed_out());
wait(delay(deterministicRandom()->random01() * 5));
TraceEvent("ClientBuggifyInFlightCommit");
TraceEvent("ClientBuggifyInFlightCommit").log();
wait(ryw->tr.commit());
}
@ -1285,9 +1285,9 @@ public:
};
ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx)
: ISingleThreadTransaction(cx->deferredError), cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0),
creationTime(now()), commitStarted(false), options(tr), versionStampFuture(tr.getVersionstamp()),
specialKeySpaceWriteMap(std::make_pair(false, Optional<Value>()), specialKeys.end) {
: ISingleThreadTransaction(cx->deferredError), tr(cx), cache(&arena), writes(&arena), retries(0), approximateSize(0),
creationTime(now()), commitStarted(false), versionStampFuture(tr.getVersionstamp()),
specialKeySpaceWriteMap(std::make_pair(false, Optional<Value>()), specialKeys.end), options(tr) {
std::copy(
cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(), std::back_inserter(persistentOptions));
applyPersistentOptions();
@ -2284,10 +2284,11 @@ void ReadYourWritesTransaction::operator=(ReadYourWritesTransaction&& r) noexcep
}
ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept
: ISingleThreadTransaction(std::move(r.deferredError)), cache(std::move(r.cache)), writes(std::move(r.writes)),
arena(std::move(r.arena)), reading(std::move(r.reading)), retries(r.retries), approximateSize(r.approximateSize),
creationTime(r.creationTime), timeoutActor(std::move(r.timeoutActor)), resetPromise(std::move(r.resetPromise)),
commitStarted(r.commitStarted), options(r.options), transactionDebugInfo(r.transactionDebugInfo) {
: ISingleThreadTransaction(std::move(r.deferredError)), arena(std::move(r.arena)), cache(std::move(r.cache)),
writes(std::move(r.writes)), resetPromise(std::move(r.resetPromise)), reading(std::move(r.reading)),
retries(r.retries), approximateSize(r.approximateSize), timeoutActor(std::move(r.timeoutActor)),
creationTime(r.creationTime), commitStarted(r.commitStarted), transactionDebugInfo(r.transactionDebugInfo),
options(r.options) {
cache.arena = &arena;
writes.arena = &arena;
tr = std::move(r.tr);

28
fdbclient/ReadYourWrites.h
View File

@ -74,20 +74,20 @@ public:
Optional<Version> getCachedReadVersion() const override { return tr.getCachedReadVersion(); }
Future<Optional<Value>> get(const Key& key, Snapshot = Snapshot::False) override;
Future<Key> getKey(const KeySelector& key, Snapshot = Snapshot::False) override;
Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
Snapshot snapshot = Snapshot::False,
Reverse reverse = Reverse::False) {
Future<RangeResult> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(const KeyRange& keys,
int limit,
Snapshot snapshot = Snapshot::False,
Reverse reverse = Reverse::False) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,

5
fdbclient/ServerKnobs.cpp
View File

@ -342,6 +342,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( ROCKSDB_PERIODIC_COMPACTION_SECONDS, 0 );
init( ROCKSDB_PREFIX_LEN, 0 );
init( ROCKSDB_BLOCK_CACHE_SIZE, 0 );
init( ROCKSDB_METRICS_DELAY, 60.0 );
// Leader election
bool longLeaderElection = randomize && BUGGIFY;
@ -366,6 +367,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET, 10.0 );
init( START_TRANSACTION_MAX_QUEUE_SIZE, 1e6 );
init( KEY_LOCATION_MAX_QUEUE_SIZE, 1e6 );
init( COMMIT_PROXY_LIVENESS_TIMEOUT, 20.0 );
init( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, 0.0005 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005;
init( COMMIT_TRANSACTION_BATCH_INTERVAL_MIN, 0.001 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_MIN = 0.1;
@ -644,6 +646,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( DBINFO_FAILED_DELAY, 1.0 );
init( ENABLE_WORKER_HEALTH_MONITOR, false );
init( WORKER_HEALTH_MONITOR_INTERVAL, 60.0 );
init( PEER_LATENCY_CHECK_MIN_POPULATION, 30 );
init( PEER_LATENCY_DEGRADATION_PERCENTILE, 0.90 );
init( PEER_LATENCY_DEGRADATION_THRESHOLD, 0.05 );
init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD, 0.1 );
@ -653,7 +656,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
// Coordination
init( COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL, 1.0 ); if( randomize && BUGGIFY ) COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL = 10.0;
init( FORWARD_REQUEST_TOO_OLD, 4*24*60*60 ); if( randomize && BUGGIFY ) FORWARD_REQUEST_TOO_OLD = 60.0;
init( ENABLE_CROSS_CLUSTER_SUPPORT, true ); if( randomize && BUGGIFY ) ENABLE_CROSS_CLUSTER_SUPPORT = false;
init( COORDINATOR_LEADER_CONNECTION_TIMEOUT, 20.0 );
// Buggification
init( BUGGIFIED_EVENTUAL_CONSISTENCY, 1.0 );

4
fdbclient/ServerKnobs.h
View File

@ -274,6 +274,7 @@ public:
int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS;
int ROCKSDB_PREFIX_LEN;
int64_t ROCKSDB_BLOCK_CACHE_SIZE;
double ROCKSDB_METRICS_DELAY;
// Leader election
int MAX_NOTIFICATIONS;
@ -297,6 +298,7 @@ public:
double START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET;
int START_TRANSACTION_MAX_QUEUE_SIZE;
int KEY_LOCATION_MAX_QUEUE_SIZE;
double COMMIT_PROXY_LIVENESS_TIMEOUT;
double COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE;
double COMMIT_TRANSACTION_BATCH_INTERVAL_MIN;
@ -593,6 +595,8 @@ public:
double COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL;
bool ENABLE_CROSS_CLUSTER_SUPPORT; // Allow a coordinator to serve requests whose connection string does not match
// the local descriptor
double FORWARD_REQUEST_TOO_OLD; // Do not forward requests older than this setting
double COORDINATOR_LEADER_CONNECTION_TIMEOUT;
// Buggification
double BUGGIFIED_EVENTUAL_CONSISTENCY;

118
fdbclient/SimpleConfigTransaction.actor.cpp
View File

@ -30,39 +30,40 @@
class SimpleConfigTransactionImpl {
ConfigTransactionCommitRequest toCommit;
Future<Version> getVersionFuture;
Future<ConfigGeneration> getGenerationFuture;
ConfigTransactionInterface cti;
int numRetries{ 0 };
bool committed{ false };
Optional<UID> dID;
Database cx;
ACTOR static Future<Version> getReadVersion(SimpleConfigTransactionImpl* self) {
ACTOR static Future<ConfigGeneration> getGeneration(SimpleConfigTransactionImpl* self) {
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGettingReadVersion", self->dID.get());
}
ConfigTransactionGetVersionRequest req;
ConfigTransactionGetVersionReply reply =
wait(self->cti.getVersion.getReply(ConfigTransactionGetVersionRequest{}));
ConfigTransactionGetGenerationRequest req;
ConfigTransactionGetGenerationReply reply =
wait(self->cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{}));
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get()).detail("Version", reply.version);
TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get())
.detail("Version", reply.generation.liveVersion);
}
return reply.version;
return reply.generation;
}
ACTOR static Future<Optional<Value>> get(SimpleConfigTransactionImpl* self, KeyRef key) {
if (!self->getVersionFuture.isValid()) {
self->getVersionFuture = getReadVersion(self);
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
state ConfigKey configKey = ConfigKey::decodeKey(key);
Version version = wait(self->getVersionFuture);
ConfigGeneration generation = wait(self->getGenerationFuture);
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGettingValue", self->dID.get())
.detail("ConfigClass", configKey.configClass)
.detail("KnobName", configKey.knobName);
}
ConfigTransactionGetReply reply =
wait(self->cti.get.getReply(ConfigTransactionGetRequest{ version, configKey }));
wait(self->cti.get.getReply(ConfigTransactionGetRequest{ generation, configKey }));
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGotValue", self->dID.get())
.detail("Value", reply.value.get().toString());
@ -70,33 +71,32 @@ class SimpleConfigTransactionImpl {
if (reply.value.present()) {
return reply.value.get().toValue();
} else {
return {};
return Optional<Value>{};
}
}
ACTOR static Future<Standalone<RangeResultRef>> getConfigClasses(SimpleConfigTransactionImpl* self) {
if (!self->getVersionFuture.isValid()) {
self->getVersionFuture = getReadVersion(self);
ACTOR static Future<RangeResult> getConfigClasses(SimpleConfigTransactionImpl* self) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
Version version = wait(self->getVersionFuture);
ConfigGeneration generation = wait(self->getGenerationFuture);
ConfigTransactionGetConfigClassesReply reply =
wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ version }));
Standalone<RangeResultRef> result;
wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ generation }));
RangeResult result;
for (const auto& configClass : reply.configClasses) {
result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
}
return result;
}
ACTOR static Future<Standalone<RangeResultRef>> getKnobs(SimpleConfigTransactionImpl* self,
Optional<Key> configClass) {
if (!self->getVersionFuture.isValid()) {
self->getVersionFuture = getReadVersion(self);
ACTOR static Future<RangeResult> getKnobs(SimpleConfigTransactionImpl* self, Optional<Key> configClass) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
Version version = wait(self->getVersionFuture);
ConfigGeneration generation = wait(self->getGenerationFuture);
ConfigTransactionGetKnobsReply reply =
wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ version, configClass }));
Standalone<RangeResultRef> result;
wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ generation, configClass }));
RangeResult result;
for (const auto& knobName : reply.knobNames) {
result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
}
@ -104,10 +104,10 @@ class SimpleConfigTransactionImpl {
}
ACTOR static Future<Void> commit(SimpleConfigTransactionImpl* self) {
if (!self->getVersionFuture.isValid()) {
self->getVersionFuture = getReadVersion(self);
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
wait(store(self->toCommit.version, self->getVersionFuture));
wait(store(self->toCommit.generation, self->getGenerationFuture));
self->toCommit.annotation.timestamp = now();
wait(self->cti.commit.getReply(self->toCommit));
self->committed = true;
@ -123,29 +123,13 @@ public:
SimpleConfigTransactionImpl(ConfigTransactionInterface const& cti) : cti(cti) {}
void set(KeyRef key, ValueRef value) {
if (key == configTransactionDescriptionKey) {
toCommit.annotation.description = KeyRef(toCommit.arena, value);
} else {
ConfigKey configKey = ConfigKeyRef::decodeKey(key);
auto knobValue = IKnobCollection::parseKnobValue(
configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST);
toCommit.mutations.emplace_back_deep(toCommit.arena, configKey, knobValue.contents());
}
}
void set(KeyRef key, ValueRef value) { toCommit.set(key, value); }
void clear(KeyRef key) {
if (key == configTransactionDescriptionKey) {
toCommit.annotation.description = ""_sr;
} else {
toCommit.mutations.emplace_back_deep(
toCommit.arena, ConfigKeyRef::decodeKey(key), Optional<KnobValueRef>{});
}
}
void clear(KeyRef key) { toCommit.clear(key); }
Future<Optional<Value>> get(KeyRef key) { return get(this, key); }
Future<Standalone<RangeResultRef>> getRange(KeyRangeRef keys) {
Future<RangeResult> getRange(KeyRangeRef keys) {
if (keys == configClassKeys) {
return getConfigClasses(this);
} else if (keys == globalConfigKnobKeys) {
@ -170,23 +154,23 @@ public:
}
Future<Version> getReadVersion() {
if (!getVersionFuture.isValid())
getVersionFuture = getReadVersion(this);
return getVersionFuture;
if (!getGenerationFuture.isValid())
getGenerationFuture = getGeneration(this);
return map(getGenerationFuture, [](auto const& gen) { return gen.committedVersion; });
}
Optional<Version> getCachedReadVersion() const {
if (getVersionFuture.isValid() && getVersionFuture.isReady() && !getVersionFuture.isError()) {
return getVersionFuture.get();
if (getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError()) {
return getGenerationFuture.get().committedVersion;
} else {
return {};
}
}
Version getCommittedVersion() const { return committed ? getVersionFuture.get() : ::invalidVersion; }
Version getCommittedVersion() const { return committed ? getGenerationFuture.get().liveVersion : ::invalidVersion; }
void reset() {
getVersionFuture = Future<Version>{};
getGenerationFuture = Future<ConfigGeneration>{};
toCommit = {};
committed = false;
}
@ -225,19 +209,25 @@ Future<Optional<Value>> SimpleConfigTransaction::get(Key const& key, Snapshot sn
return impl().get(key);
}
Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
Future<RangeResult> SimpleConfigTransaction::getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
if (reverse) {
throw client_invalid_operation();
}
return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
}
Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
Future<RangeResult> SimpleConfigTransaction::getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
if (reverse) {
throw client_invalid_operation();
}
return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
}

20
fdbclient/SimpleConfigTransaction.h
View File

@ -50,16 +50,16 @@ public:
Optional<Version> getCachedReadVersion() const override;
Future<Optional<Value>> get(Key const& key, Snapshot = Snapshot::False) override;
Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(KeySelector const& begin,
KeySelector const& end,
int limit,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<RangeResult> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;
Future<Void> commit() override;
Version getCommittedVersion() const override;
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;

2
fdbclient/SnapshotCache.h
View File

@ -311,7 +311,7 @@ public:
entries.insert(Entry(allKeys.end, afterAllKeys, VectorRef<KeyValueRef>()), NoMetric(), true);
}
// Visual Studio refuses to generate these, apparently despite the standard
SnapshotCache(SnapshotCache&& r) noexcept : entries(std::move(r.entries)), arena(r.arena) {}
SnapshotCache(SnapshotCache&& r) noexcept : arena(r.arena), entries(std::move(r.entries)) {}
SnapshotCache& operator=(SnapshotCache&& r) noexcept {
entries = std::move(r.entries);
arena = r.arena;

5
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -248,8 +248,9 @@ ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks,
}
SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool testOnly)
: range(KeyRangeRef(spaceStartKey, spaceEndKey)), readImpls(nullptr, spaceEndKey), writeImpls(nullptr, spaceEndKey),
modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey) {
: readImpls(nullptr, spaceEndKey),
modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey),
writeImpls(nullptr, spaceEndKey), range(KeyRangeRef(spaceStartKey, spaceEndKey)) {
// Default begin of KeyRangeMap is Key(), insert the range to update start key
readImpls.insert(range, nullptr);
writeImpls.insert(range, nullptr);

47
fdbclient/StorageServerInterface.cpp
View File

@ -296,6 +296,47 @@ void TSS_traceMismatch(TraceEvent& event,
ASSERT(false);
}
// range feed
template <>
bool TSS_doCompare(const RangeFeedReply& src, const RangeFeedReply& tss) {
ASSERT(false);
return true;
}
template <>
const char* TSS_mismatchTraceName(const RangeFeedRequest& req) {
ASSERT(false);
return "";
}
template <>
void TSS_traceMismatch(TraceEvent& event,
const RangeFeedRequest& req,
const RangeFeedReply& src,
const RangeFeedReply& tss) {
ASSERT(false);
}
template <>
bool TSS_doCompare(const OverlappingRangeFeedsReply& src, const OverlappingRangeFeedsReply& tss) {
ASSERT(false);
return true;
}
template <>
const char* TSS_mismatchTraceName(const OverlappingRangeFeedsRequest& req) {
ASSERT(false);
return "";
}
template <>
void TSS_traceMismatch(TraceEvent& event,
const OverlappingRangeFeedsRequest& req,
const OverlappingRangeFeedsReply& src,
const OverlappingRangeFeedsReply& tss) {
ASSERT(false);
}
// only record metrics for data reads
template <>
@ -334,6 +375,12 @@ void TSSMetrics::recordLatency(const SplitRangeRequest& req, double ssLatency, d
template <>
void TSSMetrics::recordLatency(const GetKeyValuesStreamRequest& req, double ssLatency, double tssLatency) {}
template <>
void TSSMetrics::recordLatency(const RangeFeedRequest& req, double ssLatency, double tssLatency) {}
template <>
void TSSMetrics::recordLatency(const OverlappingRangeFeedsRequest& req, double ssLatency, double tssLatency) {}
// -------------------
TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") {

105
fdbclient/StorageServerInterface.h
View File

@ -31,6 +31,7 @@
#include "fdbrpc/TimedRequest.h"
#include "fdbrpc/TSSComparison.h"
#include "fdbclient/TagThrottle.h"
#include "fdbclient/CommitTransaction.h"
#include "flow/UnitTest.h"
// Dead code, removed in the next protocol version
@ -76,6 +77,9 @@ struct StorageServerInterface {
RequestStream<struct WatchValueRequest> watchValue;
RequestStream<struct ReadHotSubRangeRequest> getReadHotRanges;
RequestStream<struct SplitRangeRequest> getRangeSplitPoints;
RequestStream<struct RangeFeedRequest> rangeFeed;
RequestStream<struct OverlappingRangeFeedsRequest> overlappingRangeFeeds;
RequestStream<struct RangeFeedPopRequest> rangeFeedPop;
RequestStream<struct GetKeyValuesStreamRequest> getKeyValuesStream;
explicit StorageServerInterface(UID uid) : uniqueID(uid) {}
@ -119,6 +123,11 @@ struct StorageServerInterface {
RequestStream<struct SplitRangeRequest>(getValue.getEndpoint().getAdjustedEndpoint(12));
getKeyValuesStream =
RequestStream<struct GetKeyValuesStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(13));
rangeFeed = RequestStream<struct RangeFeedRequest>(getValue.getEndpoint().getAdjustedEndpoint(14));
overlappingRangeFeeds =
RequestStream<struct OverlappingRangeFeedsRequest>(getValue.getEndpoint().getAdjustedEndpoint(15));
rangeFeedPop =
RequestStream<struct RangeFeedPopRequest>(getValue.getEndpoint().getAdjustedEndpoint(16));
}
} else {
ASSERT(Ar::isDeserializing);
@ -161,6 +170,9 @@ struct StorageServerInterface {
streams.push_back(getReadHotRanges.getReceiver());
streams.push_back(getRangeSplitPoints.getReceiver());
streams.push_back(getKeyValuesStream.getReceiver(TaskPriority::LoadBalancedEndpoint));
streams.push_back(rangeFeed.getReceiver());
streams.push_back(overlappingRangeFeeds.getReceiver());
streams.push_back(rangeFeedPop.getReceiver());
FlowTransport::transport().addEndpoints(streams);
}
};
@ -615,6 +627,99 @@ struct SplitRangeRequest {
}
};
struct MutationsAndVersionRef {
VectorRef<MutationRef> mutations;
Version version;
MutationsAndVersionRef() {}
explicit MutationsAndVersionRef(Version version) : version(version) {}
MutationsAndVersionRef(VectorRef<MutationRef> mutations, Version version)
: mutations(mutations), version(version) {}
MutationsAndVersionRef(Arena& to, VectorRef<MutationRef> mutations, Version version)
: mutations(to, mutations), version(version) {}
MutationsAndVersionRef(Arena& to, const MutationsAndVersionRef& from)
: mutations(to, from.mutations), version(from.version) {}
int expectedSize() const { return mutations.expectedSize(); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, mutations, version);
}
};
struct RangeFeedReply {
constexpr static FileIdentifier file_identifier = 11815134;
VectorRef<MutationsAndVersionRef> mutations;
bool cached;
Arena arena;
RangeFeedReply() : cached(false) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, mutations, arena);
}
};
struct RangeFeedRequest {
constexpr static FileIdentifier file_identifier = 10726174;
Key rangeID;
Version begin = 0;
Version end = 0;
ReplyPromise<RangeFeedReply> reply;
RangeFeedRequest() {}
explicit RangeFeedRequest(Key const& rangeID) : rangeID(rangeID) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, rangeID, begin, end, reply);
}
};
struct RangeFeedPopRequest {
constexpr static FileIdentifier file_identifier = 10726174;
Key rangeID;
Version version;
ReplyPromise<Void> reply;
RangeFeedPopRequest() {}
RangeFeedPopRequest(Key const& rangeID, Version version) : rangeID(rangeID), version(version) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, rangeID, version, reply);
}
};
struct OverlappingRangeFeedsReply {
constexpr static FileIdentifier file_identifier = 11815134;
std::vector<std::pair<Key, KeyRange>> rangeIds;
bool cached;
Arena arena;
OverlappingRangeFeedsReply() : cached(false) {}
explicit OverlappingRangeFeedsReply(std::vector<std::pair<Key, KeyRange>> const& rangeIds)
: rangeIds(rangeIds), cached(false) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, rangeIds, arena);
}
};
struct OverlappingRangeFeedsRequest {
constexpr static FileIdentifier file_identifier = 10726174;
KeyRange range;
ReplyPromise<OverlappingRangeFeedsReply> reply;
OverlappingRangeFeedsRequest() {}
explicit OverlappingRangeFeedsRequest(KeyRange const& range) : range(range) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, range, reply);
}
};
struct GetStorageMetricsReply {
constexpr static FileIdentifier file_identifier = 15491478;
StorageMetrics load;

48
fdbclient/SystemData.cpp
View File

@ -129,7 +129,7 @@ void decodeKeyServersValue(RangeResult result,
std::sort(src.begin(), src.end());
std::sort(dest.begin(), dest.end());
if (missingIsError && (src.size() != srcTag.size() || dest.size() != destTag.size())) {
TraceEvent(SevError, "AttemptedToDecodeMissingTag");
TraceEvent(SevError, "AttemptedToDecodeMissingTag").log();
for (const KeyValueRef& kv : result) {
Tag tag = decodeServerTagValue(kv.value);
UID serverID = decodeServerTagKey(kv.key);
@ -1030,6 +1030,52 @@ const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery");
const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1");
const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion");
const KeyRangeRef rangeFeedKeys(LiteralStringRef("\xff\x02/feed/"), LiteralStringRef("\xff\x02/feed0"));
const KeyRef rangeFeedPrefix = rangeFeedKeys.begin;
const KeyRef rangeFeedPrivatePrefix = LiteralStringRef("\xff\xff\x02/feed/");
const Value rangeFeedValue(KeyRangeRef const& range) {
BinaryWriter wr(IncludeVersion(ProtocolVersion::withRangeFeed()));
wr << range;
return wr.toValue();
}
KeyRange decodeRangeFeedValue(ValueRef const& value) {
KeyRange range;
BinaryReader reader(value, IncludeVersion());
reader >> range;
return range;
}
const KeyRangeRef rangeFeedDurableKeys(LiteralStringRef("\xff\xff/rf/"), LiteralStringRef("\xff\xff/rf0"));
const KeyRef rangeFeedDurablePrefix = rangeFeedDurableKeys.begin;
const Value rangeFeedDurableKey(Key const& feed, Version const& version) {
BinaryWriter wr(AssumeVersion(ProtocolVersion::withRangeFeed()));
wr.serializeBytes(rangeFeedDurablePrefix);
wr << feed;
wr << littleEndian64(version);
return wr.toValue();
}
std::pair<Key, Version> decodeRangeFeedDurableKey(ValueRef const& key) {
Key feed;
Version version;
BinaryReader reader(key.removePrefix(rangeFeedDurablePrefix), AssumeVersion(ProtocolVersion::withRangeFeed()));
reader >> feed;
reader >> version;
return std::make_pair(feed, littleEndian64(version));
}
const Value rangeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations) {
BinaryWriter wr(IncludeVersion(ProtocolVersion::withRangeFeed()));
wr << mutations;
return wr.toValue();
}
Standalone<VectorRef<MutationRef>> decodeRangeFeedDurableValue(ValueRef const& value) {
Standalone<VectorRef<MutationRef>> mutations;
BinaryReader reader(value, IncludeVersion());
reader >> mutations;
return mutations;
}
const KeyRef configTransactionDescriptionKey = "\xff\xff/description"_sr;
const KeyRange globalConfigKnobKeys = singleKeyRange("\xff\xff/globalKnobs"_sr);
const KeyRangeRef configKnobKeys("\xff\xff/knobs/"_sr, "\xff\xff/knobs0"_sr);

14
fdbclient/SystemData.h
View File

@ -496,6 +496,20 @@ extern const ValueRef writeRecoveryKeyTrue;
// Allows incremental restore to read and set starting version for consistency.
extern const KeyRef snapshotEndVersionKey;
extern const KeyRangeRef rangeFeedKeys;
const Value rangeFeedValue(KeyRangeRef const& range);
KeyRange decodeRangeFeedValue(ValueRef const& value);
extern const KeyRef rangeFeedPrefix;
extern const KeyRef rangeFeedPrivatePrefix;
extern const KeyRangeRef rangeFeedDurableKeys;
extern const KeyRef rangeFeedDurablePrefix;
const Value rangeFeedDurableKey(Key const& feed, Version const& version);
std::pair<Key, Version> decodeRangeFeedDurableKey(ValueRef const& key);
const Value rangeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations);
Standalone<VectorRef<MutationRef>> decodeRangeFeedDurableValue(ValueRef const& value);
// Configuration database special keys
extern const KeyRef configTransactionDescriptionKey;
extern const KeyRange globalConfigKnobKeys;

14
fdbclient/TaskBucket.actor.cpp
View File

@ -873,13 +873,13 @@ TaskBucket::TaskBucket(const Subspace& subspace,
AccessSystemKeys sysAccess,
PriorityBatch priorityBatch,
LockAware lockAware)
: prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), available(prefix.get(LiteralStringRef("av"))),
available_prioritized(prefix.get(LiteralStringRef("avp"))), timeouts(prefix.get(LiteralStringRef("to"))),
pauseKey(prefix.pack(LiteralStringRef("pause"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS),
system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware), cc("TaskBucket"),
dbgid(deterministicRandom()->randomUniqueID()), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc),
dispatchErrors("DispatchErrors", cc), dispatchDoTasks("DispatchDoTasks", cc),
dispatchEmptyTasks("DispatchEmptyTasks", cc), dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc) {}
: cc("TaskBucket"), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc), dispatchErrors("DispatchErrors", cc),
dispatchDoTasks("DispatchDoTasks", cc), dispatchEmptyTasks("DispatchEmptyTasks", cc),
dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc), dbgid(deterministicRandom()->randomUniqueID()),
prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), pauseKey(prefix.pack(LiteralStringRef("pause"))),
available(prefix.get(LiteralStringRef("av"))), available_prioritized(prefix.get(LiteralStringRef("avp"))),
timeouts(prefix.get(LiteralStringRef("to"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS),
system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware) {}
TaskBucket::~TaskBucket() {}

4
fdbclient/VersionedMap.h
View File

@ -58,11 +58,11 @@ struct PTree : public ReferenceCounted<PTree<T>>, FastAllocated<PTree<T>>, NonCo
Reference<PTree> left(Version at) const { return child(false, at); }
Reference<PTree> right(Version at) const { return child(true, at); }
PTree(const T& data, Version ver) : data(data), lastUpdateVersion(ver), updated(false) {
PTree(const T& data, Version ver) : lastUpdateVersion(ver), updated(false), data(data) {
priority = deterministicRandom()->randomUInt32();
}
PTree(uint32_t pri, T const& data, Reference<PTree> const& left, Reference<PTree> const& right, Version ver)
: priority(pri), data(data), lastUpdateVersion(ver), updated(false) {
: priority(pri), lastUpdateVersion(ver), updated(false), data(data) {
pointer[0] = left;
pointer[1] = right;
}

6
fdbclient/WriteMap.h
View File

@ -168,7 +168,7 @@ private:
typedef Reference<PTreeT> Tree;
public:
explicit WriteMap(Arena* arena) : arena(arena), ver(-1), scratch_iterator(this), writeMapEmpty(true) {
explicit WriteMap(Arena* arena) : arena(arena), writeMapEmpty(true), ver(-1), scratch_iterator(this) {
PTreeImpl::insert(
writes, ver, WriteMapEntry(allKeys.begin, OperationStack(), false, false, false, false, false));
PTreeImpl::insert(writes, ver, WriteMapEntry(allKeys.end, OperationStack(), false, false, false, false, false));
@ -177,8 +177,8 @@ public:
}
WriteMap(WriteMap&& r) noexcept
: writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver),
scratch_iterator(std::move(r.scratch_iterator)), arena(r.arena) {}
: arena(r.arena), writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver),
scratch_iterator(std::move(r.scratch_iterator)) {}
WriteMap& operator=(WriteMap&& r) noexcept {
writeMapEmpty = r.writeMapEmpty;
writes = std::move(r.writes);

3
fdbclient/vexillographer/fdb.options
View File

@ -57,6 +57,9 @@ description is not currently required but encouraged.
<Option name="trace_file_identifier" code="36"
paramType="String" paramDescription="The identifier that will be part of all trace file names"
description="Once provided, this string will be used to replace the port/PID in the log file names." />
<Option name="trace_partial_file_suffix" code="39"
paramType="String" paramDesciption="Append this suffix to partially written log files. When a log file is complete, it is renamed to remove the suffix. No separator is added between the file and the suffix. If you want to add a file extension, you should include the separator - e.g. '.tmp' instead of 'tmp' to add the 'tmp' extension."
description="" />
<Option name="knob" code="40"
paramType="String" paramDescription="knob_name=knob_value"
description="Set internal tuning or debugging knobs"/>

2
fdbmonitor/fdbmonitor.cpp
View File

@ -393,7 +393,7 @@ public:
Command() : argv(nullptr) {}
Command(const CSimpleIni& ini, std::string _section, ProcessID id, fdb_fd_set fds, int* maxfd)
: section(_section), argv(nullptr), fork_retry_time(-1), quiet(false), delete_envvars(nullptr), fds(fds),
: fds(fds), argv(nullptr), section(_section), fork_retry_time(-1), quiet(false), delete_envvars(nullptr),
deconfigured(false), kill_on_configuration_change(true) {
char _ssection[strlen(section.c_str()) + 22];
snprintf(_ssection, strlen(section.c_str()) + 22, "%s", id.c_str());

6
fdbrpc/AsyncFileCached.actor.h
View File

@ -298,7 +298,7 @@ private:
const std::string& filename,
int64_t length,
Reference<EvictablePageCache> pageCache)
: uncached(uncached), filename(filename), length(length), prevLength(length), pageCache(pageCache),
: filename(filename), uncached(uncached), length(length), prevLength(length), pageCache(pageCache),
currentTruncate(Void()), currentTruncateSize(0), rateControl(nullptr) {
if (!g_network->isSimulated()) {
countFileCacheWrites.init(LiteralStringRef("AsyncFile.CountFileCacheWrites"), filename);
@ -610,8 +610,8 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
}
AFCPage(AsyncFileCached* owner, int64_t offset)
: EvictablePage(owner->pageCache), owner(owner), pageOffset(offset), dirty(false), valid(false), truncated(false),
notReading(Void()), notFlushing(Void()), zeroCopyRefCount(0), flushableIndex(-1), writeThroughCount(0) {
: EvictablePage(owner->pageCache), owner(owner), pageOffset(offset), notReading(Void()), notFlushing(Void()),
dirty(false), valid(false), truncated(false), writeThroughCount(0), flushableIndex(-1), zeroCopyRefCount(0) {
pageCache->allocate(this);
}

2
fdbrpc/AsyncFileEIO.actor.h
View File

@ -277,7 +277,7 @@ private:
mutable Int64MetricHandle countLogicalReads;
AsyncFileEIO(int fd, int flags, std::string const& filename)
: fd(fd), flags(flags), filename(filename), err(new ErrorInfo) {
: fd(fd), flags(flags), err(new ErrorInfo), filename(filename) {
if (!g_network->isSimulated()) {
countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);
countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename);

55
fdbrpc/AsyncFileEncrypted.actor.cpp
View File

@ -34,16 +34,18 @@ public:
auto pos = salt.find('.');
salt = salt.substr(0, pos);
auto hash = XXH3_128bits(salt.c_str(), salt.size());
auto high = reinterpret_cast<unsigned char*>(&hash.high64);
auto low = reinterpret_cast<unsigned char*>(&hash.low64);
std::copy(high, high + 8, &iv[0]);
std::copy(low, low + 6, &iv[8]);
iv[14] = iv[15] = 0; // last 16 bits identify block
auto pHigh = reinterpret_cast<unsigned char*>(&hash.high64);
auto pLow = reinterpret_cast<unsigned char*>(&hash.low64);
std::copy(pHigh, pHigh + 8, &iv[0]);
std::copy(pLow, pLow + 4, &iv[8]);
uint32_t blockZero = 0;
auto pBlock = reinterpret_cast<unsigned char*>(&blockZero);
std::copy(pBlock, pBlock + 4, &iv[12]);
return iv;
}
// Read a single block of size ENCRYPTION_BLOCK_SIZE bytes, and decrypt.
ACTOR static Future<Standalone<StringRef>> readBlock(AsyncFileEncrypted* self, uint16_t block) {
ACTOR static Future<Standalone<StringRef>> readBlock(AsyncFileEncrypted* self, uint32_t block) {
state Arena arena;
state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE];
int bytes = wait(
@ -53,23 +55,22 @@ public:
return Standalone<StringRef>(decrypted, arena);
}
ACTOR static Future<int> read(AsyncFileEncrypted* self, void* data, int length, int offset) {
state const uint16_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
state const uint16_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
state uint16_t block;
ACTOR static Future<int> read(AsyncFileEncrypted* self, void* data, int length, int64_t offset) {
state const uint32_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
state const uint32_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
state uint32_t block;
state unsigned char* output = reinterpret_cast<unsigned char*>(data);
state int bytesRead = 0;
ASSERT(self->mode == AsyncFileEncrypted::Mode::READ_ONLY);
for (block = firstBlock; block <= lastBlock; ++block) {
state StringRef plaintext;
state Standalone<StringRef> plaintext;
auto cachedBlock = self->readBuffers.get(block);
if (cachedBlock.present()) {
plaintext = cachedBlock.get();
} else {
Standalone<StringRef> _plaintext = wait(readBlock(self, block));
self->readBuffers.insert(block, _plaintext);
plaintext = _plaintext;
wait(store(plaintext, readBlock(self, block)));
self->readBuffers.insert(block, plaintext);
}
auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)
: plaintext.begin();
@ -79,6 +80,14 @@ public:
if ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE == 0) {
end = plaintext.end();
}
// The block could be short if it includes or is after the end of the file.
end = std::min(end, plaintext.end());
// If the start position is at or after the end of the block, the read is complete.
if (start == end || start >= plaintext.end()) {
break;
}
std::copy(start, end, output);
output += (end - start);
bytesRead += (end - start);
@ -103,7 +112,7 @@ public:
if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) {
wait(self->writeLastBlockToFile());
self->offsetInBlock = 0;
ASSERT_LT(self->currentBlock, std::numeric_limits<uint16_t>::max());
ASSERT_LT(self->currentBlock, std::numeric_limits<uint32_t>::max());
++self->currentBlock;
self->encryptor = std::make_unique<EncryptionStreamCipher>(StreamCipher::Key::getKey(),
self->getIV(self->currentBlock));
@ -131,7 +140,7 @@ public:
};
AsyncFileEncrypted::AsyncFileEncrypted(Reference<IAsyncFile> file, Mode mode)
: file(file), mode(mode), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) {
: file(file), mode(mode), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS), currentBlock(0) {
firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename());
if (mode == Mode::APPEND_ONLY) {
encryptor = std::make_unique<EncryptionStreamCipher>(StreamCipher::Key::getKey(), getIV(currentBlock));
@ -196,10 +205,12 @@ int64_t AsyncFileEncrypted::debugFD() const {
return file->debugFD();
}
StreamCipher::IV AsyncFileEncrypted::getIV(uint16_t block) const {
StreamCipher::IV AsyncFileEncrypted::getIV(uint32_t block) const {
auto iv = firstBlockIV;
iv[14] = block / 256;
iv[15] = block % 256;
auto pBlock = reinterpret_cast<unsigned char*>(&block);
std::copy(pBlock, pBlock + 4, &iv[12]);
return iv;
}
@ -218,7 +229,7 @@ AsyncFileEncrypted::RandomCache::RandomCache(size_t maxSize) : maxSize(maxSize)
vec.reserve(maxSize);
}
void AsyncFileEncrypted::RandomCache::insert(uint16_t block, const Standalone<StringRef>& value) {
void AsyncFileEncrypted::RandomCache::insert(uint32_t block, const Standalone<StringRef>& value) {
auto [_, found] = hashMap.insert({ block, value });
if (found) {
return;
@ -230,7 +241,7 @@ void AsyncFileEncrypted::RandomCache::insert(uint16_t block, const Standalone<St
}
}
Optional<Standalone<StringRef>> AsyncFileEncrypted::RandomCache::get(uint16_t block) const {
Optional<Standalone<StringRef>> AsyncFileEncrypted::RandomCache::get(uint32_t block) const {
auto it = hashMap.find(block);
if (it == hashMap.end()) {
return {};
@ -255,6 +266,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") {
state Reference<IAsyncFile> file =
wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600));
state int bytesWritten = 0;
state int chunkSize;
while (bytesWritten < bytes) {
chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten);
wait(file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten));
@ -262,7 +274,6 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") {
}
wait(file->sync());
state int bytesRead = 0;
state int chunkSize;
while (bytesRead < bytes) {
chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead);
int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead));

12
fdbrpc/AsyncFileEncrypted.h
View File

@ -40,7 +40,7 @@ public:
private:
Reference<IAsyncFile> file;
StreamCipher::IV firstBlockIV;
StreamCipher::IV getIV(uint16_t block) const;
StreamCipher::IV getIV(uint32_t block) const;
Mode mode;
Future<Void> writeLastBlockToFile();
friend class AsyncFileEncryptedImpl;
@ -48,19 +48,19 @@ private:
// Reading:
class RandomCache {
size_t maxSize;
std::vector<uint16_t> vec;
std::unordered_map<uint16_t, Standalone<StringRef>> hashMap;
std::vector<uint32_t> vec;
std::unordered_map<uint32_t, Standalone<StringRef>> hashMap;
size_t evict();
public:
RandomCache(size_t maxSize);
void insert(uint16_t block, const Standalone<StringRef>& value);
Optional<Standalone<StringRef>> get(uint16_t block) const;
void insert(uint32_t block, const Standalone<StringRef>& value);
Optional<Standalone<StringRef>> get(uint32_t block) const;
} readBuffers;
// Writing (append only):
std::unique_ptr<EncryptionStreamCipher> encryptor;
uint16_t currentBlock{ 0 };
uint32_t currentBlock{ 0 };
int offsetInBlock{ 0 };
std::vector<unsigned char> writeBuffer;
Future<Void> initialize();

6
fdbrpc/AsyncFileKAIO.actor.h
View File

@ -568,8 +568,8 @@ private:
uint32_t opsIssued;
Context()
: iocx(0), evfd(-1), outstanding(0), opsIssued(0), ioStallBegin(0), fallocateSupported(true),
fallocateZeroSupported(true), submittedRequestList(nullptr) {
: iocx(0), evfd(-1), outstanding(0), ioStallBegin(0), fallocateSupported(true), fallocateZeroSupported(true),
submittedRequestList(nullptr), opsIssued(0) {
setIOTimeout(0);
}
@ -619,7 +619,7 @@ private:
static Context ctx;
explicit AsyncFileKAIO(int fd, int flags, std::string const& filename)
: fd(fd), flags(flags), filename(filename), failed(false) {
: failed(false), fd(fd), flags(flags), filename(filename) {
ASSERT(!FLOW_KNOBS->DISABLE_POSIX_KERNEL_AIO);
if (!g_network->isSimulated()) {
countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);

5
fdbrpc/AsyncFileNonDurable.actor.h
View File

@ -190,9 +190,8 @@ private:
Reference<DiskParameters> diskParameters,
NetworkAddress openedAddress,
bool aio)
: filename(filename), initialFilename(initialFilename), file(file), diskParameters(diskParameters),
openedAddress(openedAddress), pendingModifications(uint64_t(-1)), approximateSize(0), reponses(false),
aio(aio) {
: filename(filename), initialFilename(initialFilename), approximateSize(0), openedAddress(openedAddress),
aio(aio), file(file), pendingModifications(uint64_t(-1)), diskParameters(diskParameters), reponses(false) {
// This is only designed to work in simulation
ASSERT(g_network->isSimulated());

2
fdbrpc/AsyncFileReadAhead.actor.h
View File

@ -199,7 +199,7 @@ public:
int maxConcurrentReads,
int cacheSizeBlocks)
: m_f(f), m_block_size(blockSize), m_read_ahead_blocks(readAheadBlocks),
m_max_concurrent_reads(maxConcurrentReads), m_cache_block_limit(std::max<int>(1, cacheSizeBlocks)) {}
m_cache_block_limit(std::max<int>(1, cacheSizeBlocks)), m_max_concurrent_reads(maxConcurrentReads) {}
};
#include "flow/unactorcompiler.h"

4
fdbrpc/FlowTests.actor.cpp
View File

@ -234,6 +234,8 @@ struct YieldMockNetwork final : INetwork, ReferenceCounted<YieldMockNetwork> {
Future<class Void> delay(double seconds, TaskPriority taskID) override { return nextTick.getFuture(); }
Future<class Void> orderedDelay(double seconds, TaskPriority taskID) override { return nextTick.getFuture(); }
Future<class Void> yield(TaskPriority taskID) override {
if (check_yield(taskID))
return delay(0, taskID);
@ -1386,7 +1388,7 @@ TEST_CASE("/flow/DeterministicRandom/SignedOverflow") {
struct Tracker {
int copied;
bool moved;
Tracker(int copied = 0) : moved(false), copied(copied) {}
Tracker(int copied = 0) : copied(copied), moved(false) {}
Tracker(Tracker&& other) : Tracker(other.copied) {
ASSERT(!other.moved);
other.moved = true;

34
fdbrpc/FlowTransport.actor.cpp
View File

@ -51,7 +51,7 @@ constexpr UID WLTOKEN_PING_PACKET(-1, 1);
constexpr int PACKET_LEN_WIDTH = sizeof(uint32_t);
const uint64_t TOKEN_STREAM_FLAG = 1;
static constexpr int WLTOKEN_COUNTS = 20; // number of wellKnownEndpoints
static constexpr int WLTOKEN_COUNTS = 21; // number of wellKnownEndpoints
class EndpointMap : NonCopyable {
public:
@ -340,9 +340,8 @@ ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
}
TransportData::TransportData(uint64_t transportId)
: endpoints(WLTOKEN_COUNTS), endpointNotFoundReceiver(endpoints), pingReceiver(endpoints),
warnAlwaysForLargePacket(true), lastIncompatibleMessage(0), transportId(transportId),
numIncompatibleConnections(0) {
: warnAlwaysForLargePacket(true), endpoints(WLTOKEN_COUNTS), endpointNotFoundReceiver(endpoints),
pingReceiver(endpoints), numIncompatibleConnections(0), lastIncompatibleMessage(0), transportId(transportId) {
degraded = makeReference<AsyncVar<bool>>(false);
pingLogger = pingLatencyLogger(this);
}
@ -795,13 +794,14 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
}
Peer::Peer(TransportData* transport, NetworkAddress const& destination)
: transport(transport), destination(destination), outgoingConnectionIdle(true), lastConnectTime(0.0),
reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), compatible(true), outstandingReplies(0),
incompatibleProtocolVersionNewer(false), peerReferences(-1), bytesReceived(0), lastDataPacketSentTime(now()),
pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1), lastLoggedBytesReceived(0),
bytesSent(0), lastLoggedBytesSent(0), timeoutCount(0), lastLoggedTime(0.0), connectOutgoingCount(0), connectIncomingCount(0),
connectFailedCount(0), connectLatencies(destination.isPublic() ? FLOW_KNOBS->NETWORK_CONNECT_SAMPLE_AMOUNT : 1),
protocolVersion(Reference<AsyncVar<Optional<ProtocolVersion>>>(new AsyncVar<Optional<ProtocolVersion>>())) {
: transport(transport), destination(destination), compatible(true), outgoingConnectionIdle(true),
lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), peerReferences(-1),
incompatibleProtocolVersionNewer(false), bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()),
outstandingReplies(0), pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1),
lastLoggedTime(0.0), lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
protocolVersion(Reference<AsyncVar<Optional<ProtocolVersion>>>(new AsyncVar<Optional<ProtocolVersion>>())),
connectOutgoingCount(0), connectIncomingCount(0), connectFailedCount(0),
connectLatencies(destination.isPublic() ? FLOW_KNOBS->NETWORK_CONNECT_SAMPLE_AMOUNT : 1) {
IFailureMonitor::failureMonitor().setStatus(destination, FailureStatus(false));
}
@ -922,9 +922,9 @@ ACTOR static void deliver(TransportData* self,
// We want to run the task at the right priority. If the priority is higher than the current priority (which is
// ReadSocket) we can just upgrade. Otherwise we'll context switch so that we don't block other tasks that might run
// with a higher priority. ReplyPromiseStream needs to guarentee that messages are recieved in the order they were
// sent, so even in the case of local delivery those messages need to skip this delay.
if (priority < TaskPriority::ReadSocket || (priority != TaskPriority::NoDeliverDelay && !inReadSocket)) {
wait(delay(0, priority));
// sent, so we are using orderedDelay.
if (priority < TaskPriority::ReadSocket || !inReadSocket) {
wait(orderedDelay(0, priority));
} else {
g_network->setCurrentTask(priority);
}
@ -1019,7 +1019,7 @@ static void scanPackets(TransportData* transport,
BUGGIFY_WITH_PROB(0.0001)) {
g_simulator.lastConnectionFailure = g_network->now();
isBuggifyEnabled = true;
TraceEvent(SevInfo, "BitsFlip");
TraceEvent(SevInfo, "BitsFlip").log();
int flipBits = 32 - (int)floor(log2(deterministicRandom()->randomUInt32()));
uint32_t firstFlipByteLocation = deterministicRandom()->randomUInt32() % packetLen;
@ -1698,7 +1698,7 @@ Reference<AsyncVar<bool>> FlowTransport::getDegraded() {
//
// Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
// version, some other mechanism should be used to connect to that peer.
Reference<AsyncVar<Optional<ProtocolVersion>>> FlowTransport::getPeerProtocolAsyncVar(NetworkAddress addr) {
Reference<AsyncVar<Optional<ProtocolVersion>> const> FlowTransport::getPeerProtocolAsyncVar(NetworkAddress addr) {
return self->peers.at(addr)->protocolVersion;
}
@ -1723,4 +1723,4 @@ void FlowTransport::createInstance(bool isClient, uint64_t transportId) {
HealthMonitor* FlowTransport::healthMonitor() {
return &self->healthMonitor;
}
}

2
fdbrpc/FlowTransport.h
View File

@ -252,7 +252,7 @@ public:
//
// Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
// version, some other mechanism should be used to connect to that peer.
Reference<AsyncVar<Optional<ProtocolVersion>>> getPeerProtocolAsyncVar(NetworkAddress addr);
Reference<AsyncVar<Optional<ProtocolVersion>> const> getPeerProtocolAsyncVar(NetworkAddress addr);
static FlowTransport& transport() {
return *static_cast<FlowTransport*>((void*)g_network->global(INetwork::enFlowTransport));

2
fdbrpc/LoadBalance.actor.h
View File

@ -52,7 +52,7 @@ struct ModelHolder : NonCopyable, public ReferenceCounted<ModelHolder> {
double delta;
uint64_t token;
ModelHolder(QueueModel* model, uint64_t token) : model(model), token(token), released(false), startTime(now()) {
ModelHolder(QueueModel* model, uint64_t token) : model(model), released(false), startTime(now()), token(token) {
if (model) {
delta = model->addRequest(token);
}

6
fdbrpc/PerfMetric.h
View File

@ -30,11 +30,11 @@ using std::vector;
struct PerfMetric {
constexpr static FileIdentifier file_identifier = 5980618;
PerfMetric() : m_name(""), m_value(0), m_averaged(false), m_format_code("%.3g") {}
PerfMetric() : m_name(""), m_format_code("%.3g"), m_value(0), m_averaged(false) {}
PerfMetric(std::string name, double value, bool averaged)
: m_name(name), m_value(value), m_averaged(averaged), m_format_code("%.3g") {}
: m_name(name), m_format_code("%.3g"), m_value(value), m_averaged(averaged) {}
PerfMetric(std::string name, double value, bool averaged, std::string format_code)
: m_name(name), m_value(value), m_averaged(averaged), m_format_code(format_code) {}
: m_name(name), m_format_code(format_code), m_value(value), m_averaged(averaged) {}
std::string name() const { return m_name; }
double value() const { return m_value; }

2
fdbrpc/QueueModel.h
View File

@ -75,7 +75,7 @@ struct QueueData {
Optional<TSSEndpointData> tssData;
QueueData()
: latency(0.001), penalty(1.0), smoothOutstanding(FLOW_KNOBS->QUEUE_MODEL_SMOOTHING_AMOUNT), failedUntil(0),
: smoothOutstanding(FLOW_KNOBS->QUEUE_MODEL_SMOOTHING_AMOUNT), latency(0.001), penalty(1.0), failedUntil(0),
futureVersionBackoff(FLOW_KNOBS->FUTURE_VERSION_INITIAL_BACKOFF), increaseBackoffTime(0) {}
};

Some files were not shown because too many files have changed in this diff Show More