diff --git a/.gitignore b/.gitignore index 270c631eed..b486706077 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ bindings/java/foundationdb-tests*.jar bindings/java/fdb-java-*-sources.jar packaging/msi/FDBInstaller.msi build/ -cmake-build-debug/ +cmake-build-debug* # Generated source, build, and packaging files *.g.cpp *.g.h diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index dfcf279b1d..36c2464e3f 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -1,6 +1,6 @@ +add_subdirectory(c) if(NOT OPEN_FOR_IDE) # flow bindings currently doesn't support that - add_subdirectory(c) add_subdirectory(flow) endif() if(WITH_PYTHON_BINDING) diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index be4caf8240..00847c7268 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -39,6 +39,8 @@ else() strip_debug_symbols(fdb_c) endif() add_dependencies(fdb_c fdb_c_generated fdb_c_options) +add_dependencies(fdbclient fdb_c_options) +add_dependencies(fdbclient_sampling fdb_c_options) target_link_libraries(fdb_c PUBLIC $) if(APPLE) set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols) @@ -80,6 +82,10 @@ if(NOT WIN32) set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp) set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp) + set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS + test/unit/disconnected_timeout_tests.cpp + test/unit/fdb_api.cpp + test/unit/fdb_api.hpp) if(OPEN_FOR_IDE) add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h) @@ -90,6 +96,7 @@ if(NOT WIN32) add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS}) add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS}) add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) + add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS}) else() add_executable(fdb_c_performance_test test/performance_test.c test/test.h) add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h) @@ -99,6 +106,7 @@ if(NOT WIN32) add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS}) add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS}) add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS}) + add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS}) strip_debug_symbols(fdb_c_performance_test) strip_debug_symbols(fdb_c_ryw_benchmark) strip_debug_symbols(fdb_c_txn_size_test) @@ -110,13 +118,16 @@ if(NOT WIN32) add_dependencies(fdb_c_setup_tests doctest) add_dependencies(fdb_c_unit_tests doctest) add_dependencies(fdb_c_unit_tests_version_510 doctest) + add_dependencies(disconnected_timeout_unit_tests doctest) target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR}) + target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR}) target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads) target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads) target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads) target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads) + target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads) # do not set RPATH for mako set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE) @@ -130,13 +141,19 @@ if(NOT WIN32) target_link_libraries(fdb_c90_test PRIVATE fdb_c) endif() + if(OPEN_FOR_IDE) + set(FDB_C_TARGET $) + else() + set(FDB_C_TARGET $) + endif() add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so - COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so + COMMAND ${CMAKE_COMMAND} -E copy ${FDB_C_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so DEPENDS fdb_c COMMENT "Copy libfdb_c to use as external client for test") add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so) add_dependencies(fdb_c_unit_tests external_client) + add_dependencies(disconnected_timeout_unit_tests external_client) add_fdbclient_test( NAME fdb_c_setup_tests @@ -163,6 +180,17 @@ if(NOT WIN32) fdb ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so ) + add_unavailable_fdbclient_test( + NAME disconnected_timeout_unit_tests + COMMAND $ + @CLUSTER_FILE@ + ) + add_unavailable_fdbclient_test( + NAME disconnected_timeout_external_client_unit_tests + COMMAND $ + @CLUSTER_FILE@ + ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so + ) endif() set(c_workloads_srcs diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index 382c6e50ca..913564eef2 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -1330,7 +1330,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi #else /* >= 610 */ fdb_create_database(args->cluster_file, &process.database); #endif - + if (args->disable_ryw) { + fdb_database_set_option(process.database, FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE, (uint8_t*)NULL, 0); + } fprintf(debugme, "DEBUG: creating %d worker threads\n", args->num_threads); worker_threads = (pthread_t*)calloc(sizeof(pthread_t), args->num_threads); if (!worker_threads) { @@ -1444,6 +1446,7 @@ int init_args(mako_args_t* args) { for (i = 0; i < MAX_OP; i++) { args->txnspec.ops[i][OP_COUNT] = 0; } + args->disable_ryw = 0; return 0; } @@ -1608,6 +1611,7 @@ void usage() { printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs"); printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers"); printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial"); + printf("%-24s %s\n", " --disable_ryw", "Disable snapshot read-your-writes"); } /* parse benchmark paramters */ @@ -1653,6 +1657,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) { { "txntagging", required_argument, NULL, ARG_TXNTAGGING }, { "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX }, { "version", no_argument, NULL, ARG_VERSION }, + { "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW }, { NULL, 0, NULL, 0 } }; idx = 0; @@ -1800,14 +1805,16 @@ int parse_args(int argc, char* argv[], mako_args_t* args) { args->txntagging = 1000; } break; - case ARG_TXNTAGGINGPREFIX: { + case ARG_TXNTAGGINGPREFIX: if (strlen(optarg) > TAGPREFIXLENGTH_MAX) { fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX); exit(0); } memcpy(args->txntagging_prefix, optarg, strlen(optarg)); break; - } + case ARG_DISABLE_RYW: + args->disable_ryw = 1; + break; } } diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index 7b24c9cb48..a770fea857 100644 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -80,7 +80,8 @@ enum Arguments { ARG_TXNTRACE, ARG_TXNTAGGING, ARG_TXNTAGGINGPREFIX, - ARG_STREAMING_MODE + ARG_STREAMING_MODE, + ARG_DISABLE_RYW }; enum TPSChangeTypes { TPS_SIN, TPS_SQUARE, TPS_PULSE }; @@ -136,6 +137,7 @@ typedef struct { int txntagging; char txntagging_prefix[TAGPREFIXLENGTH_MAX]; FDBStreamingMode streaming_mode; + int disable_ryw; } mako_args_t; /* shared memory */ diff --git a/bindings/c/test/unit/disconnected_timeout_tests.cpp b/bindings/c/test/unit/disconnected_timeout_tests.cpp new file mode 100644 index 0000000000..36811d23d3 --- /dev/null +++ b/bindings/c/test/unit/disconnected_timeout_tests.cpp @@ -0,0 +1,292 @@ +/* + * disconnected_timeout_tests.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Unit tests that test the timeouts for a disconnected cluster + +#define FDB_API_VERSION 710 +#include + +#include +#include +#include +#include + +#define DOCTEST_CONFIG_IMPLEMENT +#include "doctest.h" +#include "fdb_api.hpp" + +void fdb_check(fdb_error_t e) { + if (e) { + std::cerr << fdb_get_error(e) << std::endl; + std::abort(); + } +} + +FDBDatabase* fdb_open_database(const char* clusterFile) { + FDBDatabase* db; + fdb_check(fdb_create_database(clusterFile, &db)); + return db; +} + +static FDBDatabase* db = nullptr; +static FDBDatabase* timeoutDb = nullptr; + +// Blocks until the given future is ready, returning an error code if there was +// an issue. +fdb_error_t wait_future(fdb::Future& f) { + fdb_check(f.block_until_ready()); + return f.get_error(); +} + +void validateTimeoutDuration(double expectedSeconds, std::chrono::time_point start) { + std::chrono::duration duration = std::chrono::steady_clock::now() - start; + double actualSeconds = duration.count(); + CHECK(actualSeconds >= expectedSeconds - 1e-6); + CHECK(actualSeconds < expectedSeconds * 2); +} + +TEST_CASE("500ms_transaction_timeout") { + auto start = std::chrono::steady_clock::now(); + + fdb::Transaction tr(db); + + int64_t timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Int64Future grvFuture = tr.get_read_version(); + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("500ms_transaction_timeout_after_op") { + auto start = std::chrono::steady_clock::now(); + + fdb::Transaction tr(db); + fdb::Int64Future grvFuture = tr.get_read_version(); + + int64_t timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("500ms_transaction_timeout_before_op_2000ms_after") { + auto start = std::chrono::steady_clock::now(); + + fdb::Transaction tr(db); + + int64_t timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Int64Future grvFuture = tr.get_read_version(); + + timeout = 2000; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("2000ms_transaction_timeout_before_op_500ms_after") { + auto start = std::chrono::steady_clock::now(); + + fdb::Transaction tr(db); + + int64_t timeout = 2000; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Int64Future grvFuture = tr.get_read_version(); + + timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("500ms_database_timeout") { + auto start = std::chrono::steady_clock::now(); + + int64_t timeout = 500; + fdb_check(fdb_database_set_option( + timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Transaction tr(timeoutDb); + + fdb::Int64Future grvFuture = tr.get_read_version(); + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("2000ms_database_timeout_500ms_transaction_timeout") { + auto start = std::chrono::steady_clock::now(); + + int64_t timeout = 2000; + fdb_check(fdb_database_set_option( + timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Transaction tr(timeoutDb); + + timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Int64Future grvFuture = tr.get_read_version(); + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(timeout / 1000.0, start); +} + +TEST_CASE("500ms_database_timeout_2000ms_transaction_timeout_with_reset") { + auto start = std::chrono::steady_clock::now(); + + int64_t dbTimeout = 500; + fdb_check(fdb_database_set_option( + timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast(&dbTimeout), sizeof(dbTimeout))); + + fdb::Transaction tr(timeoutDb); + + int64_t trTimeout = 2000; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&trTimeout), sizeof(trTimeout))); + + tr.reset(); + + fdb::Int64Future grvFuture = tr.get_read_version(); + fdb_error_t err = wait_future(grvFuture); + + CHECK(err == 1031); + validateTimeoutDuration(dbTimeout / 1000.0, start); +} + +TEST_CASE("transaction_reset_cancels_without_timeout") { + fdb::Transaction tr(db); + fdb::Int64Future grvFuture = tr.get_read_version(); + tr.reset(); + + fdb_error_t err = wait_future(grvFuture); + CHECK(err == 1025); +} + +TEST_CASE("transaction_reset_cancels_with_timeout") { + fdb::Transaction tr(db); + + int64_t timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + fdb::Int64Future grvFuture = tr.get_read_version(); + tr.reset(); + + fdb_error_t err = wait_future(grvFuture); + CHECK(err == 1025); +} + +TEST_CASE("transaction_destruction_cancels_without_timeout") { + FDBTransaction* tr; + fdb_check(fdb_database_create_transaction(db, &tr)); + + FDBFuture* grvFuture = fdb_transaction_get_read_version(tr); + fdb_transaction_destroy(tr); + + fdb_check(fdb_future_block_until_ready(grvFuture)); + fdb_error_t err = fdb_future_get_error(grvFuture); + CHECK(err == 1025); + + fdb_future_destroy(grvFuture); +} + +TEST_CASE("transaction_destruction_cancels_with_timeout") { + FDBTransaction* tr; + fdb_check(fdb_database_create_transaction(db, &tr)); + + int64_t timeout = 500; + fdb_check(fdb_transaction_set_option( + tr, FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + + FDBFuture* grvFuture = fdb_transaction_get_read_version(tr); + fdb_transaction_destroy(tr); + + fdb_check(fdb_future_block_until_ready(grvFuture)); + fdb_error_t err = fdb_future_get_error(grvFuture); + CHECK(err == 1025); + + fdb_future_destroy(grvFuture); +} + +TEST_CASE("transaction_set_timeout_and_destroy_repeatedly") { + for (int i = 0; i < 1000; ++i) { + fdb::Transaction tr(db); + int64_t timeout = 500; + fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast(&timeout), sizeof(timeout))); + } +} + +int main(int argc, char** argv) { + if (argc < 2) { + std::cout << "Disconnected timeout unit tests for the FoundationDB C API.\n" + << "Usage: disconnected_timeout_tests [externalClient] [doctest args]" + << std::endl; + return 1; + } + fdb_check(fdb_select_api_version(710)); + if (argc >= 3) { + std::string externalClientLibrary = argv[2]; + if (externalClientLibrary.substr(0, 2) != "--") { + fdb_check(fdb_network_set_option( + FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT, reinterpret_cast(""), 0)); + fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY, + reinterpret_cast(externalClientLibrary.c_str()), + externalClientLibrary.size())); + } + } + + doctest::Context context; + context.applyCommandLine(argc, argv); + + fdb_check(fdb_setup_network()); + std::thread network_thread{ &fdb_run_network }; + + db = fdb_open_database(argv[1]); + timeoutDb = fdb_open_database(argv[1]); + + int res = context.run(); + fdb_database_destroy(db); + fdb_database_destroy(timeoutDb); + + if (context.shouldExit()) { + fdb_check(fdb_stop_network()); + network_thread.join(); + return res; + } + fdb_check(fdb_stop_network()); + network_thread.join(); + + return res; +} diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt index 0af1e6cb3c..1b513a30c7 100644 --- a/bindings/go/CMakeLists.txt +++ b/bindings/go/CMakeLists.txt @@ -44,7 +44,7 @@ set(go_options_file ${GO_DEST}/src/fdb/generated.go) set(go_env GOPATH=${GOPATH} C_INCLUDE_PATH=${CMAKE_BINARY_DIR}/bindings/c/foundationdb:${CMAKE_SOURCE_DIR}/bindings/c CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/lib - GO111MODULE=off) + GO111MODULE=auto) foreach(src_file IN LISTS SRCS) set(dest_file ${GO_DEST}/${src_file}) diff --git a/bindings/python/tests/fdbcli_tests.py b/bindings/python/tests/fdbcli_tests.py index 4ca9052670..f971a1d0b6 100755 --- a/bindings/python/tests/fdbcli_tests.py +++ b/bindings/python/tests/fdbcli_tests.py @@ -385,7 +385,7 @@ def coordinators(logger): # verify now we have 5 coordinators and the description is updated output2 = run_fdbcli_command('coordinators') assert output2.split('\n')[0].split(': ')[-1] == new_cluster_description - assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(5, ','.join(addresses)) + assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(args.process_number, ','.join(addresses)) # auto change should go back to 1 coordinator run_fdbcli_command('coordinators', 'auto') assert len(get_value_from_status_json(True, 'client', 'coordinators', 'coordinators')) == 1 diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 7f0ebc049c..c68b1a5524 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -128,7 +128,8 @@ function(add_fdb_test) -n ${test_name} -b ${PROJECT_BINARY_DIR} -t ${test_type} - -O ${OLD_FDBSERVER_BINARY} + -O ${OLD_FDBSERVER_BINARY} + --config "@CTEST_CONFIGURATION_TYPE@" --crash --aggregate-traces ${TEST_AGGREGATE_TRACES} --log-format ${TEST_LOG_FORMAT} @@ -442,6 +443,40 @@ function(add_fdbclient_test) set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) endfunction() +# Creates a cluster file for a nonexistent cluster before running the specified command +# (usually a ctest test) +function(add_unavailable_fdbclient_test) + set(options DISABLED ENABLED) + set(oneValueArgs NAME TEST_TIMEOUT) + set(multiValueArgs COMMAND) + cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + if(OPEN_FOR_IDE) + return() + endif() + if(NOT T_ENABLED AND T_DISABLED) + return() + endif() + if(NOT T_NAME) + message(FATAL_ERROR "NAME is a required argument for add_unavailable_fdbclient_test") + endif() + if(NOT T_COMMAND) + message(FATAL_ERROR "COMMAND is a required argument for add_unavailable_fdbclient_test") + endif() + message(STATUS "Adding unavailable client test ${T_NAME}") + add_test(NAME "${T_NAME}" + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py + --output-dir ${CMAKE_BINARY_DIR} + -- + ${T_COMMAND}) + if (T_TEST_TIMEOUT) + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT ${T_TEST_TIMEOUT}) + else() + # default timeout + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) + endif() + set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) +endfunction() + # Creates 3 distinct clusters before running the specified command. # This is useful for testing features that require multiple clusters (like the # multi-cluster FDB client) diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake index 6d6e959fd5..7f27d11c81 100644 --- a/cmake/CompileRocksDB.cmake +++ b/cmake/CompileRocksDB.cmake @@ -1,6 +1,6 @@ # FindRocksDB -find_package(RocksDB) +find_package(RocksDB 6.22.1) include(ExternalProject) diff --git a/cmake/FindRocksDB.cmake b/cmake/FindRocksDB.cmake index e70707ce70..aec70c929b 100644 --- a/cmake/FindRocksDB.cmake +++ b/cmake/FindRocksDB.cmake @@ -4,5 +4,20 @@ find_path(ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h PATH_SUFFIXES include) +if(ROCKSDB_INCLUDE_DIR AND EXISTS "${ROCKSDB_INCLUDE_DIR}/rocksdb/version.h") + foreach(ver "MAJOR" "MINOR" "PATCH") + file(STRINGS "${ROCKSDB_INCLUDE_DIR}/rocksdb/version.h" ROCKSDB_VER_${ver}_LINE + REGEX "^#define[ \t]+ROCKSDB_${ver}[ \t]+[0-9]+$") + string(REGEX REPLACE "^#define[ \t]+ROCKSDB_${ver}[ \t]+([0-9]+)$" + "\\1" ROCKSDB_VERSION_${ver} "${ROCKSDB_VER_${ver}_LINE}") + unset(${ROCKSDB_VER_${ver}_LINE}) + endforeach() + set(ROCKSDB_VERSION_STRING + "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}") + + message(STATUS "Found RocksDB version: ${ROCKSDB_VERSION_STRING}") +endif() + find_package_handle_standard_args(RocksDB - DEFAULT_MSG ROCKSDB_INCLUDE_DIR) + REQUIRED_VARS ROCKSDB_INCLUDE_DIR + VERSION_VAR ROCKSDB_VERSION_STRING) diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index d48b0586ec..6caa871759 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -215,17 +215,17 @@ set(CPACK_COMPONENT_SERVER-DEB_DEPENDS clients-deb) set(CPACK_COMPONENT_SERVER-TGZ_DEPENDS clients-tgz) set(CPACK_COMPONENT_SERVER-VERSIONED_DEPENDS clients-versioned) set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_REQUIRES - "foundationdb-clients-${FDB_MAJOR}.${FDB_MINOR}.${FDB_PATCH} = ${FDB_MAJOR}.${FDB_MINOR}.${FDB_PATCH}") + "foundationdb${PROJECT_VERSION}-clients") set(CPACK_COMPONENT_SERVER-EL7_DISPLAY_NAME "foundationdb-server") set(CPACK_COMPONENT_SERVER-DEB_DISPLAY_NAME "foundationdb-server") set(CPACK_COMPONENT_SERVER-TGZ_DISPLAY_NAME "foundationdb-server") -set(CPACK_COMPONENT_SERVER-VERSIONED_DISPLAY_NAME "foundationdb-server-${PROJECT_VERSION}") +set(CPACK_COMPONENT_SERVER-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-server") set(CPACK_COMPONENT_CLIENTS-EL7_DISPLAY_NAME "foundationdb-clients") set(CPACK_COMPONENT_CLIENTS-DEB_DISPLAY_NAME "foundationdb-clients") set(CPACK_COMPONENT_CLIENTS-TGZ_DISPLAY_NAME "foundationdb-clients") -set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb-clients-${PROJECT_VERSION}") +set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients") # MacOS needs a file exiension for the LICENSE file @@ -246,14 +246,21 @@ else() set(prerelease_string "-1") endif() - +############# +# Filenames # +############# +set(unversioned_postfix "${PROJECT_VERSION}${prerelease_string}") # RPM filenames -set(rpm-clients-filename "foundationdb-clients-${PROJECT_VERSION}${prerelease_string}") -set(rpm-server-filename "foundationdb-server-${PROJECT_VERSION}${prerelease_string}") +set(rpm-clients-filename "foundationdb-clients-${unversioned_postfix}") +set(rpm-server-filename "foundationdb-server-${unversioned_postfix}") +set(rpm-clients-versioned-filename "foundationdb${PROJECT_VERSION}-clients${prerelease_string}") +set(rpm-server-versioned-filename "foundationdb${PROJECT_VERSION}-server${prerelease_string}") # Deb filenames -set(deb-clients-filename "foundationdb-clients_${PROJECT_VERSION}${prerelease_string}") -set(deb-server-filename "foundationdb-server_${PROJECT_VERSION}${prerelease_string}") +set(deb-clients-filename "foundationdb-clients_${unversioned_postfix}") +set(deb-server-filename "foundationdb-server_${unversioned_postfix}") +set(deb-clients-versioned-filename "foundationdb${PROJECT_VERSION}-clients${prerelease_string}") +set(deb-server-versioned-filename "foundationdb${PROJECT_VERSION}-server${prerelease_string}") ################################################################################ # Configuration for RPM @@ -264,18 +271,18 @@ set(CPACK_RPM_PACKAGE_LICENSE "Apache 2.0") set(CPACK_RPM_PACKAGE_NAME "foundationdb") set(CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME "foundationdb-clients") set(CPACK_RPM_SERVER-EL7_PACKAGE_NAME "foundationdb-server") -set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_NAME "foundationdb-server-${PROJECT_VERSION}") -set(CPACK_RPM_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb-clients-${PROJECT_VERSION}") +set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-server") +set(CPACK_RPM_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-clients") set(CPACK_RPM_CLIENTS-EL7_FILE_NAME "${rpm-clients-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm") -set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm") +set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-versioned-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm") set(CPACK_RPM_SERVER-EL7_FILE_NAME "${rpm-server-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm") -set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm") +set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-versioned-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm") set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") -set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") +set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-versioned-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") set(CPACK_RPM_SERVER-EL7_DEBUGINFO_FILE_NAME "${rpm-server-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") -set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") +set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-versioned-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm") file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir") fdb_install(DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir/" DESTINATION data COMPONENT server) @@ -347,13 +354,13 @@ set(CPACK_RPM_CLIENTS-VERSIONED_PRE_UNINSTALL_SCRIPT_FILE if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb") set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb") - set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_amd64.deb") - set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_amd64.deb") + set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-versioned-filename}.versioned_amd64.deb") + set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-versioned-filename}.versioned_amd64.deb") else() set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb") set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb") - set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") - set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") + set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-versioned-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") + set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-versioned-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") endif() set(CPACK_DEB_COMPONENT_INSTALL ON) @@ -363,8 +370,8 @@ set(CPACK_DEBIAN_ENABLE_COMPONENT_DEPENDS ON) set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_NAME "foundationdb-server") set(CPACK_DEBIAN_CLIENTS-DEB_PACKAGE_NAME "foundationdb-clients") -set(CPACK_DEBIAN_SERVER-VERSIONED_PACKAGE_NAME "foundationdb-server-${PROJECT_VERSION}") -set(CPACK_DEBIAN_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb-clients-${PROJECT_VERSION}") +set(CPACK_DEBIAN_SERVER-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-server") +set(CPACK_DEBIAN_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-clients") set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_DEPENDS "adduser, libc6 (>= 2.12), foundationdb-clients (= ${FDB_VERSION})") set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_RECOMMENDS "python (>= 2.6)") diff --git a/documentation/CMakeLists.txt b/documentation/CMakeLists.txt index e734e28e91..7899af2261 100644 --- a/documentation/CMakeLists.txt +++ b/documentation/CMakeLists.txt @@ -1,23 +1,24 @@ add_subdirectory(tutorial) -if(WIN32) - return() -endif() - # build a virtualenv set(sphinx_dir ${CMAKE_CURRENT_SOURCE_DIR}/sphinx) set(venv_dir ${CMAKE_CURRENT_BINARY_DIR}/venv) -set(EXE_SUFFIX "") if(WIN32) + set(venv_bin_dir ${CMAKE_CURRENT_BINARY_DIR}/venv/Scripts) + set(activate_script ${venv_bin_dir}/activate.bat) set(EXE_SUFFIX ".exe") +else() + set(venv_bin_dir ${CMAKE_CURRENT_BINARY_DIR}/venv/bin) + set(activate_script . ${venv_bin_dir}/activate) + set(EXE_SUFFIX "") endif() -set(pip_command ${venv_dir}/bin/pip${EXE_SUFFIX}) -set(python_command ${venv_dir}/bin/python${EXE_SUFFIX}) +set(python_command ${venv_bin_dir}/python${EXE_SUFFIX}) +set(pip_command ${venv_bin_dir}/pip${EXE_SUFFIX}) add_custom_command(OUTPUT ${venv_dir}/venv_setup COMMAND ${Python3_EXECUTABLE} -m venv venv && ${CMAKE_COMMAND} -E copy ${sphinx_dir}/.pip.conf ${venv_dir}/pip.conf && - . ${venv_dir}/bin/activate && - ${pip_command} install --upgrade pip && + ${activate_script} && + ${python_command} -m pip install --upgrade pip && ${pip_command} install --upgrade -r ${sphinx_dir}/requirements.txt && ${pip_command} install sphinx-autobuild && # somehow this is missing in requirements.txt ${CMAKE_COMMAND} -E touch ${venv_dir}/venv_setup @@ -36,9 +37,9 @@ function(add_documentation_target) message(ERROR "GENERATOR is a required argument to add_documentation_target") endif() set(target ${ADT_GENERATOR}) - set(SPHINX_COMMAND "${venv_dir}/bin/sphinx-build") + set(SPHINX_COMMAND "${venv_bin_dir}/sphinx-build${EXE_SUFFIX}") if(ADT_SPHINX_COMMAND) - set(SPHINX_COMMAND "${venv_dir}/bin/${ADT_SPHINX_COMMAND}") + set(SPHINX_COMMAND "${venv_bin_dir}/${ADT_SPHINX_COMMAND}") endif() set(doctree "doctree") if (ADT_DOCTREE) diff --git a/documentation/sphinx/source/cap-theorem.rst b/documentation/sphinx/source/cap-theorem.rst index 42942d2f8c..50c912fe87 100644 --- a/documentation/sphinx/source/cap-theorem.rst +++ b/documentation/sphinx/source/cap-theorem.rst @@ -24,7 +24,7 @@ Let's consider an **AP** database. In such a database, reads and writes would al However, the downside is stark. Imagine a simple distributed database consisting of two nodes and a network partition making them unable to communicate. To be Available, each of the two nodes must continue to accept writes from clients. -.. figure:: /images/AP_Partition.png +.. figure:: images/AP_Partition.png Data divergence in an AP system during partition @@ -62,7 +62,7 @@ Imagine that a rack-top switch fails, and A is partitioned from the network. A w However, for all other clients, the database servers can reach a majority of coordination servers, B and C. The replication configuration has ensured there is a full copy of the data available even without A. For these clients, the database will remain available for reads and writes and the web servers will continue to serve traffic. -.. figure:: /images/FDB_Partition.png +.. figure:: images/FDB_Partition.png Maintenance of availability during partition diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index c7d717d799..cacfccf56a 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu configure --------- -The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=]``. +The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [storage_migration_type={disabled|aggressive|gradual}]``. The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified. @@ -112,7 +112,24 @@ For recommendations on appropriate values for process types in large clusters, s perpetual storage wiggle ^^^^^^^^^^^^^^^^^^^^^^^^ -Set the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle. For more details, see :ref:`perpetual-storage-wiggle`. +``perpetual_storage_wiggle`` sets the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle. +``perpetual_storage_wiggle_locality`` sets the process filter for wiggling. The processes that match the given locality key and locality value are only wiggled. The value 0 will disable the locality filter and matches all the processes for wiggling. + +For more details, see :ref:`perpetual-storage-wiggle`. + +storage migration type +^^^^^^^^^^^^^^^^^^^^^^ + +Set the storage migration type, or how FDB should migrate to a new storage engine if the value is changed. +The default is ``disabled``, which means changing the storage engine will not be possible. + +* ``disabled`` +* ``gradual`` +* ``aggressive`` + +``gradual`` replaces a single storage at a time when the ``perpetual storage wiggle`` is active. This requires the perpetual storage wiggle to be set to a non-zero value to actually migrate storage servers. It is somewhat slow but very safe. This is the recommended method for all production clusters. +``aggressive`` tries to replace as many storages as it can at once, and will recruit a new storage server on the same process as the old one. This will be faster, but can potentially hit degraded performance or OOM with two storages on the same process. The main benefit over ``gradual`` is that this doesn't need to take one storage out of rotation, so it works for small or development clusters that have the same number of storage processes as the replication factor. Note that ``aggressive`` is not exclusive to running the perpetual wiggle. +``disabled`` means that if the storage engine is changed, fdb will not move the cluster over to the new storage engine. This will disable the perpetual wiggle from rewriting storage files. consistencycheck ---------------- diff --git a/documentation/sphinx/source/ha-write-path.rst b/documentation/sphinx/source/ha-write-path.rst index e5d20a2694..deea5d0b93 100644 --- a/documentation/sphinx/source/ha-write-path.rst +++ b/documentation/sphinx/source/ha-write-path.rst @@ -176,7 +176,7 @@ The *LogPushData* class is used to hold serialized mutations on a per transactio *LogPushData.writeTypedMessage* is the function that serializes each mutation and writes it to the correct binary stream to be sent to the corresponding transaction log. Each serialized mutation contains additional metadata about the message, with the format: -.. image:: /images/serialized_mutation_metadata_format.png +.. image:: images/serialized_mutation_metadata_format.png * Message size: size of the message, in bytes, excluding the four bytes used for the message size diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 2404dc180d..a8d82794e3 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -256,7 +256,7 @@ "excluded":false, "address":"1.2.3.4:1234", "disk":{ - "free_bytes":3451233456234, // an estimate of how many bytes are free to allocate to fdbservers without swapping + "free_bytes":3451233456234, "reads":{ "hz":0.0, "counter":0, @@ -268,7 +268,7 @@ "counter":0, "sectors":0 }, - "total_bytes":123412341234 // an estimate of total physical RAM + "total_bytes":123412341234 }, "uptime_seconds":1234.2345, "cpu":{ @@ -735,7 +735,14 @@ "grv_proxies":1, // this field will be absent if a value has not been explicitly set "proxies":6, // this field will be absent if a value has not been explicitly set "backup_worker_enabled":1, - "perpetual_storage_wiggle": 0 + "perpetual_storage_wiggle": 0, + "perpetual_storage_wiggle_locality":"0", + "storage_migration_type":{ + "$enum":[ + "disabled", + "gradual", + "aggressive" + ]} }, "data":{ "least_operating_space_bytes_log_server":0, diff --git a/documentation/sphinx/source/performance.rst b/documentation/sphinx/source/performance.rst index 39ee0cbb24..18fa38a6fd 100644 --- a/documentation/sphinx/source/performance.rst +++ b/documentation/sphinx/source/performance.rst @@ -9,7 +9,7 @@ Scaling FoundationDB scales linearly with the number of cores in a cluster over a wide range of sizes. -.. image:: /images/scaling.png +.. image:: images/scaling.png Here, a cluster of commodity hardware scales to **8.2 million** operations/sec doing a 90% read and 10% write workload with 16 byte keys and values between 8 and 100 bytes. @@ -24,7 +24,7 @@ Latency FoundationDB has low latencies over a broad range of workloads that only increase modestly as the cluster approaches saturation. -.. image:: /images/latency.png +.. image:: images/latency.png When run at less than **75% load**, FoundationDB typically has the following latencies: @@ -53,7 +53,7 @@ Throughput (per core) FoundationDB provides good throughput for the full range of read and write workloads, with two fully durable storage engine options. -.. image:: /images/throughput.png +.. image:: images/throughput.png FoundationDB offers two :ref:`storage engines `, optimized for distinct use cases, both of which write to disk before reporting transactions committed. For each storage engine, the graph shows throughput of a single FoundationDB process running on a **single core** with saturating read/write workloads ranging from 100% reads to 100% writes, all with 16 byte keys and values between 8 and 100 bytes. Throughput for the unmixed workloads is about: @@ -79,7 +79,7 @@ Concurrency FoundationDB is designed to achieve great performance under high concurrency from a large number of clients. -.. image:: /images/concurrency.png +.. image:: images/concurrency.png Its asynchronous design allows it to handle very high concurrency, and for a typical workload with 90% reads and 10% writes, maximum throughput is reached at about 200 concurrent operations. This number of operations was achieved with **20** concurrent transactions per FoundationDB process each running 10 operations with 16 byte keys and values between 8 and 100 bytes. diff --git a/documentation/sphinx/source/perpetual-storage-wiggle.rst b/documentation/sphinx/source/perpetual-storage-wiggle.rst index aa4741e176..43c4065362 100644 --- a/documentation/sphinx/source/perpetual-storage-wiggle.rst +++ b/documentation/sphinx/source/perpetual-storage-wiggle.rst @@ -31,6 +31,8 @@ Configuration You can configure the Perpetual Storage Wiggle via the FDB :ref:`command line interface `. +Note that to have the Perpetual Storage Wiggle change the storage engine type, you must configure ``storage_migration_type=gradual``. + Example commands ---------------- @@ -38,6 +40,10 @@ Open perpetual storage wiggle: ``configure perpetual_storage_wiggle=1``. Disable perpetual storage wiggle on the cluster: ``configure perpetual_storage_wiggle=0``. +Open perpetual storage wiggle for only processes matching the given locality key and value: ``configure perpetual_storage_wiggle=1 perpetual_storage_wiggle_locality=:``. + +Disable perpetual storage wiggle locality matching filter, which wiggles all the processes: ``configure perpetual_storage_wiggle_locality=0``. + Monitor ======= diff --git a/documentation/sphinx/source/read-write-path.rst b/documentation/sphinx/source/read-write-path.rst index c9459a03fd..8257f8905f 100644 --- a/documentation/sphinx/source/read-write-path.rst +++ b/documentation/sphinx/source/read-write-path.rst @@ -16,7 +16,7 @@ The processing order of multiple transactions is important because it affects th The content is based on FDB 6.2 and is true for FDB 6.3. A new timestamp proxy role is introduced in post FDB 6.3, which affects the read path. We will discuss the timestamp proxy role in the future version of this document. -.. image:: /images/FDB_read_path.png +.. image:: images/FDB_read_path.png Components ================= @@ -198,7 +198,7 @@ Write path of a transaction Suppose a client has a write-only transaction. Fig. 2 below shows the write path in a non-HA cluster. We will discuss how a transaction with both read and write works in the next section. -.. image:: /images/FDB_write_path.png +.. image:: images/FDB_write_path.png To simplify the explanation, the steps below do not include transaction batching on proxy, which is a typical database technique to increase transaction throughput. @@ -461,7 +461,7 @@ The ordering is enforced in the timestamp generator, the concurrency control com We use the following example and draw its swimlane diagram to illustrate how two write transactions are ordered in FDB. The diagram with notes can be viewed at `here `_. -.. image:: /images/FDB_multiple_txn_swimlane_diagram.png +.. image:: images/FDB_multiple_txn_swimlane_diagram.png Reference ============ diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 14c29f3f8a..b54258baca 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -2,9 +2,31 @@ Release Notes ############# +6.3.21 +====== +* Added a ThreadID field to all trace events for the purpose of multi-threaded client debugging. `(PR #5665) `_ +* Fixed some histograms' group name in the master proxy. `(PR #5674) `_ +* Added histograms for GRV path components in the proxy. `(PR #5689) `_ +* Fixed race condition introduced in 6.3.20 between setting timeouts and resetting or destroying transactions. `(PR #5695) `_ +* Disable detailed transaction log pop tracing by default. `(PR #5696) `_ + +6.3.20 +====== +* Several minor problems with the versioned packages have been fixed. `(PR 5607) `_ +* A client might not honor transaction timeouts when using the multi-version client if it cannot connect to the cluster. `(Issue #5595) `_ +* Fixed a very rare bug where recovery could potentially roll back a committed transaction `(PR 5461) `_ +* Added histograms for commit path components in the proxy. `(PR #5367) `_ +* Fixed a false checkRegions call that could cause unwanted primary DC failover. `(PR #5330) `_ + 6.3.19 ====== -* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) `_ +* Added the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) `_ +* Added error details in ``RemovedDeadBackupLayerStatus`` trace event. `(PR #5356) `_ +* Added RepeatableReadMultiThreadClientTest. `(PR #5212) `_ +* Added a new feature that allows FDB to detect grey failures and automatically recover from them. `(PR #5249) `_ +* Added version and timestamp to ``TimeKeeperCommit`` trace event. `(PR #5415) `_ +* Added ``RecruitFromConfigurationRetry`` trace event to improve recruitment observability. `(PR #5455) `_ +* Several fixes to pkg_tester and packaging. `(PR #5460) `_ 6.3.18 ====== diff --git a/documentation/sphinx/source/release-notes/release-notes-700.rst b/documentation/sphinx/source/release-notes/release-notes-700.rst index 072154e5a6..770c4c9af5 100644 --- a/documentation/sphinx/source/release-notes/release-notes-700.rst +++ b/documentation/sphinx/source/release-notes/release-notes-700.rst @@ -64,6 +64,7 @@ Fixes * If a restore is done using a prefix to remove and specific key ranges to restore, the key range boundaries must begin with the prefix to remove. `(PR #4684) `_ * The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5220) `_ * Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5220) `_ +* A client might not honor transaction timeouts when using the multi-version client if it cannot connect to the cluster. `(Issue #5595) `_ Status ------ diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp index 87fee7f2ce..326f0246f2 100644 --- a/documentation/tutorial/tutorial.actor.cpp +++ b/documentation/tutorial/tutorial.actor.cpp @@ -33,6 +33,12 @@ NetworkAddress serverAddress; +enum TutorialWellKnownEndpoints { + WLTOKEN_SIMPLE_KV_SERVER = WLTOKEN_FIRST_AVAILABLE, + WLTOKEN_ECHO_SERVER, + WLTOKEN_COUNT_IN_TUTORIAL +}; + // this is a simple actor that will report how long // it is already running once a second. ACTOR Future simpleTimer() { @@ -153,7 +159,7 @@ struct StreamReply : ReplyPromiseStreamReply { template void serialize(Ar& ar) { - serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, index); + serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, ReplyPromiseStreamReply::sequence, index); } }; @@ -171,7 +177,7 @@ uint64_t tokenCounter = 1; ACTOR Future echoServer() { state EchoServerInterface echoServer; - echoServer.getInterface.makeWellKnownEndpoint(UID(-1, ++tokenCounter), TaskPriority::DefaultEndpoint); + echoServer.getInterface.makeWellKnownEndpoint(WLTOKEN_ECHO_SERVER, TaskPriority::DefaultEndpoint); loop { try { choose { @@ -204,7 +210,8 @@ ACTOR Future echoServer() { ACTOR Future echoClient() { state EchoServerInterface server; - server.getInterface = RequestStream(Endpoint({ serverAddress }, UID(-1, ++tokenCounter))); + server.getInterface = + RequestStream(Endpoint::wellKnown({ serverAddress }, WLTOKEN_ECHO_SERVER)); EchoServerInterface s = wait(server.getInterface.getReply(GetInterfaceRequest())); server = s; EchoRequest echoRequest; @@ -291,7 +298,7 @@ struct ClearRequest { ACTOR Future kvStoreServer() { state SimpleKeyValueStoreInteface inf; state std::map store; - inf.connect.makeWellKnownEndpoint(UID(-1, ++tokenCounter), TaskPriority::DefaultEndpoint); + inf.connect.makeWellKnownEndpoint(WLTOKEN_SIMPLE_KV_SERVER, TaskPriority::DefaultEndpoint); loop { choose { when(GetKVInterface req = waitNext(inf.connect.getFuture())) { @@ -328,7 +335,7 @@ ACTOR Future kvStoreServer() { ACTOR Future connect() { std::cout << format("%llu: Connect...\n", uint64_t(g_network->now())); SimpleKeyValueStoreInteface c; - c.connect = RequestStream(Endpoint({ serverAddress }, UID(-1, ++tokenCounter))); + c.connect = RequestStream(Endpoint::wellKnown({ serverAddress }, WLTOKEN_SIMPLE_KV_SERVER)); SimpleKeyValueStoreInteface result = wait(c.connect.getReply(GetKVInterface())); std::cout << format("%llu: done..\n", uint64_t(g_network->now())); return result; @@ -562,7 +569,7 @@ int main(int argc, char* argv[]) { } platformInit(); g_network = newNet2(TLSConfig(), false, true); - FlowTransport::createInstance(!isServer, 0); + FlowTransport::createInstance(!isServer, 0, WLTOKEN_COUNT_IN_TUTORIAL); NetworkAddress publicAddress = NetworkAddress::parse("0.0.0.0:0"); if (isServer) { publicAddress = NetworkAddress::parse("0.0.0.0:" + port); diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 528fdc84a8..f8833c3d78 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -52,8 +52,6 @@ #include #include #include -using std::cout; -using std::endl; #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN @@ -4235,14 +4233,14 @@ int main(int argc, char* argv[]) { #ifdef ALLOC_INSTRUMENTATION { - cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " - << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " - << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " - << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " - << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " " - << FastAllocator<16384>::pageCount << endl; + std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " + << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " + << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " + << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " + << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " " + << FastAllocator<16384>::pageCount << std::endl; - vector> typeNames; + std::vector> typeNames; for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) { std::string s; diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt index db76032683..32323fb788 100644 --- a/fdbcli/CMakeLists.txt +++ b/fdbcli/CMakeLists.txt @@ -3,13 +3,19 @@ set(FDBCLI_SRCS fdbcli.actor.h AdvanceVersionCommand.actor.cpp CacheRangeCommand.actor.cpp + ConfigureCommand.actor.cpp ConsistencyCheckCommand.actor.cpp + CoordinatorsCommand.actor.cpp DataDistributionCommand.actor.cpp + ExcludeCommand.actor.cpp ExpensiveDataCheckCommand.actor.cpp + FileConfigureCommand.actor.cpp FlowLineNoise.actor.cpp FlowLineNoise.h ForceRecoveryWithDataLossCommand.actor.cpp + IncludeCommand.actor.cpp KillCommand.actor.cpp + LockCommand.actor.cpp MaintenanceCommand.actor.cpp ProfileCommand.actor.cpp SetClassCommand.actor.cpp diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp new file mode 100644 index 0000000000..ab810454a4 --- /dev/null +++ b/fdbcli/ConfigureCommand.actor.cpp @@ -0,0 +1,300 @@ +/* + * ConfigureCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/FlowLineNoise.h" +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/ManagementAPI.actor.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace fdb_cli { + +ACTOR Future configureCommandActor(Reference db, + Database localDb, + std::vector tokens, + LineNoise* linenoise, + Future warn) { + state ConfigurationResult result; + state StatusObject s; + state int startToken = 1; + state bool force = false; + if (tokens.size() < 2) + result = ConfigurationResult::NO_OPTIONS_PROVIDED; + else { + if (tokens[startToken] == LiteralStringRef("FORCE")) { + force = true; + startToken = 2; + } + + state Optional conf; + if (tokens[startToken] == LiteralStringRef("auto")) { + // get cluster status + state Reference tr = db->createTransaction(); + if (!tr->isValid()) { + StatusObject _s = wait(StatusClient::statusFetcher(localDb)); + s = _s; + } else { + state ThreadFuture> statusValueF = tr->get(LiteralStringRef("\xff\xff/status/json")); + Optional statusValue = wait(safeThreadFutureToFuture(statusValueF)); + if (!statusValue.present()) { + fprintf(stderr, "ERROR: Failed to get status json from the cluster\n"); + return false; + } + json_spirit::mValue mv; + json_spirit::read_string(statusValue.get().toString(), mv); + s = StatusObject(mv.get_obj()); + } + + if (warn.isValid()) + warn.cancel(); + + conf = parseConfig(s); + + if (!conf.get().isValid()) { + printf("Unable to provide advice for the current configuration.\n"); + return false; + } + + bool noChanges = conf.get().old_replication == conf.get().auto_replication && + conf.get().old_logs == conf.get().auto_logs && + conf.get().old_commit_proxies == conf.get().auto_commit_proxies && + conf.get().old_grv_proxies == conf.get().auto_grv_proxies && + conf.get().old_resolvers == conf.get().auto_resolvers && + conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && + conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; + + bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && + conf.get().old_commit_proxies == conf.get().desired_commit_proxies && + conf.get().old_grv_proxies == conf.get().desired_grv_proxies && + conf.get().old_resolvers == conf.get().desired_resolvers; + + std::string outputString; + + outputString += "\nYour cluster has:\n\n"; + outputString += format(" processes %d\n", conf.get().processes); + outputString += format(" machines %d\n", conf.get().machines); + + if (noDesiredChanges) + outputString += "\nConfigure recommends keeping your current configuration:\n\n"; + else if (noChanges) + outputString += + "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n"; + else + outputString += "\nConfigure recommends the following changes:\n\n"; + outputString += " ------------------------------------------------------------------- \n"; + outputString += "| parameter | old | new |\n"; + outputString += " ------------------------------------------------------------------- \n"; + outputString += format("| replication | %16s | %16s |\n", + conf.get().old_replication.c_str(), + conf.get().auto_replication.c_str()); + outputString += + format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); + outputString += conf.get().auto_logs != conf.get().desired_logs + ? format(" (manually set; would be %d)\n", conf.get().desired_logs) + : "\n"; + outputString += format("| commit_proxies | %16d | %16d |", + conf.get().old_commit_proxies, + conf.get().auto_commit_proxies); + outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) + : "\n"; + outputString += format("| grv_proxies | %16d | %16d |", + conf.get().old_grv_proxies, + conf.get().auto_grv_proxies); + outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies) + : "\n"; + outputString += format( + "| resolvers | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers); + outputString += conf.get().auto_resolvers != conf.get().desired_resolvers + ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers) + : "\n"; + outputString += format("| transaction-class processes | %16d | %16d |\n", + conf.get().old_processes_with_transaction, + conf.get().auto_processes_with_transaction); + outputString += format("| transaction-class machines | %16d | %16d |\n", + conf.get().old_machines_with_transaction, + conf.get().auto_machines_with_transaction); + outputString += " ------------------------------------------------------------------- \n\n"; + + std::printf("%s", outputString.c_str()); + + if (noChanges) + return true; + + // TODO: disable completion + Optional line = wait(linenoise->read("Would you like to make these changes? [y/n]> ")); + + if (!line.present() || (line.get() != "y" && line.get() != "Y")) { + return true; + } + } + + ConfigurationResult r = wait(ManagementAPI::changeConfig( + db, std::vector(tokens.begin() + startToken, tokens.end()), conf, force)); + result = r; + } + + // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but + // there are various results specific to changeConfig() that we need to report: + bool ret = true; + switch (result) { + case ConfigurationResult::NO_OPTIONS_PROVIDED: + case ConfigurationResult::CONFLICTING_OPTIONS: + case ConfigurationResult::UNKNOWN_OPTION: + case ConfigurationResult::INCOMPLETE_CONFIGURATION: + printUsage(LiteralStringRef("configure")); + ret = false; + break; + case ConfigurationResult::INVALID_CONFIGURATION: + fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_MIGRATION_DISABLED: + fprintf(stderr, + "ERROR: Storage engine type cannot be changed because " + "storage_migration_mode=disabled.\n"); + fprintf(stderr, + "Type `configure perpetual_storage_wiggle=1 storage_migration_type=gradual' to enable gradual " + "migration with the perpetual wiggle, or `configure " + "storage_migration_type=aggressive' for aggressive migration.\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_ALREADY_CREATED: + fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_CREATED: + printf("Database created\n"); + break; + case ConfigurationResult::DATABASE_UNAVAILABLE: + fprintf(stderr, "ERROR: The database is unavailable\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: + fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: + fprintf(stderr, + "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated " + "before changing the configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: + fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGIONS_CHANGED: + fprintf(stderr, + "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::NOT_ENOUGH_WORKERS: + fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_REPLICATION_MISMATCH: + fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::DCID_MISSING: + fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS: + printf("Configuration changed\n"); + break; + case ConfigurationResult::LOCKED_NOT_NEW: + fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL: + printf("Configuration changed, with warnings\n"); + fprintf(stderr, + "WARN: To make progress toward the desired storage type with storage_migration_type=gradual, the " + "Perpetual Wiggle must be enabled.\n"); + fprintf(stderr, + "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure " + "storage_migration_type=gradual' to set the gradual migration type.\n"); + ret = false; + break; + default: + ASSERT(false); + ret = false; + }; + return ret; +} + +CommandFactory configureFactory( + "configure", + CommandHelp( + "configure [new|tss]" + "|" + "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" + "count=|perpetual_storage_wiggle=|perpetual_storage_wiggle_locality=" + "<:|0>|storage_migration_type={disabled|gradual|aggressive}", + "change the database configuration", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " + "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead." + "When used with new to set up tss for the first time, it requires both a count and a storage engine." + "To disable the testing storage server, run \"configure tss count=0\"\n\n" + "Redundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " + "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " + "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " + "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " + "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. The proxy role is being " + "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' " + "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and " + "'grv_proxies' " + "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and " + "'commit_proxies'. " + "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the " + "default value.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. " + "Must be at least 1, or set to -1 which restores the number of commit proxies to the default " + "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " + "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " + "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " + "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " + "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n" + "perpetual_storage_wiggle=: Set the value speed (a.k.a., the number of processes that the Data " + "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the " + "perpetual storage wiggle.\n\n" + "perpetual_storage_wiggle_locality=<:|0>: Set the process filter for wiggling. " + "The processes that match the given locality key and locality value are only wiggled. The value 0 will disable " + "the locality filter and matches all the processes for wiggling.\n\n" + "See the FoundationDB Administration Guide for more information.")); + +} // namespace fdb_cli diff --git a/fdbcli/ConsistencyCheckCommand.actor.cpp b/fdbcli/ConsistencyCheckCommand.actor.cpp index 7ef135c73b..5f3f2e2cf1 100644 --- a/fdbcli/ConsistencyCheckCommand.actor.cpp +++ b/fdbcli/ConsistencyCheckCommand.actor.cpp @@ -39,7 +39,9 @@ ACTOR Future consistencyCheckCommandActor(Reference tr, // If not, the outer loop catch block(fdbcli.actor.cpp) will handle the error and print out the error message tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); if (tokens.size() == 1) { - Optional suspended = wait(safeThreadFutureToFuture(tr->get(consistencyCheckSpecialKey))); + // hold the returned standalone object's memory + state ThreadFuture> suspendedF = tr->get(consistencyCheckSpecialKey); + Optional suspended = wait(safeThreadFutureToFuture(suspendedF)); printf("ConsistencyCheck is %s\n", suspended.present() ? "off" : "on"); } else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { tr->set(consistencyCheckSpecialKey, Value()); diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.actor.cpp new file mode 100644 index 0000000000..d8bfe11f6c --- /dev/null +++ b/fdbcli/CoordinatorsCommand.actor.cpp @@ -0,0 +1,185 @@ +/* + * CoordinatorsCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/algorithm/string.hpp" + +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/Schemas.h" +#include "fdbclient/ManagementAPI.actor.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace { + +ACTOR Future printCoordinatorsInfo(Reference db) { + state Reference tr = db->createTransaction(); + loop { + try { + // Hold the reference to the standalone's memory + state ThreadFuture> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey); + Optional description = wait(safeThreadFutureToFuture(descriptionF)); + ASSERT(description.present()); + printf("Cluster description: %s\n", description.get().toString().c_str()); + // Hold the reference to the standalone's memory + state ThreadFuture> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); + Optional processes = wait(safeThreadFutureToFuture(processesF)); + ASSERT(processes.present()); + std::vector process_addresses; + boost::split(process_addresses, processes.get().toString(), [](char c) { return c == ','; }); + printf("Cluster coordinators (%zu): %s\n", process_addresses.size(), processes.get().toString().c_str()); + printf("Type `help coordinators' to learn how to change this information.\n"); + return Void(); + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future changeCoordinators(Reference db, std::vector tokens) { + state int retries = 0; + state int notEnoughMachineResults = 0; + state StringRef new_cluster_description; + state std::string auto_coordinators_str; + StringRef nameTokenBegin = LiteralStringRef("description="); + for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) + if (tok->startsWith(nameTokenBegin)) { + new_cluster_description = tok->substr(nameTokenBegin.size()); + std::copy(tok + 1, tokens.end(), tok); + tokens.resize(tokens.size() - 1); + break; + } + + state bool automatic = tokens.size() == 2 && tokens[1] == LiteralStringRef("auto"); + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + // update cluster description + if (new_cluster_description.size()) { + tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description); + } + // if auto change, read the special key to retrieve the recommended config + if (automatic) { + // if previous read failed, retry, otherwise, use the same recommened config + if (!auto_coordinators_str.size()) { + // Hold the reference to the standalone's memory + state ThreadFuture> auto_coordinatorsF = + tr->get(fdb_cli::coordinatorsAutoSpecialKey); + Optional auto_coordinators = wait(safeThreadFutureToFuture(auto_coordinatorsF)); + ASSERT(auto_coordinators.present()); + auto_coordinators_str = auto_coordinators.get().toString(); + } + tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str); + } else if (tokens.size() > 1) { + state std::set new_coordinators_addresses; + state std::vector newAddresslist; + state std::vector::iterator t; + for (t = tokens.begin() + 1; t != tokens.end(); ++t) { + try { + auto const& addr = NetworkAddress::parse(t->toString()); + if (new_coordinators_addresses.count(addr)) { + fprintf(stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str()); + return true; + } + new_coordinators_addresses.insert(addr); + newAddresslist.push_back(addr.toString()); + } catch (Error& e) { + if (e.code() == error_code_connection_string_invalid) { + fprintf( + stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str()); + return true; + } + throw; + } + } + std::string new_addresses_str = boost::algorithm::join(newAddresslist, ", "); + tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_addresses_str); + } + wait(safeThreadFutureToFuture(tr->commit())); + // commit should always fail here + // if coordinators are changed, we should get commit_unknown() error + ASSERT(false); + } catch (Error& e) { + state Error err(e); + if (e.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); + if (errorMsgStr == ManagementAPI::generateErrorMessage(CoordinatorsResult::NOT_ENOUGH_MACHINES) && + notEnoughMachineResults < 1) { + // we could get not_enough_machines if we happen to see the database while the cluster controller is + // updating the worker list, so make sure it happens twice before returning a failure + notEnoughMachineResults++; + wait(delay(1.0)); + tr->reset(); + continue; + } else if (errorMsgStr == + ManagementAPI::generateErrorMessage(CoordinatorsResult::SAME_NETWORK_ADDRESSES)) { + if (retries) + printf("Coordination state changed\n"); + else + printf("No change (existing configuration satisfies request)\n"); + return true; + } else { + fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str()); + return false; + } + } + wait(safeThreadFutureToFuture(tr->onError(err))); + ++retries; + } + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRef clusterDescriptionSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/cluster_description"); +const KeyRef coordinatorsAutoSpecialKey = LiteralStringRef("\xff\xff/management/auto_coordinators"); +const KeyRef coordinatorsProcessSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/processes"); + +ACTOR Future coordinatorsCommandActor(Reference db, std::vector tokens) { + if (tokens.size() < 2) { + wait(printCoordinatorsInfo(db)); + return true; + } else { + bool result = wait(changeCoordinators(db, tokens)); + return result; + } +} + +CommandFactory coordinatorsFactory( + "coordinators", + CommandHelp( + "coordinators auto|
+ [description=new_cluster_description]", + "change cluster coordinators or description", + "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured " + "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " + "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " + "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " + "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " + "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+.")); +} // namespace fdb_cli diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp new file mode 100644 index 0000000000..8580e6b063 --- /dev/null +++ b/fdbcli/ExcludeCommand.actor.cpp @@ -0,0 +1,397 @@ +/* + * ExcludeCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/ManagementAPI.actor.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace { + +// Exclue the given servers and localities +ACTOR Future excludeServersAndLocalities(Reference db, + std::vector servers, + std::unordered_set localities, + bool failed, + bool force) { + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + if (force && servers.size()) + tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey, + ValueRef()); + for (const auto& s : servers) { + Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) + : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); + tr->set(addr, ValueRef()); + } + if (force && localities.size()) + tr->set(failed ? fdb_cli::failedLocalityForceOptionSpecialKey + : fdb_cli::excludedLocalityForceOptionSpecialKey, + ValueRef()); + for (const auto& l : localities) { + Key addr = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) + : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); + tr->set(addr, ValueRef()); + } + wait(safeThreadFutureToFuture(tr->commit())); + return true; + } catch (Error& e) { + state Error err(e); + if (e.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); + // last character is \n + auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2); + auto last_line = errorMsgStr.substr(pos + 1); + // customized the error message for fdbcli + fprintf(stderr, + "%s\n%s\n", + errorMsgStr.substr(0, pos).c_str(), + last_line.find("free space") != std::string::npos + ? "Type `exclude FORCE ' to exclude without checking free space." + : "Type `exclude FORCE failed ' to exclude without performing safety checks."); + return false; + } + wait(safeThreadFutureToFuture(tr->onError(err))); + } + } +} + +ACTOR Future> getExcludedServers(Reference db) { + state Reference tr = db->createTransaction(); + loop { + try { + state ThreadFuture resultFuture = + tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + state ThreadFuture resultFuture2 = + tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + state RangeResult r2 = wait(safeThreadFutureToFuture(resultFuture2)); + ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector exclusions; + for (const auto& i : r) { + auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString(); + exclusions.push_back(addr); + } + for (const auto& i : r2) { + auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString(); + exclusions.push_back(addr); + } + return exclusions; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +// Get the list of excluded localities by reading the keys. +ACTOR Future> getExcludedLocalities(Reference db) { + state Reference tr = db->createTransaction(); + loop { + try { + state ThreadFuture resultFuture = + tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + state ThreadFuture resultFuture2 = + tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + state RangeResult r2 = wait(safeThreadFutureToFuture(resultFuture2)); + ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector excludedLocalities; + for (const auto& i : r) { + auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString(); + excludedLocalities.push_back(locality); + } + for (const auto& i : r2) { + auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString(); + excludedLocalities.push_back(locality); + } + return excludedLocalities; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future> checkForExcludingServers(Reference db, + std::vector excl, + bool waitForAllExcluded) { + state std::set exclusions(excl.begin(), excl.end()); + state std::set inProgressExclusion; + state Reference tr = db->createTransaction(); + loop { + inProgressExclusion.clear(); + try { + state ThreadFuture resultFuture = + tr->getRange(fdb_cli::exclusionInProgressSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult exclusionInProgress = wait(safeThreadFutureToFuture(resultFuture)); + ASSERT(!exclusionInProgress.more && exclusionInProgress.size() < CLIENT_KNOBS->TOO_MANY); + if (exclusionInProgress.empty()) + return inProgressExclusion; + for (const auto& addr : exclusionInProgress) + inProgressExclusion.insert(NetworkAddress::parse( + addr.key.removePrefix(fdb_cli::exclusionInProgressSpecialKeyRange.begin).toString())); + if (!waitForAllExcluded) + break; + + wait(delayJittered(1.0)); // SOMEDAY: watches! + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } + return inProgressExclusion; +} + +ACTOR Future checkForCoordinators(Reference db, std::vector exclusionVector) { + + state bool foundCoordinator = false; + state std::vector coordinatorList; + state Reference tr = db->createTransaction(); + loop { + try { + // Hold the reference to the standalone's memory + state ThreadFuture> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); + Optional coordinators = wait(safeThreadFutureToFuture(coordinatorsF)); + ASSERT(coordinators.present()); + coordinatorList = NetworkAddress::parseList(coordinators.get().toString()); + break; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } + for (const auto& c : coordinatorList) { + if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) || + std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) { + fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str()); + foundCoordinator = true; + } + } + if (foundCoordinator) + printf("Type `help coordinators' for information on how to change the\n" + "cluster's coordination servers before removing them.\n"); + return Void(); +} + +} // namespace + +namespace fdb_cli { + +const KeyRangeRef excludedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded/"), + LiteralStringRef("\xff\xff/management/excluded0")); +const KeyRangeRef failedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/failed/"), + LiteralStringRef("\xff\xff/management/failed0")); +const KeyRangeRef excludedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded_locality/"), + LiteralStringRef("\xff\xff/management/excluded_locality0")); +const KeyRangeRef failedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/failed_locality/"), + LiteralStringRef("\xff\xff/management/failed_locality0")); +const KeyRef excludedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/excluded/force"); +const KeyRef failedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/failed/force"); +const KeyRef excludedLocalityForceOptionSpecialKey = + LiteralStringRef("\xff\xff/management/options/excluded_locality/force"); +const KeyRef failedLocalityForceOptionSpecialKey = + LiteralStringRef("\xff\xff/management/options/failed_locality/force"); +const KeyRangeRef exclusionInProgressSpecialKeyRange(LiteralStringRef("\xff\xff/management/in_progress_exclusion/"), + LiteralStringRef("\xff\xff/management/in_progress_exclusion0")); + +ACTOR Future excludeCommandActor(Reference db, std::vector tokens, Future warn) { + if (tokens.size() <= 1) { + state std::vector excludedAddresses = wait(getExcludedServers(db)); + state std::vector excludedLocalities = wait(getExcludedLocalities(db)); + + if (!excludedAddresses.size() && !excludedLocalities.size()) { + printf("There are currently no servers or localities excluded from the database.\n" + "To learn how to exclude a server, type `help exclude'.\n"); + return true; + } + + printf("There are currently %zu servers or localities being excluded from the database:\n", + excludedAddresses.size() + excludedLocalities.size()); + for (const auto& e : excludedAddresses) + printf(" %s\n", e.c_str()); + for (const auto& e : excludedLocalities) + printf(" %s\n", e.c_str()); + + printf("To find out whether it is safe to remove one or more of these\n" + "servers from the cluster, type `exclude '.\n" + "To return one of these servers to the cluster, type `include '.\n"); + + return true; + } else { + state std::vector exclusionVector; + state std::set exclusionSet; + state std::vector exclusionAddresses; + state std::unordered_set exclusionLocalities; + state std::vector noMatchLocalities; + state bool force = false; + state bool waitForAllExcluded = true; + state bool markFailed = false; + state std::vector workers; + bool result = wait(fdb_cli::getWorkers(db, &workers)); + if (!result) + return false; + for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { + if (*t == LiteralStringRef("FORCE")) { + force = true; + } else if (*t == LiteralStringRef("no_wait")) { + waitForAllExcluded = false; + } else if (*t == LiteralStringRef("failed")) { + markFailed = true; + } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && + t->toString().find(':') != std::string::npos) { + std::set localityAddresses = getAddressesByLocality(workers, t->toString()); + if (localityAddresses.empty()) { + noMatchLocalities.push_back(t->toString()); + } else { + // add all the server ipaddresses that belong to the given localities to the exclusionSet. + exclusionVector.insert(exclusionVector.end(), localityAddresses.begin(), localityAddresses.end()); + exclusionSet.insert(localityAddresses.begin(), localityAddresses.end()); + } + exclusionLocalities.insert(t->toString()); + } else { + auto a = AddressExclusion::parse(*t); + if (!a.isValid()) { + fprintf(stderr, + "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", + t->toString().c_str()); + if (t->toString().find(":tls") != std::string::npos) + printf(" Do not include the `:tls' suffix when naming a process\n"); + return true; + } + exclusionVector.push_back(a); + exclusionSet.insert(a); + exclusionAddresses.push_back(a); + } + } + + if (exclusionAddresses.empty() && exclusionLocalities.empty()) { + fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality is not provided\n"); + return false; + } + + bool res = wait(excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force)); + if (!res) + return false; + + if (waitForAllExcluded) { + printf("Waiting for state to be removed from all excluded servers. This may take a while.\n"); + printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n"); + } + + if (warn.isValid()) + warn.cancel(); + + state std::set notExcludedServers = + wait(checkForExcludingServers(db, exclusionVector, waitForAllExcluded)); + std::map> workerPorts; + for (auto addr : workers) + workerPorts[addr.address.ip].insert(addr.address.port); + + // Print a list of all excluded addresses that don't have a corresponding worker + std::set absentExclusions; + for (const auto& addr : exclusionVector) { + auto worker = workerPorts.find(addr.ip); + if (worker == workerPorts.end()) + absentExclusions.insert(addr); + else if (addr.port > 0 && worker->second.count(addr.port) == 0) + absentExclusions.insert(addr); + } + + for (const auto& exclusion : exclusionVector) { + if (absentExclusions.find(exclusion) != absentExclusions.end()) { + if (exclusion.port == 0) { + fprintf(stderr, + " %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the " + "correct machines before removing them from the cluster!\n", + exclusion.ip.toString().c_str()); + } else { + fprintf(stderr, + " %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes " + "before removing them from the cluster!\n", + exclusion.toString().c_str()); + } + } else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(), [&](const NetworkAddress& a) { + return addressExcluded({ exclusion }, a); + })) { + if (exclusion.port == 0) { + fprintf(stderr, + " %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this " + "machine from the cluster\n", + exclusion.ip.toString().c_str()); + } else { + fprintf(stderr, + " %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the " + "cluster\n", + exclusion.toString().c_str()); + } + } else { + if (exclusion.port == 0) { + printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine " + "from the cluster.\n", + exclusion.ip.toString().c_str()); + } else { + printf( + " %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n", + exclusion.toString().c_str()); + } + } + } + + for (const auto& locality : noMatchLocalities) { + fprintf( + stderr, + " %s ---- WARNING: Currently no servers found with this locality match! Be sure that you excluded " + "the correct locality.\n", + locality.c_str()); + } + + wait(checkForCoordinators(db, exclusionVector)); + return true; + } +} + +CommandFactory excludeFactory( + "exclude", + CommandHelp( + "exclude [FORCE] [failed] [no_wait] [] [locality_dcid:] " + "[locality_zoneid:] [locality_machineid:] " + "[locality_processid:] or any locality data", + "exclude servers from the database either with IP address match or locality match", + "If no addresses or locaities are specified, lists the set of excluded addresses and localities." + "\n\nFor each IP address or IP:port pair in or any LocalityData attributes (like dcid, " + "zoneid, " + "machineid, processid), adds the address/locality to the set of excluded servers and localities then waits " + "until all database state has been safely moved away from the specified servers. If 'no_wait' is set, the " + "command returns \nimmediately without checking if the exclusions have completed successfully.\n" + "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" + "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n" + "for data movement to finish and the server cannot be included again.")); +} // namespace fdb_cli diff --git a/fdbcli/FileConfigureCommand.actor.cpp b/fdbcli/FileConfigureCommand.actor.cpp new file mode 100644 index 0000000000..e612002eb6 --- /dev/null +++ b/fdbcli/FileConfigureCommand.actor.cpp @@ -0,0 +1,181 @@ +/* + * FileConfigureCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/FlowLineNoise.h" +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/ManagementAPI.actor.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace fdb_cli { + +ACTOR Future fileConfigureCommandActor(Reference db, + std::string filePath, + bool isNewDatabase, + bool force) { + std::string contents(readFileBytes(filePath, 100000)); + json_spirit::mValue config; + if (!json_spirit::read_string(contents, config)) { + fprintf(stderr, "ERROR: Invalid JSON\n"); + return false; + } + if (config.type() != json_spirit::obj_type) { + fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n"); + return false; + } + StatusObject configJSON = config.get_obj(); + + json_spirit::mValue schema; + if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) { + ASSERT(false); + } + + std::string errorStr; + if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) { + printf("%s", errorStr.c_str()); + return false; + } + + std::string configString; + if (isNewDatabase) { + configString = "new"; + } + + for (const auto& [name, value] : configJSON) { + if (!configString.empty()) { + configString += " "; + } + if (value.type() == json_spirit::int_type) { + configString += name + ":=" + format("%d", value.get_int()); + } else if (value.type() == json_spirit::str_type) { + configString += value.get_str(); + } else if (value.type() == json_spirit::array_type) { + configString += + name + "=" + + json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none); + } else { + printUsage(LiteralStringRef("fileconfigure")); + return false; + } + } + ConfigurationResult result = wait(ManagementAPI::changeConfig(db, configString, force)); + // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but + // there are various results specific to changeConfig() that we need to report: + bool ret = true; + switch (result) { + case ConfigurationResult::NO_OPTIONS_PROVIDED: + fprintf(stderr, "ERROR: No options provided\n"); + ret = false; + break; + case ConfigurationResult::CONFLICTING_OPTIONS: + fprintf(stderr, "ERROR: Conflicting options\n"); + ret = false; + break; + case ConfigurationResult::UNKNOWN_OPTION: + fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match + ret = false; + break; + case ConfigurationResult::INCOMPLETE_CONFIGURATION: + fprintf(stderr, + "ERROR: Must specify both a replication level and a storage engine when creating a new database\n"); + ret = false; + break; + case ConfigurationResult::INVALID_CONFIGURATION: + fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_ALREADY_CREATED: + fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_CREATED: + printf("Database created\n"); + break; + case ConfigurationResult::DATABASE_UNAVAILABLE: + fprintf(stderr, "ERROR: The database is unavailable\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: + fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: + fprintf(stderr, + "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated " + "before changing the configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: + fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGIONS_CHANGED: + fprintf(stderr, + "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::NOT_ENOUGH_WORKERS: + fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_REPLICATION_MISMATCH: + fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::DCID_MISSING: + fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS: + printf("Configuration changed\n"); + break; + default: + ASSERT(false); + ret = false; + }; + return ret; +} + +CommandFactory fileconfigureFactory( + "fileconfigure", + CommandHelp( + "fileconfigure [new] ", + "change the database configuration from a file", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. Load a JSON document from the provided file, and change the database " + "configuration to match the contents of the JSON document. The format should be the same as the value of the " + "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\".")); + +} // namespace fdb_cli diff --git a/fdbcli/IncludeCommand.actor.cpp b/fdbcli/IncludeCommand.actor.cpp new file mode 100644 index 0000000000..fda54f6fcb --- /dev/null +++ b/fdbcli/IncludeCommand.actor.cpp @@ -0,0 +1,176 @@ +/* + * IncludeCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace { + +// Remove the given localities from the exclusion list. +// include localities by clearing the keys. +ACTOR Future includeLocalities(Reference db, + std::vector localities, + bool failed, + bool includeAll) { + state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + + if (includeAll) { + if (failed) { + tr->clear(fdb_cli::failedLocalitySpecialKeyRange); + } else { + tr->clear(fdb_cli::excludedLocalitySpecialKeyRange); + } + } else { + for (const auto& l : localities) { + Key locality = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) + : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); + tr->clear(locality); + } + } + wait(safeThreadFutureToFuture(tr->commit())); + return Void(); + } catch (Error& e) { + TraceEvent("IncludeLocalitiesError").error(e, true); + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +ACTOR Future includeServers(Reference db, std::vector servers, bool failed) { + state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + for (auto& s : servers) { + // include all, just clear the whole key range + if (!s.isValid()) { + if (failed) { + tr->clear(fdb_cli::failedServersSpecialKeyRange); + } else { + tr->clear(fdb_cli::excludedServersSpecialKeyRange); + } + } else { + Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) + : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); + tr->clear(addr); + // Eliminate both any ip-level exclusion (1.2.3.4) and any + // port-level exclusions (1.2.3.4:5) + // The range ['IP', 'IP;'] was originally deleted. ';' is + // char(':' + 1). This does not work, as other for all + // x between 0 and 9, 'IPx' will also be in this range. + // + // This is why we now make two clears: first only of the ip + // address, the second will delete all ports. + if (s.isWholeMachine()) + tr->clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")), + addr.withSuffix(LiteralStringRef(";")))); + } + } + wait(safeThreadFutureToFuture(tr->commit())); + return Void(); + } catch (Error& e) { + TraceEvent("IncludeServersError").error(e, true); + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +// Includes the servers that could be IP addresses or localities back to the cluster. +ACTOR Future include(Reference db, std::vector tokens) { + state std::vector addresses; + state std::vector localities; + state bool failed = false; + state bool all = false; + for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { + if (*t == LiteralStringRef("all")) { + all = true; + } else if (*t == LiteralStringRef("failed")) { + failed = true; + } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) { + // if the token starts with 'locality_' prefix. + localities.push_back(t->toString()); + } else { + auto a = AddressExclusion::parse(*t); + if (!a.isValid()) { + fprintf(stderr, + "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", + t->toString().c_str()); + if (t->toString().find(":tls") != std::string::npos) + printf(" Do not include the `:tls' suffix when naming a process\n"); + return false; + } + addresses.push_back(a); + } + } + if (all) { + std::vector includeAll; + includeAll.push_back(AddressExclusion()); + wait(includeServers(db, includeAll, failed)); + wait(includeLocalities(db, localities, failed, all)); + } else { + if (!addresses.empty()) { + wait(includeServers(db, addresses, failed)); + } + if (!localities.empty()) { + // include the servers that belong to given localities. + wait(includeLocalities(db, localities, failed, all)); + } + } + return true; +}; + +} // namespace + +namespace fdb_cli { + +ACTOR Future includeCommandActor(Reference db, std::vector tokens) { + if (tokens.size() < 2) { + printUsage(tokens[0]); + return false; + } else { + bool result = wait(include(db, tokens)); + return result; + } +} + +CommandFactory includeFactory( + "include", + CommandHelp( + "include all|[] [locality_dcid:] [locality_zoneid:] " + "[locality_machineid:] [locality_processid:] or any locality data", + "permit previously-excluded servers and localities to rejoin the database", + "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port " + "pair in or any LocalityData (like dcid, zoneid, machineid, processid), removes any " + "matching exclusions from the excluded servers and localities list. " + "(A specified IP will match all IP:* exclusion entries)")); +} // namespace fdb_cli diff --git a/fdbcli/LockCommand.actor.cpp b/fdbcli/LockCommand.actor.cpp new file mode 100644 index 0000000000..7ae69be35c --- /dev/null +++ b/fdbcli/LockCommand.actor.cpp @@ -0,0 +1,120 @@ +/* + * LockCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.actor.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +namespace { + +ACTOR Future lockDatabase(Reference db, UID id) { + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + tr->set(fdb_cli::lockSpecialKey, id.toString()); + wait(safeThreadFutureToFuture(tr->commit())); + printf("Database locked.\n"); + return true; + } catch (Error& e) { + state Error err(e); + if (e.code() == error_code_database_locked) + throw e; + else if (e.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); + fprintf(stderr, "%s\n", errorMsgStr.c_str()); + return false; + } + wait(safeThreadFutureToFuture(tr->onError(err))); + } + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRef lockSpecialKey = LiteralStringRef("\xff\xff/management/db_locked"); + +ACTOR Future lockCommandActor(Reference db, std::vector tokens) { + if (tokens.size() != 1) { + printUsage(tokens[0]); + return false; + } else { + state UID lockUID = deterministicRandom()->randomUniqueID(); + printf("Locking database with lockUID: %s\n", lockUID.toString().c_str()); + bool result = wait((lockDatabase(db, lockUID))); + return result; + } +} + +ACTOR Future unlockDatabaseActor(Reference db, UID uid) { + state Reference tr = db->createTransaction(); + loop { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + try { + state ThreadFuture> valF = tr->get(fdb_cli::lockSpecialKey); + Optional val = wait(safeThreadFutureToFuture(valF)); + + if (!val.present()) + return true; + + if (val.present() && UID::fromString(val.get().toString()) != uid) { + printf("Unable to unlock database. Make sure to unlock with the correct lock UID.\n"); + return false; + } + + tr->clear(fdb_cli::lockSpecialKey); + wait(safeThreadFutureToFuture(tr->commit())); + printf("Database unlocked.\n"); + return true; + } catch (Error& e) { + state Error err(e); + if (e.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); + fprintf(stderr, "%s\n", errorMsgStr.c_str()); + return false; + } + wait(safeThreadFutureToFuture(tr->onError(err))); + } + } +} + +CommandFactory lockFactory( + "lock", + CommandHelp("lock", + "lock the database with a randomly generated lockUID", + "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database.")); + +CommandFactory unlockFactory( + "unlock", + CommandHelp("unlock ", + "unlock the database with the provided lockUID", + "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " + "user will be asked to enter a passphrase to confirm their intent.")); +} // namespace fdb_cli diff --git a/fdbcli/ProfileCommand.actor.cpp b/fdbcli/ProfileCommand.actor.cpp index 2ab56488f7..b58c87c4dd 100644 --- a/fdbcli/ProfileCommand.actor.cpp +++ b/fdbcli/ProfileCommand.actor.cpp @@ -38,7 +38,7 @@ namespace fdb_cli { ACTOR Future profileCommandActor(Reference tr, std::vector tokens, bool intrans) { state bool result = true; if (tokens.size() == 1) { - fprintf(stderr, "ERROR: Usage: profile \n"); + printUsage(tokens[0]); result = false; } else if (tokencmp(tokens[1], "client")) { if (tokens.size() == 2) { diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp index 3760630885..c47baf38e1 100644 --- a/fdbcli/StatusCommand.actor.cpp +++ b/fdbcli/StatusCommand.actor.cpp @@ -29,7 +29,6 @@ #include "flow/FastRef.h" #include "flow/ThreadHelper.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. -#include namespace { diff --git a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp b/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp index 74e48fc148..f7ff8657a6 100644 --- a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp +++ b/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp @@ -31,7 +31,7 @@ namespace fdb_cli { -ACTOR Future triggerddteaminfologCommandActor(Reference db) { +ACTOR Future triggerddteaminfologCommandActor(Reference db) { state Reference tr = db->createTransaction(); loop { try { @@ -41,7 +41,7 @@ ACTOR Future triggerddteaminfologCommandActor(Reference db) { tr->set(triggerDDTeamInfoPrintKey, v); wait(safeThreadFutureToFuture(tr->commit())); printf("Triggered team info logging in data distribution.\n"); - return Void(); + return true; } catch (Error& e) { wait(safeThreadFutureToFuture(tr->onError(e))); } diff --git a/fdbcli/TssqCommand.actor.cpp b/fdbcli/TssqCommand.actor.cpp index e91f2b16a2..0e0f86e653 100644 --- a/fdbcli/TssqCommand.actor.cpp +++ b/fdbcli/TssqCommand.actor.cpp @@ -64,7 +64,9 @@ ACTOR Future tssQuarantine(Reference db, bool enable, UID tssId tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // Do some validation first to make sure the command is valid - Optional serverListValue = wait(safeThreadFutureToFuture(tr->get(serverListKeyFor(tssId)))); + // hold the returned standalone object's memory + state ThreadFuture> serverListValueF = tr->get(serverListKeyFor(tssId)); + Optional serverListValue = wait(safeThreadFutureToFuture(serverListValueF)); if (!serverListValue.present()) { printf("No TSS %s found in cluster!\n", tssId.toString().c_str()); return false; @@ -75,8 +77,9 @@ ACTOR Future tssQuarantine(Reference db, bool enable, UID tssId return false; } - Optional currentQuarantineValue = - wait(safeThreadFutureToFuture(tr->get(tssQuarantineKeyFor(tssId)))); + // hold the returned standalone object's memory + state ThreadFuture> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId)); + Optional currentQuarantineValue = wait(safeThreadFutureToFuture(currentQuarantineValueF)); if (enable && currentQuarantineValue.present()) { printf("TSS %s already in quarantine, doing nothing.\n", tssId.toString().c_str()); return false; diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp index 95ae5cee81..489001feee 100644 --- a/fdbcli/Util.actor.cpp +++ b/fdbcli/Util.actor.cpp @@ -25,6 +25,7 @@ #include "flow/Arena.h" +#include "flow/ThreadHelper.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. namespace fdb_cli { @@ -46,7 +47,9 @@ void printUsage(StringRef command) { } ACTOR Future getSpecialKeysFailureErrorMessage(Reference tr) { - Optional errorMsg = wait(safeThreadFutureToFuture(tr->get(fdb_cli::errorMsgSpecialKey))); + // hold the returned standalone object's memory + state ThreadFuture> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey); + Optional errorMsg = wait(safeThreadFutureToFuture(errorMsgF)); // Error message should be present ASSERT(errorMsg.present()); // Read the json string @@ -112,4 +115,49 @@ ACTOR Future getWorkerInterfaces(Reference tr, return Void(); } +ACTOR Future getWorkers(Reference db, std::vector* workers) { + state Reference tr = db->createTransaction(); + loop { + try { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + state ThreadFuture processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); + state ThreadFuture processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); + + wait(success(safeThreadFutureToFuture(processClasses)) && success(safeThreadFutureToFuture(processData))); + ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); + + state std::map>, ProcessClass> id_class; + state int i; + for (i = 0; i < processClasses.get().size(); i++) { + try { + id_class[decodeProcessClassKey(processClasses.get()[i].key)] = + decodeProcessClassValue(processClasses.get()[i].value); + } catch (Error& e) { + fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what()); + return false; + } + } + + for (i = 0; i < processData.get().size(); i++) { + ProcessData data = decodeWorkerListValue(processData.get()[i].value); + ProcessClass processClass = id_class[data.locality.processId()]; + + if (processClass.classSource() == ProcessClass::DBSource || + data.processClass.classType() == ProcessClass::UnsetClass) + data.processClass = processClass; + + if (data.processClass.classType() != ProcessClass::TesterClass) + workers->push_back(data); + } + + return true; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + } // namespace fdb_cli diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 777ef50a70..bb9c7f38d2 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -91,7 +91,8 @@ enum { OPT_BUILD_FLAGS, OPT_TRACE_FORMAT, OPT_KNOB, - OPT_DEBUG_TLS + OPT_DEBUG_TLS, + OPT_API_VERSION, }; CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, @@ -113,6 +114,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_DEBUG_TLS, "--debug-tls", SO_NONE }, + { OPT_API_VERSION, "--api-version", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS @@ -429,6 +431,8 @@ static void printProgramUsage(const char* name) { " and then exits.\n" " --no-status Disables the initial status check done when starting\n" " the CLI.\n" + " --api-version APIVERSION\n" + " Specifies the version of the API for the CLI to use.\n" #ifndef TLS_DISABLED TLS_HELP #endif @@ -470,76 +474,6 @@ void initHelp() { "clear a range of keys from the database", "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will " "succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); - helpMap["configure"] = CommandHelp( - "configure [new|tss]" - "|" - "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" - "count=|perpetual_storage_wiggle=", - "change the database configuration", - "The `new' option, if present, initializes a new database with the given configuration rather than changing " - "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " - "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead." - "When used with new to set up tss for the first time, it requires both a count and a storage engine." - "To disable the testing storage server, run \"configure tss count=0\"\n\n" - "Redundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " - "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " - "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " - "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " - "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. The proxy role is being " - "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' " - "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and " - "'grv_proxies' " - "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and " - "'commit_proxies'. " - "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the " - "default value.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. " - "Must be at least 1, or set to -1 which restores the number of commit proxies to the default " - "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " - "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " - "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " - "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " - "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n" - "perpetual_storage_wiggle=: Set the value speed (a.k.a., the number of processes that the Data " - "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the " - "perpetual storage wiggle.\n\n" - "See the FoundationDB Administration Guide for more information."); - helpMap["fileconfigure"] = CommandHelp( - "fileconfigure [new] ", - "change the database configuration from a file", - "The `new' option, if present, initializes a new database with the given configuration rather than changing " - "the configuration of an existing one. Load a JSON document from the provided file, and change the database " - "configuration to match the contents of the JSON document. The format should be the same as the value of the " - "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."); - helpMap["coordinators"] = CommandHelp( - "coordinators auto|
+ [description=new_cluster_description]", - "change cluster coordinators or description", - "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured " - "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " - "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " - "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " - "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " - "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); - helpMap["exclude"] = CommandHelp( - "exclude [FORCE] [failed] [no_wait] [] [locality_dcid:] " - "[locality_zoneid:] [locality_machineid:] " - "[locality_processid:] or any locality data", - "exclude servers from the database either with IP address match or locality match", - "If no addresses or locaities are specified, lists the set of excluded addresses and localities." - "\n\nFor each IP address or IP:port pair in or any LocalityData attributes (like dcid, zoneid, " - "machineid, processid), adds the address/locality to the set of excluded servers and localities then waits " - "until all database state has been safely moved away from the specified servers. If 'no_wait' is set, the " - "command returns \nimmediately without checking if the exclusions have completed successfully.\n" - "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" - "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n" - "for data movement to finish and the server cannot be included again."); - helpMap["include"] = CommandHelp( - "include all|[] [locality_dcid:] [locality_zoneid:] " - "[locality_machineid:] [locality_processid:] or any locality data", - "permit previously-excluded servers and localities to rejoin the database", - "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port " - "pair in or any LocalityData (like dcid, zoneid, machineid, processid), removes any " - "matching exclusions from the excluded servers and localities list. " - "(A specified IP will match all IP:* exclusion entries)"); helpMap["exit"] = CommandHelp("exit", "exit the CLI", ""); helpMap["quit"] = CommandHelp(); helpMap["waitconnected"] = CommandHelp(); @@ -588,15 +522,6 @@ void initHelp() { helpMap["writemode"] = CommandHelp("writemode ", "enables or disables sets and clears", "Setting or clearing keys from the CLI is not recommended."); - helpMap["lock"] = CommandHelp( - "lock", - "lock the database with a randomly generated lockUID", - "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database."); - helpMap["unlock"] = - CommandHelp("unlock ", - "unlock the database with the provided lockUID", - "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " - "user will be asked to enter a passphrase to confirm their intent."); helpMap["changefeed"] = CommandHelp("changefeed ", "", ""); } @@ -765,332 +690,7 @@ ACTOR Future setBlobRange(Database db, Key startKey, Key endKey, Value val } } -ACTOR Future configure(Database db, - std::vector tokens, - Reference ccf, - LineNoise* linenoise, - Future warn) { - state ConfigurationResult result; - state int startToken = 1; - state bool force = false; - if (tokens.size() < 2) - result = ConfigurationResult::NO_OPTIONS_PROVIDED; - else { - if (tokens[startToken] == LiteralStringRef("FORCE")) { - force = true; - startToken = 2; - } - - state Optional conf; - if (tokens[startToken] == LiteralStringRef("auto")) { - StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db))); - if (warn.isValid()) - warn.cancel(); - - conf = parseConfig(s); - - if (!conf.get().isValid()) { - printf("Unable to provide advice for the current configuration.\n"); - return true; - } - - bool noChanges = conf.get().old_replication == conf.get().auto_replication && - conf.get().old_logs == conf.get().auto_logs && - conf.get().old_commit_proxies == conf.get().auto_commit_proxies && - conf.get().old_grv_proxies == conf.get().auto_grv_proxies && - conf.get().old_resolvers == conf.get().auto_resolvers && - conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && - conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; - - bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && - conf.get().old_commit_proxies == conf.get().desired_commit_proxies && - conf.get().old_grv_proxies == conf.get().desired_grv_proxies && - conf.get().old_resolvers == conf.get().desired_resolvers; - - std::string outputString; - - outputString += "\nYour cluster has:\n\n"; - outputString += format(" processes %d\n", conf.get().processes); - outputString += format(" machines %d\n", conf.get().machines); - - if (noDesiredChanges) - outputString += "\nConfigure recommends keeping your current configuration:\n\n"; - else if (noChanges) - outputString += - "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n"; - else - outputString += "\nConfigure recommends the following changes:\n\n"; - outputString += " ------------------------------------------------------------------- \n"; - outputString += "| parameter | old | new |\n"; - outputString += " ------------------------------------------------------------------- \n"; - outputString += format("| replication | %16s | %16s |\n", - conf.get().old_replication.c_str(), - conf.get().auto_replication.c_str()); - outputString += - format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); - outputString += conf.get().auto_logs != conf.get().desired_logs - ? format(" (manually set; would be %d)\n", conf.get().desired_logs) - : "\n"; - outputString += format("| commit_proxies | %16d | %16d |", - conf.get().old_commit_proxies, - conf.get().auto_commit_proxies); - outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies - ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) - : "\n"; - outputString += format("| grv_proxies | %16d | %16d |", - conf.get().old_grv_proxies, - conf.get().auto_grv_proxies); - outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies - ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies) - : "\n"; - outputString += format( - "| resolvers | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers); - outputString += conf.get().auto_resolvers != conf.get().desired_resolvers - ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers) - : "\n"; - outputString += format("| transaction-class processes | %16d | %16d |\n", - conf.get().old_processes_with_transaction, - conf.get().auto_processes_with_transaction); - outputString += format("| transaction-class machines | %16d | %16d |\n", - conf.get().old_machines_with_transaction, - conf.get().auto_machines_with_transaction); - outputString += " ------------------------------------------------------------------- \n\n"; - - std::printf("%s", outputString.c_str()); - - if (noChanges) - return false; - - // TODO: disable completion - Optional line = wait(linenoise->read("Would you like to make these changes? [y/n]> ")); - - if (!line.present() || (line.get() != "y" && line.get() != "Y")) { - return false; - } - } - - ConfigurationResult r = wait(makeInterruptable( - changeConfig(db, std::vector(tokens.begin() + startToken, tokens.end()), conf, force))); - result = r; - } - - // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but - // there are various results specific to changeConfig() that we need to report: - bool ret; - switch (result) { - case ConfigurationResult::NO_OPTIONS_PROVIDED: - case ConfigurationResult::CONFLICTING_OPTIONS: - case ConfigurationResult::UNKNOWN_OPTION: - case ConfigurationResult::INCOMPLETE_CONFIGURATION: - printUsage(LiteralStringRef("configure")); - ret = true; - break; - case ConfigurationResult::INVALID_CONFIGURATION: - fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); - ret = true; - break; - case ConfigurationResult::DATABASE_ALREADY_CREATED: - fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); - ret = true; - break; - case ConfigurationResult::DATABASE_CREATED: - printf("Database created\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_UNAVAILABLE: - fprintf(stderr, "ERROR: The database is unavailable\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: - fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: - fprintf(stderr, - "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated " - "before changing the configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: - fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGIONS_CHANGED: - fprintf(stderr, - "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::NOT_ENOUGH_WORKERS: - fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGION_REPLICATION_MISMATCH: - fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::DCID_MISSING: - fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::SUCCESS: - printf("Configuration changed\n"); - ret = false; - break; - case ConfigurationResult::LOCKED_NOT_NEW: - fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n"); - ret = true; - break; - default: - ASSERT(false); - ret = true; - }; - return ret; -} - -ACTOR Future fileConfigure(Database db, std::string filePath, bool isNewDatabase, bool force) { - std::string contents(readFileBytes(filePath, 100000)); - json_spirit::mValue config; - if (!json_spirit::read_string(contents, config)) { - fprintf(stderr, "ERROR: Invalid JSON\n"); - return true; - } - if (config.type() != json_spirit::obj_type) { - fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n"); - return true; - } - StatusObject configJSON = config.get_obj(); - - json_spirit::mValue schema; - if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) { - ASSERT(false); - } - - std::string errorStr; - if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) { - printf("%s", errorStr.c_str()); - return true; - } - - std::string configString; - if (isNewDatabase) { - configString = "new"; - } - - for (const auto& [name, value] : configJSON) { - if (!configString.empty()) { - configString += " "; - } - if (value.type() == json_spirit::int_type) { - configString += name + ":=" + format("%d", value.get_int()); - } else if (value.type() == json_spirit::str_type) { - configString += value.get_str(); - } else if (value.type() == json_spirit::array_type) { - configString += - name + "=" + - json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none); - } else { - printUsage(LiteralStringRef("fileconfigure")); - return true; - } - } - ConfigurationResult result = wait(makeInterruptable(changeConfig(db, configString, force))); - // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but - // there are various results specific to changeConfig() that we need to report: - bool ret; - switch (result) { - case ConfigurationResult::NO_OPTIONS_PROVIDED: - fprintf(stderr, "ERROR: No options provided\n"); - ret = true; - break; - case ConfigurationResult::CONFLICTING_OPTIONS: - fprintf(stderr, "ERROR: Conflicting options\n"); - ret = true; - break; - case ConfigurationResult::UNKNOWN_OPTION: - fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match - ret = true; - break; - case ConfigurationResult::INCOMPLETE_CONFIGURATION: - fprintf(stderr, - "ERROR: Must specify both a replication level and a storage engine when creating a new database\n"); - ret = true; - break; - case ConfigurationResult::INVALID_CONFIGURATION: - fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); - ret = true; - break; - case ConfigurationResult::DATABASE_ALREADY_CREATED: - fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); - ret = true; - break; - case ConfigurationResult::DATABASE_CREATED: - printf("Database created\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_UNAVAILABLE: - fprintf(stderr, "ERROR: The database is unavailable\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: - fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: - fprintf(stderr, - "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated " - "before changing the configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: - fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGIONS_CHANGED: - fprintf(stderr, - "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::NOT_ENOUGH_WORKERS: - fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::REGION_REPLICATION_MISMATCH: - fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::DCID_MISSING: - fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = true; - break; - case ConfigurationResult::SUCCESS: - printf("Configuration changed\n"); - ret = false; - break; - default: - ASSERT(false); - ret = true; - }; - return ret; -} - // FIXME: Factor address parsing from coordinators, include, exclude - ACTOR Future coordinators(Database db, std::vector tokens, bool isClusterTLS) { state StringRef setName; StringRef nameTokenBegin = LiteralStringRef("description="); @@ -1225,12 +825,12 @@ ACTOR Future exclude(Database db, Reference ccf, Future warn) { if (tokens.size() <= 1) { - state Future> fexclAddresses = makeInterruptable(getExcludedServers(db)); - state Future> fexclLocalities = makeInterruptable(getExcludedLocalities(db)); + state Future> fexclAddresses = makeInterruptable(getExcludedServers(db)); + state Future> fexclLocalities = makeInterruptable(getExcludedLocalities(db)); wait(success(fexclAddresses) && success(fexclLocalities)); - vector exclAddresses = fexclAddresses.get(); - vector exclLocalities = fexclLocalities.get(); + std::vector exclAddresses = fexclAddresses.get(); + std::vector exclLocalities = fexclLocalities.get(); if (!exclAddresses.size() && !exclLocalities.size()) { printf("There are currently no servers or localities excluded from the database.\n" @@ -1630,6 +1230,8 @@ void configureGenerator(const char* text, const char* line, std::vector FDB_API_VERSION) { + // multi-version fdbcli only available after 7.0 + fprintf(stderr, + "ERROR: api version %s is not supported. (Min: 700, Max: %d)\n", + args.OptionArg(), + FDB_API_VERSION); + return 1; + } + break; + } case OPT_TRACE: trace = true; break; @@ -2017,7 +1638,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { TraceEvent::setNetworkThread(); try { - localDb = Database::createDatabase(ccf, -1, IsInternal::False); + localDb = Database::createDatabase(ccf, opt.api_version, IsInternal::False); if (!opt.exec.present()) { printf("Using cluster file `%s'.\n", ccf->getFilename().c_str()); } @@ -2209,7 +1830,8 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "waitopen")) { - wait(success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion()))); + wait(makeInterruptable( + success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())))); continue; } @@ -2241,7 +1863,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "triggerddteaminfolog")) { - wait(triggerddteaminfologCommandActor(db)); + wait(success(makeInterruptable(triggerddteaminfologCommandActor(db)))); continue; } @@ -2257,7 +1879,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - wait(changeFeedList(db)); + wait(changeFeedList(localDb)); continue; } else if (tokencmp(tokens[1], "register")) { if (tokens.size() != 5) { @@ -2265,7 +1887,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - trx = Transaction(db); + trx = Transaction(localDb); loop { try { wait(trx.registerChangeFeed(tokens[2], KeyRangeRef(tokens[3], tokens[4]))); @@ -2281,7 +1903,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - trx = Transaction(db); + trx = Transaction(localDb); loop { try { trx.destroyChangeFeed(tokens[2]); @@ -2317,7 +1939,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } } Standalone> res = - wait(db->getChangeFeedMutations(tokens[2], begin, end)); + wait(localDb->getChangeFeedMutations(tokens[2], begin, end)); printf("\n"); for (auto& it : res) { for (auto& it2 : it.mutations) { @@ -2353,7 +1975,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { warn.cancel(); } state PromiseStream>> feedResults; - state Future feed = db->getChangeFeedStream(feedResults, tokens[2], begin, end); + state Future feed = localDb->getChangeFeedStream(feedResults, tokens[2], begin, end); printf("\n"); try { state Future feedInterrupt = LineNoise::onKeyboardInterrupt(); @@ -2394,7 +2016,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { printUsage(tokens[0]); is_error = true; } else { - wait(db->popChangeFeedMutations(tokens[2], v)); + wait(localDb->popChangeFeedMutations(tokens[2], v)); } } continue; @@ -2441,8 +2063,9 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "configure")) { - bool err = wait(configure(localDb, tokens, localDb->getConnectionFile(), &linenoise, warn)); - if (err) + bool _result = + wait(makeInterruptable(configureCommandActor(db, localDb, tokens, &linenoise, warn))); + if (!_result) is_error = true; continue; } @@ -2450,11 +2073,12 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { if (tokencmp(tokens[0], "fileconfigure")) { if (tokens.size() == 2 || (tokens.size() == 3 && (tokens[1] == LiteralStringRef("new") || tokens[1] == LiteralStringRef("FORCE")))) { - bool err = wait(fileConfigure(localDb, - tokens.back().toString(), - tokens[1] == LiteralStringRef("new"), - tokens[1] == LiteralStringRef("FORCE"))); - if (err) + bool _result = + wait(makeInterruptable(fileConfigureCommandActor(db, + tokens.back().toString(), + tokens[1] == LiteralStringRef("new"), + tokens[1] == LiteralStringRef("FORCE")))); + if (!_result) is_error = true; } else { printUsage(tokens[0]); @@ -2464,57 +2088,37 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "coordinators")) { - auto cs = ClusterConnectionFile(localDb->getConnectionFile()->getFilename()).getConnectionString(); - if (tokens.size() < 2) { - printf("Cluster description: %s\n", cs.clusterKeyName().toString().c_str()); - printf("Cluster coordinators (%zu): %s\n", - cs.coordinators().size(), - describe(cs.coordinators()).c_str()); - printf("Type `help coordinators' to learn how to change this information.\n"); - } else { - bool err = wait(coordinators(localDb, tokens, cs.coordinators()[0].isTLS())); - if (err) - is_error = true; - } + bool _result = wait(makeInterruptable(coordinatorsCommandActor(db, tokens))); + if (!_result) + is_error = true; continue; } if (tokencmp(tokens[0], "exclude")) { - bool err = wait(exclude(localDb, tokens, localDb->getConnectionFile(), warn)); - if (err) + bool _result = wait(makeInterruptable(excludeCommandActor(db, tokens, warn))); + if (!_result) is_error = true; continue; } if (tokencmp(tokens[0], "include")) { - if (tokens.size() < 2) { - printUsage(tokens[0]); + bool _result = wait(makeInterruptable(includeCommandActor(db, tokens))); + if (!_result) is_error = true; - } else { - bool err = wait(include(localDb, tokens)); - if (err) - is_error = true; - } continue; } if (tokencmp(tokens[0], "snapshot")) { - bool _result = wait(snapshotCommandActor(db, tokens)); + bool _result = wait(makeInterruptable(snapshotCommandActor(db, tokens))); if (!_result) is_error = true; continue; } if (tokencmp(tokens[0], "lock")) { - if (tokens.size() != 1) { - printUsage(tokens[0]); + bool _result = wait(makeInterruptable(lockCommandActor(db, tokens))); + if (!_result) is_error = true; - } else { - state UID lockUID = deterministicRandom()->randomUniqueID(); - printf("Locking database with lockUID: %s\n", lockUID.toString().c_str()); - wait(makeInterruptable(lockDatabase(localDb, lockUID))); - printf("Database locked.\n"); - } continue; } @@ -2535,16 +2139,9 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db, localDb); if (input.present() && input.get() == passPhrase) { UID unlockUID = UID::fromString(tokens[1].toString()); - try { - wait(makeInterruptable(unlockDatabase(localDb, unlockUID))); - printf("Database unlocked.\n"); - } catch (Error& e) { - if (e.code() == error_code_database_locked) { - printf( - "Unable to unlock database. Make sure to unlock with the correct lock UID.\n"); - } - throw e; - } + bool _result = wait(makeInterruptable(unlockDatabaseActor(db, unlockUID))); + if (!_result) + is_error = true; } else { fprintf(stderr, "ERROR: Incorrect passphrase entered.\n"); is_error = true; @@ -2931,7 +2528,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "throttle")) { - bool _result = wait(throttleCommandActor(db, tokens)); + bool _result = wait(makeInterruptable(throttleCommandActor(db, tokens))); if (!_result) is_error = true; continue; @@ -3162,8 +2759,7 @@ int main(int argc, char** argv) { } try { - // Note: refactoring fdbcli, in progress - API->selectApiVersion(FDB_API_VERSION); + API->selectApiVersion(opt.api_version); API->setupNetwork(); Future cliFuture = runCli(opt); Future timeoutFuture = opt.exit_timeout ? timeExit(opt.exit_timeout) : Never(); diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h index dab31df05f..7340b3b682 100644 --- a/fdbcli/fdbcli.actor.h +++ b/fdbcli/fdbcli.actor.h @@ -28,6 +28,8 @@ #elif !defined(FDBCLI_FDBCLI_ACTOR_H) #define FDBCLI_FDBCLI_ACTOR_H +#include "fdbcli/FlowLineNoise.h" + #include "fdbclient/CoordinationInterface.h" #include "fdbclient/IClientApi.h" #include "fdbclient/StatusClient.h" @@ -63,9 +65,25 @@ struct CommandFactory { extern const KeyRef advanceVersionSpecialKey; // consistencycheck extern const KeyRef consistencyCheckSpecialKey; +// coordinators +extern const KeyRef clusterDescriptionSpecialKey; +extern const KeyRef coordinatorsAutoSpecialKey; +extern const KeyRef coordinatorsProcessSpecialKey; // datadistribution extern const KeyRef ddModeSpecialKey; extern const KeyRef ddIgnoreRebalanceSpecialKey; +// exclude/include +extern const KeyRangeRef excludedServersSpecialKeyRange; +extern const KeyRangeRef failedServersSpecialKeyRange; +extern const KeyRangeRef excludedLocalitySpecialKeyRange; +extern const KeyRangeRef failedLocalitySpecialKeyRange; +extern const KeyRef excludedForceOptionSpecialKey; +extern const KeyRef failedForceOptionSpecialKey; +extern const KeyRef excludedLocalityForceOptionSpecialKey; +extern const KeyRef failedLocalityForceOptionSpecialKey; +extern const KeyRangeRef exclusionInProgressSpecialKeyRange; +// lock/unlock +extern const KeyRef lockSpecialKey; // maintenance extern const KeyRangeRef maintenanceSpecialKeyRange; extern const KeyRef ignoreSSFailureSpecialKey; @@ -79,6 +97,8 @@ inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_messag ACTOR Future addInterface(std::map>* address_interface, Reference connectLock, KeyValue kv); +// get all workers' info +ACTOR Future getWorkers(Reference db, std::vector* workers); // compare StringRef with the given c string bool tokencmp(StringRef token, const char* command); @@ -101,29 +121,50 @@ void printStatus(StatusObjectReader statusObj, bool hideErrorMessages = false); // All fdbcli commands (alphabetically) +// All below actors return true if the command is executed successfully // advanceversion command ACTOR Future advanceVersionCommandActor(Reference db, std::vector tokens); // cache_range command ACTOR Future cacheRangeCommandActor(Reference db, std::vector tokens); +// configure command +ACTOR Future configureCommandActor(Reference db, + Database localDb, + std::vector tokens, + LineNoise* linenoise, + Future warn); // consistency command ACTOR Future consistencyCheckCommandActor(Reference tr, std::vector tokens, bool intrans); +// coordinators command +ACTOR Future coordinatorsCommandActor(Reference db, std::vector tokens); // datadistribution command ACTOR Future dataDistributionCommandActor(Reference db, std::vector tokens); +// exclude command +ACTOR Future excludeCommandActor(Reference db, std::vector tokens, Future warn); // expensive_data_check command ACTOR Future expensiveDataCheckCommandActor( Reference db, Reference tr, std::vector tokens, std::map>* address_interface); +// fileconfigure command +ACTOR Future fileConfigureCommandActor(Reference db, + std::string filePath, + bool isNewDatabase, + bool force); // force_recovery_with_data_loss command ACTOR Future forceRecoveryWithDataLossCommandActor(Reference db, std::vector tokens); +// include command +ACTOR Future includeCommandActor(Reference db, std::vector tokens); // kill command ACTOR Future killCommandActor(Reference db, Reference tr, std::vector tokens, std::map>* address_interface); +// lock/unlock command +ACTOR Future lockCommandActor(Reference db, std::vector tokens); +ACTOR Future unlockDatabaseActor(Reference db, UID uid); // maintenance command ACTOR Future setHealthyZone(Reference db, StringRef zoneId, double seconds, bool printWarning = false); ACTOR Future clearHealthyZone(Reference db, @@ -149,7 +190,7 @@ ACTOR Future suspendCommandActor(Reference db, // throttle command ACTOR Future throttleCommandActor(Reference db, std::vector tokens); // triggerteaminfolog command -ACTOR Future triggerddteaminfologCommandActor(Reference db); +ACTOR Future triggerddteaminfologCommandActor(Reference db); // tssq command ACTOR Future tssqCommandActor(Reference db, std::vector tokens); diff --git a/fdbclient/BlobGranuleCommon.h b/fdbclient/BlobGranuleCommon.h index 2c08577d57..5b162a8b50 100644 --- a/fdbclient/BlobGranuleCommon.h +++ b/fdbclient/BlobGranuleCommon.h @@ -29,15 +29,17 @@ struct MutationsAndVersionRef { VectorRef mutations; Version version; + Version knownCommittedVersion; MutationsAndVersionRef() {} - explicit MutationsAndVersionRef(Version version) : version(version) {} - MutationsAndVersionRef(VectorRef mutations, Version version) - : mutations(mutations), version(version) {} - MutationsAndVersionRef(Arena& to, VectorRef mutations, Version version) - : mutations(to, mutations), version(version) {} + explicit MutationsAndVersionRef(Version version, Version knownCommittedVersion) + : version(version), knownCommittedVersion(knownCommittedVersion) {} + MutationsAndVersionRef(VectorRef mutations, Version version, Version knownCommittedVersion) + : mutations(mutations), version(version), knownCommittedVersion(knownCommittedVersion) {} + MutationsAndVersionRef(Arena& to, VectorRef mutations, Version version, Version knownCommittedVersion) + : mutations(to, mutations), version(version), knownCommittedVersion(knownCommittedVersion) {} MutationsAndVersionRef(Arena& to, const MutationsAndVersionRef& from) - : mutations(to, from.mutations), version(from.version) {} + : mutations(to, from.mutations), version(from.version), knownCommittedVersion(from.knownCommittedVersion) {} int expectedSize() const { return mutations.expectedSize(); } struct OrderByVersion { @@ -48,7 +50,7 @@ struct MutationsAndVersionRef { template void serialize(Ar& ar) { - serializer(ar, mutations, version); + serializer(ar, mutations, version, knownCommittedVersion); } }; diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 48c6033f80..8a7b69dcd5 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -132,6 +132,7 @@ set(FDBCLIENT_SRCS VersionedMap.actor.h VersionedMap.h VersionedMap.cpp + WellKnownEndpoints.h WriteMap.h json_spirit/json_spirit_error_position.h json_spirit/json_spirit_reader_template.h @@ -187,14 +188,14 @@ if(BUILD_AZURE_BACKUP) endif() add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs}) -add_dependencies(fdbclient fdboptions fdb_c_options) +add_dependencies(fdbclient fdboptions) target_link_libraries(fdbclient PUBLIC fdbrpc msgpack) # Create a separate fdbclient library with sampling enabled. This lets # fdbserver retain sampling functionality in client code while disabling # sampling for pure clients. add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs}) -add_dependencies(fdbclient_sampling fdboptions fdb_c_options) +add_dependencies(fdbclient_sampling fdboptions) target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack) target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING) if(WIN32) diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h index 833f36ea11..8e8c2abc26 100644 --- a/fdbclient/ClusterInterface.h +++ b/fdbclient/ClusterInterface.h @@ -267,7 +267,7 @@ struct StatusRequest { struct GetClientWorkersRequest { constexpr static FileIdentifier file_identifier = 10771791; - ReplyPromise> reply; + ReplyPromise> reply; GetClientWorkersRequest() {} diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index df4df118c7..ecb745d318 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -109,12 +109,12 @@ struct CommitProxyInterface { struct ClientDBInfo { constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes - vector grvProxies; - vector commitProxies; + std::vector grvProxies; + std::vector commitProxies; Optional firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk Optional forward; - vector history; + std::vector history; ClientDBInfo() {} @@ -285,7 +285,7 @@ struct GetReadVersionRequest : TimedRequest { struct GetKeyServerLocationsReply { constexpr static FileIdentifier file_identifier = 10636023; Arena arena; - std::vector>> results; + std::vector>> results; // if any storage servers in results have a TSS pair, that mapping is in here std::vector> resultsTssMapping; @@ -499,11 +499,11 @@ struct ExclusionSafetyCheckReply { struct ExclusionSafetyCheckRequest { constexpr static FileIdentifier file_identifier = 13852702; - vector exclusions; + std::vector exclusions; ReplyPromise reply; ExclusionSafetyCheckRequest() {} - explicit ExclusionSafetyCheckRequest(vector exclusions) : exclusions(exclusions) {} + explicit ExclusionSafetyCheckRequest(std::vector exclusions) : exclusions(exclusions) {} template void serialize(Ar& ar) { diff --git a/fdbclient/ConfigTransactionInterface.cpp b/fdbclient/ConfigTransactionInterface.cpp index d6c13fed48..65f403a869 100644 --- a/fdbclient/ConfigTransactionInterface.cpp +++ b/fdbclient/ConfigTransactionInterface.cpp @@ -34,10 +34,11 @@ void ConfigTransactionInterface::setupWellKnownEndpoints() { } ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote) - : getGeneration(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)), - get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)), getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)), - getKnobs(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), commit(Endpoint({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) { -} + : getGeneration(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)), + get(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GET)), + getClasses(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)), + getKnobs(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), + commit(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {} bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rhs) const { return _id == rhs._id; diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h index a2abfa87db..c1873a0b92 100644 --- a/fdbclient/CoordinationInterface.h +++ b/fdbclient/CoordinationInterface.h @@ -27,23 +27,10 @@ #include "fdbrpc/Locality.h" #include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ClusterInterface.h" +#include "fdbclient/WellKnownEndpoints.h" const int MAX_CLUSTER_FILE_BYTES = 60000; -// well known endpoints published to the client. -constexpr UID WLTOKEN_CLIENTLEADERREG_GETLEADER(-1, 2); -constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3); - -// the value of this endpoint should be stable and not change. -constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10); -constexpr UID WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE(-1, 11); - -constexpr UID WLTOKEN_CONFIGTXN_GETGENERATION(-1, 12); -constexpr UID WLTOKEN_CONFIGTXN_GET(-1, 13); -constexpr UID WLTOKEN_CONFIGTXN_GETCLASSES(-1, 14); -constexpr UID WLTOKEN_CONFIGTXN_GETKNOBS(-1, 15); -constexpr UID WLTOKEN_CONFIGTXN_COMMIT(-1, 16); - struct ClientLeaderRegInterface { RequestStream getLeader; RequestStream openDatabase; @@ -62,8 +49,8 @@ class ClusterConnectionString { public: ClusterConnectionString() {} ClusterConnectionString(std::string const& connectionString); - ClusterConnectionString(vector, Key); - vector const& coordinators() const { return coord; } + ClusterConnectionString(std::vector, Key); + std::vector const& coordinators() const { return coord; } Key clusterKey() const { return key; } Key clusterKeyName() const { return keyDesc; @@ -74,7 +61,7 @@ public: private: void parseKey(std::string const& key); - vector coord; + std::vector coord; Key key, keyDesc; }; @@ -199,7 +186,7 @@ struct OpenDatabaseCoordRequest { Standalone> supportedVersions; UID knownClientInfoID; Key clusterKey; - vector coordinators; + std::vector coordinators; ReplyPromise> reply; template @@ -210,7 +197,7 @@ struct OpenDatabaseCoordRequest { class ClientCoordinators { public: - vector clientLeaderServers; + std::vector clientLeaderServers; Key clusterKey; Reference ccf; diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index e0817fcdf5..c2cb04bb2f 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -45,6 +45,8 @@ void DatabaseConfiguration::resetInternal() { remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0; backupWorkerEnabled = false; perpetualStorageWiggleSpeed = 0; + perpetualStorageWiggleLocality = "0"; + storageMigrationType = StorageMigrationType::DEFAULT; } void parse(int* i, ValueRef const& v) { @@ -198,7 +200,9 @@ bool DatabaseConfiguration::isValid() const { (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) && (regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") && // We cannot specify regions with three_datacenter replication - (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1))) { + (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) && + isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality) && + storageMigrationType != StorageMigrationType::UNSET)) { return false; } std::set dcIds; @@ -389,6 +393,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled; result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed; + result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality; + result["storage_migration_type"] = storageMigrationType.toString(); return result; } @@ -542,6 +548,30 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { parse(®ions, value); } else if (ck == LiteralStringRef("perpetual_storage_wiggle")) { parse(&perpetualStorageWiggleSpeed, value); + } else if (ck == LiteralStringRef("perpetual_storage_wiggle_locality")) { + if (!isValidPerpetualStorageWiggleLocality(value.toString())) { + return false; + } + perpetualStorageWiggleLocality = value.toString(); + } else if (ck == LiteralStringRef("storage_migration_type")) { + parse((&type), value); + storageMigrationType = (StorageMigrationType::MigrationType)type; + } else if (ck == LiteralStringRef("proxies")) { + int proxiesCount; + parse(&proxiesCount, value); + if (proxiesCount > 1) { + int derivedGrvProxyCount = + std::max(1, + std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES, + proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1))); + int derivedCommitProxyCount = proxiesCount - derivedGrvProxyCount; + if (grvProxyCount == -1) { + grvProxyCount = derivedGrvProxyCount; + } + if (commitProxyCount == -1) { + commitProxyCount = derivedCommitProxyCount; + } + } } else { return false; } diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 1d0ceab5c3..a390907529 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -245,6 +245,10 @@ struct DatabaseConfiguration { // Perpetual Storage Setting int32_t perpetualStorageWiggleSpeed; + std::string perpetualStorageWiggleLocality; + + // Storage Migration Type + StorageMigrationType storageMigrationType; // Excluded servers (no state should be here) bool isExcludedServer(NetworkAddressList) const; diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index a9438348f9..ad2bd97bf5 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -182,10 +182,10 @@ public: std::pair> getCachedLocation(const KeyRef&, Reverse isBackward = Reverse::False); bool getCachedLocations(const KeyRangeRef&, - vector>>&, + std::vector>>&, int limit, Reverse reverse); - Reference setCachedLocation(const KeyRangeRef&, const vector&); + Reference setCachedLocation(const KeyRangeRef&, const std::vector&); void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::False); void invalidateCache(const KeyRangeRef&); @@ -398,6 +398,8 @@ public: Counter transactionsProcessBehind; Counter transactionsThrottled; Counter transactionsExpensiveClearCostEstCount; + Counter transactionGrvFullBatches; + Counter transactionGrvTimedOutBatches; ContinuousSample latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit; @@ -408,6 +410,7 @@ public: int snapshotRywEnabled; int transactionTracingEnabled; + double verifyCausalReadsProp = 0.0; Future logger; Future throttleExpirer; diff --git a/fdbclient/FDBOptions.h b/fdbclient/FDBOptions.h index e13e44a129..284d9ab0f9 100644 --- a/fdbclient/FDBOptions.h +++ b/fdbclient/FDBOptions.h @@ -42,15 +42,20 @@ struct FDBOptionInfo { // be no cumulative effects from calling multiple times). int defaultFor; + enum class ParamType { None, String, Int, Bytes }; + + ParamType paramType; + FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, - int defaultFor) + int defaultFor, + ParamType paramType) : name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), - persistent(persistent), defaultFor(defaultFor) {} + persistent(persistent), defaultFor(defaultFor), paramType(paramType) {} FDBOptionInfo() {} }; @@ -103,8 +108,9 @@ public: typename OptionList::const_iterator end() const { return options.cend(); } }; -#define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \ +#define ADD_OPTION_INFO( \ + type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor, paramType) \ type::optionInfo.insert( \ - var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor)); + var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor, paramType)); #endif diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 0a1e69ab36..17ad22b93e 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -864,6 +864,14 @@ struct StorageBytes { used / 1e6, temp / 1e6); } + + void toTraceEvent(TraceEvent& e) const { + e.detail("StorageBytesUsed", used) + .detail("StorageBytesTemp", temp) + .detail("StorageBytesTotal", total) + .detail("StorageBytesFree", free) + .detail("StorageBytesAvailable", available); + } }; struct LogMessageVersion { // Each message pushed into the log system has a unique, totally ordered LogMessageVersion @@ -1126,4 +1134,47 @@ inline const char* transactionPriorityToString(TransactionPriority priority, boo throw internal_error(); } +struct StorageMigrationType { + // These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones + // just before END. + enum MigrationType { DEFAULT = 1, UNSET = 0, DISABLED = 1, AGGRESSIVE = 2, GRADUAL = 3, END = 4 }; + + StorageMigrationType() : type(UNSET) {} + StorageMigrationType(MigrationType type) : type(type) { + if ((uint32_t)type >= END) { + this->type = UNSET; + } + } + operator MigrationType() const { return MigrationType(type); } + + template + void serialize(Ar& ar) { + serializer(ar, type); + } + + std::string toString() const { + switch (type) { + case DISABLED: + return "disabled"; + case AGGRESSIVE: + return "aggressive"; + case GRADUAL: + return "gradual"; + case UNSET: + return "unset"; + default: + ASSERT(false); + } + return ""; + } + + uint32_t type; +}; + +inline bool isValidPerpetualStorageWiggleLocality(std::string locality) { + int pos = locality.find(':'); + // locality should be either 0 or in the format ':' + return ((pos > 0 && pos < locality.size() - 1) || locality == "0"); +} + #endif diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 2f04e2c154..b42b192435 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -2743,7 +2743,7 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase { if (!backupWorkerEnabled && partitionedLog.get().present() && partitionedLog.get().get()) { // Change configuration only when we set to use partitioned logs and // the flag was not set before. - wait(success(changeConfig(cx, "backup_worker_enabled:=1", true))); + wait(success(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true))); backupWorkerEnabled = true; } diff --git a/fdbclient/GrvProxyInterface.h b/fdbclient/GrvProxyInterface.h index 85ad4d16bc..d4b3b78bcb 100644 --- a/fdbclient/GrvProxyInterface.h +++ b/fdbclient/GrvProxyInterface.h @@ -22,6 +22,9 @@ #ifndef FDBCLIENT_GRVPROXYINTERFACE_H #define FDBCLIENT_GRVPROXYINTERFACE_H #pragma once +#include "flow/FileIdentifier.h" +#include "fdbrpc/fdbrpc.h" +#include "fdbclient/FDBTypes.h" // GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it // communicates with RateKeeper to gather health information of the cluster. diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.actor.cpp index 607bc56d97..152377cb95 100644 --- a/fdbclient/KeyRangeMap.actor.cpp +++ b/fdbclient/KeyRangeMap.actor.cpp @@ -25,7 +25,7 @@ #include "fdbclient/ReadYourWrites.h" #include "flow/actorcompiler.h" // has to be last include -void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, vector& affectedRanges) { +void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector& affectedRanges) { auto s = map.rangeContaining(keys.begin); if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady()) affectedRanges.push_back(KeyRangeRef(s.begin(), keys.begin)); @@ -176,7 +176,7 @@ static Future krmSetRangeCoalescing_(Transaction* tr, state KeyRange maxWithPrefix = KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString()); - state vector> keys; + state std::vector> keys; keys.push_back( tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::True)); keys.push_back( diff --git a/fdbclient/KeyRangeMap.h b/fdbclient/KeyRangeMap.h index 7016dcfc4d..6fd9bfe5b7 100644 --- a/fdbclient/KeyRangeMap.h +++ b/fdbclient/KeyRangeMap.h @@ -111,7 +111,7 @@ public: class KeyRangeActorMap { public: - void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector& affectedRanges); + void getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector& affectedRanges); void insert(const KeyRangeRef& keys, const Future& value) { map.insert(keys, value); } void cancel(const KeyRangeRef& keys) { insert(keys, Future()); } bool liveActorAt(const KeyRef& key) { diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index f061ada4b3..cd6ed05dab 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -150,6 +150,28 @@ std::map configForToken(std::string const& mode) { } out[p + key] = value; } + if (key == "perpetual_storage_wiggle_locality") { + if (!isValidPerpetualStorageWiggleLocality(value)) { + printf("Error: perpetual_storage_wiggle_locality should be in : " + "format or enter 0 to disable the locality match for wiggling.\n"); + return out; + } + out[p + key] = value; + } + if (key == "storage_migration_type") { + StorageMigrationType type; + if (value == "disabled") { + type = StorageMigrationType::DISABLED; + } else if (value == "aggressive") { + type = StorageMigrationType::AGGRESSIVE; + } else if (value == "gradual") { + type = StorageMigrationType::GRADUAL; + } else { + printf("Error: Only disabled|aggressive|gradual are valid for storage_migration_mode.\n"); + return out; + } + out[p + key] = format("%d", type); + } return out; } @@ -413,266 +435,6 @@ ACTOR Future getDatabaseConfiguration(Database cx) { } } -ACTOR Future changeConfig(Database cx, std::map m, bool force) { - state StringRef initIdKey = LiteralStringRef("\xff/init_id"); - state Transaction tr(cx); - - if (!m.size()) { - return ConfigurationResult::NO_OPTIONS_PROVIDED; - } - - // make sure we have essential configuration options - std::string initKey = configKeysPrefix.toString() + "initialized"; - state bool creating = m.count(initKey) != 0; - state Optional locked; - { - auto iter = m.find(databaseLockedKey.toString()); - if (iter != m.end()) { - if (!creating) { - return ConfigurationResult::LOCKED_NOT_NEW; - } - locked = UID::fromString(iter->second); - m.erase(iter); - } - } - if (creating) { - m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString(); - if (!isCompleteConfiguration(m)) { - return ConfigurationResult::INCOMPLETE_CONFIGURATION; - } - } - - state Future tooLong = delay(60); - state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned()); - state bool oldReplicationUsesDcId = false; - loop { - try { - tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); - - if (!creating && !force) { - state Future fConfig = tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY); - state Future> fWorkers = getWorkers(&tr); - wait(success(fConfig) || tooLong); - - if (!fConfig.isReady()) { - return ConfigurationResult::DATABASE_UNAVAILABLE; - } - - if (fConfig.isReady()) { - ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY); - state DatabaseConfiguration oldConfig; - oldConfig.fromKeyValues((VectorRef)fConfig.get()); - state DatabaseConfiguration newConfig = oldConfig; - for (auto kv : m) { - newConfig.set(kv.first, kv.second); - } - if (!newConfig.isValid()) { - return ConfigurationResult::INVALID_CONFIGURATION; - } - - if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) { - return ConfigurationResult::REGION_REPLICATION_MISMATCH; - } - - oldReplicationUsesDcId = - oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid"); - - if (oldConfig.usableRegions != newConfig.usableRegions) { - // cannot change region configuration - std::map dcId_priority; - for (auto& it : newConfig.regions) { - dcId_priority[it.dcId] = it.priority; - } - for (auto& it : oldConfig.regions) { - if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) { - return ConfigurationResult::REGIONS_CHANGED; - } - } - - // must only have one region with priority >= 0 - int activeRegionCount = 0; - for (auto& it : newConfig.regions) { - if (it.priority >= 0) { - activeRegionCount++; - } - } - if (activeRegionCount > 1) { - return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS; - } - } - - state Future fServerList = (newConfig.regions.size()) - ? tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY) - : Future(); - - if (newConfig.usableRegions == 2) { - if (oldReplicationUsesDcId) { - state Future fLocalityList = - tr.getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY); - wait(success(fLocalityList) || tooLong); - if (!fLocalityList.isReady()) { - return ConfigurationResult::DATABASE_UNAVAILABLE; - } - RangeResult localityList = fLocalityList.get(); - ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY); - - std::set localityDcIds; - for (auto& s : localityList) { - auto dc = decodeTagLocalityListKey(s.key); - if (dc.present()) { - localityDcIds.insert(dc.get()); - } - } - - for (auto& it : newConfig.regions) { - if (localityDcIds.count(it.dcId) == 0) { - return ConfigurationResult::DCID_MISSING; - } - } - } else { - // all regions with priority >= 0 must be fully replicated - state std::vector>> replicasFutures; - for (auto& it : newConfig.regions) { - if (it.priority >= 0) { - replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(it.dcId))); - } - } - wait(waitForAll(replicasFutures) || tooLong); - - for (auto& it : replicasFutures) { - if (!it.isReady()) { - return ConfigurationResult::DATABASE_UNAVAILABLE; - } - if (!it.get().present()) { - return ConfigurationResult::REGION_NOT_FULLY_REPLICATED; - } - } - } - } - - if (newConfig.regions.size()) { - // all storage servers must be in one of the regions - wait(success(fServerList) || tooLong); - if (!fServerList.isReady()) { - return ConfigurationResult::DATABASE_UNAVAILABLE; - } - RangeResult serverList = fServerList.get(); - ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY); - - std::set newDcIds; - for (auto& it : newConfig.regions) { - newDcIds.insert(it.dcId); - } - std::set> missingDcIds; - for (auto& s : serverList) { - auto ssi = decodeServerListValue(s.value); - if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) { - missingDcIds.insert(ssi.locality.dcId()); - } - } - if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) { - return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID; - } - } - - wait(success(fWorkers) || tooLong); - if (!fWorkers.isReady()) { - return ConfigurationResult::DATABASE_UNAVAILABLE; - } - - if (newConfig.regions.size()) { - std::map, std::set>> dcId_zoneIds; - for (auto& it : fWorkers.get()) { - if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) { - dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId()); - } - } - for (auto& region : newConfig.regions) { - if (dcId_zoneIds[region.dcId].size() < - std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) { - return ConfigurationResult::NOT_ENOUGH_WORKERS; - } - if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) { - int totalSatelliteProcesses = 0; - for (auto& sat : region.satellites) { - totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size(); - } - if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) { - return ConfigurationResult::NOT_ENOUGH_WORKERS; - } - } - } - } else { - std::set> zoneIds; - for (auto& it : fWorkers.get()) { - if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) { - zoneIds.insert(it.locality.zoneId()); - } - } - if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) { - return ConfigurationResult::NOT_ENOUGH_WORKERS; - } - } - } - } - if (creating) { - tr.setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE); - tr.addReadConflictRange(singleKeyRange(initIdKey)); - } else if (m.size()) { - // might be used in an emergency transaction, so make sure it is retry-self-conflicting and - // CAUSAL_WRITE_RISKY - tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY); - tr.addReadConflictRange(singleKeyRange(m.begin()->first)); - } - - if (locked.present()) { - ASSERT(creating); - tr.atomicOp(databaseLockedKey, - BinaryWriter::toValue(locked.get(), Unversioned()) - .withPrefix(LiteralStringRef("0123456789")) - .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), - MutationRef::SetVersionstampedValue); - } - - for (auto i = m.begin(); i != m.end(); ++i) { - tr.set(StringRef(i->first), StringRef(i->second)); - } - - tr.addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey)); - tr.set(moveKeysLockOwnerKey, versionKey); - - wait(tr.commit()); - break; - } catch (Error& e) { - state Error e1(e); - if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) { - // The database now exists. Determine whether we created it or it was already existing/created by - // someone else. The latter is an error. - tr.reset(); - loop { - try { - tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); - - Optional v = wait(tr.get(initIdKey)); - if (v != m[initIdKey.toString()]) - return ConfigurationResult::DATABASE_ALREADY_CREATED; - else - return ConfigurationResult::DATABASE_CREATED; - } catch (Error& e2) { - wait(tr.onError(e2)); - } - } - } - wait(tr.onError(e1)); - } - } - return ConfigurationResult::SUCCESS; -} - ConfigureAutoResult parseConfig(StatusObject const& status) { ConfigureAutoResult result; StatusObjectReader statusObj(status); @@ -942,97 +704,7 @@ ConfigureAutoResult parseConfig(StatusObject const& status) { return result; } -ACTOR Future autoConfig(Database cx, ConfigureAutoResult conf) { - state Transaction tr(cx); - state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned()); - - if (!conf.address_class.size()) - return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type - - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); - - vector workers = wait(getWorkers(&tr)); - std::map>> address_processId; - for (auto& w : workers) { - address_processId[w.address] = w.locality.processId(); - } - - for (auto& it : conf.address_class) { - if (it.second.classSource() == ProcessClass::CommandLineSource) { - tr.clear(processClassKeyFor(address_processId[it.first].get())); - } else { - tr.set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second)); - } - } - - if (conf.address_class.size()) - tr.set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString()); - - if (conf.auto_logs != conf.old_logs) - tr.set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs)); - - if (conf.auto_commit_proxies != conf.old_commit_proxies) - tr.set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies)); - - if (conf.auto_grv_proxies != conf.old_grv_proxies) - tr.set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies)); - - if (conf.auto_resolvers != conf.old_resolvers) - tr.set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers)); - - if (conf.auto_replication != conf.old_replication) { - std::vector modes; - modes.push_back(conf.auto_replication); - std::map m; - auto r = buildConfiguration(modes, m); - if (r != ConfigurationResult::SUCCESS) - return r; - - for (auto& kv : m) - tr.set(kv.first, kv.second); - } - - tr.addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey)); - tr.set(moveKeysLockOwnerKey, versionKey); - - wait(tr.commit()); - return ConfigurationResult::SUCCESS; - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -Future changeConfig(Database const& cx, - std::vector const& modes, - Optional const& conf, - bool force) { - if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) { - return autoConfig(cx, conf.get()); - } - - std::map m; - auto r = buildConfiguration(modes, m); - if (r != ConfigurationResult::SUCCESS) - return r; - return changeConfig(cx, m, force); -} - -Future changeConfig(Database const& cx, std::string const& modes, bool force) { - TraceEvent("ChangeConfig").detail("Mode", modes); - std::map m; - auto r = buildConfiguration(modes, m); - if (r != ConfigurationResult::SUCCESS) - return r; - return changeConfig(cx, m, force); -} - -ACTOR Future> getWorkers(Transaction* tr) { +ACTOR Future> getWorkers(Transaction* tr) { state Future processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); state Future processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); @@ -1063,14 +735,14 @@ ACTOR Future> getWorkers(Transaction* tr) { return results; } -ACTOR Future> getWorkers(Database cx) { +ACTOR Future> getWorkers(Database cx) { state Transaction tr(cx); loop { try { tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary? tr.setOption(FDBTransactionOptions::LOCK_AWARE); - vector workers = wait(getWorkers(&tr)); + std::vector workers = wait(getWorkers(&tr)); return workers; } catch (Error& e) { wait(tr.onError(e)); @@ -1148,7 +820,7 @@ ACTOR Future> changeQuorumChecker(Transaction* tr, } } - vector>> leaderServers; + std::vector>> leaderServers; ClientCoordinators coord(Reference(new ClusterConnectionFile(conn))); leaderServers.reserve(coord.clientLeaderServers.size()); @@ -1234,7 +906,7 @@ ACTOR Future changeQuorum(Database cx, Reference>> leaderServers; + state std::vector>> leaderServers; state ClientCoordinators coord(Reference(new ClusterConnectionFile(conn))); // check if allowed to modify the cluster descriptor if (!change->getDesiredClusterKeyName().empty()) { @@ -1266,24 +938,24 @@ ACTOR Future changeQuorum(Database cx, Reference desired; - explicit SpecifiedQuorumChange(vector const& desired) : desired(desired) {} - Future> getDesiredCoordinators(Transaction* tr, - vector oldCoordinators, - Reference, - CoordinatorsResult&) override { + std::vector desired; + explicit SpecifiedQuorumChange(std::vector const& desired) : desired(desired) {} + Future> getDesiredCoordinators(Transaction* tr, + std::vector oldCoordinators, + Reference, + CoordinatorsResult&) override { return desired; } }; -Reference specifiedQuorumChange(vector const& addresses) { +Reference specifiedQuorumChange(std::vector const& addresses) { return Reference(new SpecifiedQuorumChange(addresses)); } struct NoQuorumChange final : IQuorumChange { - Future> getDesiredCoordinators(Transaction* tr, - vector oldCoordinators, - Reference, - CoordinatorsResult&) override { + Future> getDesiredCoordinators(Transaction* tr, + std::vector oldCoordinators, + Reference, + CoordinatorsResult&) override { return oldCoordinators; } }; @@ -1296,10 +968,10 @@ struct NameQuorumChange final : IQuorumChange { Reference otherChange; explicit NameQuorumChange(std::string const& newName, Reference const& otherChange) : newName(newName), otherChange(otherChange) {} - Future> getDesiredCoordinators(Transaction* tr, - vector oldCoordinators, - Reference cf, - CoordinatorsResult& t) override { + Future> getDesiredCoordinators(Transaction* tr, + std::vector oldCoordinators, + Reference cf, + CoordinatorsResult& t) override { return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t); } std::string getDesiredClusterKeyName() const override { return newName; } @@ -1312,10 +984,10 @@ struct AutoQuorumChange final : IQuorumChange { int desired; explicit AutoQuorumChange(int desired) : desired(desired) {} - Future> getDesiredCoordinators(Transaction* tr, - vector oldCoordinators, - Reference ccf, - CoordinatorsResult& err) override { + Future> getDesiredCoordinators(Transaction* tr, + std::vector oldCoordinators, + Reference ccf, + CoordinatorsResult& err) override { return getDesired(Reference::addRef(this), tr, oldCoordinators, ccf, &err); } @@ -1333,7 +1005,7 @@ struct AutoQuorumChange final : IQuorumChange { ACTOR static Future isAcceptable(AutoQuorumChange* self, Transaction* tr, - vector oldCoordinators, + std::vector oldCoordinators, Reference ccf, int desiredCount, std::set* excluded) { @@ -1345,14 +1017,14 @@ struct AutoQuorumChange final : IQuorumChange { // Check availability ClientCoordinators coord(ccf); - vector>> leaderServers; + std::vector>> leaderServers; leaderServers.reserve(coord.clientLeaderServers.size()); for (int i = 0; i < coord.clientLeaderServers.size(); i++) { leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader, GetLeaderRequest(coord.clusterKey, UID()), TaskPriority::CoordinationReply)); } - Optional>> results = + Optional>> results = wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY)); if (!results.present()) { return false; @@ -1379,11 +1051,11 @@ struct AutoQuorumChange final : IQuorumChange { return true; // The status quo seems fine } - ACTOR static Future> getDesired(Reference self, - Transaction* tr, - vector oldCoordinators, - Reference ccf, - CoordinatorsResult* err) { + ACTOR static Future> getDesired(Reference self, + Transaction* tr, + std::vector oldCoordinators, + Reference ccf, + CoordinatorsResult* err) { state int desiredCount = self->desired; if (desiredCount == -1) { @@ -1394,8 +1066,8 @@ struct AutoQuorumChange final : IQuorumChange { std::vector excl = wait(getExcludedServers(tr)); state std::set excluded(excl.begin(), excl.end()); - vector _workers = wait(getWorkers(tr)); - state vector workers = _workers; + std::vector _workers = wait(getWorkers(tr)); + state std::vector workers = _workers; std::map addr_locality; for (auto w : workers) @@ -1431,7 +1103,7 @@ struct AutoQuorumChange final : IQuorumChange { .detail("DesiredCoordinators", desiredCount) .detail("CurrentCoordinators", oldCoordinators.size()); *err = CoordinatorsResult::NOT_ENOUGH_MACHINES; - return vector(); + return std::vector(); } chosen.resize((chosen.size() - 1) | 1); } @@ -1443,11 +1115,11 @@ struct AutoQuorumChange final : IQuorumChange { // (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and // (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute // workers. - void addDesiredWorkers(vector& chosen, - const vector& workers, + void addDesiredWorkers(std::vector& chosen, + const std::vector& workers, int desiredCount, const std::set& excluded) { - vector remainingWorkers(workers); + std::vector remainingWorkers(workers); deterministicRandom()->randomShuffle(remainingWorkers); std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) { @@ -1470,10 +1142,10 @@ struct AutoQuorumChange final : IQuorumChange { std::map> currentCounts; std::map hardLimits; - vector fields({ LiteralStringRef("dcid"), - LiteralStringRef("data_hall"), - LiteralStringRef("zoneid"), - LiteralStringRef("machineid") }); + std::vector fields({ LiteralStringRef("dcid"), + LiteralStringRef("data_hall"), + LiteralStringRef("zoneid"), + LiteralStringRef("machineid") }); for (auto field = fields.begin(); field != fields.end(); field++) { if (field->toString() == "zoneid") { @@ -1537,7 +1209,7 @@ Reference autoQuorumChange(int desired) { return Reference(new AutoQuorumChange(desired)); } -void excludeServers(Transaction& tr, vector& servers, bool failed) { +void excludeServers(Transaction& tr, std::vector& servers, bool failed) { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr.setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1556,7 +1228,7 @@ void excludeServers(Transaction& tr, vector& servers, bool fai TraceEvent("ExcludeServersCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed); } -ACTOR Future excludeServers(Database cx, vector servers, bool failed) { +ACTOR Future excludeServers(Database cx, std::vector servers, bool failed) { if (cx->apiVersionAtLeast(700)) { state ReadYourWritesTransaction ryw(cx); loop { @@ -1659,7 +1331,7 @@ ACTOR Future excludeLocalities(Database cx, std::unordered_set includeServers(Database cx, vector servers, bool failed) { +ACTOR Future includeServers(Database cx, std::vector servers, bool failed) { state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); if (cx->apiVersionAtLeast(700)) { state ReadYourWritesTransaction ryw(cx); @@ -1762,7 +1434,7 @@ ACTOR Future includeServers(Database cx, vector servers, // Remove the given localities from the exclusion list. // include localities by clearing the keys. -ACTOR Future includeLocalities(Database cx, vector localities, bool failed, bool includeAll) { +ACTOR Future includeLocalities(Database cx, std::vector localities, bool failed, bool includeAll) { state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); if (cx->apiVersionAtLeast(700)) { state ReadYourWritesTransaction ryw(cx); @@ -1856,7 +1528,7 @@ ACTOR Future setClass(Database cx, AddressExclusion server, ProcessClass p tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); - vector workers = wait(getWorkers(&tr)); + std::vector workers = wait(getWorkers(&tr)); bool foundChange = false; for (int i = 0; i < workers.size(); i++) { @@ -1881,13 +1553,13 @@ ACTOR Future setClass(Database cx, AddressExclusion server, ProcessClass p } } -ACTOR Future> getExcludedServers(Transaction* tr) { +ACTOR Future> getExcludedServers(Transaction* tr) { state RangeResult r = wait(tr->getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY)); ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); state RangeResult r2 = wait(tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY)); ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY); - vector exclusions; + std::vector exclusions; for (auto i = r.begin(); i != r.end(); ++i) { auto a = decodeExcludedServersKey(i->key); if (a.isValid()) @@ -1902,14 +1574,14 @@ ACTOR Future> getExcludedServers(Transaction* tr) { return exclusions; } -ACTOR Future> getExcludedServers(Database cx) { +ACTOR Future> getExcludedServers(Database cx) { state Transaction tr(cx); loop { try { tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary? tr.setOption(FDBTransactionOptions::LOCK_AWARE); - vector exclusions = wait(getExcludedServers(&tr)); + std::vector exclusions = wait(getExcludedServers(&tr)); return exclusions; } catch (Error& e) { wait(tr.onError(e)); @@ -1918,13 +1590,13 @@ ACTOR Future> getExcludedServers(Database cx) { } // Get the current list of excluded localities by reading the keys. -ACTOR Future> getExcludedLocalities(Transaction* tr) { +ACTOR Future> getExcludedLocalities(Transaction* tr) { state RangeResult r = wait(tr->getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY)); ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); state RangeResult r2 = wait(tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY)); ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY); - vector excludedLocalities; + std::vector excludedLocalities; for (const auto& i : r) { auto a = decodeExcludedLocalityKey(i.key); excludedLocalities.push_back(a); @@ -1938,14 +1610,14 @@ ACTOR Future> getExcludedLocalities(Transaction* tr) { } // Get the list of excluded localities by reading the keys. -ACTOR Future> getExcludedLocalities(Database cx) { +ACTOR Future> getExcludedLocalities(Database cx) { state Transaction tr(cx); loop { try { tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr.setOption(FDBTransactionOptions::LOCK_AWARE); - vector exclusions = wait(getExcludedLocalities(&tr)); + std::vector exclusions = wait(getExcludedLocalities(&tr)); return exclusions; } catch (Error& e) { wait(tr.onError(e)); @@ -2175,7 +1847,7 @@ ACTOR Future checkForExcludingServersTxActor(ReadYourWritesTransaction* tr } ACTOR Future> checkForExcludingServers(Database cx, - vector excl, + std::vector excl, bool waitForAllExcluded) { state std::set exclusions(excl.begin(), excl.end()); state std::set inProgressExclusion; @@ -2626,6 +2298,40 @@ bool schemaMatch(json_spirit::mValue const& schemaValue, } } +std::string ManagementAPI::generateErrorMessage(const CoordinatorsResult& res) { + // Note: the error message here should not be changed if possible + // If you do change the message here, + // please update the corresponding fdbcli code to support both the old and the new message + + std::string msg; + switch (res) { + case CoordinatorsResult::INVALID_NETWORK_ADDRESSES: + msg = "The specified network addresses are invalid"; + break; + case CoordinatorsResult::SAME_NETWORK_ADDRESSES: + msg = "No change (existing configuration satisfies request)"; + break; + case CoordinatorsResult::NOT_COORDINATORS: + msg = "Coordination servers are not running on the specified network addresses"; + break; + case CoordinatorsResult::DATABASE_UNREACHABLE: + msg = "Database unreachable"; + break; + case CoordinatorsResult::BAD_DATABASE_STATE: + msg = "The database is in an unexpected state from which changing coordinators might be unsafe"; + break; + case CoordinatorsResult::COORDINATOR_UNREACHABLE: + msg = "One of the specified coordinators is unreachable"; + break; + case CoordinatorsResult::NOT_ENOUGH_MACHINES: + msg = "Too few fdbserver machines to provide coordination at the current redundancy level"; + break; + default: + break; + } + return msg; +} + TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") { wait(Future(Void())); diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h index 281e2df74f..021dfde2bf 100644 --- a/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/ManagementAPI.actor.h @@ -49,6 +49,7 @@ enum class ConfigurationResult { UNKNOWN_OPTION, INCOMPLETE_CONFIGURATION, INVALID_CONFIGURATION, + STORAGE_MIGRATION_DISABLED, DATABASE_ALREADY_CREATED, DATABASE_CREATED, DATABASE_UNAVAILABLE, @@ -60,6 +61,7 @@ enum class ConfigurationResult { REGION_REPLICATION_MISMATCH, DCID_MISSING, LOCKED_NOT_NEW, + SUCCESS_WARN_PPW_GRADUAL, SUCCESS, }; @@ -119,31 +121,17 @@ ConfigurationResult buildConfiguration( bool isCompleteConfiguration(std::map const& options); -// All versions of changeConfig apply the given set of configuration tokens to the database, and return a -// ConfigurationResult (or error). -Future changeConfig(Database const& cx, - std::string const& configMode, - bool force); // Accepts tokens separated by spaces in a single string - ConfigureAutoResult parseConfig(StatusObject const& status); -Future changeConfig(Database const& cx, - std::vector const& modes, - Optional const& conf, - bool force); // Accepts a vector of configuration tokens -ACTOR Future changeConfig( - Database cx, - std::map m, - bool force); // Accepts a full configuration in key/value format (from buildConfiguration) ACTOR Future getDatabaseConfiguration(Database cx); ACTOR Future waitForFullReplication(Database cx); struct IQuorumChange : ReferenceCounted { virtual ~IQuorumChange() {} - virtual Future> getDesiredCoordinators(Transaction* tr, - vector oldCoordinators, - Reference, - CoordinatorsResult&) = 0; + virtual Future> getDesiredCoordinators(Transaction* tr, + std::vector oldCoordinators, + Reference, + CoordinatorsResult&) = 0; virtual std::string getDesiredClusterKeyName() const { return std::string(); } }; @@ -154,14 +142,14 @@ ACTOR Future> changeQuorumChecker(Transaction* tr, ACTOR Future changeQuorum(Database cx, Reference change); Reference autoQuorumChange(int desired = -1); Reference noQuorumChange(); -Reference specifiedQuorumChange(vector const&); +Reference specifiedQuorumChange(std::vector const&); Reference nameQuorumChange(std::string const& name, Reference const& other); // Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without // necessarily waiting for the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the // given IP. -ACTOR Future excludeServers(Database cx, vector servers, bool failed = false); -void excludeServers(Transaction& tr, vector& servers, bool failed = false); +ACTOR Future excludeServers(Database cx, std::vector servers, bool failed = false); +void excludeServers(Transaction& tr, std::vector& servers, bool failed = false); // Exclude the servers matching the given set of localities from use as state servers. Returns as soon as the change // is durable, without necessarily waiting for the servers to be evacuated. @@ -170,11 +158,11 @@ void excludeLocalities(Transaction& tr, std::unordered_set localiti // Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given // IP. A NetworkAddress() means all servers (don't exclude anything) -ACTOR Future includeServers(Database cx, vector servers, bool failed = false); +ACTOR Future includeServers(Database cx, std::vector servers, bool failed = false); // Remove the given localities from the exclusion list. ACTOR Future includeLocalities(Database cx, - vector localities, + std::vector localities, bool failed = false, bool includeAll = false); @@ -183,12 +171,12 @@ ACTOR Future includeLocalities(Database cx, ACTOR Future setClass(Database cx, AddressExclusion server, ProcessClass processClass); // Get the current list of excluded servers -ACTOR Future> getExcludedServers(Database cx); -ACTOR Future> getExcludedServers(Transaction* tr); +ACTOR Future> getExcludedServers(Database cx); +ACTOR Future> getExcludedServers(Transaction* tr); // Get the current list of excluded localities -ACTOR Future> getExcludedLocalities(Database cx); -ACTOR Future> getExcludedLocalities(Transaction* tr); +ACTOR Future> getExcludedLocalities(Database cx); +ACTOR Future> getExcludedLocalities(Transaction* tr); std::set getAddressesByLocality(const std::vector& workers, const std::string& locality); @@ -196,15 +184,15 @@ std::set getAddressesByLocality(const std::vector // true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and // unless any of them are explicitly included with includeServers() ACTOR Future> checkForExcludingServers(Database cx, - vector servers, + std::vector servers, bool waitForAllExcluded); ACTOR Future checkForExcludingServersTxActor(ReadYourWritesTransaction* tr, std::set* exclusions, std::set* inProgressExclusion); // Gets a list of all workers in the cluster (excluding testers) -ACTOR Future> getWorkers(Database cx); -ACTOR Future> getWorkers(Transaction* tr); +ACTOR Future> getWorkers(Database cx); +ACTOR Future> getWorkers(Transaction* tr); ACTOR Future timeKeeperSetDisable(Database cx); @@ -322,6 +310,436 @@ Future removeCachedRange(Reference db, KeyRangeRef range) { return changeCachedRange(db, range, false); } +ACTOR template +Future> getWorkers(Reference tr, + typename Tr::template FutureT processClassesF, + typename Tr::template FutureT processDataF) { + // processClassesF and processDataF are used to hold standalone memory + processClassesF = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); + processDataF = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); + state Future processClasses = safeThreadFutureToFuture(processClassesF); + state Future processData = safeThreadFutureToFuture(processDataF); + + wait(success(processClasses) && success(processData)); + ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); + + std::map>, ProcessClass> id_class; + for (int i = 0; i < processClasses.get().size(); i++) { + id_class[decodeProcessClassKey(processClasses.get()[i].key)] = + decodeProcessClassValue(processClasses.get()[i].value); + } + + std::vector results; + + for (int i = 0; i < processData.get().size(); i++) { + ProcessData data = decodeWorkerListValue(processData.get()[i].value); + ProcessClass processClass = id_class[data.locality.processId()]; + + if (processClass.classSource() == ProcessClass::DBSource || + data.processClass.classType() == ProcessClass::UnsetClass) + data.processClass = processClass; + + if (data.processClass.classType() != ProcessClass::TesterClass) + results.push_back(data); + } + + return results; +} + +// All versions of changeConfig apply the given set of configuration tokens to the database, and return a +// ConfigurationResult (or error). + +// Accepts a full configuration in key/value format (from buildConfiguration) +ACTOR template +Future changeConfig(Reference db, std::map m, bool force) { + state StringRef initIdKey = LiteralStringRef("\xff/init_id"); + state Reference tr = db->createTransaction(); + + if (!m.size()) { + return ConfigurationResult::NO_OPTIONS_PROVIDED; + } + + // make sure we have essential configuration options + std::string initKey = configKeysPrefix.toString() + "initialized"; + state bool creating = m.count(initKey) != 0; + state Optional locked; + { + auto iter = m.find(databaseLockedKey.toString()); + if (iter != m.end()) { + if (!creating) { + return ConfigurationResult::LOCKED_NOT_NEW; + } + locked = UID::fromString(iter->second); + m.erase(iter); + } + } + if (creating) { + m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString(); + if (!isCompleteConfiguration(m)) { + return ConfigurationResult::INCOMPLETE_CONFIGURATION; + } + } + + state Future tooLong = delay(60); + state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned()); + state bool oldReplicationUsesDcId = false; + state bool warnPPWGradual = false; + state bool warnChangeStorageNoMigrate = false; + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); + + if (!creating && !force) { + state typename DB::TransactionT::template FutureT fConfigF = + tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY); + state Future fConfig = safeThreadFutureToFuture(fConfigF); + state typename DB::TransactionT::template FutureT processClassesF; + state typename DB::TransactionT::template FutureT processDataF; + state Future> fWorkers = getWorkers(tr, processClassesF, processDataF); + wait(success(fConfig) || tooLong); + + if (!fConfig.isReady()) { + return ConfigurationResult::DATABASE_UNAVAILABLE; + } + + if (fConfig.isReady()) { + ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY); + state DatabaseConfiguration oldConfig; + oldConfig.fromKeyValues((VectorRef)fConfig.get()); + state DatabaseConfiguration newConfig = oldConfig; + for (auto kv : m) { + newConfig.set(kv.first, kv.second); + } + if (!newConfig.isValid()) { + return ConfigurationResult::INVALID_CONFIGURATION; + } + + if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) { + return ConfigurationResult::REGION_REPLICATION_MISMATCH; + } + + oldReplicationUsesDcId = + oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid"); + + if (oldConfig.usableRegions != newConfig.usableRegions) { + // cannot change region configuration + std::map dcId_priority; + for (auto& it : newConfig.regions) { + dcId_priority[it.dcId] = it.priority; + } + for (auto& it : oldConfig.regions) { + if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) { + return ConfigurationResult::REGIONS_CHANGED; + } + } + + // must only have one region with priority >= 0 + int activeRegionCount = 0; + for (auto& it : newConfig.regions) { + if (it.priority >= 0) { + activeRegionCount++; + } + } + if (activeRegionCount > 1) { + return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS; + } + } + + state typename DB::TransactionT::template FutureT fServerListF = + tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY); + state Future fServerList = + (newConfig.regions.size()) ? safeThreadFutureToFuture(fServerListF) : Future(); + + if (newConfig.usableRegions == 2) { + if (oldReplicationUsesDcId) { + state typename DB::TransactionT::template FutureT fLocalityListF = + tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY); + state Future fLocalityList = safeThreadFutureToFuture(fLocalityListF); + wait(success(fLocalityList) || tooLong); + if (!fLocalityList.isReady()) { + return ConfigurationResult::DATABASE_UNAVAILABLE; + } + RangeResult localityList = fLocalityList.get(); + ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY); + + std::set localityDcIds; + for (auto& s : localityList) { + auto dc = decodeTagLocalityListKey(s.key); + if (dc.present()) { + localityDcIds.insert(dc.get()); + } + } + + for (auto& it : newConfig.regions) { + if (localityDcIds.count(it.dcId) == 0) { + return ConfigurationResult::DCID_MISSING; + } + } + } else { + // all regions with priority >= 0 must be fully replicated + state std::vector>> + replicasFuturesF; + state std::vector>> replicasFutures; + for (auto& it : newConfig.regions) { + if (it.priority >= 0) { + replicasFuturesF.push_back(tr->get(datacenterReplicasKeyFor(it.dcId))); + replicasFutures.push_back(safeThreadFutureToFuture(replicasFuturesF.back())); + } + } + wait(waitForAll(replicasFutures) || tooLong); + + for (auto& it : replicasFutures) { + if (!it.isReady()) { + return ConfigurationResult::DATABASE_UNAVAILABLE; + } + if (!it.get().present()) { + return ConfigurationResult::REGION_NOT_FULLY_REPLICATED; + } + } + } + } + + if (newConfig.regions.size()) { + // all storage servers must be in one of the regions + wait(success(fServerList) || tooLong); + if (!fServerList.isReady()) { + return ConfigurationResult::DATABASE_UNAVAILABLE; + } + RangeResult serverList = fServerList.get(); + ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY); + + std::set newDcIds; + for (auto& it : newConfig.regions) { + newDcIds.insert(it.dcId); + } + std::set> missingDcIds; + for (auto& s : serverList) { + auto ssi = decodeServerListValue(s.value); + if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) { + missingDcIds.insert(ssi.locality.dcId()); + } + } + if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) { + return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID; + } + } + + wait(success(fWorkers) || tooLong); + if (!fWorkers.isReady()) { + return ConfigurationResult::DATABASE_UNAVAILABLE; + } + + if (newConfig.regions.size()) { + std::map, std::set>> dcId_zoneIds; + for (auto& it : fWorkers.get()) { + if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) { + dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId()); + } + } + for (auto& region : newConfig.regions) { + if (dcId_zoneIds[region.dcId].size() < + std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) { + return ConfigurationResult::NOT_ENOUGH_WORKERS; + } + if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) { + int totalSatelliteProcesses = 0; + for (auto& sat : region.satellites) { + totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size(); + } + if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) { + return ConfigurationResult::NOT_ENOUGH_WORKERS; + } + } + } + } else { + std::set> zoneIds; + for (auto& it : fWorkers.get()) { + if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) { + zoneIds.insert(it.locality.zoneId()); + } + } + if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) { + return ConfigurationResult::NOT_ENOUGH_WORKERS; + } + } + + if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType && + newConfig.storageMigrationType == StorageMigrationType::DISABLED) { + return ConfigurationResult::STORAGE_MIGRATION_DISABLED; + } else if (newConfig.storageMigrationType == StorageMigrationType::GRADUAL && + newConfig.perpetualStorageWiggleSpeed == 0) { + warnPPWGradual = true; + } + } + } + if (creating) { + tr->setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE); + tr->addReadConflictRange(singleKeyRange(initIdKey)); + } else if (m.size()) { + // might be used in an emergency transaction, so make sure it is retry-self-conflicting and + // CAUSAL_WRITE_RISKY + tr->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY); + tr->addReadConflictRange(singleKeyRange(m.begin()->first)); + } + + if (locked.present()) { + ASSERT(creating); + tr->atomicOp(databaseLockedKey, + BinaryWriter::toValue(locked.get(), Unversioned()) + .withPrefix(LiteralStringRef("0123456789")) + .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), + MutationRef::SetVersionstampedValue); + } + + for (auto i = m.begin(); i != m.end(); ++i) { + tr->set(StringRef(i->first), StringRef(i->second)); + } + + tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey)); + tr->set(moveKeysLockOwnerKey, versionKey); + + wait(safeThreadFutureToFuture(tr->commit())); + break; + } catch (Error& e) { + state Error e1(e); + if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) { + // The database now exists. Determine whether we created it or it was already existing/created by + // someone else. The latter is an error. + tr->reset(); + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); + + state typename DB::TransactionT::template FutureT> vF = tr->get(initIdKey); + Optional v = wait(safeThreadFutureToFuture(vF)); + if (v != m[initIdKey.toString()]) + return ConfigurationResult::DATABASE_ALREADY_CREATED; + else + return ConfigurationResult::DATABASE_CREATED; + } catch (Error& e2) { + wait(safeThreadFutureToFuture(tr->onError(e2))); + } + } + } + wait(safeThreadFutureToFuture(tr->onError(e1))); + } + } + + if (warnPPWGradual) { + return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL; + } else { + return ConfigurationResult::SUCCESS; + } +} + +ACTOR template +Future autoConfig(Reference db, ConfigureAutoResult conf) { + state Reference tr = db->createTransaction(); + state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned()); + + if (!conf.address_class.size()) + return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type + + loop { + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); + + state typename DB::TransactionT::template FutureT processClassesF; + state typename DB::TransactionT::template FutureT processDataF; + std::vector workers = wait(getWorkers(tr, processClassesF, processDataF)); + std::map>> address_processId; + for (auto& w : workers) { + address_processId[w.address] = w.locality.processId(); + } + + for (auto& it : conf.address_class) { + if (it.second.classSource() == ProcessClass::CommandLineSource) { + tr->clear(processClassKeyFor(address_processId[it.first].get())); + } else { + tr->set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second)); + } + } + + if (conf.address_class.size()) + tr->set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString()); + + if (conf.auto_logs != conf.old_logs) + tr->set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs)); + + if (conf.auto_commit_proxies != conf.old_commit_proxies) + tr->set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies)); + + if (conf.auto_grv_proxies != conf.old_grv_proxies) + tr->set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies)); + + if (conf.auto_resolvers != conf.old_resolvers) + tr->set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers)); + + if (conf.auto_replication != conf.old_replication) { + std::vector modes; + modes.push_back(conf.auto_replication); + std::map m; + auto r = buildConfiguration(modes, m); + if (r != ConfigurationResult::SUCCESS) + return r; + + for (auto& kv : m) + tr->set(kv.first, kv.second); + } + + tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey)); + tr->set(moveKeysLockOwnerKey, versionKey); + + wait(safeThreadFutureToFuture(tr->commit())); + return ConfigurationResult::SUCCESS; + } catch (Error& e) { + wait(safeThreadFutureToFuture(tr->onError(e))); + } + } +} + +// Accepts tokens separated by spaces in a single string +template +Future changeConfig(Reference db, std::string const& modes, bool force) { + TraceEvent("ChangeConfig").detail("Mode", modes); + std::map m; + auto r = buildConfiguration(modes, m); + if (r != ConfigurationResult::SUCCESS) + return r; + return changeConfig(db, m, force); +} + +// Accepts a vector of configuration tokens +template +Future changeConfig(Reference db, + std::vector const& modes, + Optional const& conf, + bool force) { + if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) { + return autoConfig(db, conf.get()); + } + + std::map m; + auto r = buildConfiguration(modes, m); + if (r != ConfigurationResult::SUCCESS) + return r; + return changeConfig(db, m, force); +} + +// return the corresponding error message for the CoordinatorsResult +// used by special keys and fdbcli +std::string generateErrorMessage(const CoordinatorsResult& res); + } // namespace ManagementAPI #include "flow/unactorcompiler.h" diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index f4ff31a8b4..7047935b0c 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -328,7 +328,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") { return Void(); } -ClusterConnectionString::ClusterConnectionString(vector servers, Key key) : coord(servers) { +ClusterConnectionString::ClusterConnectionString(std::vector servers, Key key) : coord(servers) { parseKey(key.toString()); } @@ -383,9 +383,9 @@ ClientCoordinators::ClientCoordinators(Key clusterKey, std::vector monitorNominee(Key key, ClientLeaderRegInterface coord, AsyncTrigger* nomineeChange, @@ -428,13 +427,13 @@ ACTOR Future monitorNominee(Key key, // Also used in fdbserver/LeaderElection.actor.cpp! // bool represents if the LeaderInfo is a majority answer or not. // This function also masks the first 7 bits of changeId of the nominees and returns the Leader with masked changeId -Optional> getLeader(const vector>& nominees) { +Optional> getLeader(const std::vector>& nominees) { // If any coordinator says that the quorum is forwarded, then it is for (int i = 0; i < nominees.size(); i++) if (nominees[i].present() && nominees[i].get().forward) return std::pair(nominees[i].get(), true); - vector> maskedNominees; + std::vector> maskedNominees; maskedNominees.reserve(nominees.size()); for (int i = 0; i < nominees.size(); i++) { if (nominees[i].present()) { @@ -529,18 +528,6 @@ ACTOR Future monitorLeaderOneGeneration(Reference monitorLeaderRemotelyInternal(Reference const& connFile, - Reference> const& outSerializedLeaderInfo); - -template -Future monitorLeaderRemotely(Reference const& connFile, - Reference>> const& outKnownLeader) { - LeaderDeserializer deserializer; - auto serializedInfo = makeReference>(); - Future m = monitorLeaderRemotelyInternal(connFile, serializedInfo); - return m || deserializer(serializedInfo, outKnownLeader); -} - ACTOR Future monitorLeaderInternal(Reference connFile, Reference> outSerializedLeaderInfo) { state MonitorLeaderInfo info(connFile); @@ -656,7 +643,7 @@ ACTOR Future getClientInfoFromLeader(Referenceget().get().clientInterface.openDatabase.getReply(req)))) { - TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()) + TraceEvent("GetClientInfoFromLeaderGotClientInfo", knownLeader->get().get().clientInterface.id()) .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID()) .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) .detail("ClientID", ni.id); @@ -667,11 +654,11 @@ ACTOR Future getClientInfoFromLeader(Reference monitorLeaderForProxies(Key clusterKey, - vector coordinators, - ClientData* clientData, - Reference>> leaderInfo) { - state vector clientLeaderServers; +ACTOR Future monitorLeaderAndGetClientInfo(Key clusterKey, + std::vector coordinators, + ClientData* clientData, + Reference>> leaderInfo) { + state std::vector clientLeaderServers; state AsyncTrigger nomineeChange; state std::vector> nominees; state Future allActors; @@ -695,7 +682,7 @@ ACTOR Future monitorLeaderForProxies(Key clusterKey, loop { Optional> leader = getLeader(nominees); - TraceEvent("MonitorLeaderForProxiesChange") + TraceEvent("MonitorLeaderAndGetClientInfoLeaderChange") .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1)) .detail("Key", clusterKey.printable()); if (leader.present()) { @@ -705,7 +692,7 @@ ACTOR Future monitorLeaderForProxies(Key clusterKey, outInfo.forward = leader.get().first.serializedInfo; clientData->clientInfo->set(CachedSerialization(outInfo)); leaderInfo->set(leader.get().first); - TraceEvent("MonitorLeaderForProxiesForwarding") + TraceEvent("MonitorLeaderAndGetClientInfoForwarding") .detail("NewConnStr", leader.get().first.serializedInfo.toString()); return Void(); } @@ -762,7 +749,6 @@ void shrinkProxyList(ClientDBInfo& ni, } } -// Leader is the process that will be elected by coordinators as the cluster controller ACTOR Future monitorProxiesOneGeneration( Reference connFile, Reference> clientInfo, @@ -771,9 +757,9 @@ ACTOR Future monitorProxiesOneGeneration( Reference>>> supportedVersions, Key traceLogGroup) { state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString(); - state vector addrs = cs.coordinators(); + state std::vector addrs = cs.coordinators(); state int idx = 0; - state int successIdx = 0; + state int successIndex = 0; state Optional incorrectTime; state std::vector lastCommitProxyUIDs; state std::vector lastCommitProxies; @@ -840,11 +826,11 @@ ACTOR Future monitorProxiesOneGeneration( auto& ni = rep.get().mutate(); shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies); clientInfo->set(ni); - successIdx = idx; + successIndex = idx; } else { TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller idx = (idx + 1) % addrs.size(); - if (idx == successIdx) { + if (idx == successIndex) { wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY)); } } diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index f57e1ccb4f..b9c58de422 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -61,17 +61,23 @@ struct MonitorLeaderInfo { : hasConnected(false), intermediateConnFile(intermediateConnFile) {} }; -// Monitors the given coordination group's leader election process and provides a best current guess -// of the current leader. If a leader is elected for long enough and communication with a quorum of -// coordinators is possible, eventually outKnownLeader will be that leader's interface. +Optional> getLeader(const std::vector>& nominees); + +// This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect +// nominees, the nominee with the most nomination is the leader. This function also monitors the change of the leader. +// If a leader is elected for long enough and communication with a quorum of coordinators is possible, eventually +// outKnownLeader will be that leader's interface. template Future monitorLeader(Reference const& connFile, Reference>> const& outKnownLeader); -Future monitorLeaderForProxies(Value const& key, - vector const& coordinators, - ClientData* const& clientData, - Reference>> const& leaderInfo); +// This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect +// nominees, the nominee with the most nomination is the leader, and collects client data from the leader. This function +// also monitors the change of the leader. +Future monitorLeaderAndGetClientInfo(Value const& key, + std::vector const& coordinators, + ClientData* const& clientData, + Reference>> const& leaderInfo); Future monitorProxies( Reference>> const& connFile, diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 61425401c2..c0a4317b93 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -606,7 +606,7 @@ void DLApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParame // MultiVersionTransaction MultiVersionTransaction::MultiVersionTransaction(Reference db, UniqueOrderedOptionList defaultOptions) - : db(db) { + : db(db), startTime(timer_monotonic()), timeoutTsav(new ThreadSingleAssignmentVar()) { setDefaultOptions(defaultOptions); updateTransaction(); } @@ -622,20 +622,23 @@ void MultiVersionTransaction::updateTransaction() { TransactionInfo newTr; if (currentDb.value) { newTr.transaction = currentDb.value->createTransaction(); + } - Optional timeout; - for (auto option : persistentOptions) { - if (option.first == FDBTransactionOptions::TIMEOUT) { - timeout = option.second.castTo(); - } else { - newTr.transaction->setOption(option.first, option.second.castTo()); - } + Optional timeout; + for (auto option : persistentOptions) { + if (option.first == FDBTransactionOptions::TIMEOUT) { + timeout = option.second.castTo(); + } else if (currentDb.value) { + newTr.transaction->setOption(option.first, option.second.castTo()); } + } - // Setting a timeout can immediately cause a transaction to fail. The only timeout - // that matters is the one most recently set, so we ignore any earlier set timeouts - // that might inadvertently fail the transaction. - if (timeout.present()) { + // Setting a timeout can immediately cause a transaction to fail. The only timeout + // that matters is the one most recently set, so we ignore any earlier set timeouts + // that might inadvertently fail the transaction. + if (timeout.present()) { + setTimeout(timeout); + if (currentDb.value) { newTr.transaction->setOption(FDBTransactionOptions::TIMEOUT, timeout); } } @@ -670,19 +673,19 @@ void MultiVersionTransaction::setVersion(Version v) { } ThreadFuture MultiVersionTransaction::getReadVersion() { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getReadVersion() : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getReadVersion() : makeTimeout(); return abortableFuture(f, tr.onChange); } ThreadFuture> MultiVersionTransaction::get(const KeyRef& key, bool snapshot) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->get(key, snapshot) : ThreadFuture>(Never()); + auto f = tr.transaction ? tr.transaction->get(key, snapshot) : makeTimeout>(); return abortableFuture(f, tr.onChange); } ThreadFuture MultiVersionTransaction::getKey(const KeySelectorRef& key, bool snapshot) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getKey(key, snapshot) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getKey(key, snapshot) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -692,8 +695,8 @@ ThreadFuture MultiVersionTransaction::getRange(const KeySelectorRef bool snapshot, bool reverse) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse) - : ThreadFuture(Never()); + auto f = + tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -703,8 +706,8 @@ ThreadFuture MultiVersionTransaction::getRange(const KeySelectorRef bool snapshot, bool reverse) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse) - : ThreadFuture(Never()); + auto f = + tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -713,8 +716,7 @@ ThreadFuture MultiVersionTransaction::getRange(const KeyRangeRef& k bool snapshot, bool reverse) { auto tr = getTransaction(); - auto f = - tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -723,21 +725,20 @@ ThreadFuture MultiVersionTransaction::getRange(const KeyRangeRef& k bool snapshot, bool reverse) { auto tr = getTransaction(); - auto f = - tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse) : makeTimeout(); return abortableFuture(f, tr.onChange); } ThreadFuture> MultiVersionTransaction::getVersionstamp() { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getVersionstamp() : ThreadFuture>(Never()); + auto f = tr.transaction ? tr.transaction->getVersionstamp() : makeTimeout>(); return abortableFuture(f, tr.onChange); } ThreadFuture>> MultiVersionTransaction::getAddressesForKey(const KeyRef& key) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getAddressesForKey(key) - : ThreadFuture>>(Never()); + auto f = + tr.transaction ? tr.transaction->getAddressesForKey(key) : makeTimeout>>(); return abortableFuture(f, tr.onChange); } @@ -750,7 +751,7 @@ void MultiVersionTransaction::addReadConflictRange(const KeyRangeRef& keys) { ThreadFuture MultiVersionTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -758,7 +759,7 @@ ThreadFuture>> MultiVersionTransaction::getRangeSpl int64_t chunkSize) { auto tr = getTransaction(); auto f = tr.transaction ? tr.transaction->getRangeSplitPoints(range, chunkSize) - : ThreadFuture>>(Never()); + : makeTimeout>>(); return abortableFuture(f, tr.onChange); } @@ -799,7 +800,7 @@ void MultiVersionTransaction::clear(const KeyRef& key) { ThreadFuture MultiVersionTransaction::watch(const KeyRef& key) { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->watch(key) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->watch(key) : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -812,7 +813,7 @@ void MultiVersionTransaction::addWriteConflictRange(const KeyRangeRef& keys) { ThreadFuture MultiVersionTransaction::commit() { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->commit() : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->commit() : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -827,7 +828,7 @@ Version MultiVersionTransaction::getCommittedVersion() { ThreadFuture MultiVersionTransaction::getApproximateSize() { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->getApproximateSize() : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->getApproximateSize() : makeTimeout(); return abortableFuture(f, tr.onChange); } @@ -841,6 +842,11 @@ void MultiVersionTransaction::setOption(FDBTransactionOptions::Option option, Op if (MultiVersionApi::apiVersionAtLeast(610) && itr->second.persistent) { persistentOptions.emplace_back(option, value.castTo>()); } + + if (itr->first == FDBTransactionOptions::TIMEOUT) { + setTimeout(value); + } + auto tr = getTransaction(); if (tr.transaction) { tr.transaction->setOption(option, value); @@ -853,7 +859,7 @@ ThreadFuture MultiVersionTransaction::onError(Error const& e) { return ThreadFuture(Void()); } else { auto tr = getTransaction(); - auto f = tr.transaction ? tr.transaction->onError(e) : ThreadFuture(Never()); + auto f = tr.transaction ? tr.transaction->onError(e) : makeTimeout(); f = abortableFuture(f, tr.onChange); return flatMapThreadFuture(f, [this, e](ErrorOr ready) { @@ -871,12 +877,95 @@ ThreadFuture MultiVersionTransaction::onError(Error const& e) { } } +// Waits for the specified duration and signals the assignment variable with a timed out error +// This will be canceled if a new timeout is set, in which case the tsav will not be signaled. +ACTOR Future timeoutImpl(Reference> tsav, double duration) { + wait(delay(duration)); + + tsav->trySendError(transaction_timed_out()); + return Void(); +} + +// Configure a timeout based on the options set for this transaction. This timeout only applies +// if we don't have an underlying database object to connect with. +void MultiVersionTransaction::setTimeout(Optional value) { + double timeoutDuration = extractIntOption(value, 0, std::numeric_limits::max()) / 1000.0; + + ThreadFuture prevTimeout; + double transactionStartTime = startTime; + + { // lock scope + ThreadSpinLockHolder holder(timeoutLock); + + Reference> tsav = timeoutTsav; + ThreadFuture newTimeout = onMainThread([transactionStartTime, tsav, timeoutDuration]() { + return timeoutImpl(tsav, timeoutDuration - std::max(0.0, now() - transactionStartTime)); + }); + + prevTimeout = currentTimeout; + currentTimeout = newTimeout; + } + + // Cancel the previous timeout now that we have a new one. This means that changing the timeout + // affects in-flight operations, which is consistent with the behavior in RYW. + if (prevTimeout.isValid()) { + prevTimeout.cancel(); + } +} + +// Creates a ThreadFuture that will signal an error if the transaction times out. +template +ThreadFuture MultiVersionTransaction::makeTimeout() { + ThreadFuture f; + + { // lock scope + ThreadSpinLockHolder holder(timeoutLock); + + // Our ThreadFuture holds a reference to this TSAV, + // but the ThreadFuture does not increment the ref count + timeoutTsav->addref(); + f = ThreadFuture(timeoutTsav.getPtr()); + } + + // When our timeoutTsav gets set, map it to the appropriate type + return mapThreadFuture(f, [](ErrorOr v) { + ASSERT(v.isError()); + return ErrorOr(v.getError()); + }); +} + void MultiVersionTransaction::reset() { persistentOptions.clear(); + + // Reset the timeout state + Reference> prevTimeoutTsav; + ThreadFuture prevTimeout; + startTime = timer_monotonic(); + + { // lock scope + ThreadSpinLockHolder holder(timeoutLock); + + prevTimeoutTsav = timeoutTsav; + timeoutTsav = makeReference>(); + + prevTimeout = currentTimeout; + currentTimeout = ThreadFuture(); + } + + // Cancel any outstanding operations if they don't have an underlying transaction object to cancel them + prevTimeoutTsav->trySendError(transaction_cancelled()); + if (prevTimeout.isValid()) { + prevTimeout.cancel(); + } + setDefaultOptions(db->dbState->transactionDefaultOptions); updateTransaction(); } +MultiVersionTransaction::~MultiVersionTransaction() { + timeoutTsav->trySendError(transaction_cancelled()); +} + bool MultiVersionTransaction::isValid() { auto tr = getTransaction(); return tr.transaction.isValid(); @@ -1896,8 +1985,28 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() { std::string valueStr; try { if (platform::getEnvironmentVar(("FDB_NETWORK_OPTION_" + option.second.name).c_str(), valueStr)) { + FDBOptionInfo::ParamType curParamType = option.second.paramType; for (auto value : parseOptionValues(valueStr)) { - Standalone currentValue = StringRef(value); + Standalone currentValue; + int64_t intParamVal; + if (curParamType == FDBOptionInfo::ParamType::Int) { + try { + size_t nextIdx; + intParamVal = std::stoll(value, &nextIdx); + if (nextIdx != value.length()) { + throw invalid_option_value(); + } + } catch (std::exception e) { + TraceEvent(SevError, "EnvironmentVariableParseIntegerFailed") + .detail("Option", option.second.name) + .detail("Value", valueStr) + .detail("Error", e.what()); + throw invalid_option_value(); + } + currentValue = StringRef(reinterpret_cast(&intParamVal), 8); + } else { + currentValue = StringRef(value); + } { // lock scope MutexHolder holder(lock); if (setEnvOptions[option.first].count(currentValue) == 0) { diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index f3d7f5d8ce..95d9a8b14c 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -334,6 +334,8 @@ public: MultiVersionTransaction(Reference db, UniqueOrderedOptionList defaultOptions); + ~MultiVersionTransaction() override; + void cancel() override; void setVersion(Version v) override; ThreadFuture getReadVersion() override; @@ -400,6 +402,29 @@ private: ThreadFuture onChange; }; + // Timeout related variables for MultiVersionTransaction objects that do not have an underlying ITransaction + + // The time when the MultiVersionTransaction was last created or reset + std::atomic startTime; + + // A lock that needs to be held if using timeoutTsav or currentTimeout + ThreadSpinLock timeoutLock; + + // A single assignment var (i.e. promise) that gets set with an error when the timeout elapses or the transaction + // is reset or destroyed. + Reference> timeoutTsav; + + // A reference to the current actor waiting for the timeout. This actor will set the timeoutTsav promise. + ThreadFuture currentTimeout; + + // Configure a timeout based on the options set for this transaction. This timeout only applies + // if we don't have an underlying database object to connect with. + void setTimeout(Optional value); + + // Creates a ThreadFuture that will signal an error if the transaction times out. + template + ThreadFuture makeTimeout(); + TransactionInfo transaction; TransactionInfo getTransaction(); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index b050ec67b5..7a6f89c1b0 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -55,6 +55,7 @@ #include "fdbclient/SystemData.h" #include "fdbclient/TransactionLineage.h" #include "fdbclient/versions.h" +#include "fdbclient/WellKnownEndpoints.h" #include "fdbrpc/LoadBalance.h" #include "fdbrpc/Net2FileSystem.h" #include "fdbrpc/simulator.h" @@ -85,10 +86,6 @@ extern const char* getSourceVersion(); -using std::max; -using std::min; -using std::pair; - namespace { TransactionLineageCollector transactionLineageCollector; @@ -458,10 +455,10 @@ ACTOR Future databaseLogger(DatabaseContext* cx) { .detail("TSSGetKeyLatency90", it.second->TSSgetKeyLatency.percentile(0.90)) .detail("TSSGetKeyLatency99", it.second->TSSgetKeyLatency.percentile(0.99)); - tssEv.detail("MeanSSGetKeyValuesLatency", it.second->SSgetKeyLatency.mean()) - .detail("MedianSSGetKeyValuesLatency", it.second->SSgetKeyLatency.median()) - .detail("SSGetKeyValuesLatency90", it.second->SSgetKeyLatency.percentile(0.90)) - .detail("SSGetKeyValuesLatency99", it.second->SSgetKeyLatency.percentile(0.99)); + tssEv.detail("MeanSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.mean()) + .detail("MedianSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.median()) + .detail("SSGetKeyValuesLatency90", it.second->SSgetKeyValuesLatency.percentile(0.90)) + .detail("SSGetKeyValuesLatency99", it.second->SSgetKeyValuesLatency.percentile(0.99)); tssEv.detail("MeanTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.mean()) .detail("MedianTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.median()) @@ -672,19 +669,82 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext* cx) { } } -ACTOR static Future monitorProxiesChange(Reference const> clientDBInfo, +ACTOR Future assertFailure(GrvProxyInterface remote, Future> reply) { + try { + ErrorOr res = wait(reply); + if (!res.isError()) { + TraceEvent(SevError, "GotStaleReadVersion") + .detail("Remote", remote.getConsistentReadVersion.getEndpoint().addresses.address.toString()) + .detail("Provisional", remote.provisional) + .detail("ReadVersion", res.get().version); + ASSERT_WE_THINK(false); + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + throw; + } + // we want this to fail -- so getting here is good, we'll just ignore the error. + } + return Void(); +} + +Future attemptGRVFromOldProxies(std::vector oldProxies, + std::vector newProxies) { + Span span(deterministicRandom()->randomUniqueID(), "VerifyCausalReadRisky"_loc); + std::vector> replies; + replies.reserve(oldProxies.size()); + GetReadVersionRequest req( + span.context, 1, TransactionPriority::IMMEDIATE, GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY); + TraceEvent evt("AttemptGRVFromOldProxies"); + evt.detail("NumOldProxies", oldProxies.size()).detail("NumNewProxies", newProxies.size()); + auto traceProxies = [&](std::vector& proxies, std::string const& key) { + for (int i = 0; i < proxies.size(); ++i) { + auto k = key + std::to_string(i); + evt.detail(k.c_str(), proxies[i].id()); + } + }; + traceProxies(oldProxies, "OldProxy"s); + traceProxies(newProxies, "NewProxy"s); + evt.log(); + for (auto& i : oldProxies) { + req.reply = ReplyPromise(); + replies.push_back(assertFailure(i, i.getConsistentReadVersion.tryGetReply(req))); + } + return waitForAll(replies); +} + +ACTOR static Future monitorProxiesChange(DatabaseContext* cx, + Reference const> clientDBInfo, AsyncTrigger* triggerVar) { - state vector curCommitProxies; - state vector curGrvProxies; + state std::vector curCommitProxies; + state std::vector curGrvProxies; + state ActorCollection actors(false); curCommitProxies = clientDBInfo->get().commitProxies; curGrvProxies = clientDBInfo->get().grvProxies; loop { - wait(clientDBInfo->onChange()); - if (clientDBInfo->get().commitProxies != curCommitProxies || clientDBInfo->get().grvProxies != curGrvProxies) { - curCommitProxies = clientDBInfo->get().commitProxies; - curGrvProxies = clientDBInfo->get().grvProxies; - triggerVar->trigger(); + choose { + when(wait(clientDBInfo->onChange())) { + if (clientDBInfo->get().commitProxies != curCommitProxies || + clientDBInfo->get().grvProxies != curGrvProxies) { + // This condition is a bit complicated. Here we want to verify that we're unable to receive a read + // version from a proxy of an old generation after a successful recovery. The conditions are: + // 1. We only do this with a configured probability. + // 2. If the old set of Grv proxies is empty, there's nothing to do + // 3. If the new set of Grv proxies is empty, it means the recovery is not complete. So if an old + // Grv proxy still gives out read versions, this would be correct behavior. + // 4. If we see a provisional proxy, it means the recovery didn't complete yet, so the same as (3) + // applies. + if (deterministicRandom()->random01() < cx->verifyCausalReadsProp && !curGrvProxies.empty() && + !clientDBInfo->get().grvProxies.empty() && !clientDBInfo->get().grvProxies[0].provisional) { + actors.add(attemptGRVFromOldProxies(curGrvProxies, clientDBInfo->get().grvProxies)); + } + curCommitProxies = clientDBInfo->get().commitProxies; + curGrvProxies = clientDBInfo->get().grvProxies; + triggerVar->trigger(); + } + } + when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); } } } } @@ -1145,11 +1205,13 @@ DatabaseContext::DatabaseContext(ReferenceSHARD_STAT_SMOOTH_AMOUNT), + transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), + transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc), + latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), + bytesPerCommit(1000), outstandingWatches(0), transactionTracingEnabled(true), taskID(taskID), + clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), + mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0), + smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT), specialKeySpace(std::make_unique(specialKeys.begin, specialKeys.end, /* test */ false)) { dbId = deterministicRandom()->randomUniqueID(); connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) @@ -1168,7 +1230,7 @@ DatabaseContext::DatabaseContext(ReferenceSHARD_STAT_SMOOTH_AMOUNT) {} + transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), + transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc), + latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), + bytesPerCommit(1000), transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT) {} // Static constructor used by server processes to create a DatabaseContext // For internal (fdbserver) use only @@ -1435,7 +1498,7 @@ DatabaseContext::~DatabaseContext() { locationCache.insert(allKeys, Reference()); } -pair> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) { +std::pair> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) { if (isBackward) { auto range = locationCache.rangeContainingKeyBefore(key); return std::make_pair(range->range(), range->value()); @@ -1446,7 +1509,7 @@ pair> DatabaseContext::getCachedLocation(const } bool DatabaseContext::getCachedLocations(const KeyRangeRef& range, - vector>>& result, + std::vector>>& result, int limit, Reverse reverse) { result.clear(); @@ -1476,8 +1539,8 @@ bool DatabaseContext::getCachedLocations(const KeyRangeRef& range, } Reference DatabaseContext::setCachedLocation(const KeyRangeRef& keys, - const vector& servers) { - vector>> serverRefs; + const std::vector& servers) { + std::vector>> serverRefs; serverRefs.reserve(servers.size()); for (const auto& interf : servers) { serverRefs.push_back(StorageServerInfo::getInterface(this, interf, clientLocality)); @@ -1610,6 +1673,9 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, OptionaladdStopCallback(Net2FileSystem::stop); g_network->addStopCallback(TLS::DestroyOpenSSLGlobalState); - FlowTransport::createInstance(true, transportId); + FlowTransport::createInstance(true, transportId, WLTOKEN_RESERVED_COUNT); Net2FileSystem::newFileSystem(); uncancellable(monitorNetworkBusyness()); @@ -2251,23 +2317,23 @@ ACTOR Future> fetchServerInterface(Database cx, return decodeServerListValue(val.get()); } -ACTOR Future>> transactionalGetServerInterfaces(Future ver, - Database cx, - TransactionInfo info, - vector ids, - TagSet tags) { - state vector>> serverListEntries; +ACTOR Future>> transactionalGetServerInterfaces(Future ver, + Database cx, + TransactionInfo info, + std::vector ids, + TagSet tags) { + state std::vector>> serverListEntries; serverListEntries.reserve(ids.size()); for (int s = 0; s < ids.size(); s++) { serverListEntries.push_back(fetchServerInterface(cx, info, ids[s], tags, ver)); } - vector> serverListValues = wait(getAll(serverListEntries)); - vector serverInterfaces; + std::vector> serverListValues = wait(getAll(serverListEntries)); + std::vector serverInterfaces; for (int s = 0; s < serverListValues.size(); s++) { if (!serverListValues[s].present()) { // A storage server has been removed from ServerList since we read keyServers - return Optional>(); + return Optional>(); } serverInterfaces.push_back(serverListValues[s].get()); } @@ -2299,10 +2365,8 @@ void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) { // If isBackward == true, returns the shard containing the key before 'key' (an infinitely long, inexpressible key). // Otherwise returns the shard containing key -ACTOR Future>> getKeyLocation_internal(Database cx, - Key key, - TransactionInfo info, - Reverse isBackward = Reverse::False) { +ACTOR Future>> +getKeyLocation_internal(Database cx, Key key, TransactionInfo info, Reverse isBackward = Reverse::False) { state Span span("NAPI:getKeyLocation"_loc, info.spanID); if (isBackward) { ASSERT(key != allKeys.begin && key <= allKeys.end); @@ -2337,11 +2401,11 @@ ACTOR Future>> getKeyLocation_internal(Da } template -Future>> getKeyLocation(Database const& cx, - Key const& key, - F StorageServerInterface::*member, - TransactionInfo const& info, - Reverse isBackward = Reverse::False) { +Future>> getKeyLocation(Database const& cx, + Key const& key, + F StorageServerInterface::*member, + TransactionInfo const& info, + Reverse isBackward = Reverse::False) { // we first check whether this range is cached auto ssi = cx->getCachedLocation(key, isBackward); if (!ssi.second) { @@ -2359,11 +2423,8 @@ Future>> getKeyLocation(Database const& c return ssi; } -ACTOR Future>>> getKeyRangeLocations_internal(Database cx, - KeyRange keys, - int limit, - Reverse reverse, - TransactionInfo info) { +ACTOR Future>>> +getKeyRangeLocations_internal(Database cx, KeyRange keys, int limit, Reverse reverse, TransactionInfo info) { state Span span("NAPI:getKeyRangeLocations"_loc, info.spanID); if (info.debugID.present()) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.Before"); @@ -2384,7 +2445,7 @@ ACTOR Future>>> getKeyRangeLocatio "TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.After"); ASSERT(rep.results.size()); - state vector>> results; + state std::vector>> results; state int shard = 0; for (; shard < rep.results.size(); shard++) { // FIXME: these shards are being inserted into the map sequentially, it would be much more CPU @@ -2408,15 +2469,16 @@ ACTOR Future>>> getKeyRangeLocatio // Example: If query the function with key range (b, d), the returned list of pairs could be something like: // [([a, b1), locationInfo), ([b1, c), locationInfo), ([c, d1), locationInfo)]. template -Future>>> getKeyRangeLocations(Database const& cx, - KeyRange const& keys, - int limit, - Reverse reverse, - F StorageServerInterface::*member, - TransactionInfo const& info) { +Future>>> getKeyRangeLocations( + Database const& cx, + KeyRange const& keys, + int limit, + Reverse reverse, + F StorageServerInterface::*member, + TransactionInfo const& info) { ASSERT(!keys.empty()); - vector>> locations; + std::vector>> locations; if (!cx->getCachedLocations(keys, locations, limit, reverse)) { return getKeyRangeLocations_internal(cx, keys, limit, reverse, info); } @@ -2448,7 +2510,7 @@ ACTOR Future warmRange_impl(Transaction* self, Database cx, KeyRange keys) state int totalRanges = 0; state int totalRequests = 0; loop { - vector>> locations = wait( + std::vector>> locations = wait( getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, Reverse::False, self->info)); totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT; totalRequests++; @@ -2493,7 +2555,7 @@ ACTOR Future> getValue(Future version, cx->validateVersion(ver); loop { - state pair> ssi = + state std::pair> ssi = wait(getKeyLocation(cx, key, &StorageServerInterface::getValue, info)); state Optional getValueID = Optional(); state uint64_t startTime; @@ -2618,7 +2680,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr } Key locationKey(k.getKey(), k.arena()); - state pair> ssi = + state std::pair> ssi = wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, Reverse{ k.isBackward() })); try { @@ -2739,7 +2801,7 @@ ACTOR Future watchValue(Future version, ASSERT(ver != latestVersion); loop { - state pair> ssi = + state std::pair> ssi = wait(getKeyLocation(cx, key, &StorageServerInterface::watchValue, info)); try { @@ -2992,7 +3054,7 @@ ACTOR Future getExactRange(Database cx, // printf("getExactRange( '%s', '%s' )\n", keys.begin.toString().c_str(), keys.end.toString().c_str()); loop { - state vector>> locations = wait(getKeyRangeLocations( + state std::vector>> locations = wait(getKeyRangeLocations( cx, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValues, info)); ASSERT(locations.size()); state int shard = 0; @@ -3315,7 +3377,7 @@ ACTOR Future getRange(Database cx, Key locationKey = reverse ? Key(end.getKey(), end.arena()) : Key(begin.getKey(), begin.arena()); Reverse locationBackward{ reverse ? (end - 1).isBackward() : begin.isBackward() }; - state pair> beginServer = + state std::pair> beginServer = wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKeyValues, info, locationBackward)); state KeyRange shard = beginServer.first; state bool modifiedSelectors = false; @@ -3745,7 +3807,7 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* TagSet tags, SpanID spanContext) { loop { - state vector>> locations = wait(getKeyRangeLocations( + state std::vector>> locations = wait(getKeyRangeLocations( cx, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValuesStream, info)); ASSERT(locations.size()); state int shard = 0; @@ -3806,7 +3868,7 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* break; } - vector> ok(locations[shard].second->size()); + std::vector> ok(locations[shard].second->size()); for (int i = 0; i < ok.size(); i++) { ok[i] = IFailureMonitor::failureMonitor().onStateEqual( locations[shard].second->get(i, &StorageServerInterface::getKeyValuesStream).getEndpoint(), @@ -4048,7 +4110,7 @@ ACTOR Future getRangeStream(PromiseStream _results, state std::vector> outstandingRequests; while (b < e) { - state pair> ssi = + state std::pair> ssi = wait(getKeyLocation(cx, reverse ? e : b, &StorageServerInterface::getKeyValuesStream, info, reverse)); state KeyRange shardIntersection = intersect(ssi.first, KeyRangeRef(b, e)); state Standalone> splitPoints = @@ -4309,7 +4371,7 @@ ACTOR Future>> getAddressesForKeyActor(Key key Database cx, TransactionInfo info, TransactionOptions options) { - state vector ssi; + state std::vector ssi; // If key >= allKeys.end, then getRange will return a kv-pair with an empty value. This will result in our // serverInterfaces vector being empty, which will cause us to return an empty addresses list. @@ -4336,12 +4398,12 @@ ACTOR Future>> getAddressesForKeyActor(Key key ASSERT(serverUids.size()); // every shard needs to have a team - vector src; - vector ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in the - // case where this key is being relocated. But 'src' is the canonical location until the move is - // finished, because it could be cancelled at any time. + std::vector src; + std::vector ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in + // the case where this key is being relocated. But 'src' is the canonical location until + // the move is finished, because it could be cancelled at any time. decodeKeyServersValue(serverTagResult, serverUids[0].value, src, ignore); - Optional> serverInterfaces = + Optional> serverInterfaces = wait(transactionalGetServerInterfaces(ver, cx, info, src, options.readTags)); ASSERT(serverInterfaces.present()); // since this is happening transactionally, /FF/keyServers and /FF/serverList @@ -4709,8 +4771,9 @@ double Transaction::getBackoff(int errCode) { auto tagItr = priorityItr->second.find(tag); if (tagItr != priorityItr->second.end()) { TEST(true); // Returning throttle backoff - returnedBackoff = std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL, - std::max(returnedBackoff, tagItr->second.throttleDuration())); + returnedBackoff = std::max( + returnedBackoff, + std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL, tagItr->second.throttleDuration())); if (returnedBackoff == CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL) { break; } @@ -4988,7 +5051,7 @@ ACTOR Future> estimateCommitCosts(Transac ++trCommitCosts.expensiveCostEstCount; ++self->getDatabase()->transactionsExpensiveClearCostEstCount; } else { - std::vector>> locations = + std::vector>> locations = wait(getKeyRangeLocations(self->getDatabase(), keyRange, CLIENT_KNOBS->TOO_MANY, @@ -5602,6 +5665,20 @@ ACTOR Future readVersionBatcher(DatabaseContext* cx, state Future timeout; state Optional debugID; state bool send_batch; + state Reference batchSizeDist = Histogram::getHistogram(LiteralStringRef("GrvBatcher"), + LiteralStringRef("ClientGrvBatchSize"), + Histogram::Unit::countLinear, + 0, + CLIENT_KNOBS->MAX_BATCH_SIZE * 2); + state Reference batchIntervalDist = + Histogram::getHistogram(LiteralStringRef("GrvBatcher"), + LiteralStringRef("ClientGrvBatchInterval"), + Histogram::Unit::microseconds, + 0, + CLIENT_KNOBS->GRV_BATCH_TIMEOUT * 1000000 * 2); + state Reference grvReplyLatencyDist = Histogram::getHistogram( + LiteralStringRef("GrvBatcher"), LiteralStringRef("ClientGrvReplyLatency"), Histogram::Unit::microseconds); + state double lastRequestTime = now(); state TransactionTagMap tags; @@ -5626,22 +5703,34 @@ ACTOR Future readVersionBatcher(DatabaseContext* cx, ++tags[tag]; } - if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE) + if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE) { send_batch = true; - else if (!timeout.isValid()) + ++cx->transactionGrvFullBatches; + } else if (!timeout.isValid()) { timeout = delay(batchTime, TaskPriority::GetConsistentReadVersion); + } + } + when(wait(timeout.isValid() ? timeout : Never())) { + send_batch = true; + ++cx->transactionGrvTimedOutBatches; } - when(wait(timeout.isValid() ? timeout : Never())) { send_batch = true; } // dynamic batching monitors reply latencies when(double reply_latency = waitNext(replyTimes.getFuture())) { double target_latency = reply_latency * 0.5; - batchTime = min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT); + batchTime = std::min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT); + grvReplyLatencyDist->sampleSeconds(reply_latency); } when(wait(collection)) {} // for errors } if (send_batch) { int count = requests.size(); ASSERT(count); + + batchSizeDist->sampleRecordCounter(count); + auto requestTime = now(); + batchIntervalDist->sampleSeconds(requestTime - lastRequestTime); + lastRequestTime = requestTime; + // dynamic batching Promise GRVReply; requests.push_back(GRVReply); @@ -5827,7 +5916,8 @@ Future> Transaction::getVersionstamp() { // Gets the protocol version reported by a coordinator via the protocol info interface ACTOR Future getCoordinatorProtocol(NetworkAddressList coordinatorAddresses) { - RequestStream requestStream{ Endpoint{ { coordinatorAddresses }, WLTOKEN_PROTOCOL_INFO } }; + RequestStream requestStream{ Endpoint::wellKnown({ coordinatorAddresses }, + WLTOKEN_PROTOCOL_INFO) }; ProtocolInfoReply reply = wait(retryBrokenPromise(requestStream, ProtocolInfoRequest{})); return reply.version; @@ -5988,7 +6078,7 @@ ACTOR Future doGetStorageMetrics(Database cx, KeyRange keys, Ref ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRange keys) { state Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc); - vector>> locations = + std::vector>> locations = wait(getKeyRangeLocations(cx, keys, std::numeric_limits::max(), @@ -5996,7 +6086,7 @@ ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRang &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); state int nLocs = locations.size(); - state vector> fx(nLocs); + state std::vector> fx(nLocs); state StorageMetrics total; KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { @@ -6030,15 +6120,15 @@ ACTOR Future trackBoundedStorageMetrics(KeyRange keys, } ACTOR Future waitStorageMetricsMultipleLocations( - vector>> locations, + std::vector>> locations, StorageMetrics min, StorageMetrics max, StorageMetrics permittedError) { state int nLocs = locations.size(); - state vector> fx(nLocs); + state std::vector> fx(nLocs); state StorageMetrics total; state PromiseStream deltas; - state vector> wx(fx.size()); + state std::vector> wx(fx.size()); state StorageMetrics halfErrorPerMachine = permittedError * (0.5 / nLocs); state StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs - 1); state StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs - 1); @@ -6087,7 +6177,7 @@ ACTOR Future>> getReadHotRanges(Da loop { int64_t shardLimit = 100; // Shard limit here does not really matter since this function is currently only used // to find the read-hot sub ranges within a read-hot shard. - vector>> locations = + std::vector>> locations = wait(getKeyRangeLocations(cx, keys, shardLimit, @@ -6106,7 +6196,7 @@ ACTOR Future>> getReadHotRanges(Da // .detail("KeysBegin", keys.begin.printable().c_str()) // .detail("KeysEnd", keys.end.printable().c_str()); // } - state vector> fReplies(nLocs); + state std::vector> fReplies(nLocs); KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { partBegin = (i == 0) ? keys.begin : locations[i].first.begin; @@ -6155,7 +6245,7 @@ ACTOR Future, int>> waitStorageMetrics(Databa int expectedShardCount) { state Span span("NAPI:WaitStorageMetrics"_loc); loop { - vector>> locations = + std::vector>> locations = wait(getKeyRangeLocations(cx, keys, shardLimit, @@ -6247,7 +6337,7 @@ Future>> Transaction::getReadHotRa ACTOR Future>> getRangeSplitPoints(Database cx, KeyRange keys, int64_t chunkSize) { state Span span("NAPI:GetRangeSplitPoints"_loc); loop { - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->TOO_MANY, @@ -6256,7 +6346,7 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key TransactionInfo(TaskPriority::DataDistribution, span.context))); try { state int nLocs = locations.size(); - state vector> fReplies(nLocs); + state std::vector> fReplies(nLocs); KeyRef partBegin, partEnd; for (int i = 0; i < nLocs; i++) { partBegin = (i == 0) ? keys.begin : locations[i].first.begin; @@ -6308,7 +6398,7 @@ ACTOR Future>> splitStorageMetrics(Database cx, StorageMetrics estimated) { state Span span("NAPI:SplitStorageMetrics"_loc); loop { - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT, @@ -6431,7 +6521,7 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn } } -ACTOR Future checkSafeExclusions(Database cx, vector exclusions) { +ACTOR Future checkSafeExclusions(Database cx, std::vector exclusions) { TraceEvent("ExclusionSafetyCheckBegin") .detail("NumExclusion", exclusions.size()) .detail("Exclusions", describe(exclusions)); @@ -6462,7 +6552,7 @@ ACTOR Future checkSafeExclusions(Database cx, vector exc } TraceEvent("ExclusionSafetyCheckCoordinators").log(); state ClientCoordinators coordinatorList(cx->getConnectionFile()); - state vector>> leaderServers; + state std::vector>> leaderServers; leaderServers.reserve(coordinatorList.clientLeaderServers.size()); for (int i = 0; i < coordinatorList.clientLeaderServers.size(); i++) { leaderServers.push_back(retryBrokenPromise(coordinatorList.clientLeaderServers[i].getLeader, @@ -6597,7 +6687,7 @@ ACTOR Future>> getChangeFeedMutatio throw unsupported_operation(); } state KeyRange keys = std::get<0>(decodeChangeFeedValue(val.get())) & range; - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, keys, 100, @@ -6662,7 +6752,7 @@ ACTOR Future singleChangeFeedStream(StorageServerInterface interf, begin); }*/ state int resultLoc = 0; - // TODO TELL EVAN about fix + // FIXME: fix better while (resultLoc < rep.mutations.size()) { if (rep.mutations[resultLoc].mutations.size() || rep.mutations[resultLoc].version + 1 == end || (rep.mutations[resultLoc].mutations.empty() && @@ -6678,12 +6768,9 @@ ACTOR Future singleChangeFeedStream(StorageServerInterface interf, } } } catch (Error& e) { - // FIXME shouldn't this also just send to the stream so it throws in mergeChangeFeedStream and retries in - /*if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed || - e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed || - e.code() == error_code_actor_cancelled) { - throw; - }*/ + if (e.code() == error_code_actor_cancelled) { + throw; + } results.sendError(e); return Void(); } @@ -6759,18 +6846,17 @@ ACTOR Future getChangeFeedStreamActor(Reference db, Version end, KeyRange range) { state Database cx(db); - state Transaction tr(cx); state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix); state Span span("NAPI:GetChangeFeedStream"_loc); state KeyRange keys; loop { + state Transaction tr(cx); loop { try { Version readVer = wait(tr.getReadVersion()); if (readVer < begin) { wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); - tr.reset(); } else { Optional val = wait(tr.get(rangeIDKey)); if (!val.present()) { @@ -6786,7 +6872,7 @@ ACTOR Future getChangeFeedStreamActor(Reference db, } try { - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, keys, 1000, @@ -6824,7 +6910,7 @@ ACTOR Future getChangeFeedStreamActor(Reference db, continue; } - vector> ok(locations[loc].second->size()); + std::vector> ok(locations[loc].second->size()); for (int i = 0; i < ok.size(); i++) { ok[i] = IFailureMonitor::failureMonitor().onStateEqual( locations[loc].second->get(i, &StorageServerInterface::changeFeedStream).getEndpoint(), @@ -6876,10 +6962,10 @@ ACTOR Future getChangeFeedStreamActor(Reference db, throw; } if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed || - e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed) { + e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed || + e.code() == error_code_broken_promise) { cx->invalidateCache(keys); wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY)); - tr.reset(); } else { results.sendError(e); return Void(); @@ -6929,7 +7015,7 @@ ACTOR Future>> getOverlappingChangeFeedsAct loop { try { - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, range, 1000, @@ -6998,24 +7084,24 @@ ACTOR static Future popChangeFeedBackup(Database cx, Key rangeID, Version ACTOR Future popChangeFeedMutationsActor(Reference db, Key rangeID, Version version) { state Database cx(db); - state Transaction tr(cx); state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix); state Span span("NAPI:PopChangeFeedMutations"_loc); - state Optional val; + + state Transaction tr(cx); + state KeyRange keys; loop { try { - Optional _val = wait(tr.get(rangeIDKey)); - val = _val; + Optional val = wait(tr.get(rangeIDKey)); + if (!val.present()) { + throw unsupported_operation(); + } + keys = std::get<0>(decodeChangeFeedValue(val.get())); break; } catch (Error& e) { wait(tr.onError(e)); } } - if (!val.present()) { - throw unsupported_operation(); - } - state KeyRange keys = std::get<0>(decodeChangeFeedValue(val.get())); - state vector>> locations = + state std::vector>> locations = wait(getKeyRangeLocations(cx, keys, 3, diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 7f04a8930c..085435b535 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -415,7 +415,7 @@ public: void setTransactionID(uint64_t id); void setToken(uint64_t token); - const vector>>& getExtraReadConflictRanges() const { return extraConflictRanges; } + const std::vector>>& getExtraReadConflictRanges() const { return extraConflictRanges; } Standalone> readConflictRanges() const { return Standalone>(tr.transaction.read_conflict_ranges, tr.arena); } @@ -432,7 +432,7 @@ private: CommitTransactionRequest tr; Future readVersion; Promise> metadataVersion; - vector>> extraConflictRanges; + std::vector>> extraConflictRanges; Promise commitResult; Future committing; }; @@ -453,7 +453,7 @@ int64_t extractIntOption(Optional value, ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID snapUID); // Checks with Data Distributor that it is safe to mark all servers in exclusions as failed -ACTOR Future checkSafeExclusions(Database cx, vector exclusions); +ACTOR Future checkSafeExclusions(Database cx, std::vector exclusions); inline uint64_t getWriteOperationCost(uint64_t bytes) { return bytes / std::max(1, CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR) + 1; diff --git a/fdbclient/ProcessInterface.h b/fdbclient/ProcessInterface.h index 466f91b307..2c471d28da 100644 --- a/fdbclient/ProcessInterface.h +++ b/fdbclient/ProcessInterface.h @@ -21,8 +21,7 @@ #include "fdbclient/AnnotateActor.h" #include "fdbclient/FDBTypes.h" #include "fdbrpc/fdbrpc.h" - -constexpr UID WLTOKEN_PROCESS(-1, 21); +#include "fdbclient/WellKnownEndpoints.h" struct ProcessInterface { constexpr static FileIdentifier file_identifier = 985636; diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 77dcf2ef05..dffb84db62 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1342,7 +1342,7 @@ ACTOR Future getWorkerInterfaces(Reference c loop { choose { - when(vector workers = + when(std::vector workers = wait(clusterInterface->get().present() ? brokenPromiseToNever( clusterInterface->get().get().getClientWorkers.getReply(GetClientWorkersRequest())) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index e3ef1e3081..145f9058a5 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -763,7 +763,14 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "grv_proxies":1, "proxies":6, "backup_worker_enabled":1, - "perpetual_storage_wiggle":0 + "perpetual_storage_wiggle":0, + "perpetual_storage_wiggle_locality":"0", + "storage_migration_type": { + "$enum":[ + "disabled", + "aggressive", + "gradual" + ]} }, "data":{ "least_operating_space_bytes_log_server":0, diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 691ced7756..f246ae9db6 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -102,7 +102,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( PUSH_STATS_SLOW_RATIO, 0.5 ); init( TLOG_POP_BATCH_SIZE, 1000 ); if ( randomize && BUGGIFY ) TLOG_POP_BATCH_SIZE = 10; init( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE, 250e6 ); - init( ENABLE_DETAILED_TLOG_POP_TRACE, true ); + init( ENABLE_DETAILED_TLOG_POP_TRACE, false ); if ( randomize && BUGGIFY ) ENABLE_DETAILED_TLOG_POP_TRACE = true; // disk snapshot max timeout, to be put in TLog, storage and coordinator nodes init( MAX_FORKED_PROCESS_OUTPUT, 1024 ); @@ -444,7 +444,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( BACKUP_TIMEOUT, 0.4 ); init( BACKUP_NOOP_POP_DELAY, 5.0 ); init( BACKUP_FILE_BLOCK_BYTES, 1024 * 1024 ); - init( BACKUP_LOCK_BYTES, 3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 1024; + init( BACKUP_LOCK_BYTES, 3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 15 * 1024; init( BACKUP_UPLOAD_DELAY, 10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60; //Cluster Controller @@ -467,6 +467,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( CHECK_OUTSTANDING_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001; init( VERSION_LAG_METRIC_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0; init( MAX_VERSION_DIFFERENCE, 20 * VERSIONS_PER_SECOND ); + init( INITIAL_UPDATE_CROSS_DC_INFO_DELAY, 300 ); + init( CHECK_REMOTE_HEALTH_INTERVAL, 60 ); init( FORCE_RECOVERY_CHECK_DELAY, 5.0 ); init( RATEKEEPER_FAILURE_TIME, 1.0 ); init( BLOB_MANAGER_FAILURE_TIME, 1.0 ); @@ -482,7 +484,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( CC_MAX_EXCLUSION_DUE_TO_HEALTH, 2 ); init( CC_HEALTH_TRIGGER_RECOVERY, false ); init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL, 3600.0 ); - init( CC_MAX_HEALTH_RECOVERY_COUNT, 2 ); + init( CC_MAX_HEALTH_RECOVERY_COUNT, 5 ); + init( CC_HEALTH_TRIGGER_FAILOVER, false ); + init( CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION, 5 ); + init( CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION, 10 ); init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0; init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 2be6757624..e04bfa9903 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -391,6 +391,10 @@ public: double INCOMPATIBLE_PEERS_LOGGING_INTERVAL; double VERSION_LAG_METRIC_INTERVAL; int64_t MAX_VERSION_DIFFERENCE; + double INITIAL_UPDATE_CROSS_DC_INFO_DELAY; // The intial delay in a new Cluster Controller just started to refresh + // the info of remote DC, such as remote DC health, and whether we need + // to take remote DC health info when making failover decision. + double CHECK_REMOTE_HEALTH_INTERVAL; // Remote DC health refresh interval. double FORCE_RECOVERY_CHECK_DELAY; double RATEKEEPER_FAILURE_TIME; double BLOB_MANAGER_FAILURE_TIME; @@ -414,7 +418,13 @@ public: double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed // CC_MAX_HEALTH_RECOVERY_COUNT within // CC_TRACKING_HEALTH_RECOVERY_INTERVAL. - int CC_MAX_HEALTH_RECOVERY_COUNT; + int CC_MAX_HEALTH_RECOVERY_COUNT; // The max number of recoveries can be triggered due to worker health within + // CC_TRACKING_HEALTH_RECOVERY_INTERVAL + bool CC_HEALTH_TRIGGER_FAILOVER; // Whether to enable health triggered failover in CC. + int CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION; // The minimum number of degraded servers that can trigger a + // failover. + int CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION; // The maximum number of degraded servers that can trigger a + // failover. // Knobs used to select the best policy (via monte carlo) int POLICY_RATING_TESTS; // number of tests per policy (in order to compare) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 4d1af8aac3..d2fcbf96ec 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1145,7 +1145,7 @@ Future ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransac } ACTOR Future getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) { - vector _workers = wait(getWorkers(&ryw->getTransaction())); + std::vector _workers = wait(getWorkers(&ryw->getTransaction())); auto workers = _workers; // strip const // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { @@ -1168,7 +1168,7 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); ryw->setOption(FDBTransactionOptions::LOCK_AWARE); ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); - vector workers = wait( + std::vector workers = wait( getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit() auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); @@ -1259,7 +1259,7 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& } ACTOR Future getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) { - vector _workers = wait(getWorkers(&ryw->getTransaction())); + std::vector _workers = wait(getWorkers(&ryw->getTransaction())); auto workers = _workers; // strip const // Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5 std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) { @@ -1322,7 +1322,7 @@ ACTOR Future> lockDatabaseCommitActor(ReadYourWritesTransa if (val.present() && BinaryReader::fromStringRef(val.get().substr(10), Unversioned()) != uid) { // check database not locked // if locked already, throw error - msg = ManagementAPIError::toJsonString(false, "lock", "Database has already been locked"); + throw database_locked(); } else if (!val.present()) { // lock database ryw->getTransaction().atomicOp(databaseLockedKey, @@ -1623,7 +1623,7 @@ ACTOR static Future> coordinatorsCommitActor(ReadYourWrite state int index; state bool parse_error = false; - // check update for cluster_description + // check update for coordinators Key processes_key = LiteralStringRef("processes").withPrefix(kr.begin); auto processes_entry = ryw->getSpecialKeySpaceWriteMap()[processes_key]; if (processes_entry.first) { @@ -1691,29 +1691,14 @@ ACTOR static Future> coordinatorsCommitActor(ReadYourWrite .detail("Result", r.present() ? static_cast(r.get()) : -1); // -1 means success if (r.present()) { auto res = r.get(); - std::string error_msg; bool retriable = false; - if (res == CoordinatorsResult::INVALID_NETWORK_ADDRESSES) { - error_msg = "The specified network addresses are invalid"; - } else if (res == CoordinatorsResult::SAME_NETWORK_ADDRESSES) { - error_msg = "No change (existing configuration satisfies request)"; - } else if (res == CoordinatorsResult::NOT_COORDINATORS) { - error_msg = "Coordination servers are not running on the specified network addresses"; - } else if (res == CoordinatorsResult::DATABASE_UNREACHABLE) { - error_msg = "Database unreachable"; - } else if (res == CoordinatorsResult::BAD_DATABASE_STATE) { - error_msg = "The database is in an unexpected state from which changing coordinators might be unsafe"; - } else if (res == CoordinatorsResult::COORDINATOR_UNREACHABLE) { - error_msg = "One of the specified coordinators is unreachable"; + if (res == CoordinatorsResult::COORDINATOR_UNREACHABLE) { retriable = true; - } else if (res == CoordinatorsResult::NOT_ENOUGH_MACHINES) { - error_msg = "Too few fdbserver machines to provide coordination at the current redundancy level"; } else if (res == CoordinatorsResult::SUCCESS) { TraceEvent(SevError, "SpecialKeysForCoordinators").detail("UnexpectedSuccessfulResult", ""); - } else { ASSERT(false); } - msg = ManagementAPIError::toJsonString(retriable, "coordinators", error_msg); + msg = ManagementAPIError::toJsonString(retriable, "coordinators", ManagementAPI::generateErrorMessage(res)); } return msg; } @@ -1758,7 +1743,9 @@ ACTOR static Future CoordinatorsAutoImplActor(ReadYourWritesTransac // we could get not_enough_machines if we happen to see the database while the cluster controller is updating // the worker list, so make sure it happens twice before returning a failure ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString( - true, "auto_coordinators", "The auto change attempt did not get enough machines, please try again")); + true, + "auto_coordinators", + "Too few fdbserver machines to provide coordination at the current redundancy level")); throw special_keys_api_failure(); } @@ -2105,7 +2092,7 @@ ACTOR static Future actorLineageGetRangeActor(ReadYourWritesTransac // Open endpoint to target process on each call. This can be optimized at // some point... state ProcessInterface process; - process.getInterface = RequestStream(Endpoint({ host }, WLTOKEN_PROCESS)); + process.getInterface = RequestStream(Endpoint::wellKnown({ host }, WLTOKEN_PROCESS)); ProcessInterface p = wait(retryBrokenPromise(process.getInterface, GetProcessInterfaceRequest{})); process = p; diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp index 0250ce9aff..01f8bedf0b 100644 --- a/fdbclient/StatusClient.actor.cpp +++ b/fdbclient/StatusClient.actor.cpp @@ -309,18 +309,18 @@ ACTOR Future> clientCoordinatorsStatusFetcher(Reference>> leaderServers; + state std::vector>> leaderServers; leaderServers.reserve(coord.clientLeaderServers.size()); for (int i = 0; i < coord.clientLeaderServers.size(); i++) leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader, GetLeaderRequest(coord.clusterKey, UID()), TaskPriority::CoordinationReply)); - state vector> coordProtocols; + state std::vector> coordProtocols; coordProtocols.reserve(coord.clientLeaderServers.size()); for (int i = 0; i < coord.clientLeaderServers.size(); i++) { - RequestStream requestStream{ Endpoint{ - { coord.clientLeaderServers[i].getLeader.getEndpoint().addresses }, WLTOKEN_PROTOCOL_INFO } }; + RequestStream requestStream{ Endpoint::wellKnown( + { coord.clientLeaderServers[i].getLeader.getEndpoint().addresses }, WLTOKEN_PROTOCOL_INFO) }; coordProtocols.push_back(retryBrokenPromise(requestStream, ProtocolInfoRequest{})); } diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 1c8a8a8004..f47e9b7aa7 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -330,7 +330,14 @@ struct GetKeyValuesStreamReply : public ReplyPromiseStreamReply { template void serialize(Ar& ar) { - serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, data, version, more, cached, arena); + serializer(ar, + ReplyPromiseStreamReply::acknowledgeToken, + ReplyPromiseStreamReply::sequence, + data, + version, + more, + cached, + arena); } }; @@ -674,7 +681,7 @@ struct ChangeFeedStreamReply : public ReplyPromiseStreamReply { template void serialize(Ar& ar) { - serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, mutations, arena); + serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, ReplyPromiseStreamReply::sequence, mutations, arena); } }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 5a41111091..daedfa740d 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -242,13 +242,13 @@ const Key storageCacheKey(const KeyRef& k) { return k.withPrefix(storageCachePrefix); } -const Value storageCacheValue(const vector& serverIndices) { +const Value storageCacheValue(const std::vector& serverIndices) { BinaryWriter wr((IncludeVersion(ProtocolVersion::withStorageCacheValue()))); wr << serverIndices; return wr.toValue(); } -void decodeStorageCacheValue(const ValueRef& value, vector& serverIndices) { +void decodeStorageCacheValue(const ValueRef& value, std::vector& serverIndices) { serverIndices.clear(); if (value.size()) { BinaryReader rd(value, IncludeVersion()); @@ -256,25 +256,26 @@ void decodeStorageCacheValue(const ValueRef& value, vector& serverIndi } } -const Value logsValue(const vector>& logs, - const vector>& oldLogs) { +const Value logsValue(const std::vector>& logs, + const std::vector>& oldLogs) { BinaryWriter wr(IncludeVersion(ProtocolVersion::withLogsValue())); wr << logs; wr << oldLogs; return wr.toValue(); } -std::pair>, vector>> decodeLogsValue( +std::pair>, std::vector>> decodeLogsValue( const ValueRef& value) { - vector> logs; - vector> oldLogs; + std::vector> logs; + std::vector> oldLogs; BinaryReader reader(value, IncludeVersion()); reader >> logs; reader >> oldLogs; return std::make_pair(logs, oldLogs); } -const KeyRef serverKeysPrefix = LiteralStringRef("\xff/serverKeys/"); -const ValueRef serverKeysTrue = LiteralStringRef("1"), // compatible with what was serverKeysTrue +const KeyRef serverKeysPrefix = "\xff/serverKeys/"_sr; +const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTrue + serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty. serverKeysFalse; const Key serverKeysKey(UID serverID, const KeyRef& key) { @@ -299,7 +300,7 @@ UID serverKeysDecodeServer(const KeyRef& key) { return server_id; } bool serverHasKey(ValueRef storedValue) { - return storedValue == serverKeysTrue; + return storedValue == serverKeysTrue || storedValue == serverKeysTrueEmptyRange; } const KeyRef cacheKeysPrefix = LiteralStringRef("\xff\x02/cacheKeys/"); @@ -629,6 +630,7 @@ const KeyRangeRef configKeys(LiteralStringRef("\xff/conf/"), LiteralStringRef("\ const KeyRef configKeysPrefix = configKeys.begin; const KeyRef perpetualStorageWiggleKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle")); +const KeyRef perpetualStorageWiggleLocalityKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle_locality")); const KeyRef wigglingStorageServerKey(LiteralStringRef("\xff/storageWigglePID")); const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint")); @@ -1056,7 +1058,7 @@ std::tuple decodeChangeFeedValue(ValueRef const& value) const KeyRangeRef changeFeedDurableKeys(LiteralStringRef("\xff\xff/cf/"), LiteralStringRef("\xff\xff/cf0")); const KeyRef changeFeedDurablePrefix = changeFeedDurableKeys.begin; -const Value changeFeedDurableKey(Key const& feed, Version const& version) { +const Value changeFeedDurableKey(Key const& feed, Version version) { BinaryWriter wr(AssumeVersion(ProtocolVersion::withChangeFeed())); wr.serializeBytes(changeFeedDurablePrefix); wr << feed; @@ -1071,16 +1073,19 @@ std::pair decodeChangeFeedDurableKey(ValueRef const& key) { reader >> version; return std::make_pair(feed, bigEndian64(version)); } -const Value changeFeedDurableValue(Standalone> const& mutations) { +const Value changeFeedDurableValue(Standalone> const& mutations, Version knownCommittedVersion) { BinaryWriter wr(IncludeVersion(ProtocolVersion::withChangeFeed())); wr << mutations; + wr << knownCommittedVersion; return wr.toValue(); } -Standalone> decodeChangeFeedDurableValue(ValueRef const& value) { +std::pair>, Version> decodeChangeFeedDurableValue(ValueRef const& value) { Standalone> mutations; + Version knownCommittedVersion; BinaryReader reader(value, IncludeVersion()); reader >> mutations; - return mutations; + reader >> knownCommittedVersion; + return std::make_pair(mutations, knownCommittedVersion); } const KeyRef configTransactionDescriptionKey = "\xff\xff/description"_sr; diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index cd3fb5635d..bc8d1c591d 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -44,7 +44,7 @@ extern const KeyRangeRef specialKeys; // [FF][FF] to [FF][FF][FF], some client f // using these special keys, see pr#2662 extern const KeyRef afterAllKeys; -// "\xff/keyServers/[[begin]]" := "[[vector, vector]|[vector, vector]]" +// "\xff/keyServers/[[begin]]" := "[[vector, std::vector]|[vector, std::vector]]" // An internal mapping of where shards are located in the database. [[begin]] is the start of the shard range // and the result is a list of serverIDs or Tags where these shards are located. These values can be changed // as data movement occurs. @@ -90,7 +90,7 @@ void decodeStorageCacheValue(const ValueRef& value, std::vector& serve // as the key, the value indicates whether the shard does or does not exist on the server. // These values can be changed as data movement occurs. extern const KeyRef serverKeysPrefix; -extern const ValueRef serverKeysTrue, serverKeysFalse; +extern const ValueRef serverKeysTrue, serverKeysTrueEmptyRange, serverKeysFalse; const Key serverKeysKey(UID serverID, const KeyRef& keys); const Key serverKeysPrefixFor(UID serverID); UID serverKeysDecodeServer(const KeyRef& key); @@ -211,6 +211,7 @@ extern const KeyRangeRef configKeys; extern const KeyRef configKeysPrefix; extern const KeyRef perpetualStorageWiggleKey; +extern const KeyRef perpetualStorageWiggleLocalityKey; extern const KeyRef wigglingStorageServerKey; // Change the value of this key to anything and that will trigger detailed data distribution team info log. extern const KeyRef triggerDDTeamInfoPrintKey; @@ -331,9 +332,9 @@ extern const KeyRef logsKey; // Used during backup/recovery to restrict version requirements extern const KeyRef minRequiredCommitVersionKey; -const Value logsValue(const vector>& logs, - const vector>& oldLogs); -std::pair>, vector>> decodeLogsValue( +const Value logsValue(const std::vector>& logs, + const std::vector>& oldLogs); +std::pair>, std::vector>> decodeLogsValue( const ValueRef& value); // The "global keys" are sent to each storage server any time they are changed @@ -505,10 +506,10 @@ extern const KeyRef changeFeedPrivatePrefix; extern const KeyRangeRef changeFeedDurableKeys; extern const KeyRef changeFeedDurablePrefix; -const Value changeFeedDurableKey(Key const& feed, Version const& version); +const Value changeFeedDurableKey(Key const& feed, Version version); std::pair decodeChangeFeedDurableKey(ValueRef const& key); -const Value changeFeedDurableValue(Standalone> const& mutations); -Standalone> decodeChangeFeedDurableValue(ValueRef const& value); +const Value changeFeedDurableValue(Standalone> const& mutations, Version knownCommittedVersion); +std::pair>, Version> decodeChangeFeedDurableValue(ValueRef const& value); // Configuration database special keys extern const KeyRef configTransactionDescriptionKey; diff --git a/fdbclient/TagThrottle.actor.h b/fdbclient/TagThrottle.actor.h index 4946830ffc..10440e0d45 100644 --- a/fdbclient/TagThrottle.actor.h +++ b/fdbclient/TagThrottle.actor.h @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "flow/Arena.h" #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TAG_THROTTLE_ACTOR_G_H) #define FDBCLIENT_TAG_THROTTLE_ACTOR_G_H #include "fdbclient/TagThrottle.actor.g.h" @@ -247,7 +248,9 @@ ACTOR template Future getValidAutoEnabled(Reference tr) { state bool result; loop { - Optional value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey))); + // hold the returned standalone object's memory + state typename Tr::template FutureT> valueF = tr->get(tagThrottleAutoEnabledKey); + Optional value = wait(safeThreadFutureToFuture(valueF)); if (!value.present()) { tr->reset(); wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF)); @@ -466,10 +469,12 @@ Future unthrottleTags(Reference db, loop { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { + state std::vector>> valueFutures; state std::vector>> values; values.reserve(keys.size()); for (auto key : keys) { - values.push_back(safeThreadFutureToFuture(tr->get(key))); + valueFutures.push_back(tr->get(key)); + values.push_back(safeThreadFutureToFuture(valueFutures.back())); } wait(waitForAll(values)); @@ -535,7 +540,9 @@ Future throttleTags(Reference db, tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { if (throttleType == TagThrottleType::MANUAL) { - Optional oldThrottle = wait(safeThreadFutureToFuture(tr->get(key))); + // hold the returned standalone object's memory + state typename DB::TransactionT::template FutureT> oldThrottleF = tr->get(key); + Optional oldThrottle = wait(safeThreadFutureToFuture(oldThrottleF)); if (!oldThrottle.present()) { wait(updateThrottleCount(tr, 1)); } @@ -562,7 +569,10 @@ Future enableAuto(Reference db, bool enabled) { loop { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); try { - Optional value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey))); + // hold the returned standalone object's memory + state typename DB::TransactionT::template FutureT> valueF = + tr->get(tagThrottleAutoEnabledKey); + Optional value = wait(safeThreadFutureToFuture(valueF)); if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) || (!enabled && value.get() != LiteralStringRef("0"))) { tr->set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0")); diff --git a/fdbclient/WellKnownEndpoints.h b/fdbclient/WellKnownEndpoints.h new file mode 100644 index 0000000000..9d152bb75f --- /dev/null +++ b/fdbclient/WellKnownEndpoints.h @@ -0,0 +1,54 @@ +/* + * WellKnownEndpoints.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FDBCLIENT_WELLKNOWNENDPOINTS_H +#define FDBCLIENT_WELLKNOWNENDPOINTS_H +#pragma once + +#include + +/* + * All well-known endpoints of FDB must be listed here to guarantee their uniqueness + */ +enum WellKnownEndpoints { + WLTOKEN_CLIENTLEADERREG_GETLEADER = WLTOKEN_FIRST_AVAILABLE, // 2 + WLTOKEN_CLIENTLEADERREG_OPENDATABASE, // 3 + WLTOKEN_LEADERELECTIONREG_CANDIDACY, // 4 + WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, // 5 + WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, // 6 + WLTOKEN_LEADERELECTIONREG_FORWARD, // 7 + WLTOKEN_GENERATIONREG_READ, // 8 + WLTOKEN_GENERATIONREG_WRITE, // 9 + WLTOKEN_PROTOCOL_INFO, // 10 : the value of this endpoint should be stable and not change. + WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE, // 11 + WLTOKEN_CONFIGTXN_GETGENERATION, // 12 + WLTOKEN_CONFIGTXN_GET, // 13 + WLTOKEN_CONFIGTXN_GETCLASSES, // 14 + WLTOKEN_CONFIGTXN_GETKNOBS, // 15 + WLTOKEN_CONFIGTXN_COMMIT, // 16 + WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES, // 17 + WLTOKEN_CONFIGFOLLOWER_GETCHANGES, // 18 + WLTOKEN_CONFIGFOLLOWER_COMPACT, // 19 + WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION, // 20 + WLTOKEN_PROCESS, // 21 + WLTOKEN_RESERVED_COUNT // 22 +}; + +#endif \ No newline at end of file diff --git a/fdbclient/rapidjson/document.h b/fdbclient/rapidjson/document.h index 18958dea8f..b94ec4319f 100644 --- a/fdbclient/rapidjson/document.h +++ b/fdbclient/rapidjson/document.h @@ -862,8 +862,7 @@ public: Helper function to enable support for common swap implementation pattern based on \c std::swap: \code void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.value, b.value); + std::swap(a.value, b.value); // ... } \endcode @@ -2326,8 +2325,7 @@ public: Helper function to enable support for common swap implementation pattern based on \c std::swap: \code void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.doc, b.doc); + std::swap(a.doc, b.doc); // ... } \endcode diff --git a/fdbclient/vexillographer/cpp.cs b/fdbclient/vexillographer/cpp.cs index 21a372cb42..100a1bc8e2 100644 --- a/fdbclient/vexillographer/cpp.cs +++ b/fdbclient/vexillographer/cpp.cs @@ -47,8 +47,9 @@ namespace vexillographer private static string getCInfoLine(Option o, string indent, string structName) { - return String.Format("{0}ADD_OPTION_INFO({1}, {2}, \"{2}\", \"{3}\", \"{4}\", {5}, {6}, {7}, {8})", - indent, structName, o.name.ToUpper(), o.comment, o.getParameterComment(), (o.paramDesc != null).ToString().ToLower(), o.hidden.ToString().ToLower(), o.persistent.ToString().ToLower(), o.defaultFor); + return String.Format("{0}ADD_OPTION_INFO({1}, {2}, \"{2}\", \"{3}\", \"{4}\", {5}, {6}, {7}, {8}, FDBOptionInfo::ParamType::{9})", + indent, structName, o.name.ToUpper(), o.comment, o.getParameterComment(), (o.paramDesc != null).ToString().ToLower(), + o.hidden.ToString().ToLower(), o.persistent.ToString().ToLower(), o.defaultFor, o.paramType); } private static void writeCppInfo(TextWriter outFile, Scope scope, IEnumerable