Merge branch 'feature-range-feed' into blob_full

.gitignore

@@ -9,7 +9,7 @@ bindings/java/foundationdb-tests*.jar
 bindings/java/fdb-java-*-sources.jar
 packaging/msi/FDBInstaller.msi
 build/
-cmake-build-debug/
+cmake-build-debug*
 # Generated source, build, and packaging files
 *.g.cpp
 *.g.h

bindings/CMakeLists.txt

@@ -1,6 +1,6 @@
+add_subdirectory(c)
 if(NOT OPEN_FOR_IDE)
   # flow bindings currently doesn't support that
-  add_subdirectory(c)
   add_subdirectory(flow)
 endif()
 if(WITH_PYTHON_BINDING)

bindings/c/CMakeLists.txt

@@ -39,6 +39,8 @@ else()
   strip_debug_symbols(fdb_c)
 endif()
 add_dependencies(fdb_c fdb_c_generated fdb_c_options)
+add_dependencies(fdbclient fdb_c_options)
+add_dependencies(fdbclient_sampling fdb_c_options)
 target_link_libraries(fdb_c PUBLIC $<BUILD_INTERFACE:fdbclient>)
 if(APPLE)
   set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
@@ -80,6 +82,10 @@ if(NOT WIN32)
 
   set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
   set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
+  set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS 
+    test/unit/disconnected_timeout_tests.cpp
+    test/unit/fdb_api.cpp
+    test/unit/fdb_api.hpp)
 
   if(OPEN_FOR_IDE)
     add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
@@ -90,6 +96,7 @@ if(NOT WIN32)
     add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
     add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
     add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
+    add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
   else()
     add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
     add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
@@ -99,6 +106,7 @@ if(NOT WIN32)
     add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
     add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
     add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
+    add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
     strip_debug_symbols(fdb_c_performance_test)
     strip_debug_symbols(fdb_c_ryw_benchmark)
     strip_debug_symbols(fdb_c_txn_size_test)
@@ -110,13 +118,16 @@ if(NOT WIN32)
   add_dependencies(fdb_c_setup_tests doctest)
   add_dependencies(fdb_c_unit_tests doctest)
   add_dependencies(fdb_c_unit_tests_version_510 doctest)
+  add_dependencies(disconnected_timeout_unit_tests doctest)
   target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
   target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
   target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
+  target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
   target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
   target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
   target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
   target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
 
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
@@ -130,13 +141,19 @@ if(NOT WIN32)
     target_link_libraries(fdb_c90_test PRIVATE fdb_c)
   endif()
 
+  if(OPEN_FOR_IDE)
+    set(FDB_C_TARGET $<TARGET_OBJECTS:fdb_c>)
+  else()
+    set(FDB_C_TARGET $<TARGET_FILE:fdb_c>)
+  endif()
   add_custom_command(
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:fdb_c> ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
+    COMMAND ${CMAKE_COMMAND} -E copy ${FDB_C_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
     DEPENDS fdb_c
     COMMENT "Copy libfdb_c to use as external client for test")
   add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so)
   add_dependencies(fdb_c_unit_tests external_client)
+  add_dependencies(disconnected_timeout_unit_tests external_client)
 
   add_fdbclient_test(
     NAME fdb_c_setup_tests
@@ -163,6 +180,17 @@ if(NOT WIN32)
             fdb
             ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
             )
+  add_unavailable_fdbclient_test(
+    NAME disconnected_timeout_unit_tests
+    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+            @CLUSTER_FILE@
+            )
+  add_unavailable_fdbclient_test(
+    NAME disconnected_timeout_external_client_unit_tests
+    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
+            @CLUSTER_FILE@
+            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
+            )
 endif()
 
 set(c_workloads_srcs

bindings/c/test/mako/mako.c

@@ -1330,7 +1330,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
 #else /* >= 610 */
 	fdb_create_database(args->cluster_file, &process.database);
 #endif
-
+	if (args->disable_ryw) {
+		fdb_database_set_option(process.database, FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE, (uint8_t*)NULL, 0);
+	}
 	fprintf(debugme, "DEBUG: creating %d worker threads\n", args->num_threads);
 	worker_threads = (pthread_t*)calloc(sizeof(pthread_t), args->num_threads);
 	if (!worker_threads) {
@@ -1444,6 +1446,7 @@ int init_args(mako_args_t* args) {
 	for (i = 0; i < MAX_OP; i++) {
 		args->txnspec.ops[i][OP_COUNT] = 0;
 	}
+	args->disable_ryw = 0;
 	return 0;
 }
 
@@ -1608,6 +1611,7 @@ void usage() {
 	printf("%-24s %s\n", "    --knobs=KNOBS", "Set client knobs");
 	printf("%-24s %s\n", "    --flatbuffers", "Use flatbuffers");
 	printf("%-24s %s\n", "    --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
+	printf("%-24s %s\n", "    --disable_ryw", "Disable snapshot read-your-writes");
 }
 
 /* parse benchmark paramters */
@@ -1653,6 +1657,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 			                                    { "txntagging", required_argument, NULL, ARG_TXNTAGGING },
 			                                    { "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX },
 			                                    { "version", no_argument, NULL, ARG_VERSION },
+			                                    { "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW },
 			                                    { NULL, 0, NULL, 0 }
 		};
 		idx = 0;
@@ -1800,14 +1805,16 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 				args->txntagging = 1000;
 			}
 			break;
-		case ARG_TXNTAGGINGPREFIX: {
+		case ARG_TXNTAGGINGPREFIX:
 			if (strlen(optarg) > TAGPREFIXLENGTH_MAX) {
 				fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX);
 				exit(0);
 			}
 			memcpy(args->txntagging_prefix, optarg, strlen(optarg));
 			break;
-		}
+		case ARG_DISABLE_RYW:
+			args->disable_ryw = 1;
+			break;
 		}
 	}
 

bindings/c/test/mako/mako.h

@@ -80,7 +80,8 @@ enum Arguments {
 	ARG_TXNTRACE,
 	ARG_TXNTAGGING,
 	ARG_TXNTAGGINGPREFIX,
-	ARG_STREAMING_MODE
+	ARG_STREAMING_MODE,
+	ARG_DISABLE_RYW
 };
 
 enum TPSChangeTypes { TPS_SIN, TPS_SQUARE, TPS_PULSE };
@@ -136,6 +137,7 @@ typedef struct {
 	int txntagging;
 	char txntagging_prefix[TAGPREFIXLENGTH_MAX];
 	FDBStreamingMode streaming_mode;
+	int disable_ryw;
 } mako_args_t;
 
 /* shared memory */

bindings/c/test/unit/disconnected_timeout_tests.cpp

@@ -0,0 +1,292 @@
+/*
+ * disconnected_timeout_tests.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Unit tests that test the timeouts for a disconnected cluster
+
+#define FDB_API_VERSION 710
+#include <foundationdb/fdb_c.h>
+
+#include <chrono>
+#include <iostream>
+#include <string.h>
+#include <thread>
+
+#define DOCTEST_CONFIG_IMPLEMENT
+#include "doctest.h"
+#include "fdb_api.hpp"
+
+void fdb_check(fdb_error_t e) {
+	if (e) {
+		std::cerr << fdb_get_error(e) << std::endl;
+		std::abort();
+	}
+}
+
+FDBDatabase* fdb_open_database(const char* clusterFile) {
+	FDBDatabase* db;
+	fdb_check(fdb_create_database(clusterFile, &db));
+	return db;
+}
+
+static FDBDatabase* db = nullptr;
+static FDBDatabase* timeoutDb = nullptr;
+
+// Blocks until the given future is ready, returning an error code if there was
+// an issue.
+fdb_error_t wait_future(fdb::Future& f) {
+	fdb_check(f.block_until_ready());
+	return f.get_error();
+}
+
+void validateTimeoutDuration(double expectedSeconds, std::chrono::time_point<std::chrono::steady_clock> start) {
+	std::chrono::duration<double> duration = std::chrono::steady_clock::now() - start;
+	double actualSeconds = duration.count();
+	CHECK(actualSeconds >= expectedSeconds - 1e-6);
+	CHECK(actualSeconds < expectedSeconds * 2);
+}
+
+TEST_CASE("500ms_transaction_timeout") {
+	auto start = std::chrono::steady_clock::now();
+
+	fdb::Transaction tr(db);
+
+	int64_t timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("500ms_transaction_timeout_after_op") {
+	auto start = std::chrono::steady_clock::now();
+
+	fdb::Transaction tr(db);
+	fdb::Int64Future grvFuture = tr.get_read_version();
+
+	int64_t timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("500ms_transaction_timeout_before_op_2000ms_after") {
+	auto start = std::chrono::steady_clock::now();
+
+	fdb::Transaction tr(db);
+
+	int64_t timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+
+	timeout = 2000;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("2000ms_transaction_timeout_before_op_500ms_after") {
+	auto start = std::chrono::steady_clock::now();
+
+	fdb::Transaction tr(db);
+
+	int64_t timeout = 2000;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+
+	timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("500ms_database_timeout") {
+	auto start = std::chrono::steady_clock::now();
+
+	int64_t timeout = 500;
+	fdb_check(fdb_database_set_option(
+	    timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Transaction tr(timeoutDb);
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("2000ms_database_timeout_500ms_transaction_timeout") {
+	auto start = std::chrono::steady_clock::now();
+
+	int64_t timeout = 2000;
+	fdb_check(fdb_database_set_option(
+	    timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Transaction tr(timeoutDb);
+
+	timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(timeout / 1000.0, start);
+}
+
+TEST_CASE("500ms_database_timeout_2000ms_transaction_timeout_with_reset") {
+	auto start = std::chrono::steady_clock::now();
+
+	int64_t dbTimeout = 500;
+	fdb_check(fdb_database_set_option(
+	    timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&dbTimeout), sizeof(dbTimeout)));
+
+	fdb::Transaction tr(timeoutDb);
+
+	int64_t trTimeout = 2000;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&trTimeout), sizeof(trTimeout)));
+
+	tr.reset();
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	fdb_error_t err = wait_future(grvFuture);
+
+	CHECK(err == 1031);
+	validateTimeoutDuration(dbTimeout / 1000.0, start);
+}
+
+TEST_CASE("transaction_reset_cancels_without_timeout") {
+	fdb::Transaction tr(db);
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	tr.reset();
+
+	fdb_error_t err = wait_future(grvFuture);
+	CHECK(err == 1025);
+}
+
+TEST_CASE("transaction_reset_cancels_with_timeout") {
+	fdb::Transaction tr(db);
+
+	int64_t timeout = 500;
+	fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	fdb::Int64Future grvFuture = tr.get_read_version();
+	tr.reset();
+
+	fdb_error_t err = wait_future(grvFuture);
+	CHECK(err == 1025);
+}
+
+TEST_CASE("transaction_destruction_cancels_without_timeout") {
+	FDBTransaction* tr;
+	fdb_check(fdb_database_create_transaction(db, &tr));
+
+	FDBFuture* grvFuture = fdb_transaction_get_read_version(tr);
+	fdb_transaction_destroy(tr);
+
+	fdb_check(fdb_future_block_until_ready(grvFuture));
+	fdb_error_t err = fdb_future_get_error(grvFuture);
+	CHECK(err == 1025);
+
+	fdb_future_destroy(grvFuture);
+}
+
+TEST_CASE("transaction_destruction_cancels_with_timeout") {
+	FDBTransaction* tr;
+	fdb_check(fdb_database_create_transaction(db, &tr));
+
+	int64_t timeout = 500;
+	fdb_check(fdb_transaction_set_option(
+	    tr, FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+
+	FDBFuture* grvFuture = fdb_transaction_get_read_version(tr);
+	fdb_transaction_destroy(tr);
+
+	fdb_check(fdb_future_block_until_ready(grvFuture));
+	fdb_error_t err = fdb_future_get_error(grvFuture);
+	CHECK(err == 1025);
+
+	fdb_future_destroy(grvFuture);
+}
+
+TEST_CASE("transaction_set_timeout_and_destroy_repeatedly") {
+	for (int i = 0; i < 1000; ++i) {
+		fdb::Transaction tr(db);
+		int64_t timeout = 500;
+		fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
+	}
+}
+
+int main(int argc, char** argv) {
+	if (argc < 2) {
+		std::cout << "Disconnected timeout unit tests for the FoundationDB C API.\n"
+		          << "Usage: disconnected_timeout_tests <unavailableClusterFile> [externalClient] [doctest args]"
+		          << std::endl;
+		return 1;
+	}
+	fdb_check(fdb_select_api_version(710));
+	if (argc >= 3) {
+		std::string externalClientLibrary = argv[2];
+		if (externalClientLibrary.substr(0, 2) != "--") {
+			fdb_check(fdb_network_set_option(
+			    FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT, reinterpret_cast<const uint8_t*>(""), 0));
+			fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
+			                                 reinterpret_cast<const uint8_t*>(externalClientLibrary.c_str()),
+			                                 externalClientLibrary.size()));
+		}
+	}
+
+	doctest::Context context;
+	context.applyCommandLine(argc, argv);
+
+	fdb_check(fdb_setup_network());
+	std::thread network_thread{ &fdb_run_network };
+
+	db = fdb_open_database(argv[1]);
+	timeoutDb = fdb_open_database(argv[1]);
+
+	int res = context.run();
+	fdb_database_destroy(db);
+	fdb_database_destroy(timeoutDb);
+
+	if (context.shouldExit()) {
+		fdb_check(fdb_stop_network());
+		network_thread.join();
+		return res;
+	}
+	fdb_check(fdb_stop_network());
+	network_thread.join();
+
+	return res;
+}

bindings/go/CMakeLists.txt

@@ -44,7 +44,7 @@ set(go_options_file ${GO_DEST}/src/fdb/generated.go)
 set(go_env GOPATH=${GOPATH}
   C_INCLUDE_PATH=${CMAKE_BINARY_DIR}/bindings/c/foundationdb:${CMAKE_SOURCE_DIR}/bindings/c
   CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/lib
-  GO111MODULE=off)
+  GO111MODULE=auto)
 
 foreach(src_file IN LISTS SRCS)
   set(dest_file ${GO_DEST}/${src_file})

bindings/python/tests/fdbcli_tests.py

@@ -385,7 +385,7 @@ def coordinators(logger):
     # verify now we have 5 coordinators and the description is updated
     output2 = run_fdbcli_command('coordinators')
     assert output2.split('\n')[0].split(': ')[-1] == new_cluster_description
-    assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(5, ','.join(addresses))
+    assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(args.process_number, ','.join(addresses))
     # auto change should go back to 1 coordinator
     run_fdbcli_command('coordinators', 'auto')
     assert len(get_value_from_status_json(True, 'client', 'coordinators', 'coordinators')) == 1

cmake/AddFdbTest.cmake

@@ -128,7 +128,8 @@ function(add_fdb_test)
       -n ${test_name}
       -b ${PROJECT_BINARY_DIR}
       -t ${test_type}
-      -O ${OLD_FDBSERVER_BINARY}
+      -O ${OLD_FDBSERVER_BINARY}  
+      --config "@CTEST_CONFIGURATION_TYPE@"
       --crash
       --aggregate-traces ${TEST_AGGREGATE_TRACES}
       --log-format ${TEST_LOG_FORMAT}
@@ -442,6 +443,40 @@ function(add_fdbclient_test)
   set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
 endfunction()
 
+# Creates a cluster file for a nonexistent cluster before running the specified command 
+# (usually a ctest test)
+function(add_unavailable_fdbclient_test)
+  set(options DISABLED ENABLED)
+  set(oneValueArgs NAME TEST_TIMEOUT)
+  set(multiValueArgs COMMAND)
+  cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
+  if(OPEN_FOR_IDE)
+    return()
+  endif()
+  if(NOT T_ENABLED AND T_DISABLED)
+    return()
+  endif()
+  if(NOT T_NAME)
+    message(FATAL_ERROR "NAME is a required argument for add_unavailable_fdbclient_test")
+  endif()
+  if(NOT T_COMMAND)
+    message(FATAL_ERROR "COMMAND is a required argument for add_unavailable_fdbclient_test")
+  endif()
+  message(STATUS "Adding unavailable client test ${T_NAME}")
+  add_test(NAME "${T_NAME}"
+  COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
+          --output-dir ${CMAKE_BINARY_DIR}
+          --
+          ${T_COMMAND})
+  if (T_TEST_TIMEOUT)
+    set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT ${T_TEST_TIMEOUT})
+  else()
+    # default timeout
+    set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60)
+  endif()
+  set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
+endfunction()
+
 # Creates 3 distinct clusters before running the specified command.
 # This is useful for testing features that require multiple clusters (like the
 # multi-cluster FDB client)

cmake/CompileRocksDB.cmake

@@ -1,6 +1,6 @@
 # FindRocksDB
 
-find_package(RocksDB)
+find_package(RocksDB 6.22.1)
 
 include(ExternalProject)
 

cmake/FindRocksDB.cmake

@@ -4,5 +4,20 @@ find_path(ROCKSDB_INCLUDE_DIR
   NAMES rocksdb/db.h
   PATH_SUFFIXES include)
 
+if(ROCKSDB_INCLUDE_DIR AND EXISTS "${ROCKSDB_INCLUDE_DIR}/rocksdb/version.h")
+  foreach(ver "MAJOR" "MINOR" "PATCH")
+    file(STRINGS "${ROCKSDB_INCLUDE_DIR}/rocksdb/version.h" ROCKSDB_VER_${ver}_LINE
+      REGEX "^#define[ \t]+ROCKSDB_${ver}[ \t]+[0-9]+$")
+    string(REGEX REPLACE "^#define[ \t]+ROCKSDB_${ver}[ \t]+([0-9]+)$"
+      "\\1" ROCKSDB_VERSION_${ver} "${ROCKSDB_VER_${ver}_LINE}")
+    unset(${ROCKSDB_VER_${ver}_LINE})
+  endforeach()
+  set(ROCKSDB_VERSION_STRING
+    "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}")
+
+  message(STATUS "Found RocksDB version: ${ROCKSDB_VERSION_STRING}")
+endif()
+
 find_package_handle_standard_args(RocksDB
-  DEFAULT_MSG ROCKSDB_INCLUDE_DIR)
+  REQUIRED_VARS ROCKSDB_INCLUDE_DIR
+  VERSION_VAR ROCKSDB_VERSION_STRING)

cmake/InstallLayout.cmake

@@ -215,17 +215,17 @@ set(CPACK_COMPONENT_SERVER-DEB_DEPENDS clients-deb)
 set(CPACK_COMPONENT_SERVER-TGZ_DEPENDS clients-tgz)
 set(CPACK_COMPONENT_SERVER-VERSIONED_DEPENDS clients-versioned)
 set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_REQUIRES
-  "foundationdb-clients-${FDB_MAJOR}.${FDB_MINOR}.${FDB_PATCH} = ${FDB_MAJOR}.${FDB_MINOR}.${FDB_PATCH}")
+  "foundationdb${PROJECT_VERSION}-clients")
 
 set(CPACK_COMPONENT_SERVER-EL7_DISPLAY_NAME "foundationdb-server")
 set(CPACK_COMPONENT_SERVER-DEB_DISPLAY_NAME "foundationdb-server")
 set(CPACK_COMPONENT_SERVER-TGZ_DISPLAY_NAME "foundationdb-server")
-set(CPACK_COMPONENT_SERVER-VERSIONED_DISPLAY_NAME "foundationdb-server-${PROJECT_VERSION}")
+set(CPACK_COMPONENT_SERVER-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-server")
 
 set(CPACK_COMPONENT_CLIENTS-EL7_DISPLAY_NAME "foundationdb-clients")
 set(CPACK_COMPONENT_CLIENTS-DEB_DISPLAY_NAME "foundationdb-clients")
 set(CPACK_COMPONENT_CLIENTS-TGZ_DISPLAY_NAME "foundationdb-clients")
-set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb-clients-${PROJECT_VERSION}")
+set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients")
 
 
 # MacOS needs a file exiension for the LICENSE file
@@ -246,14 +246,21 @@ else()
   set(prerelease_string "-1")
 endif()
 
-
+#############
+# Filenames #
+#############
+set(unversioned_postfix "${PROJECT_VERSION}${prerelease_string}")
 # RPM filenames
-set(rpm-clients-filename "foundationdb-clients-${PROJECT_VERSION}${prerelease_string}")
-set(rpm-server-filename "foundationdb-server-${PROJECT_VERSION}${prerelease_string}")
+set(rpm-clients-filename "foundationdb-clients-${unversioned_postfix}")
+set(rpm-server-filename "foundationdb-server-${unversioned_postfix}")
+set(rpm-clients-versioned-filename "foundationdb${PROJECT_VERSION}-clients${prerelease_string}")
+set(rpm-server-versioned-filename "foundationdb${PROJECT_VERSION}-server${prerelease_string}")
 
 # Deb filenames
-set(deb-clients-filename "foundationdb-clients_${PROJECT_VERSION}${prerelease_string}")
-set(deb-server-filename "foundationdb-server_${PROJECT_VERSION}${prerelease_string}")
+set(deb-clients-filename "foundationdb-clients_${unversioned_postfix}")
+set(deb-server-filename "foundationdb-server_${unversioned_postfix}")
+set(deb-clients-versioned-filename "foundationdb${PROJECT_VERSION}-clients${prerelease_string}")
+set(deb-server-versioned-filename "foundationdb${PROJECT_VERSION}-server${prerelease_string}")
 
 ################################################################################
 # Configuration for RPM
@@ -264,18 +271,18 @@ set(CPACK_RPM_PACKAGE_LICENSE "Apache 2.0")
 set(CPACK_RPM_PACKAGE_NAME "foundationdb")
 set(CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME "foundationdb-clients")
 set(CPACK_RPM_SERVER-EL7_PACKAGE_NAME "foundationdb-server")
-set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_NAME "foundationdb-server-${PROJECT_VERSION}")
-set(CPACK_RPM_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb-clients-${PROJECT_VERSION}")
+set(CPACK_RPM_SERVER-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-server")
+set(CPACK_RPM_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-clients")
 
 set(CPACK_RPM_CLIENTS-EL7_FILE_NAME "${rpm-clients-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
+set(CPACK_RPM_CLIENTS-VERSIONED_FILE_NAME "${rpm-clients-versioned-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 set(CPACK_RPM_SERVER-EL7_FILE_NAME "${rpm-server-filename}.el7.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
+set(CPACK_RPM_SERVER-VERSIONED_FILE_NAME "${rpm-server-versioned-filename}.versioned.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 
 set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
+set(CPACK_RPM_CLIENTS-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-clients-versioned-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 set(CPACK_RPM_SERVER-EL7_DEBUGINFO_FILE_NAME "${rpm-server-filename}.el7-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
-set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
+set(CPACK_RPM_SERVER-VERSIONED_DEBUGINFO_FILE_NAME "${rpm-server-versioned-filename}.versioned-debuginfo.${CMAKE_SYSTEM_PROCESSOR}.rpm")
 
 file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir")
 fdb_install(DIRECTORY "${CMAKE_BINARY_DIR}/packaging/emptydir/" DESTINATION data COMPONENT server)
@@ -347,13 +354,13 @@ set(CPACK_RPM_CLIENTS-VERSIONED_PRE_UNINSTALL_SCRIPT_FILE
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
   set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb")
   set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb")
-  set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_amd64.deb")
-  set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_amd64.deb")
+  set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-versioned-filename}.versioned_amd64.deb")
+  set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-versioned-filename}.versioned_amd64.deb")
 else()
   set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
   set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
-  set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
-  set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
+  set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-versioned-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
+  set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-versioned-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
 endif()
 
 set(CPACK_DEB_COMPONENT_INSTALL ON)
@@ -363,8 +370,8 @@ set(CPACK_DEBIAN_ENABLE_COMPONENT_DEPENDS ON)
 
 set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_NAME "foundationdb-server")
 set(CPACK_DEBIAN_CLIENTS-DEB_PACKAGE_NAME "foundationdb-clients")
-set(CPACK_DEBIAN_SERVER-VERSIONED_PACKAGE_NAME "foundationdb-server-${PROJECT_VERSION}")
-set(CPACK_DEBIAN_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb-clients-${PROJECT_VERSION}")
+set(CPACK_DEBIAN_SERVER-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-server")
+set(CPACK_DEBIAN_CLIENTS-VERSIONED_PACKAGE_NAME "foundationdb${PROJECT_VERSION}-clients")
 
 set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_DEPENDS "adduser, libc6 (>= 2.12), foundationdb-clients (= ${FDB_VERSION})")
 set(CPACK_DEBIAN_SERVER-DEB_PACKAGE_RECOMMENDS "python (>= 2.6)")

documentation/CMakeLists.txt

@@ -1,23 +1,24 @@
 add_subdirectory(tutorial)
-if(WIN32)
-  return()
-endif()
-
 # build a virtualenv
 set(sphinx_dir ${CMAKE_CURRENT_SOURCE_DIR}/sphinx)
 set(venv_dir ${CMAKE_CURRENT_BINARY_DIR}/venv)
-set(EXE_SUFFIX "")
 if(WIN32)
+  set(venv_bin_dir ${CMAKE_CURRENT_BINARY_DIR}/venv/Scripts)
+  set(activate_script ${venv_bin_dir}/activate.bat)
   set(EXE_SUFFIX ".exe")
+else()
+  set(venv_bin_dir ${CMAKE_CURRENT_BINARY_DIR}/venv/bin)
+  set(activate_script . ${venv_bin_dir}/activate)
+  set(EXE_SUFFIX "")
 endif()
-set(pip_command ${venv_dir}/bin/pip${EXE_SUFFIX})
-set(python_command ${venv_dir}/bin/python${EXE_SUFFIX})
+set(python_command ${venv_bin_dir}/python${EXE_SUFFIX})
+set(pip_command ${venv_bin_dir}/pip${EXE_SUFFIX})
 
 add_custom_command(OUTPUT ${venv_dir}/venv_setup
   COMMAND ${Python3_EXECUTABLE} -m venv venv &&
           ${CMAKE_COMMAND} -E copy ${sphinx_dir}/.pip.conf ${venv_dir}/pip.conf &&
-          . ${venv_dir}/bin/activate &&
-          ${pip_command} install --upgrade pip &&
+          ${activate_script} &&
+          ${python_command} -m pip install --upgrade pip &&
           ${pip_command} install --upgrade -r ${sphinx_dir}/requirements.txt &&
           ${pip_command} install sphinx-autobuild && # somehow this is missing in requirements.txt
           ${CMAKE_COMMAND} -E touch ${venv_dir}/venv_setup
@@ -36,9 +37,9 @@ function(add_documentation_target)
     message(ERROR "GENERATOR is a required argument to add_documentation_target")
   endif()
   set(target ${ADT_GENERATOR})
-  set(SPHINX_COMMAND "${venv_dir}/bin/sphinx-build")
+  set(SPHINX_COMMAND "${venv_bin_dir}/sphinx-build${EXE_SUFFIX}")
   if(ADT_SPHINX_COMMAND)
-    set(SPHINX_COMMAND "${venv_dir}/bin/${ADT_SPHINX_COMMAND}")
+    set(SPHINX_COMMAND "${venv_bin_dir}/${ADT_SPHINX_COMMAND}")
   endif()
   set(doctree "doctree")
   if (ADT_DOCTREE)

documentation/sphinx/source/cap-theorem.rst

@@ -24,7 +24,7 @@ Let's consider an **AP** database. In such a database, reads and writes would al
 
 However, the downside is stark. Imagine a simple distributed database consisting of two nodes and a network partition making them unable to communicate. To be Available, each of the two nodes must continue to accept writes from clients.
 
-.. figure:: /images/AP_Partition.png
+.. figure:: images/AP_Partition.png
 
    Data divergence in an AP system during partition
 
@@ -62,7 +62,7 @@ Imagine that a rack-top switch fails, and A is partitioned from the network. A w
 
 However, for all other clients, the database servers can reach a majority of coordination servers, B and C. The replication configuration has ensured there is a full copy of the data available even without A. For these clients, the database will remain available for reads and writes and the web servers will continue to serve traffic.
 
-.. figure:: /images/FDB_Partition.png
+.. figure:: images/FDB_Partition.png
 
     Maintenance of availability during partition
     

documentation/sphinx/source/command-line-interface.rst

@@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu
 configure
 ---------
 
-The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>]``.
+The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>] [perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>] [storage_migration_type={disabled|aggressive|gradual}]``.
 
 The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified.
 
@@ -112,7 +112,24 @@ For recommendations on appropriate values for process types in large clusters, s
 perpetual storage wiggle
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-Set the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle. For more details, see :ref:`perpetual-storage-wiggle`.
+``perpetual_storage_wiggle`` sets the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle.
+``perpetual_storage_wiggle_locality`` sets the process filter for wiggling. The processes that match the given locality key and locality value are only wiggled. The value 0 will disable the locality filter and matches all the processes for wiggling.
+
+For more details, see :ref:`perpetual-storage-wiggle`.
+
+storage migration type
+^^^^^^^^^^^^^^^^^^^^^^
+
+Set the storage migration type, or how FDB should migrate to a new storage engine if the value is changed.
+The default is ``disabled``, which means changing the storage engine will not be possible.
+
+* ``disabled``
+* ``gradual``
+* ``aggressive``
+
+``gradual`` replaces a single storage at a time when the ``perpetual storage wiggle`` is active. This requires the perpetual storage wiggle to be set to a non-zero value to actually migrate storage servers. It is somewhat slow but very safe. This is the recommended method for all production clusters.
+``aggressive`` tries to replace as many storages as it can at once, and will recruit a new storage server on the same process as the old one. This will be faster, but can potentially hit degraded performance or OOM with two storages on the same process. The main benefit over ``gradual`` is that this doesn't need to take one storage out of rotation, so it works for small or development clusters that have the same number of storage processes as the replication factor. Note that ``aggressive`` is not exclusive to running the perpetual wiggle.
+``disabled`` means that if the storage engine is changed, fdb will not move the cluster over to the new storage engine. This will disable the perpetual wiggle from rewriting storage files.
 
 consistencycheck
 ----------------

documentation/sphinx/source/ha-write-path.rst

@@ -176,7 +176,7 @@ The *LogPushData* class is used to hold serialized mutations on a per transactio
 
 *LogPushData.writeTypedMessage* is the function that serializes each mutation and writes it to the correct binary stream to be sent to the corresponding transaction log. Each serialized mutation contains additional metadata about the message, with the format:
 
-.. image:: /images/serialized_mutation_metadata_format.png
+.. image:: images/serialized_mutation_metadata_format.png
 
 * Message size: size of the message, in bytes, excluding the four bytes used for the message size
 

documentation/sphinx/source/mr-status-json-schemas.rst.inc

@@ -256,7 +256,7 @@
             "excluded":false,
             "address":"1.2.3.4:1234",
             "disk":{
-               "free_bytes":3451233456234, // an estimate of how many bytes are free to allocate to fdbservers without swapping
+               "free_bytes":3451233456234,
                "reads":{
                   "hz":0.0,
                   "counter":0,
@@ -268,7 +268,7 @@
                   "counter":0,
                   "sectors":0
                },
-               "total_bytes":123412341234 // an estimate of total physical RAM
+               "total_bytes":123412341234
             },
             "uptime_seconds":1234.2345,
             "cpu":{
@@ -735,7 +735,14 @@
          "grv_proxies":1, // this field will be absent if a value has not been explicitly set
          "proxies":6, // this field will be absent if a value has not been explicitly set
          "backup_worker_enabled":1,
-         "perpetual_storage_wiggle": 0
+         "perpetual_storage_wiggle": 0,
+         "perpetual_storage_wiggle_locality":"0",
+         "storage_migration_type":{
+         "$enum":[
+             "disabled",
+             "gradual",
+             "aggressive"
+         ]}
       },
       "data":{
          "least_operating_space_bytes_log_server":0,

documentation/sphinx/source/performance.rst

@@ -9,7 +9,7 @@ Scaling
 
 FoundationDB scales linearly with the number of cores in a cluster over a wide range of sizes.
 
-.. image:: /images/scaling.png
+.. image:: images/scaling.png
 
 Here, a cluster of commodity hardware scales to **8.2 million** operations/sec doing a 90% read and 10% write workload with 16 byte keys and values between 8 and 100 bytes.
 
@@ -24,7 +24,7 @@ Latency
 
 FoundationDB has low latencies over a broad range of workloads that only increase modestly as the cluster approaches saturation.
 
-.. image:: /images/latency.png
+.. image:: images/latency.png
 
 When run at less than **75% load**, FoundationDB typically has the following latencies:
 
@@ -53,7 +53,7 @@ Throughput (per core)
 
 FoundationDB provides good throughput for the full range of read and write workloads, with two fully durable storage engine options.
 
-.. image:: /images/throughput.png
+.. image:: images/throughput.png
 
 FoundationDB offers two :ref:`storage engines <configuration-storage-engine>`, optimized for distinct use cases, both of which write to disk before reporting transactions committed. For each storage engine, the graph shows throughput of a single FoundationDB process running on a **single core** with saturating read/write workloads ranging from 100% reads to 100% writes, all with 16 byte keys and values between 8 and 100 bytes. Throughput for the unmixed workloads is about:
 
@@ -79,7 +79,7 @@ Concurrency
 
 FoundationDB is designed to achieve great performance under high concurrency from a large number of clients.
 
-.. image:: /images/concurrency.png
+.. image:: images/concurrency.png
 
 Its asynchronous design allows it to handle very high concurrency, and for a typical workload with 90% reads and 10% writes, maximum throughput is reached at about 200 concurrent operations. This number of operations was achieved with **20** concurrent transactions per FoundationDB process each running 10 operations with 16 byte keys and values between 8 and 100 bytes.
 

documentation/sphinx/source/perpetual-storage-wiggle.rst

@@ -31,6 +31,8 @@ Configuration
 
 You can configure the Perpetual Storage Wiggle via the FDB :ref:`command line interface <command-line-interface>`.
 
+Note that to have the Perpetual Storage Wiggle change the storage engine type, you must configure ``storage_migration_type=gradual``.
+
 Example commands
 ----------------
 
@@ -38,6 +40,10 @@ Open perpetual storage wiggle: ``configure perpetual_storage_wiggle=1``.
 
 Disable perpetual storage wiggle on the cluster: ``configure perpetual_storage_wiggle=0``.
 
+Open perpetual storage wiggle for only processes matching the given locality key and value: ``configure perpetual_storage_wiggle=1 perpetual_storage_wiggle_locality=<LOCALITY_KEY>:<LOCALITY_VALUE>``.
+
+Disable perpetual storage wiggle locality matching filter, which wiggles all the processes: ``configure perpetual_storage_wiggle_locality=0``.
+
 Monitor
 =======
 

documentation/sphinx/source/read-write-path.rst

@@ -16,7 +16,7 @@ The processing order of multiple transactions is important because it affects th
 The content is based on FDB 6.2 and is true for FDB 6.3.  A new timestamp proxy role is introduced in post FDB 6.3,
 which affects the read path. We will discuss the timestamp proxy role in the future version of this document.
 
-.. image:: /images/FDB_read_path.png
+.. image:: images/FDB_read_path.png
 
 Components
 =================
@@ -198,7 +198,7 @@ Write path of a transaction
 Suppose a client has a write-only transaction. Fig. 2 below shows the write path in a non-HA cluster.
 We will discuss how a transaction with both read and write works in the next section.
 
-.. image:: /images/FDB_write_path.png
+.. image:: images/FDB_write_path.png
 
 To simplify the explanation, the steps below do not include transaction batching on proxy,
 which is a typical database technique to increase transaction throughput.
@@ -461,7 +461,7 @@ The ordering is enforced in the timestamp generator, the concurrency control com
 We use the following example and draw its swimlane diagram to illustrate how two write transactions are ordered in FDB.
 The diagram with notes can be viewed at `here <https://lucid.app/lucidchart/6336dbe3-cff4-4c46-995a-4ca3d9260696/view?page=0_0#?folder_id=home&browser=icon>`_.
 
-.. image:: /images/FDB_multiple_txn_swimlane_diagram.png
+.. image:: images/FDB_multiple_txn_swimlane_diagram.png
 
 Reference
 ============

documentation/sphinx/source/release-notes/release-notes-630.rst

@@ -2,9 +2,31 @@
 Release Notes
 #############
 
+6.3.21
+======
+* Added a ThreadID field to all trace events for the purpose of multi-threaded client debugging. `(PR #5665) <https://github.com/apple/foundationdb/pull/5665>`_
+* Fixed some histograms' group name in the master proxy. `(PR #5674) <https://github.com/apple/foundationdb/pull/5674>`_
+* Added histograms for GRV path components in the proxy. `(PR #5689) <https://github.com/apple/foundationdb/pull/5689>`_
+* Fixed race condition introduced in 6.3.20 between setting timeouts and resetting or destroying transactions. `(PR #5695) <https://github.com/apple/foundationdb/pull/5695>`_
+* Disable detailed transaction log pop tracing by default. `(PR #5696) <https://github.com/apple/foundationdb/pull/5696>`_
+
+6.3.20
+======
+* Several minor problems with the versioned packages have been fixed. `(PR 5607) <https://github.com/apple/foundationdb/pull/5607>`_
+* A client might not honor transaction timeouts when using the multi-version client if it cannot connect to the cluster. `(Issue #5595) <https://github.com/apple/foundationdb/issues/5595>`_
+* Fixed a very rare bug where recovery could potentially roll back a committed transaction `(PR 5461) <https://github.com/apple/foundationdb/pull/5461>`_
+* Added histograms for commit path components in the proxy. `(PR #5367) <https://github.com/apple/foundationdb/pull/5367>`_
+* Fixed a false checkRegions call that could cause unwanted primary DC failover. `(PR #5330) <https://github.com/apple/foundationdb/pull/5330>`_
+
 6.3.19
 ======
-* Add the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) <https://github.com/apple/foundationdb/pull/5330>`_
+* Added the ``trace_partial_file_suffix`` network option. This option will give unfinished trace files a special suffix to indicate they're not complete yet. When the trace file is complete, it is renamed to remove the suffix. `(PR #5330) <https://github.com/apple/foundationdb/pull/5330>`_
+* Added error details in ``RemovedDeadBackupLayerStatus`` trace event. `(PR #5356) <https://github.com/apple/foundationdb/pull/5356>`_
+* Added RepeatableReadMultiThreadClientTest. `(PR #5212) <https://github.com/apple/foundationdb/pull/5212>`_
+* Added a new feature that allows FDB to detect grey failures and automatically recover from them. `(PR #5249) <https://github.com/apple/foundationdb/pull/5249>`_
+* Added version and timestamp to ``TimeKeeperCommit`` trace event. `(PR #5415) <https://github.com/apple/foundationdb/pull/5415>`_
+* Added ``RecruitFromConfigurationRetry`` trace event to improve recruitment observability. `(PR #5455) <https://github.com/apple/foundationdb/pull/5455>`_
+* Several fixes to pkg_tester and packaging. `(PR #5460) <https://github.com/apple/foundationdb/pull/5460>`_
 
 6.3.18
 ======

documentation/sphinx/source/release-notes/release-notes-700.rst

@@ -64,6 +64,7 @@ Fixes
 * If a restore is done using a prefix to remove and specific key ranges to restore, the key range boundaries must begin with the prefix to remove. `(PR #4684) <https://github.com/apple/foundationdb/pull/4684>`_
 * The multi-version client API would not propagate errors that occurred when creating databases on external clients. This could result in a invalid memory accesses. `(PR #5220) <https://github.com/apple/foundationdb/pull/5220>`_
 * Fixed a race between the multi-version client connecting to a cluster and destroying the database that could cause an assertion failure. `(PR #5220) <https://github.com/apple/foundationdb/pull/5220>`_
+* A client might not honor transaction timeouts when using the multi-version client if it cannot connect to the cluster. `(Issue #5595) <https://github.com/apple/foundationdb/issues/5595>`_
 
 Status
 ------

documentation/tutorial/tutorial.actor.cpp

@@ -33,6 +33,12 @@
 
 NetworkAddress serverAddress;
 
+enum TutorialWellKnownEndpoints {
+	WLTOKEN_SIMPLE_KV_SERVER = WLTOKEN_FIRST_AVAILABLE,
+	WLTOKEN_ECHO_SERVER,
+	WLTOKEN_COUNT_IN_TUTORIAL
+};
+
 // this is a simple actor that will report how long
 // it is already running once a second.
 ACTOR Future<Void> simpleTimer() {
@@ -153,7 +159,7 @@ struct StreamReply : ReplyPromiseStreamReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, index);
+		serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, ReplyPromiseStreamReply::sequence, index);
 	}
 };
 
@@ -171,7 +177,7 @@ uint64_t tokenCounter = 1;
 
 ACTOR Future<Void> echoServer() {
 	state EchoServerInterface echoServer;
-	echoServer.getInterface.makeWellKnownEndpoint(UID(-1, ++tokenCounter), TaskPriority::DefaultEndpoint);
+	echoServer.getInterface.makeWellKnownEndpoint(WLTOKEN_ECHO_SERVER, TaskPriority::DefaultEndpoint);
 	loop {
 		try {
 			choose {
@@ -204,7 +210,8 @@ ACTOR Future<Void> echoServer() {
 
 ACTOR Future<Void> echoClient() {
 	state EchoServerInterface server;
-	server.getInterface = RequestStream<GetInterfaceRequest>(Endpoint({ serverAddress }, UID(-1, ++tokenCounter)));
+	server.getInterface =
+	    RequestStream<GetInterfaceRequest>(Endpoint::wellKnown({ serverAddress }, WLTOKEN_ECHO_SERVER));
 	EchoServerInterface s = wait(server.getInterface.getReply(GetInterfaceRequest()));
 	server = s;
 	EchoRequest echoRequest;
@@ -291,7 +298,7 @@ struct ClearRequest {
 ACTOR Future<Void> kvStoreServer() {
 	state SimpleKeyValueStoreInteface inf;
 	state std::map<std::string, std::string> store;
-	inf.connect.makeWellKnownEndpoint(UID(-1, ++tokenCounter), TaskPriority::DefaultEndpoint);
+	inf.connect.makeWellKnownEndpoint(WLTOKEN_SIMPLE_KV_SERVER, TaskPriority::DefaultEndpoint);
 	loop {
 		choose {
 			when(GetKVInterface req = waitNext(inf.connect.getFuture())) {
@@ -328,7 +335,7 @@ ACTOR Future<Void> kvStoreServer() {
 ACTOR Future<SimpleKeyValueStoreInteface> connect() {
 	std::cout << format("%llu: Connect...\n", uint64_t(g_network->now()));
 	SimpleKeyValueStoreInteface c;
-	c.connect = RequestStream<GetKVInterface>(Endpoint({ serverAddress }, UID(-1, ++tokenCounter)));
+	c.connect = RequestStream<GetKVInterface>(Endpoint::wellKnown({ serverAddress }, WLTOKEN_SIMPLE_KV_SERVER));
 	SimpleKeyValueStoreInteface result = wait(c.connect.getReply(GetKVInterface()));
 	std::cout << format("%llu: done..\n", uint64_t(g_network->now()));
 	return result;
@@ -562,7 +569,7 @@ int main(int argc, char* argv[]) {
 	}
 	platformInit();
 	g_network = newNet2(TLSConfig(), false, true);
-	FlowTransport::createInstance(!isServer, 0);
+	FlowTransport::createInstance(!isServer, 0, WLTOKEN_COUNT_IN_TUTORIAL);
 	NetworkAddress publicAddress = NetworkAddress::parse("0.0.0.0:0");
 	if (isServer) {
 		publicAddress = NetworkAddress::parse("0.0.0.0:" + port);

fdbbackup/backup.actor.cpp

@@ -52,8 +52,6 @@
 #include <string>
 #include <iostream>
 #include <ctime>
-using std::cout;
-using std::endl;
 
 #ifdef _WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -4235,14 +4233,14 @@ int main(int argc, char* argv[]) {
 
 #ifdef ALLOC_INSTRUMENTATION
 		{
-			cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
-			     << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " "
-			     << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " "
-			     << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " "
-			     << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " "
-			     << FastAllocator<16384>::pageCount << endl;
+			std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
+			          << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " "
+			          << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " "
+			          << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " "
+			          << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " "
+			          << FastAllocator<16384>::pageCount << std::endl;
 
-			vector<std::pair<std::string, const char*>> typeNames;
+			std::vector<std::pair<std::string, const char*>> typeNames;
 			for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) {
 				std::string s;
 

fdbcli/CMakeLists.txt

@@ -3,13 +3,19 @@ set(FDBCLI_SRCS
   fdbcli.actor.h
   AdvanceVersionCommand.actor.cpp
   CacheRangeCommand.actor.cpp
+  ConfigureCommand.actor.cpp
   ConsistencyCheckCommand.actor.cpp
+  CoordinatorsCommand.actor.cpp
   DataDistributionCommand.actor.cpp
+  ExcludeCommand.actor.cpp
   ExpensiveDataCheckCommand.actor.cpp
+  FileConfigureCommand.actor.cpp
   FlowLineNoise.actor.cpp
   FlowLineNoise.h
   ForceRecoveryWithDataLossCommand.actor.cpp
+  IncludeCommand.actor.cpp
   KillCommand.actor.cpp
+  LockCommand.actor.cpp
   MaintenanceCommand.actor.cpp
   ProfileCommand.actor.cpp
   SetClassCommand.actor.cpp

fdbcli/ConfigureCommand.actor.cpp

@@ -0,0 +1,300 @@
+/*
+ * ConfigureCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/FlowLineNoise.h"
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/ManagementAPI.actor.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
+                                         Database localDb,
+                                         std::vector<StringRef> tokens,
+                                         LineNoise* linenoise,
+                                         Future<Void> warn) {
+	state ConfigurationResult result;
+	state StatusObject s;
+	state int startToken = 1;
+	state bool force = false;
+	if (tokens.size() < 2)
+		result = ConfigurationResult::NO_OPTIONS_PROVIDED;
+	else {
+		if (tokens[startToken] == LiteralStringRef("FORCE")) {
+			force = true;
+			startToken = 2;
+		}
+
+		state Optional<ConfigureAutoResult> conf;
+		if (tokens[startToken] == LiteralStringRef("auto")) {
+			// get cluster status
+			state Reference<ITransaction> tr = db->createTransaction();
+			if (!tr->isValid()) {
+				StatusObject _s = wait(StatusClient::statusFetcher(localDb));
+				s = _s;
+			} else {
+				state ThreadFuture<Optional<Value>> statusValueF = tr->get(LiteralStringRef("\xff\xff/status/json"));
+				Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
+				if (!statusValue.present()) {
+					fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
+					return false;
+				}
+				json_spirit::mValue mv;
+				json_spirit::read_string(statusValue.get().toString(), mv);
+				s = StatusObject(mv.get_obj());
+			}
+
+			if (warn.isValid())
+				warn.cancel();
+
+			conf = parseConfig(s);
+
+			if (!conf.get().isValid()) {
+				printf("Unable to provide advice for the current configuration.\n");
+				return false;
+			}
+
+			bool noChanges = conf.get().old_replication == conf.get().auto_replication &&
+			                 conf.get().old_logs == conf.get().auto_logs &&
+			                 conf.get().old_commit_proxies == conf.get().auto_commit_proxies &&
+			                 conf.get().old_grv_proxies == conf.get().auto_grv_proxies &&
+			                 conf.get().old_resolvers == conf.get().auto_resolvers &&
+			                 conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction &&
+			                 conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction;
+
+			bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs &&
+			                        conf.get().old_commit_proxies == conf.get().desired_commit_proxies &&
+			                        conf.get().old_grv_proxies == conf.get().desired_grv_proxies &&
+			                        conf.get().old_resolvers == conf.get().desired_resolvers;
+
+			std::string outputString;
+
+			outputString += "\nYour cluster has:\n\n";
+			outputString += format("  processes %d\n", conf.get().processes);
+			outputString += format("  machines  %d\n", conf.get().machines);
+
+			if (noDesiredChanges)
+				outputString += "\nConfigure recommends keeping your current configuration:\n\n";
+			else if (noChanges)
+				outputString +=
+				    "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n";
+			else
+				outputString += "\nConfigure recommends the following changes:\n\n";
+			outputString += " ------------------------------------------------------------------- \n";
+			outputString += "| parameter                   | old              | new              |\n";
+			outputString += " ------------------------------------------------------------------- \n";
+			outputString += format("| replication                 | %16s | %16s |\n",
+			                       conf.get().old_replication.c_str(),
+			                       conf.get().auto_replication.c_str());
+			outputString +=
+			    format("| logs                        | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs);
+			outputString += conf.get().auto_logs != conf.get().desired_logs
+			                    ? format(" (manually set; would be %d)\n", conf.get().desired_logs)
+			                    : "\n";
+			outputString += format("| commit_proxies              | %16d | %16d |",
+			                       conf.get().old_commit_proxies,
+			                       conf.get().auto_commit_proxies);
+			outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies
+			                    ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies)
+			                    : "\n";
+			outputString += format("| grv_proxies                 | %16d | %16d |",
+			                       conf.get().old_grv_proxies,
+			                       conf.get().auto_grv_proxies);
+			outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies
+			                    ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies)
+			                    : "\n";
+			outputString += format(
+			    "| resolvers                   | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers);
+			outputString += conf.get().auto_resolvers != conf.get().desired_resolvers
+			                    ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers)
+			                    : "\n";
+			outputString += format("| transaction-class processes | %16d | %16d |\n",
+			                       conf.get().old_processes_with_transaction,
+			                       conf.get().auto_processes_with_transaction);
+			outputString += format("| transaction-class machines  | %16d | %16d |\n",
+			                       conf.get().old_machines_with_transaction,
+			                       conf.get().auto_machines_with_transaction);
+			outputString += " ------------------------------------------------------------------- \n\n";
+
+			std::printf("%s", outputString.c_str());
+
+			if (noChanges)
+				return true;
+
+			// TODO: disable completion
+			Optional<std::string> line = wait(linenoise->read("Would you like to make these changes? [y/n]> "));
+
+			if (!line.present() || (line.get() != "y" && line.get() != "Y")) {
+				return true;
+			}
+		}
+
+		ConfigurationResult r = wait(ManagementAPI::changeConfig(
+		    db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force));
+		result = r;
+	}
+
+	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
+	// there are various results specific to changeConfig() that we need to report:
+	bool ret = true;
+	switch (result) {
+	case ConfigurationResult::NO_OPTIONS_PROVIDED:
+	case ConfigurationResult::CONFLICTING_OPTIONS:
+	case ConfigurationResult::UNKNOWN_OPTION:
+	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
+		printUsage(LiteralStringRef("configure"));
+		ret = false;
+		break;
+	case ConfigurationResult::INVALID_CONFIGURATION:
+		fprintf(stderr, "ERROR: These changes would make the configuration invalid\n");
+		ret = false;
+		break;
+	case ConfigurationResult::STORAGE_MIGRATION_DISABLED:
+		fprintf(stderr,
+		        "ERROR: Storage engine type cannot be changed because "
+		        "storage_migration_mode=disabled.\n");
+		fprintf(stderr,
+		        "Type `configure perpetual_storage_wiggle=1 storage_migration_type=gradual' to enable gradual "
+		        "migration with the perpetual wiggle, or `configure "
+		        "storage_migration_type=aggressive' for aggressive migration.\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DATABASE_ALREADY_CREATED:
+		fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DATABASE_CREATED:
+		printf("Database created\n");
+		break;
+	case ConfigurationResult::DATABASE_UNAVAILABLE:
+		fprintf(stderr, "ERROR: The database is unavailable\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
+		fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
+		fprintf(stderr,
+		        "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated "
+		        "before changing the configuration\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
+		fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGIONS_CHANGED:
+		fprintf(stderr,
+		        "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::NOT_ENOUGH_WORKERS:
+		fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGION_REPLICATION_MISMATCH:
+		fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DCID_MISSING:
+		fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n");
+		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::SUCCESS:
+		printf("Configuration changed\n");
+		break;
+	case ConfigurationResult::LOCKED_NOT_NEW:
+		fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n");
+		ret = false;
+		break;
+	case ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL:
+		printf("Configuration changed, with warnings\n");
+		fprintf(stderr,
+		        "WARN: To make progress toward the desired storage type with storage_migration_type=gradual, the "
+		        "Perpetual Wiggle must be enabled.\n");
+		fprintf(stderr,
+		        "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure "
+		        "storage_migration_type=gradual' to set the gradual migration type.\n");
+		ret = false;
+		break;
+	default:
+		ASSERT(false);
+		ret = false;
+	};
+	return ret;
+}
+
+CommandFactory configureFactory(
+    "configure",
+    CommandHelp(
+        "configure [new|tss]"
+        "<single|double|triple|three_data_hall|three_datacenter|ssd|memory|memory-radixtree-beta|proxies=<PROXIES>|"
+        "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*|"
+        "count=<TSS_COUNT>|perpetual_storage_wiggle=<WIGGLE_SPEED>|perpetual_storage_wiggle_locality="
+        "<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>|storage_migration_type={disabled|gradual|aggressive}",
+        "change the database configuration",
+        "The `new' option, if present, initializes a new database with the given configuration rather than changing "
+        "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be "
+        "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead."
+        "When used with new to set up tss for the first time, it requires both a count and a storage engine."
+        "To disable the testing storage server, run \"configure tss count=0\"\n\n"
+        "Redundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies "
+        "of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - "
+        "See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage "
+        "engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small "
+        "datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. The proxy role is being "
+        "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' "
+        "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and "
+        "'grv_proxies' "
+        "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and "
+        "'commit_proxies'. "
+        "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the "
+        "default value.\n\ncommit_proxies=<COMMIT_PROXIES>: Sets the desired number of commit proxies in the cluster. "
+        "Must be at least 1, or set to -1 which restores the number of commit proxies to the default "
+        "value.\n\ngrv_proxies=<GRV_PROXIES>: Sets the desired number of GRV proxies in the cluster. Must be at least "
+        "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=<LOGS>: Sets the "
+        "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of "
+        "logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. "
+        "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n"
+        "perpetual_storage_wiggle=<WIGGLE_SPEED>: Set the value speed (a.k.a., the number of processes that the Data "
+        "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the "
+        "perpetual storage wiggle.\n\n"
+        "perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>: Set the process filter for wiggling. "
+        "The processes that match the given locality key and locality value are only wiggled. The value 0 will disable "
+        "the locality filter and matches all the processes for wiggling.\n\n"
+        "See the FoundationDB Administration Guide for more information."));
+
+} // namespace fdb_cli

fdbcli/ConsistencyCheckCommand.actor.cpp

@@ -39,7 +39,9 @@ ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
 	// If not, the outer loop catch block(fdbcli.actor.cpp) will handle the error and print out the error message
 	tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 	if (tokens.size() == 1) {
-		Optional<Value> suspended = wait(safeThreadFutureToFuture(tr->get(consistencyCheckSpecialKey)));
+		// hold the returned standalone object's memory
+		state ThreadFuture<Optional<Value>> suspendedF = tr->get(consistencyCheckSpecialKey);
+		Optional<Value> suspended = wait(safeThreadFutureToFuture(suspendedF));
 		printf("ConsistencyCheck is %s\n", suspended.present() ? "off" : "on");
 	} else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) {
 		tr->set(consistencyCheckSpecialKey, Value());

fdbcli/CoordinatorsCommand.actor.cpp

@@ -0,0 +1,185 @@
+/*
+ * CoordinatorsCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "boost/algorithm/string.hpp"
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/Schemas.h"
+#include "fdbclient/ManagementAPI.actor.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+ACTOR Future<Void> printCoordinatorsInfo(Reference<IDatabase> db) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		try {
+			// Hold the reference to the standalone's memory
+			state ThreadFuture<Optional<Value>> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey);
+			Optional<Value> description = wait(safeThreadFutureToFuture(descriptionF));
+			ASSERT(description.present());
+			printf("Cluster description: %s\n", description.get().toString().c_str());
+			// Hold the reference to the standalone's memory
+			state ThreadFuture<Optional<Value>> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
+			Optional<Value> processes = wait(safeThreadFutureToFuture(processesF));
+			ASSERT(processes.present());
+			std::vector<std::string> process_addresses;
+			boost::split(process_addresses, processes.get().toString(), [](char c) { return c == ','; });
+			printf("Cluster coordinators (%zu): %s\n", process_addresses.size(), processes.get().toString().c_str());
+			printf("Type `help coordinators' to learn how to change this information.\n");
+			return Void();
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	state int retries = 0;
+	state int notEnoughMachineResults = 0;
+	state StringRef new_cluster_description;
+	state std::string auto_coordinators_str;
+	StringRef nameTokenBegin = LiteralStringRef("description=");
+	for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok)
+		if (tok->startsWith(nameTokenBegin)) {
+			new_cluster_description = tok->substr(nameTokenBegin.size());
+			std::copy(tok + 1, tokens.end(), tok);
+			tokens.resize(tokens.size() - 1);
+			break;
+		}
+
+	state bool automatic = tokens.size() == 2 && tokens[1] == LiteralStringRef("auto");
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+			// update cluster description
+			if (new_cluster_description.size()) {
+				tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description);
+			}
+			// if auto change, read the special key to retrieve the recommended config
+			if (automatic) {
+				// if previous read failed, retry, otherwise, use the same recommened config
+				if (!auto_coordinators_str.size()) {
+					// Hold the reference to the standalone's memory
+					state ThreadFuture<Optional<Value>> auto_coordinatorsF =
+					    tr->get(fdb_cli::coordinatorsAutoSpecialKey);
+					Optional<Value> auto_coordinators = wait(safeThreadFutureToFuture(auto_coordinatorsF));
+					ASSERT(auto_coordinators.present());
+					auto_coordinators_str = auto_coordinators.get().toString();
+				}
+				tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str);
+			} else if (tokens.size() > 1) {
+				state std::set<NetworkAddress> new_coordinators_addresses;
+				state std::vector<std::string> newAddresslist;
+				state std::vector<StringRef>::iterator t;
+				for (t = tokens.begin() + 1; t != tokens.end(); ++t) {
+					try {
+						auto const& addr = NetworkAddress::parse(t->toString());
+						if (new_coordinators_addresses.count(addr)) {
+							fprintf(stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
+							return true;
+						}
+						new_coordinators_addresses.insert(addr);
+						newAddresslist.push_back(addr.toString());
+					} catch (Error& e) {
+						if (e.code() == error_code_connection_string_invalid) {
+							fprintf(
+							    stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str());
+							return true;
+						}
+						throw;
+					}
+				}
+				std::string new_addresses_str = boost::algorithm::join(newAddresslist, ", ");
+				tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_addresses_str);
+			}
+			wait(safeThreadFutureToFuture(tr->commit()));
+			// commit should always fail here
+			// if coordinators are changed, we should get commit_unknown() error
+			ASSERT(false);
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
+				if (errorMsgStr == ManagementAPI::generateErrorMessage(CoordinatorsResult::NOT_ENOUGH_MACHINES) &&
+				    notEnoughMachineResults < 1) {
+					// we could get not_enough_machines if we happen to see the database while the cluster controller is
+					// updating the worker list, so make sure it happens twice before returning a failure
+					notEnoughMachineResults++;
+					wait(delay(1.0));
+					tr->reset();
+					continue;
+				} else if (errorMsgStr ==
+				           ManagementAPI::generateErrorMessage(CoordinatorsResult::SAME_NETWORK_ADDRESSES)) {
+					if (retries)
+						printf("Coordination state changed\n");
+					else
+						printf("No change (existing configuration satisfies request)\n");
+					return true;
+				} else {
+					fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
+					return false;
+				}
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+			++retries;
+		}
+	}
+}
+
+} // namespace
+
+namespace fdb_cli {
+
+const KeyRef clusterDescriptionSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/cluster_description");
+const KeyRef coordinatorsAutoSpecialKey = LiteralStringRef("\xff\xff/management/auto_coordinators");
+const KeyRef coordinatorsProcessSpecialKey = LiteralStringRef("\xff\xff/configuration/coordinators/processes");
+
+ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 2) {
+		wait(printCoordinatorsInfo(db));
+		return true;
+	} else {
+		bool result = wait(changeCoordinators(db, tokens));
+		return result;
+	}
+}
+
+CommandFactory coordinatorsFactory(
+    "coordinators",
+    CommandHelp(
+        "coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
+        "change cluster coordinators or description",
+        "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured "
+        "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, "
+        "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by "
+        "<ADDRESS>+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators "
+        "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in "
+        "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."));
+} // namespace fdb_cli

fdbcli/ExcludeCommand.actor.cpp

@@ -0,0 +1,397 @@
+/*
+ * ExcludeCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/Schemas.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+// Exclue the given servers and localities
+ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
+                                               std::vector<AddressExclusion> servers,
+                                               std::unordered_set<std::string> localities,
+                                               bool failed,
+                                               bool force) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+			if (force && servers.size())
+				tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey,
+				        ValueRef());
+			for (const auto& s : servers) {
+				Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString())
+				                  : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString());
+				tr->set(addr, ValueRef());
+			}
+			if (force && localities.size())
+				tr->set(failed ? fdb_cli::failedLocalityForceOptionSpecialKey
+				               : fdb_cli::excludedLocalityForceOptionSpecialKey,
+				        ValueRef());
+			for (const auto& l : localities) {
+				Key addr = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l)
+				                  : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l);
+				tr->set(addr, ValueRef());
+			}
+			wait(safeThreadFutureToFuture(tr->commit()));
+			return true;
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
+				// last character is \n
+				auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2);
+				auto last_line = errorMsgStr.substr(pos + 1);
+				// customized the error message for fdbcli
+				fprintf(stderr,
+				        "%s\n%s\n",
+				        errorMsgStr.substr(0, pos).c_str(),
+				        last_line.find("free space") != std::string::npos
+				            ? "Type `exclude FORCE <ADDRESS...>' to exclude without checking free space."
+				            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
+				return false;
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+		}
+	}
+}
+
+ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		try {
+			state ThreadFuture<RangeResult> resultFuture =
+			    tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
+			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
+			state ThreadFuture<RangeResult> resultFuture2 =
+			    tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
+			state RangeResult r2 = wait(safeThreadFutureToFuture(resultFuture2));
+			ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
+
+			std::vector<std::string> exclusions;
+			for (const auto& i : r) {
+				auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString();
+				exclusions.push_back(addr);
+			}
+			for (const auto& i : r2) {
+				auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString();
+				exclusions.push_back(addr);
+			}
+			return exclusions;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+// Get the list of excluded localities by reading the keys.
+ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		try {
+			state ThreadFuture<RangeResult> resultFuture =
+			    tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
+			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
+			state ThreadFuture<RangeResult> resultFuture2 =
+			    tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
+			state RangeResult r2 = wait(safeThreadFutureToFuture(resultFuture2));
+			ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
+
+			std::vector<std::string> excludedLocalities;
+			for (const auto& i : r) {
+				auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString();
+				excludedLocalities.push_back(locality);
+			}
+			for (const auto& i : r2) {
+				auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString();
+				excludedLocalities.push_back(locality);
+			}
+			return excludedLocalities;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatabase> db,
+                                                                std::vector<AddressExclusion> excl,
+                                                                bool waitForAllExcluded) {
+	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
+	state std::set<NetworkAddress> inProgressExclusion;
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		inProgressExclusion.clear();
+		try {
+			state ThreadFuture<RangeResult> resultFuture =
+			    tr->getRange(fdb_cli::exclusionInProgressSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
+			RangeResult exclusionInProgress = wait(safeThreadFutureToFuture(resultFuture));
+			ASSERT(!exclusionInProgress.more && exclusionInProgress.size() < CLIENT_KNOBS->TOO_MANY);
+			if (exclusionInProgress.empty())
+				return inProgressExclusion;
+			for (const auto& addr : exclusionInProgress)
+				inProgressExclusion.insert(NetworkAddress::parse(
+				    addr.key.removePrefix(fdb_cli::exclusionInProgressSpecialKeyRange.begin).toString()));
+			if (!waitForAllExcluded)
+				break;
+
+			wait(delayJittered(1.0)); // SOMEDAY: watches!
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+	return inProgressExclusion;
+}
+
+ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::vector<AddressExclusion> exclusionVector) {
+
+	state bool foundCoordinator = false;
+	state std::vector<NetworkAddress> coordinatorList;
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		try {
+			// Hold the reference to the standalone's memory
+			state ThreadFuture<Optional<Value>> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
+			Optional<Value> coordinators = wait(safeThreadFutureToFuture(coordinatorsF));
+			ASSERT(coordinators.present());
+			coordinatorList = NetworkAddress::parseList(coordinators.get().toString());
+			break;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+	for (const auto& c : coordinatorList) {
+		if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
+		    std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
+			fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str());
+			foundCoordinator = true;
+		}
+	}
+	if (foundCoordinator)
+		printf("Type `help coordinators' for information on how to change the\n"
+		       "cluster's coordination servers before removing them.\n");
+	return Void();
+}
+
+} // namespace
+
+namespace fdb_cli {
+
+const KeyRangeRef excludedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded/"),
+                                                 LiteralStringRef("\xff\xff/management/excluded0"));
+const KeyRangeRef failedServersSpecialKeyRange(LiteralStringRef("\xff\xff/management/failed/"),
+                                               LiteralStringRef("\xff\xff/management/failed0"));
+const KeyRangeRef excludedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/excluded_locality/"),
+                                                  LiteralStringRef("\xff\xff/management/excluded_locality0"));
+const KeyRangeRef failedLocalitySpecialKeyRange(LiteralStringRef("\xff\xff/management/failed_locality/"),
+                                                LiteralStringRef("\xff\xff/management/failed_locality0"));
+const KeyRef excludedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/excluded/force");
+const KeyRef failedForceOptionSpecialKey = LiteralStringRef("\xff\xff/management/options/failed/force");
+const KeyRef excludedLocalityForceOptionSpecialKey =
+    LiteralStringRef("\xff\xff/management/options/excluded_locality/force");
+const KeyRef failedLocalityForceOptionSpecialKey =
+    LiteralStringRef("\xff\xff/management/options/failed_locality/force");
+const KeyRangeRef exclusionInProgressSpecialKeyRange(LiteralStringRef("\xff\xff/management/in_progress_exclusion/"),
+                                                     LiteralStringRef("\xff\xff/management/in_progress_exclusion0"));
+
+ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn) {
+	if (tokens.size() <= 1) {
+		state std::vector<std::string> excludedAddresses = wait(getExcludedServers(db));
+		state std::vector<std::string> excludedLocalities = wait(getExcludedLocalities(db));
+
+		if (!excludedAddresses.size() && !excludedLocalities.size()) {
+			printf("There are currently no servers or localities excluded from the database.\n"
+			       "To learn how to exclude a server, type `help exclude'.\n");
+			return true;
+		}
+
+		printf("There are currently %zu servers or localities being excluded from the database:\n",
+		       excludedAddresses.size() + excludedLocalities.size());
+		for (const auto& e : excludedAddresses)
+			printf("  %s\n", e.c_str());
+		for (const auto& e : excludedLocalities)
+			printf("  %s\n", e.c_str());
+
+		printf("To find out whether it is safe to remove one or more of these\n"
+		       "servers from the cluster, type `exclude <addresses>'.\n"
+		       "To return one of these servers to the cluster, type `include <addresses>'.\n");
+
+		return true;
+	} else {
+		state std::vector<AddressExclusion> exclusionVector;
+		state std::set<AddressExclusion> exclusionSet;
+		state std::vector<AddressExclusion> exclusionAddresses;
+		state std::unordered_set<std::string> exclusionLocalities;
+		state std::vector<std::string> noMatchLocalities;
+		state bool force = false;
+		state bool waitForAllExcluded = true;
+		state bool markFailed = false;
+		state std::vector<ProcessData> workers;
+		bool result = wait(fdb_cli::getWorkers(db, &workers));
+		if (!result)
+			return false;
+		for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
+			if (*t == LiteralStringRef("FORCE")) {
+				force = true;
+			} else if (*t == LiteralStringRef("no_wait")) {
+				waitForAllExcluded = false;
+			} else if (*t == LiteralStringRef("failed")) {
+				markFailed = true;
+			} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) &&
+			           t->toString().find(':') != std::string::npos) {
+				std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, t->toString());
+				if (localityAddresses.empty()) {
+					noMatchLocalities.push_back(t->toString());
+				} else {
+					// add all the server ipaddresses that belong to the given localities to the exclusionSet.
+					exclusionVector.insert(exclusionVector.end(), localityAddresses.begin(), localityAddresses.end());
+					exclusionSet.insert(localityAddresses.begin(), localityAddresses.end());
+				}
+				exclusionLocalities.insert(t->toString());
+			} else {
+				auto a = AddressExclusion::parse(*t);
+				if (!a.isValid()) {
+					fprintf(stderr,
+					        "ERROR: '%s' is neither a valid network endpoint address nor a locality\n",
+					        t->toString().c_str());
+					if (t->toString().find(":tls") != std::string::npos)
+						printf("        Do not include the `:tls' suffix when naming a process\n");
+					return true;
+				}
+				exclusionVector.push_back(a);
+				exclusionSet.insert(a);
+				exclusionAddresses.push_back(a);
+			}
+		}
+
+		if (exclusionAddresses.empty() && exclusionLocalities.empty()) {
+			fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality is not provided\n");
+			return false;
+		}
+
+		bool res = wait(excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force));
+		if (!res)
+			return false;
+
+		if (waitForAllExcluded) {
+			printf("Waiting for state to be removed from all excluded servers. This may take a while.\n");
+			printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n");
+		}
+
+		if (warn.isValid())
+			warn.cancel();
+
+		state std::set<NetworkAddress> notExcludedServers =
+		    wait(checkForExcludingServers(db, exclusionVector, waitForAllExcluded));
+		std::map<IPAddress, std::set<uint16_t>> workerPorts;
+		for (auto addr : workers)
+			workerPorts[addr.address.ip].insert(addr.address.port);
+
+		// Print a list of all excluded addresses that don't have a corresponding worker
+		std::set<AddressExclusion> absentExclusions;
+		for (const auto& addr : exclusionVector) {
+			auto worker = workerPorts.find(addr.ip);
+			if (worker == workerPorts.end())
+				absentExclusions.insert(addr);
+			else if (addr.port > 0 && worker->second.count(addr.port) == 0)
+				absentExclusions.insert(addr);
+		}
+
+		for (const auto& exclusion : exclusionVector) {
+			if (absentExclusions.find(exclusion) != absentExclusions.end()) {
+				if (exclusion.port == 0) {
+					fprintf(stderr,
+					        "  %s(Whole machine)  ---- WARNING: Missing from cluster!Be sure that you excluded the "
+					        "correct machines before removing them from the cluster!\n",
+					        exclusion.ip.toString().c_str());
+				} else {
+					fprintf(stderr,
+					        "  %s  ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes "
+					        "before removing them from the cluster!\n",
+					        exclusion.toString().c_str());
+				}
+			} else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(), [&](const NetworkAddress& a) {
+				           return addressExcluded({ exclusion }, a);
+			           })) {
+				if (exclusion.port == 0) {
+					fprintf(stderr,
+					        "  %s(Whole machine)  ---- WARNING: Exclusion in progress! It is not safe to remove this "
+					        "machine from the cluster\n",
+					        exclusion.ip.toString().c_str());
+				} else {
+					fprintf(stderr,
+					        "  %s  ---- WARNING: Exclusion in progress! It is not safe to remove this process from the "
+					        "cluster\n",
+					        exclusion.toString().c_str());
+				}
+			} else {
+				if (exclusion.port == 0) {
+					printf("  %s(Whole machine)  ---- Successfully excluded. It is now safe to remove this machine "
+					       "from the cluster.\n",
+					       exclusion.ip.toString().c_str());
+				} else {
+					printf(
+					    "  %s  ---- Successfully excluded. It is now safe to remove this process from the cluster.\n",
+					    exclusion.toString().c_str());
+				}
+			}
+		}
+
+		for (const auto& locality : noMatchLocalities) {
+			fprintf(
+			    stderr,
+			    "  %s  ---- WARNING: Currently no servers found with this locality match! Be sure that you excluded "
+			    "the correct locality.\n",
+			    locality.c_str());
+		}
+
+		wait(checkForCoordinators(db, exclusionVector));
+		return true;
+	}
+}
+
+CommandFactory excludeFactory(
+    "exclude",
+    CommandHelp(
+        "exclude [FORCE] [failed] [no_wait] [<ADDRESS...>] [locality_dcid:<excludedcid>] "
+        "[locality_zoneid:<excludezoneid>] [locality_machineid:<excludemachineid>] "
+        "[locality_processid:<excludeprocessid>] or any locality data",
+        "exclude servers from the database either with IP address match or locality match",
+        "If no addresses or locaities are specified, lists the set of excluded addresses and localities."
+        "\n\nFor each IP address or IP:port pair in <ADDRESS...> or any LocalityData attributes (like dcid, "
+        "zoneid, "
+        "machineid, processid), adds the address/locality to the set of excluded servers and localities then waits "
+        "until all database state has been safely moved away from the specified servers. If 'no_wait' is set, the "
+        "command returns \nimmediately without checking if the exclusions have completed successfully.\n"
+        "If 'FORCE' is set, the command does not perform safety checks before excluding.\n"
+        "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n"
+        "for data movement to finish and the server cannot be included again."));
+} // namespace fdb_cli

fdbcli/FileConfigureCommand.actor.cpp

@@ -0,0 +1,181 @@
+/*
+ * FileConfigureCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/FlowLineNoise.h"
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/Schemas.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
+                                             std::string filePath,
+                                             bool isNewDatabase,
+                                             bool force) {
+	std::string contents(readFileBytes(filePath, 100000));
+	json_spirit::mValue config;
+	if (!json_spirit::read_string(contents, config)) {
+		fprintf(stderr, "ERROR: Invalid JSON\n");
+		return false;
+	}
+	if (config.type() != json_spirit::obj_type) {
+		fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n");
+		return false;
+	}
+	StatusObject configJSON = config.get_obj();
+
+	json_spirit::mValue schema;
+	if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) {
+		ASSERT(false);
+	}
+
+	std::string errorStr;
+	if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) {
+		printf("%s", errorStr.c_str());
+		return false;
+	}
+
+	std::string configString;
+	if (isNewDatabase) {
+		configString = "new";
+	}
+
+	for (const auto& [name, value] : configJSON) {
+		if (!configString.empty()) {
+			configString += " ";
+		}
+		if (value.type() == json_spirit::int_type) {
+			configString += name + ":=" + format("%d", value.get_int());
+		} else if (value.type() == json_spirit::str_type) {
+			configString += value.get_str();
+		} else if (value.type() == json_spirit::array_type) {
+			configString +=
+			    name + "=" +
+			    json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none);
+		} else {
+			printUsage(LiteralStringRef("fileconfigure"));
+			return false;
+		}
+	}
+	ConfigurationResult result = wait(ManagementAPI::changeConfig(db, configString, force));
+	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
+	// there are various results specific to changeConfig() that we need to report:
+	bool ret = true;
+	switch (result) {
+	case ConfigurationResult::NO_OPTIONS_PROVIDED:
+		fprintf(stderr, "ERROR: No options provided\n");
+		ret = false;
+		break;
+	case ConfigurationResult::CONFLICTING_OPTIONS:
+		fprintf(stderr, "ERROR: Conflicting options\n");
+		ret = false;
+		break;
+	case ConfigurationResult::UNKNOWN_OPTION:
+		fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match
+		ret = false;
+		break;
+	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
+		fprintf(stderr,
+		        "ERROR: Must specify both a replication level and a storage engine when creating a new database\n");
+		ret = false;
+		break;
+	case ConfigurationResult::INVALID_CONFIGURATION:
+		fprintf(stderr, "ERROR: These changes would make the configuration invalid\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DATABASE_ALREADY_CREATED:
+		fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DATABASE_CREATED:
+		printf("Database created\n");
+		break;
+	case ConfigurationResult::DATABASE_UNAVAILABLE:
+		fprintf(stderr, "ERROR: The database is unavailable\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
+		fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
+		fprintf(stderr,
+		        "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated "
+		        "before changing the configuration\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
+		fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGIONS_CHANGED:
+		fprintf(stderr,
+		        "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::NOT_ENOUGH_WORKERS:
+		fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n");
+		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::REGION_REPLICATION_MISMATCH:
+		fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n");
+		printf("Type `fileconfigure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::DCID_MISSING:
+		fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n");
+		printf("Type `fileconfigure FORCE <TOKEN...>' to configure without this check\n");
+		ret = false;
+		break;
+	case ConfigurationResult::SUCCESS:
+		printf("Configuration changed\n");
+		break;
+	default:
+		ASSERT(false);
+		ret = false;
+	};
+	return ret;
+}
+
+CommandFactory fileconfigureFactory(
+    "fileconfigure",
+    CommandHelp(
+        "fileconfigure [new] <FILENAME>",
+        "change the database configuration from a file",
+        "The `new' option, if present, initializes a new database with the given configuration rather than changing "
+        "the configuration of an existing one. Load a JSON document from the provided file, and change the database "
+        "configuration to match the contents of the JSON document. The format should be the same as the value of the "
+        "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."));
+
+} // namespace fdb_cli

fdbcli/IncludeCommand.actor.cpp

@@ -0,0 +1,176 @@
+/*
+ * IncludeCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+// Remove the given localities from the exclusion list.
+// include localities by clearing the keys.
+ACTOR Future<Void> includeLocalities(Reference<IDatabase> db,
+                                     std::vector<std::string> localities,
+                                     bool failed,
+                                     bool includeAll) {
+	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+
+			if (includeAll) {
+				if (failed) {
+					tr->clear(fdb_cli::failedLocalitySpecialKeyRange);
+				} else {
+					tr->clear(fdb_cli::excludedLocalitySpecialKeyRange);
+				}
+			} else {
+				for (const auto& l : localities) {
+					Key locality = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l)
+					                      : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l);
+					tr->clear(locality);
+				}
+			}
+			wait(safeThreadFutureToFuture(tr->commit()));
+			return Void();
+		} catch (Error& e) {
+			TraceEvent("IncludeLocalitiesError").error(e, true);
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+ACTOR Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressExclusion> servers, bool failed) {
+	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+			for (auto& s : servers) {
+				// include all, just clear the whole key range
+				if (!s.isValid()) {
+					if (failed) {
+						tr->clear(fdb_cli::failedServersSpecialKeyRange);
+					} else {
+						tr->clear(fdb_cli::excludedServersSpecialKeyRange);
+					}
+				} else {
+					Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString())
+					                  : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString());
+					tr->clear(addr);
+					// Eliminate both any ip-level exclusion (1.2.3.4) and any
+					// port-level exclusions (1.2.3.4:5)
+					// The range ['IP', 'IP;'] was originally deleted. ';' is
+					// char(':' + 1). This does not work, as other for all
+					// x between 0 and 9, 'IPx' will also be in this range.
+					//
+					// This is why we now make two clears: first only of the ip
+					// address, the second will delete all ports.
+					if (s.isWholeMachine())
+						tr->clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")),
+						                      addr.withSuffix(LiteralStringRef(";"))));
+				}
+			}
+			wait(safeThreadFutureToFuture(tr->commit()));
+			return Void();
+		} catch (Error& e) {
+			TraceEvent("IncludeServersError").error(e, true);
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+// Includes the servers that could be IP addresses or localities back to the cluster.
+ACTOR Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	state std::vector<AddressExclusion> addresses;
+	state std::vector<std::string> localities;
+	state bool failed = false;
+	state bool all = false;
+	for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
+		if (*t == LiteralStringRef("all")) {
+			all = true;
+		} else if (*t == LiteralStringRef("failed")) {
+			failed = true;
+		} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) {
+			// if the token starts with 'locality_' prefix.
+			localities.push_back(t->toString());
+		} else {
+			auto a = AddressExclusion::parse(*t);
+			if (!a.isValid()) {
+				fprintf(stderr,
+				        "ERROR: '%s' is neither a valid network endpoint address nor a locality\n",
+				        t->toString().c_str());
+				if (t->toString().find(":tls") != std::string::npos)
+					printf("        Do not include the `:tls' suffix when naming a process\n");
+				return false;
+			}
+			addresses.push_back(a);
+		}
+	}
+	if (all) {
+		std::vector<AddressExclusion> includeAll;
+		includeAll.push_back(AddressExclusion());
+		wait(includeServers(db, includeAll, failed));
+		wait(includeLocalities(db, localities, failed, all));
+	} else {
+		if (!addresses.empty()) {
+			wait(includeServers(db, addresses, failed));
+		}
+		if (!localities.empty()) {
+			// include the servers that belong to given localities.
+			wait(includeLocalities(db, localities, failed, all));
+		}
+	}
+	return true;
+};
+
+} // namespace
+
+namespace fdb_cli {
+
+ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() < 2) {
+		printUsage(tokens[0]);
+		return false;
+	} else {
+		bool result = wait(include(db, tokens));
+		return result;
+	}
+}
+
+CommandFactory includeFactory(
+    "include",
+    CommandHelp(
+        "include all|[<ADDRESS...>] [locality_dcid:<excludedcid>] [locality_zoneid:<excludezoneid>] "
+        "[locality_machineid:<excludemachineid>] [locality_processid:<excludeprocessid>] or any locality data",
+        "permit previously-excluded servers and localities to rejoin the database",
+        "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port "
+        "pair in <ADDRESS...> or any LocalityData (like dcid, zoneid, machineid, processid), removes any "
+        "matching exclusions from the excluded servers and localities list. "
+        "(A specified IP will match all IP:* exclusion entries)"));
+} // namespace fdb_cli

fdbcli/LockCommand.actor.cpp

@@ -0,0 +1,120 @@
+/*
+ * LockCommand.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/Schemas.h"
+
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+ACTOR Future<bool> lockDatabase(Reference<IDatabase> db, UID id) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+			tr->set(fdb_cli::lockSpecialKey, id.toString());
+			wait(safeThreadFutureToFuture(tr->commit()));
+			printf("Database locked.\n");
+			return true;
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_database_locked)
+				throw e;
+			else if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
+				fprintf(stderr, "%s\n", errorMsgStr.c_str());
+				return false;
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+		}
+	}
+}
+
+} // namespace
+
+namespace fdb_cli {
+
+const KeyRef lockSpecialKey = LiteralStringRef("\xff\xff/management/db_locked");
+
+ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	if (tokens.size() != 1) {
+		printUsage(tokens[0]);
+		return false;
+	} else {
+		state UID lockUID = deterministicRandom()->randomUniqueID();
+		printf("Locking database with lockUID: %s\n", lockUID.toString().c_str());
+		bool result = wait((lockDatabase(db, lockUID)));
+		return result;
+	}
+}
+
+ACTOR Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		try {
+			state ThreadFuture<Optional<Value>> valF = tr->get(fdb_cli::lockSpecialKey);
+			Optional<Value> val = wait(safeThreadFutureToFuture(valF));
+
+			if (!val.present())
+				return true;
+
+			if (val.present() && UID::fromString(val.get().toString()) != uid) {
+				printf("Unable to unlock database. Make sure to unlock with the correct lock UID.\n");
+				return false;
+			}
+
+			tr->clear(fdb_cli::lockSpecialKey);
+			wait(safeThreadFutureToFuture(tr->commit()));
+			printf("Database unlocked.\n");
+			return true;
+		} catch (Error& e) {
+			state Error err(e);
+			if (e.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
+				fprintf(stderr, "%s\n", errorMsgStr.c_str());
+				return false;
+			}
+			wait(safeThreadFutureToFuture(tr->onError(err)));
+		}
+	}
+}
+
+CommandFactory lockFactory(
+    "lock",
+    CommandHelp("lock",
+                "lock the database with a randomly generated lockUID",
+                "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database."));
+
+CommandFactory unlockFactory(
+    "unlock",
+    CommandHelp("unlock <UID>",
+                "unlock the database with the provided lockUID",
+                "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the "
+                "user will be asked to enter a passphrase to confirm their intent."));
+} // namespace fdb_cli

fdbcli/ProfileCommand.actor.cpp

@@ -38,7 +38,7 @@ namespace fdb_cli {
 ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans) {
 	state bool result = true;
 	if (tokens.size() == 1) {
-		fprintf(stderr, "ERROR: Usage: profile <client|list|flow|heap>\n");
+		printUsage(tokens[0]);
 		result = false;
 	} else if (tokencmp(tokens[1], "client")) {
 		if (tokens.size() == 2) {

fdbcli/StatusCommand.actor.cpp

@@ -29,7 +29,6 @@
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
-#include <cstdio>
 
 namespace {
 

fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp

@@ -31,7 +31,7 @@
 
 namespace fdb_cli {
 
-ACTOR Future<Void> triggerddteaminfologCommandActor(Reference<IDatabase> db) {
+ACTOR Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db) {
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		try {
@@ -41,7 +41,7 @@ ACTOR Future<Void> triggerddteaminfologCommandActor(Reference<IDatabase> db) {
 			tr->set(triggerDDTeamInfoPrintKey, v);
 			wait(safeThreadFutureToFuture(tr->commit()));
 			printf("Triggered team info logging in data distribution.\n");
-			return Void();
+			return true;
 		} catch (Error& e) {
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}

fdbcli/TssqCommand.actor.cpp

@@ -64,7 +64,9 @@ ACTOR Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			// Do some validation first to make sure the command is valid
-			Optional<Value> serverListValue = wait(safeThreadFutureToFuture(tr->get(serverListKeyFor(tssId))));
+			// hold the returned standalone object's memory
+			state ThreadFuture<Optional<Value>> serverListValueF = tr->get(serverListKeyFor(tssId));
+			Optional<Value> serverListValue = wait(safeThreadFutureToFuture(serverListValueF));
 			if (!serverListValue.present()) {
 				printf("No TSS %s found in cluster!\n", tssId.toString().c_str());
 				return false;
@@ -75,8 +77,9 @@ ACTOR Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId
 				return false;
 			}
 
-			Optional<Value> currentQuarantineValue =
-			    wait(safeThreadFutureToFuture(tr->get(tssQuarantineKeyFor(tssId))));
+			// hold the returned standalone object's memory
+			state ThreadFuture<Optional<Value>> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId));
+			Optional<Value> currentQuarantineValue = wait(safeThreadFutureToFuture(currentQuarantineValueF));
 			if (enable && currentQuarantineValue.present()) {
 				printf("TSS %s already in quarantine, doing nothing.\n", tssId.toString().c_str());
 				return false;

fdbcli/Util.actor.cpp

@@ -25,6 +25,7 @@
 
 #include "flow/Arena.h"
 
+#include "flow/ThreadHelper.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
@@ -46,7 +47,9 @@ void printUsage(StringRef command) {
 }
 
 ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr) {
-	Optional<Value> errorMsg = wait(safeThreadFutureToFuture(tr->get(fdb_cli::errorMsgSpecialKey)));
+	// hold the returned standalone object's memory
+	state ThreadFuture<Optional<Value>> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey);
+	Optional<Value> errorMsg = wait(safeThreadFutureToFuture(errorMsgF));
 	// Error message should be present
 	ASSERT(errorMsg.present());
 	// Read the json string
@@ -112,4 +115,49 @@ ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
 	return Void();
 }
 
+ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			state ThreadFuture<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
+			state ThreadFuture<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
+
+			wait(success(safeThreadFutureToFuture(processClasses)) && success(safeThreadFutureToFuture(processData)));
+			ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
+			ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
+
+			state std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
+			state int i;
+			for (i = 0; i < processClasses.get().size(); i++) {
+				try {
+					id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
+					    decodeProcessClassValue(processClasses.get()[i].value);
+				} catch (Error& e) {
+					fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what());
+					return false;
+				}
+			}
+
+			for (i = 0; i < processData.get().size(); i++) {
+				ProcessData data = decodeWorkerListValue(processData.get()[i].value);
+				ProcessClass processClass = id_class[data.locality.processId()];
+
+				if (processClass.classSource() == ProcessClass::DBSource ||
+				    data.processClass.classType() == ProcessClass::UnsetClass)
+					data.processClass = processClass;
+
+				if (data.processClass.classType() != ProcessClass::TesterClass)
+					workers->push_back(data);
+			}
+
+			return true;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
 } // namespace fdb_cli

fdbcli/fdbcli.actor.cpp

@@ -91,7 +91,8 @@ enum {
 	OPT_BUILD_FLAGS,
 	OPT_TRACE_FORMAT,
 	OPT_KNOB,
-	OPT_DEBUG_TLS
+	OPT_DEBUG_TLS,
+	OPT_API_VERSION,
 };
 
 CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
@@ -113,6 +114,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
 	                                  { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
 	                                  { OPT_KNOB, "--knob_", SO_REQ_SEP },
 	                                  { OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
+	                                  { OPT_API_VERSION, "--api-version", SO_REQ_SEP },
 
 #ifndef TLS_DISABLED
 	                                  TLS_OPTION_FLAGS
@@ -429,6 +431,8 @@ static void printProgramUsage(const char* name) {
 	       "                 and then exits.\n"
 	       "  --no-status    Disables the initial status check done when starting\n"
 	       "                 the CLI.\n"
+	       "  --api-version  APIVERSION\n"
+	       "                 Specifies the version of the API for the CLI to use.\n"
 #ifndef TLS_DISABLED
 	       TLS_HELP
 #endif
@@ -470,76 +474,6 @@ void initHelp() {
 	    "clear a range of keys from the database",
 	    "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will "
 	    "succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK);
-	helpMap["configure"] = CommandHelp(
-	    "configure [new|tss]"
-	    "<single|double|triple|three_data_hall|three_datacenter|ssd|memory|memory-radixtree-beta|proxies=<PROXIES>|"
-	    "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*|"
-	    "count=<TSS_COUNT>|perpetual_storage_wiggle=<WIGGLE_SPEED>",
-	    "change the database configuration",
-	    "The `new' option, if present, initializes a new database with the given configuration rather than changing "
-	    "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be "
-	    "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead."
-	    "When used with new to set up tss for the first time, it requires both a count and a storage engine."
-	    "To disable the testing storage server, run \"configure tss count=0\"\n\n"
-	    "Redundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies "
-	    "of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - "
-	    "See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage "
-	    "engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small "
-	    "datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. The proxy role is being "
-	    "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' "
-	    "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and "
-	    "'grv_proxies' "
-	    "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and "
-	    "'commit_proxies'. "
-	    "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the "
-	    "default value.\n\ncommit_proxies=<COMMIT_PROXIES>: Sets the desired number of commit proxies in the cluster. "
-	    "Must be at least 1, or set to -1 which restores the number of commit proxies to the default "
-	    "value.\n\ngrv_proxies=<GRV_PROXIES>: Sets the desired number of GRV proxies in the cluster. Must be at least "
-	    "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=<LOGS>: Sets the "
-	    "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of "
-	    "logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. "
-	    "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n"
-	    "perpetual_storage_wiggle=<WIGGLE_SPEED>: Set the value speed (a.k.a., the number of processes that the Data "
-	    "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the "
-	    "perpetual storage wiggle.\n\n"
-	    "See the FoundationDB Administration Guide for more information.");
-	helpMap["fileconfigure"] = CommandHelp(
-	    "fileconfigure [new] <FILENAME>",
-	    "change the database configuration from a file",
-	    "The `new' option, if present, initializes a new database with the given configuration rather than changing "
-	    "the configuration of an existing one. Load a JSON document from the provided file, and change the database "
-	    "configuration to match the contents of the JSON document. The format should be the same as the value of the "
-	    "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\".");
-	helpMap["coordinators"] = CommandHelp(
-	    "coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
-	    "change cluster coordinators or description",
-	    "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured "
-	    "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, "
-	    "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by "
-	    "<ADDRESS>+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators "
-	    "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in "
-	    "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+.");
-	helpMap["exclude"] = CommandHelp(
-	    "exclude [FORCE] [failed] [no_wait] [<ADDRESS...>] [locality_dcid:<excludedcid>] "
-	    "[locality_zoneid:<excludezoneid>] [locality_machineid:<excludemachineid>] "
-	    "[locality_processid:<excludeprocessid>] or any locality data",
-	    "exclude servers from the database either with IP address match or locality match",
-	    "If no addresses or locaities are specified, lists the set of excluded addresses and localities."
-	    "\n\nFor each IP address or IP:port pair in <ADDRESS...> or any LocalityData attributes (like dcid, zoneid, "
-	    "machineid, processid), adds the address/locality to the set of excluded servers and localities then waits "
-	    "until all database state has been safely moved away from the specified servers. If 'no_wait' is set, the "
-	    "command returns \nimmediately without checking if the exclusions have completed successfully.\n"
-	    "If 'FORCE' is set, the command does not perform safety checks before excluding.\n"
-	    "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n"
-	    "for data movement to finish and the server cannot be included again.");
-	helpMap["include"] = CommandHelp(
-	    "include all|[<ADDRESS...>] [locality_dcid:<excludedcid>] [locality_zoneid:<excludezoneid>] "
-	    "[locality_machineid:<excludemachineid>] [locality_processid:<excludeprocessid>] or any locality data",
-	    "permit previously-excluded servers and localities to rejoin the database",
-	    "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port "
-	    "pair in <ADDRESS...> or any LocalityData (like dcid, zoneid, machineid, processid), removes any "
-	    "matching exclusions from the excluded servers and localities list. "
-	    "(A specified IP will match all IP:* exclusion entries)");
 	helpMap["exit"] = CommandHelp("exit", "exit the CLI", "");
 	helpMap["quit"] = CommandHelp();
 	helpMap["waitconnected"] = CommandHelp();
@@ -588,15 +522,6 @@ void initHelp() {
 	helpMap["writemode"] = CommandHelp("writemode <on|off>",
 	                                   "enables or disables sets and clears",
 	                                   "Setting or clearing keys from the CLI is not recommended.");
-	helpMap["lock"] = CommandHelp(
-	    "lock",
-	    "lock the database with a randomly generated lockUID",
-	    "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database.");
-	helpMap["unlock"] =
-	    CommandHelp("unlock <UID>",
-	                "unlock the database with the provided lockUID",
-	                "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the "
-	                "user will be asked to enter a passphrase to confirm their intent.");
 	helpMap["changefeed"] =
 	    CommandHelp("changefeed <register|destroy|get|stream|pop|list> <RANGEID> <BEGIN> <END>", "", "");
 }
@@ -765,332 +690,7 @@ ACTOR Future<Void> setBlobRange(Database db, Key startKey, Key endKey, Value val
 	}
 }
 
-ACTOR Future<bool> configure(Database db,
-                             std::vector<StringRef> tokens,
-                             Reference<ClusterConnectionFile> ccf,
-                             LineNoise* linenoise,
-                             Future<Void> warn) {
-	state ConfigurationResult result;
-	state int startToken = 1;
-	state bool force = false;
-	if (tokens.size() < 2)
-		result = ConfigurationResult::NO_OPTIONS_PROVIDED;
-	else {
-		if (tokens[startToken] == LiteralStringRef("FORCE")) {
-			force = true;
-			startToken = 2;
-		}
-
-		state Optional<ConfigureAutoResult> conf;
-		if (tokens[startToken] == LiteralStringRef("auto")) {
-			StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db)));
-			if (warn.isValid())
-				warn.cancel();
-
-			conf = parseConfig(s);
-
-			if (!conf.get().isValid()) {
-				printf("Unable to provide advice for the current configuration.\n");
-				return true;
-			}
-
-			bool noChanges = conf.get().old_replication == conf.get().auto_replication &&
-			                 conf.get().old_logs == conf.get().auto_logs &&
-			                 conf.get().old_commit_proxies == conf.get().auto_commit_proxies &&
-			                 conf.get().old_grv_proxies == conf.get().auto_grv_proxies &&
-			                 conf.get().old_resolvers == conf.get().auto_resolvers &&
-			                 conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction &&
-			                 conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction;
-
-			bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs &&
-			                        conf.get().old_commit_proxies == conf.get().desired_commit_proxies &&
-			                        conf.get().old_grv_proxies == conf.get().desired_grv_proxies &&
-			                        conf.get().old_resolvers == conf.get().desired_resolvers;
-
-			std::string outputString;
-
-			outputString += "\nYour cluster has:\n\n";
-			outputString += format("  processes %d\n", conf.get().processes);
-			outputString += format("  machines  %d\n", conf.get().machines);
-
-			if (noDesiredChanges)
-				outputString += "\nConfigure recommends keeping your current configuration:\n\n";
-			else if (noChanges)
-				outputString +=
-				    "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n";
-			else
-				outputString += "\nConfigure recommends the following changes:\n\n";
-			outputString += " ------------------------------------------------------------------- \n";
-			outputString += "| parameter                   | old              | new              |\n";
-			outputString += " ------------------------------------------------------------------- \n";
-			outputString += format("| replication                 | %16s | %16s |\n",
-			                       conf.get().old_replication.c_str(),
-			                       conf.get().auto_replication.c_str());
-			outputString +=
-			    format("| logs                        | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs);
-			outputString += conf.get().auto_logs != conf.get().desired_logs
-			                    ? format(" (manually set; would be %d)\n", conf.get().desired_logs)
-			                    : "\n";
-			outputString += format("| commit_proxies              | %16d | %16d |",
-			                       conf.get().old_commit_proxies,
-			                       conf.get().auto_commit_proxies);
-			outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies
-			                    ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies)
-			                    : "\n";
-			outputString += format("| grv_proxies                 | %16d | %16d |",
-			                       conf.get().old_grv_proxies,
-			                       conf.get().auto_grv_proxies);
-			outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies
-			                    ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies)
-			                    : "\n";
-			outputString += format(
-			    "| resolvers                   | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers);
-			outputString += conf.get().auto_resolvers != conf.get().desired_resolvers
-			                    ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers)
-			                    : "\n";
-			outputString += format("| transaction-class processes | %16d | %16d |\n",
-			                       conf.get().old_processes_with_transaction,
-			                       conf.get().auto_processes_with_transaction);
-			outputString += format("| transaction-class machines  | %16d | %16d |\n",
-			                       conf.get().old_machines_with_transaction,
-			                       conf.get().auto_machines_with_transaction);
-			outputString += " ------------------------------------------------------------------- \n\n";
-
-			std::printf("%s", outputString.c_str());
-
-			if (noChanges)
-				return false;
-
-			// TODO: disable completion
-			Optional<std::string> line = wait(linenoise->read("Would you like to make these changes? [y/n]> "));
-
-			if (!line.present() || (line.get() != "y" && line.get() != "Y")) {
-				return false;
-			}
-		}
-
-		ConfigurationResult r = wait(makeInterruptable(
-		    changeConfig(db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force)));
-		result = r;
-	}
-
-	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
-	// there are various results specific to changeConfig() that we need to report:
-	bool ret;
-	switch (result) {
-	case ConfigurationResult::NO_OPTIONS_PROVIDED:
-	case ConfigurationResult::CONFLICTING_OPTIONS:
-	case ConfigurationResult::UNKNOWN_OPTION:
-	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
-		printUsage(LiteralStringRef("configure"));
-		ret = true;
-		break;
-	case ConfigurationResult::INVALID_CONFIGURATION:
-		fprintf(stderr, "ERROR: These changes would make the configuration invalid\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DATABASE_ALREADY_CREATED:
-		fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DATABASE_CREATED:
-		printf("Database created\n");
-		ret = false;
-		break;
-	case ConfigurationResult::DATABASE_UNAVAILABLE:
-		fprintf(stderr, "ERROR: The database is unavailable\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
-		fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
-		fprintf(stderr,
-		        "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated "
-		        "before changing the configuration\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
-		fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGIONS_CHANGED:
-		fprintf(stderr,
-		        "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::NOT_ENOUGH_WORKERS:
-		fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGION_REPLICATION_MISMATCH:
-		fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DCID_MISSING:
-		fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n");
-		fprintf(stderr, "Type `configure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::SUCCESS:
-		printf("Configuration changed\n");
-		ret = false;
-		break;
-	case ConfigurationResult::LOCKED_NOT_NEW:
-		fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n");
-		ret = true;
-		break;
-	default:
-		ASSERT(false);
-		ret = true;
-	};
-	return ret;
-}
-
-ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDatabase, bool force) {
-	std::string contents(readFileBytes(filePath, 100000));
-	json_spirit::mValue config;
-	if (!json_spirit::read_string(contents, config)) {
-		fprintf(stderr, "ERROR: Invalid JSON\n");
-		return true;
-	}
-	if (config.type() != json_spirit::obj_type) {
-		fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n");
-		return true;
-	}
-	StatusObject configJSON = config.get_obj();
-
-	json_spirit::mValue schema;
-	if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) {
-		ASSERT(false);
-	}
-
-	std::string errorStr;
-	if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) {
-		printf("%s", errorStr.c_str());
-		return true;
-	}
-
-	std::string configString;
-	if (isNewDatabase) {
-		configString = "new";
-	}
-
-	for (const auto& [name, value] : configJSON) {
-		if (!configString.empty()) {
-			configString += " ";
-		}
-		if (value.type() == json_spirit::int_type) {
-			configString += name + ":=" + format("%d", value.get_int());
-		} else if (value.type() == json_spirit::str_type) {
-			configString += value.get_str();
-		} else if (value.type() == json_spirit::array_type) {
-			configString +=
-			    name + "=" +
-			    json_spirit::write_string(json_spirit::mValue(value.get_array()), json_spirit::Output_options::none);
-		} else {
-			printUsage(LiteralStringRef("fileconfigure"));
-			return true;
-		}
-	}
-	ConfigurationResult result = wait(makeInterruptable(changeConfig(db, configString, force)));
-	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
-	// there are various results specific to changeConfig() that we need to report:
-	bool ret;
-	switch (result) {
-	case ConfigurationResult::NO_OPTIONS_PROVIDED:
-		fprintf(stderr, "ERROR: No options provided\n");
-		ret = true;
-		break;
-	case ConfigurationResult::CONFLICTING_OPTIONS:
-		fprintf(stderr, "ERROR: Conflicting options\n");
-		ret = true;
-		break;
-	case ConfigurationResult::UNKNOWN_OPTION:
-		fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match
-		ret = true;
-		break;
-	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
-		fprintf(stderr,
-		        "ERROR: Must specify both a replication level and a storage engine when creating a new database\n");
-		ret = true;
-		break;
-	case ConfigurationResult::INVALID_CONFIGURATION:
-		fprintf(stderr, "ERROR: These changes would make the configuration invalid\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DATABASE_ALREADY_CREATED:
-		fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DATABASE_CREATED:
-		printf("Database created\n");
-		ret = false;
-		break;
-	case ConfigurationResult::DATABASE_UNAVAILABLE:
-		fprintf(stderr, "ERROR: The database is unavailable\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
-		fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
-		fprintf(stderr,
-		        "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated "
-		        "before changing the configuration\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
-		fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGIONS_CHANGED:
-		fprintf(stderr,
-		        "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::NOT_ENOUGH_WORKERS:
-		fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n");
-		printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::REGION_REPLICATION_MISMATCH:
-		fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n");
-		printf("Type `fileconfigure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::DCID_MISSING:
-		fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n");
-		printf("Type `fileconfigure FORCE <TOKEN...>' to configure without this check\n");
-		ret = true;
-		break;
-	case ConfigurationResult::SUCCESS:
-		printf("Configuration changed\n");
-		ret = false;
-		break;
-	default:
-		ASSERT(false);
-		ret = true;
-	};
-	return ret;
-}
-
 // FIXME: Factor address parsing from coordinators, include, exclude
-
 ACTOR Future<bool> coordinators(Database db, std::vector<StringRef> tokens, bool isClusterTLS) {
 	state StringRef setName;
 	StringRef nameTokenBegin = LiteralStringRef("description=");
@@ -1225,12 +825,12 @@ ACTOR Future<bool> exclude(Database db,
                            Reference<ClusterConnectionFile> ccf,
                            Future<Void> warn) {
 	if (tokens.size() <= 1) {
-		state Future<vector<AddressExclusion>> fexclAddresses = makeInterruptable(getExcludedServers(db));
-		state Future<vector<std::string>> fexclLocalities = makeInterruptable(getExcludedLocalities(db));
+		state Future<std::vector<AddressExclusion>> fexclAddresses = makeInterruptable(getExcludedServers(db));
+		state Future<std::vector<std::string>> fexclLocalities = makeInterruptable(getExcludedLocalities(db));
 
 		wait(success(fexclAddresses) && success(fexclLocalities));
-		vector<AddressExclusion> exclAddresses = fexclAddresses.get();
-		vector<std::string> exclLocalities = fexclLocalities.get();
+		std::vector<AddressExclusion> exclAddresses = fexclAddresses.get();
+		std::vector<std::string> exclLocalities = fexclLocalities.get();
 
 		if (!exclAddresses.size() && !exclLocalities.size()) {
 			printf("There are currently no servers or localities excluded from the database.\n"
@@ -1630,6 +1230,8 @@ void configureGenerator(const char* text, const char* line, std::vector<std::str
 		                   "logs=",
 		                   "resolvers=",
 		                   "perpetual_storage_wiggle=",
+		                   "perpetual_storage_wiggle_locality=",
+		                   "storage_migration_type=",
 		                   nullptr };
 	arrayGenerator(text, line, opts, lc);
 }
@@ -1828,6 +1430,9 @@ struct CLIOptions {
 
 	std::vector<std::pair<std::string, std::string>> knobs;
 
+	// api version, using the latest version by default
+	int api_version = FDB_API_VERSION;
+
 	CLIOptions(int argc, char* argv[]) {
 		program_name = argv[0];
 		for (int a = 0; a < argc; a++) {
@@ -1890,6 +1495,22 @@ struct CLIOptions {
 		case OPT_CONNFILE:
 			clusterFile = args.OptionArg();
 			break;
+		case OPT_API_VERSION: {
+			char* endptr;
+			api_version = strtoul((char*)args.OptionArg(), &endptr, 10);
+			if (*endptr != '\0') {
+				fprintf(stderr, "ERROR: invalid client version %s\n", args.OptionArg());
+				return 1;
+			} else if (api_version < 700 || api_version > FDB_API_VERSION) {
+				// multi-version fdbcli only available after 7.0
+				fprintf(stderr,
+				        "ERROR: api version %s is not supported. (Min: 700, Max: %d)\n",
+				        args.OptionArg(),
+				        FDB_API_VERSION);
+				return 1;
+			}
+			break;
+		}
 		case OPT_TRACE:
 			trace = true;
 			break;
@@ -2017,7 +1638,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	TraceEvent::setNetworkThread();
 
 	try {
-		localDb = Database::createDatabase(ccf, -1, IsInternal::False);
+		localDb = Database::createDatabase(ccf, opt.api_version, IsInternal::False);
 		if (!opt.exec.present()) {
 			printf("Using cluster file `%s'.\n", ccf->getFilename().c_str());
 		}
@@ -2209,7 +1830,8 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "waitopen")) {
-					wait(success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
+					wait(makeInterruptable(
+					    success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion()))));
 					continue;
 				}
 
@@ -2241,7 +1863,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "triggerddteaminfolog")) {
-					wait(triggerddteaminfologCommandActor(db));
+					wait(success(makeInterruptable(triggerddteaminfologCommandActor(db))));
 					continue;
 				}
 
@@ -2257,7 +1879,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							is_error = true;
 							continue;
 						}
-						wait(changeFeedList(db));
+						wait(changeFeedList(localDb));
 						continue;
 					} else if (tokencmp(tokens[1], "register")) {
 						if (tokens.size() != 5) {
@@ -2265,7 +1887,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							is_error = true;
 							continue;
 						}
-						trx = Transaction(db);
+						trx = Transaction(localDb);
 						loop {
 							try {
 								wait(trx.registerChangeFeed(tokens[2], KeyRangeRef(tokens[3], tokens[4])));
@@ -2281,7 +1903,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							is_error = true;
 							continue;
 						}
-						trx = Transaction(db);
+						trx = Transaction(localDb);
 						loop {
 							try {
 								trx.destroyChangeFeed(tokens[2]);
@@ -2317,7 +1939,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							}
 						}
 						Standalone<VectorRef<MutationsAndVersionRef>> res =
-						    wait(db->getChangeFeedMutations(tokens[2], begin, end));
+						    wait(localDb->getChangeFeedMutations(tokens[2], begin, end));
 						printf("\n");
 						for (auto& it : res) {
 							for (auto& it2 : it.mutations) {
@@ -2353,7 +1975,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							warn.cancel();
 						}
 						state PromiseStream<Standalone<VectorRef<MutationsAndVersionRef>>> feedResults;
-						state Future<Void> feed = db->getChangeFeedStream(feedResults, tokens[2], begin, end);
+						state Future<Void> feed = localDb->getChangeFeedStream(feedResults, tokens[2], begin, end);
 						printf("\n");
 						try {
 							state Future<Void> feedInterrupt = LineNoise::onKeyboardInterrupt();
@@ -2394,7 +2016,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 							printUsage(tokens[0]);
 							is_error = true;
 						} else {
-							wait(db->popChangeFeedMutations(tokens[2], v));
+							wait(localDb->popChangeFeedMutations(tokens[2], v));
 						}
 					}
 					continue;
@@ -2441,8 +2063,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "configure")) {
-					bool err = wait(configure(localDb, tokens, localDb->getConnectionFile(), &linenoise, warn));
-					if (err)
+					bool _result =
+					    wait(makeInterruptable(configureCommandActor(db, localDb, tokens, &linenoise, warn)));
+					if (!_result)
 						is_error = true;
 					continue;
 				}
@@ -2450,11 +2073,12 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				if (tokencmp(tokens[0], "fileconfigure")) {
 					if (tokens.size() == 2 || (tokens.size() == 3 && (tokens[1] == LiteralStringRef("new") ||
 					                                                  tokens[1] == LiteralStringRef("FORCE")))) {
-						bool err = wait(fileConfigure(localDb,
-						                              tokens.back().toString(),
-						                              tokens[1] == LiteralStringRef("new"),
-						                              tokens[1] == LiteralStringRef("FORCE")));
-						if (err)
+						bool _result =
+						    wait(makeInterruptable(fileConfigureCommandActor(db,
+						                                                     tokens.back().toString(),
+						                                                     tokens[1] == LiteralStringRef("new"),
+						                                                     tokens[1] == LiteralStringRef("FORCE"))));
+						if (!_result)
 							is_error = true;
 					} else {
 						printUsage(tokens[0]);
@@ -2464,57 +2088,37 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "coordinators")) {
-					auto cs = ClusterConnectionFile(localDb->getConnectionFile()->getFilename()).getConnectionString();
-					if (tokens.size() < 2) {
-						printf("Cluster description: %s\n", cs.clusterKeyName().toString().c_str());
-						printf("Cluster coordinators (%zu): %s\n",
-						       cs.coordinators().size(),
-						       describe(cs.coordinators()).c_str());
-						printf("Type `help coordinators' to learn how to change this information.\n");
-					} else {
-						bool err = wait(coordinators(localDb, tokens, cs.coordinators()[0].isTLS()));
-						if (err)
-							is_error = true;
-					}
+					bool _result = wait(makeInterruptable(coordinatorsCommandActor(db, tokens)));
+					if (!_result)
+						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "exclude")) {
-					bool err = wait(exclude(localDb, tokens, localDb->getConnectionFile(), warn));
-					if (err)
+					bool _result = wait(makeInterruptable(excludeCommandActor(db, tokens, warn)));
+					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "include")) {
-					if (tokens.size() < 2) {
-						printUsage(tokens[0]);
+					bool _result = wait(makeInterruptable(includeCommandActor(db, tokens)));
+					if (!_result)
 						is_error = true;
-					} else {
-						bool err = wait(include(localDb, tokens));
-						if (err)
-							is_error = true;
-					}
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "snapshot")) {
-					bool _result = wait(snapshotCommandActor(db, tokens));
+					bool _result = wait(makeInterruptable(snapshotCommandActor(db, tokens)));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "lock")) {
-					if (tokens.size() != 1) {
-						printUsage(tokens[0]);
+					bool _result = wait(makeInterruptable(lockCommandActor(db, tokens)));
+					if (!_result)
 						is_error = true;
-					} else {
-						state UID lockUID = deterministicRandom()->randomUniqueID();
-						printf("Locking database with lockUID: %s\n", lockUID.toString().c_str());
-						wait(makeInterruptable(lockDatabase(localDb, lockUID)));
-						printf("Database locked.\n");
-					}
 					continue;
 				}
 
@@ -2535,16 +2139,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 						    checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db, localDb);
 						if (input.present() && input.get() == passPhrase) {
 							UID unlockUID = UID::fromString(tokens[1].toString());
-							try {
-								wait(makeInterruptable(unlockDatabase(localDb, unlockUID)));
-								printf("Database unlocked.\n");
-							} catch (Error& e) {
-								if (e.code() == error_code_database_locked) {
-									printf(
-									    "Unable to unlock database. Make sure to unlock with the correct lock UID.\n");
-								}
-								throw e;
-							}
+							bool _result = wait(makeInterruptable(unlockDatabaseActor(db, unlockUID)));
+							if (!_result)
+								is_error = true;
 						} else {
 							fprintf(stderr, "ERROR: Incorrect passphrase entered.\n");
 							is_error = true;
@@ -2931,7 +2528,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "throttle")) {
-					bool _result = wait(throttleCommandActor(db, tokens));
+					bool _result = wait(makeInterruptable(throttleCommandActor(db, tokens)));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -3162,8 +2759,7 @@ int main(int argc, char** argv) {
 	}
 
 	try {
-		// Note: refactoring fdbcli, in progress
-		API->selectApiVersion(FDB_API_VERSION);
+		API->selectApiVersion(opt.api_version);
 		API->setupNetwork();
 		Future<int> cliFuture = runCli(opt);
 		Future<Void> timeoutFuture = opt.exit_timeout ? timeExit(opt.exit_timeout) : Never();

fdbcli/fdbcli.actor.h

@@ -28,6 +28,8 @@
 #elif !defined(FDBCLI_FDBCLI_ACTOR_H)
 #define FDBCLI_FDBCLI_ACTOR_H
 
+#include "fdbcli/FlowLineNoise.h"
+
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/StatusClient.h"
@@ -63,9 +65,25 @@ struct CommandFactory {
 extern const KeyRef advanceVersionSpecialKey;
 // consistencycheck
 extern const KeyRef consistencyCheckSpecialKey;
+// coordinators
+extern const KeyRef clusterDescriptionSpecialKey;
+extern const KeyRef coordinatorsAutoSpecialKey;
+extern const KeyRef coordinatorsProcessSpecialKey;
 // datadistribution
 extern const KeyRef ddModeSpecialKey;
 extern const KeyRef ddIgnoreRebalanceSpecialKey;
+// exclude/include
+extern const KeyRangeRef excludedServersSpecialKeyRange;
+extern const KeyRangeRef failedServersSpecialKeyRange;
+extern const KeyRangeRef excludedLocalitySpecialKeyRange;
+extern const KeyRangeRef failedLocalitySpecialKeyRange;
+extern const KeyRef excludedForceOptionSpecialKey;
+extern const KeyRef failedForceOptionSpecialKey;
+extern const KeyRef excludedLocalityForceOptionSpecialKey;
+extern const KeyRef failedLocalityForceOptionSpecialKey;
+extern const KeyRangeRef exclusionInProgressSpecialKeyRange;
+// lock/unlock
+extern const KeyRef lockSpecialKey;
 // maintenance
 extern const KeyRangeRef maintenanceSpecialKeyRange;
 extern const KeyRef ignoreSSFailureSpecialKey;
@@ -79,6 +97,8 @@ inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_messag
 ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
                                 Reference<FlowLock> connectLock,
                                 KeyValue kv);
+// get all workers' info
+ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers);
 
 // compare StringRef with the given c string
 bool tokencmp(StringRef token, const char* command);
@@ -101,29 +121,50 @@ void printStatus(StatusObjectReader statusObj,
                  bool hideErrorMessages = false);
 
 // All fdbcli commands (alphabetically)
+// All below actors return true if the command is executed successfully
 // advanceversion command
 ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // cache_range command
 ACTOR Future<bool> cacheRangeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// configure command
+ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
+                                         Database localDb,
+                                         std::vector<StringRef> tokens,
+                                         LineNoise* linenoise,
+                                         Future<Void> warn);
 // consistency command
 ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
                                                 std::vector<StringRef> tokens,
                                                 bool intrans);
+// coordinators command
+ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // datadistribution command
 ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// exclude command
+ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn);
 // expensive_data_check command
 ACTOR Future<bool> expensiveDataCheckCommandActor(
     Reference<IDatabase> db,
     Reference<ITransaction> tr,
     std::vector<StringRef> tokens,
     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// fileconfigure command
+ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
+                                             std::string filePath,
+                                             bool isNewDatabase,
+                                             bool force);
 // force_recovery_with_data_loss command
 ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// include command
+ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // kill command
 ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
                                     Reference<ITransaction> tr,
                                     std::vector<StringRef> tokens,
                                     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// lock/unlock command
+ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+ACTOR Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid);
 // maintenance command
 ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning = false);
 ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
@@ -149,7 +190,7 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 // throttle command
 ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // triggerteaminfolog command
-ACTOR Future<Void> triggerddteaminfologCommandActor(Reference<IDatabase> db);
+ACTOR Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db);
 // tssq command
 ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 

fdbclient/BlobGranuleCommon.h

@@ -29,15 +29,17 @@
 struct MutationsAndVersionRef {
 	VectorRef<MutationRef> mutations;
 	Version version;
+	Version knownCommittedVersion;
 
 	MutationsAndVersionRef() {}
-	explicit MutationsAndVersionRef(Version version) : version(version) {}
-	MutationsAndVersionRef(VectorRef<MutationRef> mutations, Version version)
-	  : mutations(mutations), version(version) {}
-	MutationsAndVersionRef(Arena& to, VectorRef<MutationRef> mutations, Version version)
-	  : mutations(to, mutations), version(version) {}
+	explicit MutationsAndVersionRef(Version version, Version knownCommittedVersion)
+	  : version(version), knownCommittedVersion(knownCommittedVersion) {}
+	MutationsAndVersionRef(VectorRef<MutationRef> mutations, Version version, Version knownCommittedVersion)
+	  : mutations(mutations), version(version), knownCommittedVersion(knownCommittedVersion) {}
+	MutationsAndVersionRef(Arena& to, VectorRef<MutationRef> mutations, Version version, Version knownCommittedVersion)
+	  : mutations(to, mutations), version(version), knownCommittedVersion(knownCommittedVersion) {}
 	MutationsAndVersionRef(Arena& to, const MutationsAndVersionRef& from)
-	  : mutations(to, from.mutations), version(from.version) {}
+	  : mutations(to, from.mutations), version(from.version), knownCommittedVersion(from.knownCommittedVersion) {}
 	int expectedSize() const { return mutations.expectedSize(); }
 
 	struct OrderByVersion {
@@ -48,7 +50,7 @@ struct MutationsAndVersionRef {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, mutations, version);
+		serializer(ar, mutations, version, knownCommittedVersion);
 	}
 };
 

fdbclient/CMakeLists.txt

@@ -132,6 +132,7 @@ set(FDBCLIENT_SRCS
   VersionedMap.actor.h
   VersionedMap.h
   VersionedMap.cpp
+  WellKnownEndpoints.h
   WriteMap.h
   json_spirit/json_spirit_error_position.h
   json_spirit/json_spirit_reader_template.h
@@ -187,14 +188,14 @@ if(BUILD_AZURE_BACKUP)
 endif()
 
 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
-add_dependencies(fdbclient fdboptions fdb_c_options)
+add_dependencies(fdbclient fdboptions)
 target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
 
 # Create a separate fdbclient library with sampling enabled. This lets
 # fdbserver retain sampling functionality in client code while disabling
 # sampling for pure clients.
 add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
-add_dependencies(fdbclient_sampling fdboptions fdb_c_options)
+add_dependencies(fdbclient_sampling fdboptions)
 target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack)
 target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)
 if(WIN32)

fdbclient/ClusterInterface.h

@@ -267,7 +267,7 @@ struct StatusRequest {
 
 struct GetClientWorkersRequest {
 	constexpr static FileIdentifier file_identifier = 10771791;
-	ReplyPromise<vector<ClientWorkerInterface>> reply;
+	ReplyPromise<std::vector<ClientWorkerInterface>> reply;
 
 	GetClientWorkersRequest() {}
 

fdbclient/CommitProxyInterface.h

@@ -109,12 +109,12 @@ struct CommitProxyInterface {
 struct ClientDBInfo {
 	constexpr static FileIdentifier file_identifier = 5355080;
 	UID id; // Changes each time anything else changes
-	vector<GrvProxyInterface> grvProxies;
-	vector<CommitProxyInterface> commitProxies;
+	std::vector<GrvProxyInterface> grvProxies;
+	std::vector<CommitProxyInterface> commitProxies;
 	Optional<CommitProxyInterface>
 	    firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk
 	Optional<Value> forward;
-	vector<VersionHistory> history;
+	std::vector<VersionHistory> history;
 
 	ClientDBInfo() {}
 
@@ -285,7 +285,7 @@ struct GetReadVersionRequest : TimedRequest {
 struct GetKeyServerLocationsReply {
 	constexpr static FileIdentifier file_identifier = 10636023;
 	Arena arena;
-	std::vector<std::pair<KeyRangeRef, vector<StorageServerInterface>>> results;
+	std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>> results;
 
 	// if any storage servers in results have a TSS pair, that mapping is in here
 	std::vector<std::pair<UID, StorageServerInterface>> resultsTssMapping;
@@ -499,11 +499,11 @@ struct ExclusionSafetyCheckReply {
 
 struct ExclusionSafetyCheckRequest {
 	constexpr static FileIdentifier file_identifier = 13852702;
-	vector<AddressExclusion> exclusions;
+	std::vector<AddressExclusion> exclusions;
 	ReplyPromise<ExclusionSafetyCheckReply> reply;
 
 	ExclusionSafetyCheckRequest() {}
-	explicit ExclusionSafetyCheckRequest(vector<AddressExclusion> exclusions) : exclusions(exclusions) {}
+	explicit ExclusionSafetyCheckRequest(std::vector<AddressExclusion> exclusions) : exclusions(exclusions) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {

fdbclient/ConfigTransactionInterface.cpp

@@ -34,10 +34,11 @@ void ConfigTransactionInterface::setupWellKnownEndpoints() {
 }
 
 ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote)
-  : getGeneration(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)),
-    get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)), getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
-    getKnobs(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), commit(Endpoint({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {
-}
+  : getGeneration(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETGENERATION)),
+    get(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GET)),
+    getClasses(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
+    getKnobs(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)),
+    commit(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {}
 
 bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rhs) const {
 	return _id == rhs._id;

fdbclient/CoordinationInterface.h

@@ -27,23 +27,10 @@
 #include "fdbrpc/Locality.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/ClusterInterface.h"
+#include "fdbclient/WellKnownEndpoints.h"
 
 const int MAX_CLUSTER_FILE_BYTES = 60000;
 
-// well known endpoints published to the client.
-constexpr UID WLTOKEN_CLIENTLEADERREG_GETLEADER(-1, 2);
-constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
-
-// the value of this endpoint should be stable and not change.
-constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10);
-constexpr UID WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE(-1, 11);
-
-constexpr UID WLTOKEN_CONFIGTXN_GETGENERATION(-1, 12);
-constexpr UID WLTOKEN_CONFIGTXN_GET(-1, 13);
-constexpr UID WLTOKEN_CONFIGTXN_GETCLASSES(-1, 14);
-constexpr UID WLTOKEN_CONFIGTXN_GETKNOBS(-1, 15);
-constexpr UID WLTOKEN_CONFIGTXN_COMMIT(-1, 16);
-
 struct ClientLeaderRegInterface {
 	RequestStream<struct GetLeaderRequest> getLeader;
 	RequestStream<struct OpenDatabaseCoordRequest> openDatabase;
@@ -62,8 +49,8 @@ class ClusterConnectionString {
 public:
 	ClusterConnectionString() {}
 	ClusterConnectionString(std::string const& connectionString);
-	ClusterConnectionString(vector<NetworkAddress>, Key);
-	vector<NetworkAddress> const& coordinators() const { return coord; }
+	ClusterConnectionString(std::vector<NetworkAddress>, Key);
+	std::vector<NetworkAddress> const& coordinators() const { return coord; }
 	Key clusterKey() const { return key; }
 	Key clusterKeyName() const {
 		return keyDesc;
@@ -74,7 +61,7 @@ public:
 private:
 	void parseKey(std::string const& key);
 
-	vector<NetworkAddress> coord;
+	std::vector<NetworkAddress> coord;
 	Key key, keyDesc;
 };
 
@@ -199,7 +186,7 @@ struct OpenDatabaseCoordRequest {
 	Standalone<VectorRef<ClientVersionRef>> supportedVersions;
 	UID knownClientInfoID;
 	Key clusterKey;
-	vector<NetworkAddress> coordinators;
+	std::vector<NetworkAddress> coordinators;
 	ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
 
 	template <class Ar>
@@ -210,7 +197,7 @@ struct OpenDatabaseCoordRequest {
 
 class ClientCoordinators {
 public:
-	vector<ClientLeaderRegInterface> clientLeaderServers;
+	std::vector<ClientLeaderRegInterface> clientLeaderServers;
 	Key clusterKey;
 	Reference<ClusterConnectionFile> ccf;
 

fdbclient/DatabaseConfiguration.cpp

@@ -45,6 +45,8 @@ void DatabaseConfiguration::resetInternal() {
 	remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0;
 	backupWorkerEnabled = false;
 	perpetualStorageWiggleSpeed = 0;
+	perpetualStorageWiggleLocality = "0";
+	storageMigrationType = StorageMigrationType::DEFAULT;
 }
 
 void parse(int* i, ValueRef const& v) {
@@ -198,7 +200,9 @@ bool DatabaseConfiguration::isValid() const {
 	      (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
 	      (regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") &&
 	      // We cannot specify regions with three_datacenter replication
-	      (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1))) {
+	      (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) &&
+	      isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality) &&
+	      storageMigrationType != StorageMigrationType::UNSET)) {
 		return false;
 	}
 	std::set<Key> dcIds;
@@ -389,6 +393,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 
 	result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
 	result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed;
+	result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality;
+	result["storage_migration_type"] = storageMigrationType.toString();
 	return result;
 }
 
@@ -542,6 +548,30 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 		parse(&regions, value);
 	} else if (ck == LiteralStringRef("perpetual_storage_wiggle")) {
 		parse(&perpetualStorageWiggleSpeed, value);
+	} else if (ck == LiteralStringRef("perpetual_storage_wiggle_locality")) {
+		if (!isValidPerpetualStorageWiggleLocality(value.toString())) {
+			return false;
+		}
+		perpetualStorageWiggleLocality = value.toString();
+	} else if (ck == LiteralStringRef("storage_migration_type")) {
+		parse((&type), value);
+		storageMigrationType = (StorageMigrationType::MigrationType)type;
+	} else if (ck == LiteralStringRef("proxies")) {
+		int proxiesCount;
+		parse(&proxiesCount, value);
+		if (proxiesCount > 1) {
+			int derivedGrvProxyCount =
+			    std::max(1,
+			             std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES,
+			                      proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1)));
+			int derivedCommitProxyCount = proxiesCount - derivedGrvProxyCount;
+			if (grvProxyCount == -1) {
+				grvProxyCount = derivedGrvProxyCount;
+			}
+			if (commitProxyCount == -1) {
+				commitProxyCount = derivedCommitProxyCount;
+			}
+		}
 	} else {
 		return false;
 	}

fdbclient/DatabaseConfiguration.h

@@ -245,6 +245,10 @@ struct DatabaseConfiguration {
 
 	// Perpetual Storage Setting
 	int32_t perpetualStorageWiggleSpeed;
+	std::string perpetualStorageWiggleLocality;
+
+	// Storage Migration Type
+	StorageMigrationType storageMigrationType;
 
 	// Excluded servers (no state should be here)
 	bool isExcludedServer(NetworkAddressList) const;

fdbclient/DatabaseContext.h

@@ -182,10 +182,10 @@ public:
 
 	std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, Reverse isBackward = Reverse::False);
 	bool getCachedLocations(const KeyRangeRef&,
-	                        vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
+	                        std::vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
 	                        int limit,
 	                        Reverse reverse);
-	Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
+	Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const std::vector<struct StorageServerInterface>&);
 	void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::False);
 	void invalidateCache(const KeyRangeRef&);
 
@@ -398,6 +398,8 @@ public:
 	Counter transactionsProcessBehind;
 	Counter transactionsThrottled;
 	Counter transactionsExpensiveClearCostEstCount;
+	Counter transactionGrvFullBatches;
+	Counter transactionGrvTimedOutBatches;
 
 	ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
 	    bytesPerCommit;
@@ -408,6 +410,7 @@ public:
 	int snapshotRywEnabled;
 
 	int transactionTracingEnabled;
+	double verifyCausalReadsProp = 0.0;
 
 	Future<Void> logger;
 	Future<Void> throttleExpirer;

fdbclient/FDBOptions.h

@@ -42,15 +42,20 @@ struct FDBOptionInfo {
 	// be no cumulative effects from calling multiple times).
 	int defaultFor;
 
+	enum class ParamType { None, String, Int, Bytes };
+
+	ParamType paramType;
+
 	FDBOptionInfo(std::string name,
 	              std::string comment,
 	              std::string parameterComment,
 	              bool hasParameter,
 	              bool hidden,
 	              bool persistent,
-	              int defaultFor)
+	              int defaultFor,
+	              ParamType paramType)
 	  : name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
-	    persistent(persistent), defaultFor(defaultFor) {}
+	    persistent(persistent), defaultFor(defaultFor), paramType(paramType) {}
 
 	FDBOptionInfo() {}
 };
@@ -103,8 +108,9 @@ public:
 	typename OptionList::const_iterator end() const { return options.cend(); }
 };
 
-#define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor)      \
+#define ADD_OPTION_INFO(                                                                                               \
+    type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor, paramType)               \
 	type::optionInfo.insert(                                                                                           \
-	    var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
+	    var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor, paramType));
 
 #endif

fdbclient/FDBTypes.h

@@ -864,6 +864,14 @@ struct StorageBytes {
 		              used / 1e6,
 		              temp / 1e6);
 	}
+
+	void toTraceEvent(TraceEvent& e) const {
+		e.detail("StorageBytesUsed", used)
+		    .detail("StorageBytesTemp", temp)
+		    .detail("StorageBytesTotal", total)
+		    .detail("StorageBytesFree", free)
+		    .detail("StorageBytesAvailable", available);
+	}
 };
 struct LogMessageVersion {
 	// Each message pushed into the log system has a unique, totally ordered LogMessageVersion
@@ -1126,4 +1134,47 @@ inline const char* transactionPriorityToString(TransactionPriority priority, boo
 	throw internal_error();
 }
 
+struct StorageMigrationType {
+	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones
+	// just before END.
+	enum MigrationType { DEFAULT = 1, UNSET = 0, DISABLED = 1, AGGRESSIVE = 2, GRADUAL = 3, END = 4 };
+
+	StorageMigrationType() : type(UNSET) {}
+	StorageMigrationType(MigrationType type) : type(type) {
+		if ((uint32_t)type >= END) {
+			this->type = UNSET;
+		}
+	}
+	operator MigrationType() const { return MigrationType(type); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, type);
+	}
+
+	std::string toString() const {
+		switch (type) {
+		case DISABLED:
+			return "disabled";
+		case AGGRESSIVE:
+			return "aggressive";
+		case GRADUAL:
+			return "gradual";
+		case UNSET:
+			return "unset";
+		default:
+			ASSERT(false);
+		}
+		return "";
+	}
+
+	uint32_t type;
+};
+
+inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
+	int pos = locality.find(':');
+	// locality should be either 0 or in the format '<non_empty_string>:<non_empty_string>'
+	return ((pos > 0 && pos < locality.size() - 1) || locality == "0");
+}
+
 #endif

fdbclient/FileBackupAgent.actor.cpp

@@ -2743,7 +2743,7 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 		if (!backupWorkerEnabled && partitionedLog.get().present() && partitionedLog.get().get()) {
 			// Change configuration only when we set to use partitioned logs and
 			// the flag was not set before.
-			wait(success(changeConfig(cx, "backup_worker_enabled:=1", true)));
+			wait(success(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true)));
 			backupWorkerEnabled = true;
 		}
 

fdbclient/GrvProxyInterface.h

@@ -22,6 +22,9 @@
 #ifndef FDBCLIENT_GRVPROXYINTERFACE_H
 #define FDBCLIENT_GRVPROXYINTERFACE_H
 #pragma once
+#include "flow/FileIdentifier.h"
+#include "fdbrpc/fdbrpc.h"
+#include "fdbclient/FDBTypes.h"
 
 // GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it
 // communicates with RateKeeper to gather health information of the cluster.

fdbclient/KeyRangeMap.actor.cpp

@@ -25,7 +25,7 @@
 #include "fdbclient/ReadYourWrites.h"
 #include "flow/actorcompiler.h" // has to be last include
 
-void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges) {
+void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector<KeyRange>& affectedRanges) {
 	auto s = map.rangeContaining(keys.begin);
 	if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady())
 		affectedRanges.push_back(KeyRangeRef(s.begin(), keys.begin));
@@ -176,7 +176,7 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	state KeyRange maxWithPrefix =
 	    KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
 
-	state vector<Future<RangeResult>> keys;
+	state std::vector<Future<RangeResult>> keys;
 	keys.push_back(
 	    tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::True));
 	keys.push_back(

fdbclient/KeyRangeMap.h

@@ -111,7 +111,7 @@ public:
 
 class KeyRangeActorMap {
 public:
-	void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
+	void getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector<KeyRange>& affectedRanges);
 	void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
 	void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
 	bool liveActorAt(const KeyRef& key) {

fdbclient/ManagementAPI.actor.cpp

@@ -150,6 +150,28 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			}
 			out[p + key] = value;
 		}
+		if (key == "perpetual_storage_wiggle_locality") {
+			if (!isValidPerpetualStorageWiggleLocality(value)) {
+				printf("Error: perpetual_storage_wiggle_locality should be in <locality_key>:<locality_value> "
+				       "format or enter 0 to disable the locality match for wiggling.\n");
+				return out;
+			}
+			out[p + key] = value;
+		}
+		if (key == "storage_migration_type") {
+			StorageMigrationType type;
+			if (value == "disabled") {
+				type = StorageMigrationType::DISABLED;
+			} else if (value == "aggressive") {
+				type = StorageMigrationType::AGGRESSIVE;
+			} else if (value == "gradual") {
+				type = StorageMigrationType::GRADUAL;
+			} else {
+				printf("Error: Only disabled|aggressive|gradual are valid for storage_migration_mode.\n");
+				return out;
+			}
+			out[p + key] = format("%d", type);
+		}
 		return out;
 	}
 
@@ -413,266 +435,6 @@ ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx) {
 	}
 }
 
-ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string, std::string> m, bool force) {
-	state StringRef initIdKey = LiteralStringRef("\xff/init_id");
-	state Transaction tr(cx);
-
-	if (!m.size()) {
-		return ConfigurationResult::NO_OPTIONS_PROVIDED;
-	}
-
-	// make sure we have essential configuration options
-	std::string initKey = configKeysPrefix.toString() + "initialized";
-	state bool creating = m.count(initKey) != 0;
-	state Optional<UID> locked;
-	{
-		auto iter = m.find(databaseLockedKey.toString());
-		if (iter != m.end()) {
-			if (!creating) {
-				return ConfigurationResult::LOCKED_NOT_NEW;
-			}
-			locked = UID::fromString(iter->second);
-			m.erase(iter);
-		}
-	}
-	if (creating) {
-		m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString();
-		if (!isCompleteConfiguration(m)) {
-			return ConfigurationResult::INCOMPLETE_CONFIGURATION;
-		}
-	}
-
-	state Future<Void> tooLong = delay(60);
-	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
-	state bool oldReplicationUsesDcId = false;
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-
-			if (!creating && !force) {
-				state Future<RangeResult> fConfig = tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<vector<ProcessData>> fWorkers = getWorkers(&tr);
-				wait(success(fConfig) || tooLong);
-
-				if (!fConfig.isReady()) {
-					return ConfigurationResult::DATABASE_UNAVAILABLE;
-				}
-
-				if (fConfig.isReady()) {
-					ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY);
-					state DatabaseConfiguration oldConfig;
-					oldConfig.fromKeyValues((VectorRef<KeyValueRef>)fConfig.get());
-					state DatabaseConfiguration newConfig = oldConfig;
-					for (auto kv : m) {
-						newConfig.set(kv.first, kv.second);
-					}
-					if (!newConfig.isValid()) {
-						return ConfigurationResult::INVALID_CONFIGURATION;
-					}
-
-					if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) {
-						return ConfigurationResult::REGION_REPLICATION_MISMATCH;
-					}
-
-					oldReplicationUsesDcId =
-					    oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
-
-					if (oldConfig.usableRegions != newConfig.usableRegions) {
-						// cannot change region configuration
-						std::map<Key, int32_t> dcId_priority;
-						for (auto& it : newConfig.regions) {
-							dcId_priority[it.dcId] = it.priority;
-						}
-						for (auto& it : oldConfig.regions) {
-							if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) {
-								return ConfigurationResult::REGIONS_CHANGED;
-							}
-						}
-
-						// must only have one region with priority >= 0
-						int activeRegionCount = 0;
-						for (auto& it : newConfig.regions) {
-							if (it.priority >= 0) {
-								activeRegionCount++;
-							}
-						}
-						if (activeRegionCount > 1) {
-							return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
-						}
-					}
-
-					state Future<RangeResult> fServerList = (newConfig.regions.size())
-					                                            ? tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY)
-					                                            : Future<RangeResult>();
-
-					if (newConfig.usableRegions == 2) {
-						if (oldReplicationUsesDcId) {
-							state Future<RangeResult> fLocalityList =
-							    tr.getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
-							wait(success(fLocalityList) || tooLong);
-							if (!fLocalityList.isReady()) {
-								return ConfigurationResult::DATABASE_UNAVAILABLE;
-							}
-							RangeResult localityList = fLocalityList.get();
-							ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY);
-
-							std::set<Key> localityDcIds;
-							for (auto& s : localityList) {
-								auto dc = decodeTagLocalityListKey(s.key);
-								if (dc.present()) {
-									localityDcIds.insert(dc.get());
-								}
-							}
-
-							for (auto& it : newConfig.regions) {
-								if (localityDcIds.count(it.dcId) == 0) {
-									return ConfigurationResult::DCID_MISSING;
-								}
-							}
-						} else {
-							// all regions with priority >= 0 must be fully replicated
-							state std::vector<Future<Optional<Value>>> replicasFutures;
-							for (auto& it : newConfig.regions) {
-								if (it.priority >= 0) {
-									replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(it.dcId)));
-								}
-							}
-							wait(waitForAll(replicasFutures) || tooLong);
-
-							for (auto& it : replicasFutures) {
-								if (!it.isReady()) {
-									return ConfigurationResult::DATABASE_UNAVAILABLE;
-								}
-								if (!it.get().present()) {
-									return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
-								}
-							}
-						}
-					}
-
-					if (newConfig.regions.size()) {
-						// all storage servers must be in one of the regions
-						wait(success(fServerList) || tooLong);
-						if (!fServerList.isReady()) {
-							return ConfigurationResult::DATABASE_UNAVAILABLE;
-						}
-						RangeResult serverList = fServerList.get();
-						ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
-
-						std::set<Key> newDcIds;
-						for (auto& it : newConfig.regions) {
-							newDcIds.insert(it.dcId);
-						}
-						std::set<Optional<Key>> missingDcIds;
-						for (auto& s : serverList) {
-							auto ssi = decodeServerListValue(s.value);
-							if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) {
-								missingDcIds.insert(ssi.locality.dcId());
-							}
-						}
-						if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) {
-							return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
-						}
-					}
-
-					wait(success(fWorkers) || tooLong);
-					if (!fWorkers.isReady()) {
-						return ConfigurationResult::DATABASE_UNAVAILABLE;
-					}
-
-					if (newConfig.regions.size()) {
-						std::map<Optional<Key>, std::set<Optional<Key>>> dcId_zoneIds;
-						for (auto& it : fWorkers.get()) {
-							if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
-								dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId());
-							}
-						}
-						for (auto& region : newConfig.regions) {
-							if (dcId_zoneIds[region.dcId].size() <
-							    std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
-								return ConfigurationResult::NOT_ENOUGH_WORKERS;
-							}
-							if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
-								int totalSatelliteProcesses = 0;
-								for (auto& sat : region.satellites) {
-									totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size();
-								}
-								if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) {
-									return ConfigurationResult::NOT_ENOUGH_WORKERS;
-								}
-							}
-						}
-					} else {
-						std::set<Optional<Key>> zoneIds;
-						for (auto& it : fWorkers.get()) {
-							if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
-								zoneIds.insert(it.locality.zoneId());
-							}
-						}
-						if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
-							return ConfigurationResult::NOT_ENOUGH_WORKERS;
-						}
-					}
-				}
-			}
-			if (creating) {
-				tr.setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
-				tr.addReadConflictRange(singleKeyRange(initIdKey));
-			} else if (m.size()) {
-				// might be used in an emergency transaction, so make sure it is retry-self-conflicting and
-				// CAUSAL_WRITE_RISKY
-				tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
-				tr.addReadConflictRange(singleKeyRange(m.begin()->first));
-			}
-
-			if (locked.present()) {
-				ASSERT(creating);
-				tr.atomicOp(databaseLockedKey,
-				            BinaryWriter::toValue(locked.get(), Unversioned())
-				                .withPrefix(LiteralStringRef("0123456789"))
-				                .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
-				            MutationRef::SetVersionstampedValue);
-			}
-
-			for (auto i = m.begin(); i != m.end(); ++i) {
-				tr.set(StringRef(i->first), StringRef(i->second));
-			}
-
-			tr.addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
-			tr.set(moveKeysLockOwnerKey, versionKey);
-
-			wait(tr.commit());
-			break;
-		} catch (Error& e) {
-			state Error e1(e);
-			if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
-				// The database now exists.  Determine whether we created it or it was already existing/created by
-				// someone else.  The latter is an error.
-				tr.reset();
-				loop {
-					try {
-						tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-						tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-
-						Optional<Value> v = wait(tr.get(initIdKey));
-						if (v != m[initIdKey.toString()])
-							return ConfigurationResult::DATABASE_ALREADY_CREATED;
-						else
-							return ConfigurationResult::DATABASE_CREATED;
-					} catch (Error& e2) {
-						wait(tr.onError(e2));
-					}
-				}
-			}
-			wait(tr.onError(e1));
-		}
-	}
-	return ConfigurationResult::SUCCESS;
-}
-
 ConfigureAutoResult parseConfig(StatusObject const& status) {
 	ConfigureAutoResult result;
 	StatusObjectReader statusObj(status);
@@ -942,97 +704,7 @@ ConfigureAutoResult parseConfig(StatusObject const& status) {
 	return result;
 }
 
-ACTOR Future<ConfigurationResult> autoConfig(Database cx, ConfigureAutoResult conf) {
-	state Transaction tr(cx);
-	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
-
-	if (!conf.address_class.size())
-		return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
-
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-
-			vector<ProcessData> workers = wait(getWorkers(&tr));
-			std::map<NetworkAddress, Optional<Standalone<StringRef>>> address_processId;
-			for (auto& w : workers) {
-				address_processId[w.address] = w.locality.processId();
-			}
-
-			for (auto& it : conf.address_class) {
-				if (it.second.classSource() == ProcessClass::CommandLineSource) {
-					tr.clear(processClassKeyFor(address_processId[it.first].get()));
-				} else {
-					tr.set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second));
-				}
-			}
-
-			if (conf.address_class.size())
-				tr.set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString());
-
-			if (conf.auto_logs != conf.old_logs)
-				tr.set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs));
-
-			if (conf.auto_commit_proxies != conf.old_commit_proxies)
-				tr.set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies));
-
-			if (conf.auto_grv_proxies != conf.old_grv_proxies)
-				tr.set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies));
-
-			if (conf.auto_resolvers != conf.old_resolvers)
-				tr.set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
-
-			if (conf.auto_replication != conf.old_replication) {
-				std::vector<StringRef> modes;
-				modes.push_back(conf.auto_replication);
-				std::map<std::string, std::string> m;
-				auto r = buildConfiguration(modes, m);
-				if (r != ConfigurationResult::SUCCESS)
-					return r;
-
-				for (auto& kv : m)
-					tr.set(kv.first, kv.second);
-			}
-
-			tr.addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
-			tr.set(moveKeysLockOwnerKey, versionKey);
-
-			wait(tr.commit());
-			return ConfigurationResult::SUCCESS;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-}
-
-Future<ConfigurationResult> changeConfig(Database const& cx,
-                                         std::vector<StringRef> const& modes,
-                                         Optional<ConfigureAutoResult> const& conf,
-                                         bool force) {
-	if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
-		return autoConfig(cx, conf.get());
-	}
-
-	std::map<std::string, std::string> m;
-	auto r = buildConfiguration(modes, m);
-	if (r != ConfigurationResult::SUCCESS)
-		return r;
-	return changeConfig(cx, m, force);
-}
-
-Future<ConfigurationResult> changeConfig(Database const& cx, std::string const& modes, bool force) {
-	TraceEvent("ChangeConfig").detail("Mode", modes);
-	std::map<std::string, std::string> m;
-	auto r = buildConfiguration(modes, m);
-	if (r != ConfigurationResult::SUCCESS)
-		return r;
-	return changeConfig(cx, m, force);
-}
-
-ACTOR Future<vector<ProcessData>> getWorkers(Transaction* tr) {
+ACTOR Future<std::vector<ProcessData>> getWorkers(Transaction* tr) {
 	state Future<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
 	state Future<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
 
@@ -1063,14 +735,14 @@ ACTOR Future<vector<ProcessData>> getWorkers(Transaction* tr) {
 	return results;
 }
 
-ACTOR Future<vector<ProcessData>> getWorkers(Database cx) {
+ACTOR Future<std::vector<ProcessData>> getWorkers(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			vector<ProcessData> workers = wait(getWorkers(&tr));
+			std::vector<ProcessData> workers = wait(getWorkers(&tr));
 			return workers;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -1148,7 +820,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		}
 	}
 
-	vector<Future<Optional<LeaderInfo>>> leaderServers;
+	std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
 
 	leaderServers.reserve(coord.clientLeaderServers.size());
@@ -1234,7 +906,7 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 			TEST(old.clusterKeyName() != conn.clusterKeyName()); // Quorum change with new name
 			TEST(old.clusterKeyName() == conn.clusterKeyName()); // Quorum change with unchanged name
 
-			state vector<Future<Optional<LeaderInfo>>> leaderServers;
+			state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 			state ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
 			// check if allowed to modify the cluster descriptor
 			if (!change->getDesiredClusterKeyName().empty()) {
@@ -1266,24 +938,24 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 }
 
 struct SpecifiedQuorumChange final : IQuorumChange {
-	vector<NetworkAddress> desired;
-	explicit SpecifiedQuorumChange(vector<NetworkAddress> const& desired) : desired(desired) {}
-	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                      vector<NetworkAddress> oldCoordinators,
-	                                                      Reference<ClusterConnectionFile>,
-	                                                      CoordinatorsResult&) override {
+	std::vector<NetworkAddress> desired;
+	explicit SpecifiedQuorumChange(std::vector<NetworkAddress> const& desired) : desired(desired) {}
+	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                           std::vector<NetworkAddress> oldCoordinators,
+	                                                           Reference<ClusterConnectionFile>,
+	                                                           CoordinatorsResult&) override {
 		return desired;
 	}
 };
-Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) {
+Reference<IQuorumChange> specifiedQuorumChange(std::vector<NetworkAddress> const& addresses) {
 	return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses));
 }
 
 struct NoQuorumChange final : IQuorumChange {
-	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                      vector<NetworkAddress> oldCoordinators,
-	                                                      Reference<ClusterConnectionFile>,
-	                                                      CoordinatorsResult&) override {
+	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                           std::vector<NetworkAddress> oldCoordinators,
+	                                                           Reference<ClusterConnectionFile>,
+	                                                           CoordinatorsResult&) override {
 		return oldCoordinators;
 	}
 };
@@ -1296,10 +968,10 @@ struct NameQuorumChange final : IQuorumChange {
 	Reference<IQuorumChange> otherChange;
 	explicit NameQuorumChange(std::string const& newName, Reference<IQuorumChange> const& otherChange)
 	  : newName(newName), otherChange(otherChange) {}
-	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                      vector<NetworkAddress> oldCoordinators,
-	                                                      Reference<ClusterConnectionFile> cf,
-	                                                      CoordinatorsResult& t) override {
+	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                           std::vector<NetworkAddress> oldCoordinators,
+	                                                           Reference<ClusterConnectionFile> cf,
+	                                                           CoordinatorsResult& t) override {
 		return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t);
 	}
 	std::string getDesiredClusterKeyName() const override { return newName; }
@@ -1312,10 +984,10 @@ struct AutoQuorumChange final : IQuorumChange {
 	int desired;
 	explicit AutoQuorumChange(int desired) : desired(desired) {}
 
-	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                      vector<NetworkAddress> oldCoordinators,
-	                                                      Reference<ClusterConnectionFile> ccf,
-	                                                      CoordinatorsResult& err) override {
+	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                           std::vector<NetworkAddress> oldCoordinators,
+	                                                           Reference<ClusterConnectionFile> ccf,
+	                                                           CoordinatorsResult& err) override {
 		return getDesired(Reference<AutoQuorumChange>::addRef(this), tr, oldCoordinators, ccf, &err);
 	}
 
@@ -1333,7 +1005,7 @@ struct AutoQuorumChange final : IQuorumChange {
 
 	ACTOR static Future<bool> isAcceptable(AutoQuorumChange* self,
 	                                       Transaction* tr,
-	                                       vector<NetworkAddress> oldCoordinators,
+	                                       std::vector<NetworkAddress> oldCoordinators,
 	                                       Reference<ClusterConnectionFile> ccf,
 	                                       int desiredCount,
 	                                       std::set<AddressExclusion>* excluded) {
@@ -1345,14 +1017,14 @@ struct AutoQuorumChange final : IQuorumChange {
 
 		// Check availability
 		ClientCoordinators coord(ccf);
-		vector<Future<Optional<LeaderInfo>>> leaderServers;
+		std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 		leaderServers.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
 			leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
 			                                           GetLeaderRequest(coord.clusterKey, UID()),
 			                                           TaskPriority::CoordinationReply));
 		}
-		Optional<vector<Optional<LeaderInfo>>> results =
+		Optional<std::vector<Optional<LeaderInfo>>> results =
 		    wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY));
 		if (!results.present()) {
 			return false;
@@ -1379,11 +1051,11 @@ struct AutoQuorumChange final : IQuorumChange {
 		return true; // The status quo seems fine
 	}
 
-	ACTOR static Future<vector<NetworkAddress>> getDesired(Reference<AutoQuorumChange> self,
-	                                                       Transaction* tr,
-	                                                       vector<NetworkAddress> oldCoordinators,
-	                                                       Reference<ClusterConnectionFile> ccf,
-	                                                       CoordinatorsResult* err) {
+	ACTOR static Future<std::vector<NetworkAddress>> getDesired(Reference<AutoQuorumChange> self,
+	                                                            Transaction* tr,
+	                                                            std::vector<NetworkAddress> oldCoordinators,
+	                                                            Reference<ClusterConnectionFile> ccf,
+	                                                            CoordinatorsResult* err) {
 		state int desiredCount = self->desired;
 
 		if (desiredCount == -1) {
@@ -1394,8 +1066,8 @@ struct AutoQuorumChange final : IQuorumChange {
 		std::vector<AddressExclusion> excl = wait(getExcludedServers(tr));
 		state std::set<AddressExclusion> excluded(excl.begin(), excl.end());
 
-		vector<ProcessData> _workers = wait(getWorkers(tr));
-		state vector<ProcessData> workers = _workers;
+		std::vector<ProcessData> _workers = wait(getWorkers(tr));
+		state std::vector<ProcessData> workers = _workers;
 
 		std::map<NetworkAddress, LocalityData> addr_locality;
 		for (auto w : workers)
@@ -1431,7 +1103,7 @@ struct AutoQuorumChange final : IQuorumChange {
 				    .detail("DesiredCoordinators", desiredCount)
 				    .detail("CurrentCoordinators", oldCoordinators.size());
 				*err = CoordinatorsResult::NOT_ENOUGH_MACHINES;
-				return vector<NetworkAddress>();
+				return std::vector<NetworkAddress>();
 			}
 			chosen.resize((chosen.size() - 1) | 1);
 		}
@@ -1443,11 +1115,11 @@ struct AutoQuorumChange final : IQuorumChange {
 	// (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and
 	// (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute
 	// workers.
-	void addDesiredWorkers(vector<NetworkAddress>& chosen,
-	                       const vector<ProcessData>& workers,
+	void addDesiredWorkers(std::vector<NetworkAddress>& chosen,
+	                       const std::vector<ProcessData>& workers,
 	                       int desiredCount,
 	                       const std::set<AddressExclusion>& excluded) {
-		vector<ProcessData> remainingWorkers(workers);
+		std::vector<ProcessData> remainingWorkers(workers);
 		deterministicRandom()->randomShuffle(remainingWorkers);
 
 		std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) {
@@ -1470,10 +1142,10 @@ struct AutoQuorumChange final : IQuorumChange {
 		std::map<StringRef, std::map<StringRef, int>> currentCounts;
 		std::map<StringRef, int> hardLimits;
 
-		vector<StringRef> fields({ LiteralStringRef("dcid"),
-		                           LiteralStringRef("data_hall"),
-		                           LiteralStringRef("zoneid"),
-		                           LiteralStringRef("machineid") });
+		std::vector<StringRef> fields({ LiteralStringRef("dcid"),
+		                                LiteralStringRef("data_hall"),
+		                                LiteralStringRef("zoneid"),
+		                                LiteralStringRef("machineid") });
 
 		for (auto field = fields.begin(); field != fields.end(); field++) {
 			if (field->toString() == "zoneid") {
@@ -1537,7 +1209,7 @@ Reference<IQuorumChange> autoQuorumChange(int desired) {
 	return Reference<IQuorumChange>(new AutoQuorumChange(desired));
 }
 
-void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed) {
+void excludeServers(Transaction& tr, std::vector<AddressExclusion>& servers, bool failed) {
 	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -1556,7 +1228,7 @@ void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool fai
 	TraceEvent("ExcludeServersCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed);
 }
 
-ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed) {
+ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
 	if (cx->apiVersionAtLeast(700)) {
 		state ReadYourWritesTransaction ryw(cx);
 		loop {
@@ -1659,7 +1331,7 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 	}
 }
 
-ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed) {
+ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
 	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	if (cx->apiVersionAtLeast(700)) {
 		state ReadYourWritesTransaction ryw(cx);
@@ -1762,7 +1434,7 @@ ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers,
 
 // Remove the given localities from the exclusion list.
 // include localities by clearing the keys.
-ACTOR Future<Void> includeLocalities(Database cx, vector<std::string> localities, bool failed, bool includeAll) {
+ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> localities, bool failed, bool includeAll) {
 	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	if (cx->apiVersionAtLeast(700)) {
 		state ReadYourWritesTransaction ryw(cx);
@@ -1856,7 +1528,7 @@ ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass p
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
-			vector<ProcessData> workers = wait(getWorkers(&tr));
+			std::vector<ProcessData> workers = wait(getWorkers(&tr));
 
 			bool foundChange = false;
 			for (int i = 0; i < workers.size(); i++) {
@@ -1881,13 +1553,13 @@ ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass p
 	}
 }
 
-ACTOR Future<vector<AddressExclusion>> getExcludedServers(Transaction* tr) {
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr) {
 	state RangeResult r = wait(tr->getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 	state RangeResult r2 = wait(tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
 
-	vector<AddressExclusion> exclusions;
+	std::vector<AddressExclusion> exclusions;
 	for (auto i = r.begin(); i != r.end(); ++i) {
 		auto a = decodeExcludedServersKey(i->key);
 		if (a.isValid())
@@ -1902,14 +1574,14 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers(Transaction* tr) {
 	return exclusions;
 }
 
-ACTOR Future<vector<AddressExclusion>> getExcludedServers(Database cx) {
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			vector<AddressExclusion> exclusions = wait(getExcludedServers(&tr));
+			std::vector<AddressExclusion> exclusions = wait(getExcludedServers(&tr));
 			return exclusions;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -1918,13 +1590,13 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers(Database cx) {
 }
 
 // Get the current list of excluded localities by reading the keys.
-ACTOR Future<vector<std::string>> getExcludedLocalities(Transaction* tr) {
+ACTOR Future<std::vector<std::string>> getExcludedLocalities(Transaction* tr) {
 	state RangeResult r = wait(tr->getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 	state RangeResult r2 = wait(tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
 
-	vector<std::string> excludedLocalities;
+	std::vector<std::string> excludedLocalities;
 	for (const auto& i : r) {
 		auto a = decodeExcludedLocalityKey(i.key);
 		excludedLocalities.push_back(a);
@@ -1938,14 +1610,14 @@ ACTOR Future<vector<std::string>> getExcludedLocalities(Transaction* tr) {
 }
 
 // Get the list of excluded localities by reading the keys.
-ACTOR Future<vector<std::string>> getExcludedLocalities(Database cx) {
+ACTOR Future<std::vector<std::string>> getExcludedLocalities(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			vector<std::string> exclusions = wait(getExcludedLocalities(&tr));
+			std::vector<std::string> exclusions = wait(getExcludedLocalities(&tr));
 			return exclusions;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -2175,7 +1847,7 @@ ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr
 }
 
 ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
-                                                                vector<AddressExclusion> excl,
+                                                                std::vector<AddressExclusion> excl,
                                                                 bool waitForAllExcluded) {
 	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
 	state std::set<NetworkAddress> inProgressExclusion;
@@ -2626,6 +2298,40 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 	}
 }
 
+std::string ManagementAPI::generateErrorMessage(const CoordinatorsResult& res) {
+	// Note: the error message here should not be changed if possible
+	// If you do change the message here,
+	// please update the corresponding fdbcli code to support both the old and the new message
+
+	std::string msg;
+	switch (res) {
+	case CoordinatorsResult::INVALID_NETWORK_ADDRESSES:
+		msg = "The specified network addresses are invalid";
+		break;
+	case CoordinatorsResult::SAME_NETWORK_ADDRESSES:
+		msg = "No change (existing configuration satisfies request)";
+		break;
+	case CoordinatorsResult::NOT_COORDINATORS:
+		msg = "Coordination servers are not running on the specified network addresses";
+		break;
+	case CoordinatorsResult::DATABASE_UNREACHABLE:
+		msg = "Database unreachable";
+		break;
+	case CoordinatorsResult::BAD_DATABASE_STATE:
+		msg = "The database is in an unexpected state from which changing coordinators might be unsafe";
+		break;
+	case CoordinatorsResult::COORDINATOR_UNREACHABLE:
+		msg = "One of the specified coordinators is unreachable";
+		break;
+	case CoordinatorsResult::NOT_ENOUGH_MACHINES:
+		msg = "Too few fdbserver machines to provide coordination at the current redundancy level";
+		break;
+	default:
+		break;
+	}
+	return msg;
+}
+
 TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
 	wait(Future<Void>(Void()));
 

fdbclient/ManagementAPI.actor.h

@@ -49,6 +49,7 @@ enum class ConfigurationResult {
 	UNKNOWN_OPTION,
 	INCOMPLETE_CONFIGURATION,
 	INVALID_CONFIGURATION,
+	STORAGE_MIGRATION_DISABLED,
 	DATABASE_ALREADY_CREATED,
 	DATABASE_CREATED,
 	DATABASE_UNAVAILABLE,
@@ -60,6 +61,7 @@ enum class ConfigurationResult {
 	REGION_REPLICATION_MISMATCH,
 	DCID_MISSING,
 	LOCKED_NOT_NEW,
+	SUCCESS_WARN_PPW_GRADUAL,
 	SUCCESS,
 };
 
@@ -119,31 +121,17 @@ ConfigurationResult buildConfiguration(
 
 bool isCompleteConfiguration(std::map<std::string, std::string> const& options);
 
-// All versions of changeConfig apply the given set of configuration tokens to the database, and return a
-// ConfigurationResult (or error).
-Future<ConfigurationResult> changeConfig(Database const& cx,
-                                         std::string const& configMode,
-                                         bool force); // Accepts tokens separated by spaces in a single string
-
 ConfigureAutoResult parseConfig(StatusObject const& status);
-Future<ConfigurationResult> changeConfig(Database const& cx,
-                                         std::vector<StringRef> const& modes,
-                                         Optional<ConfigureAutoResult> const& conf,
-                                         bool force); // Accepts a vector of configuration tokens
-ACTOR Future<ConfigurationResult> changeConfig(
-    Database cx,
-    std::map<std::string, std::string> m,
-    bool force); // Accepts a full configuration in key/value format (from buildConfiguration)
 
 ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx);
 ACTOR Future<Void> waitForFullReplication(Database cx);
 
 struct IQuorumChange : ReferenceCounted<IQuorumChange> {
 	virtual ~IQuorumChange() {}
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                              vector<NetworkAddress> oldCoordinators,
-	                                                              Reference<ClusterConnectionFile>,
-	                                                              CoordinatorsResult&) = 0;
+	virtual Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                                   std::vector<NetworkAddress> oldCoordinators,
+	                                                                   Reference<ClusterConnectionFile>,
+	                                                                   CoordinatorsResult&) = 0;
 	virtual std::string getDesiredClusterKeyName() const { return std::string(); }
 };
 
@@ -154,14 +142,14 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
 Reference<IQuorumChange> autoQuorumChange(int desired = -1);
 Reference<IQuorumChange> noQuorumChange();
-Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&);
+Reference<IQuorumChange> specifiedQuorumChange(std::vector<NetworkAddress> const&);
 Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
 
 // Exclude the given set of servers from use as state servers.  Returns as soon as the change is durable, without
 // necessarily waiting for the servers to be evacuated.  A NetworkAddress with a port of 0 means all servers on the
 // given IP.
-ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
-void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed = false);
+ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
+void excludeServers(Transaction& tr, std::vector<AddressExclusion>& servers, bool failed = false);
 
 // Exclude the servers matching the given set of localities from use as state servers.  Returns as soon as the change
 // is durable, without necessarily waiting for the servers to be evacuated.
@@ -170,11 +158,11 @@ void excludeLocalities(Transaction& tr, std::unordered_set<std::string> localiti
 
 // Remove the given servers from the exclusion list.  A NetworkAddress with a port of 0 means all servers on the given
 // IP.  A NetworkAddress() means all servers (don't exclude anything)
-ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
+ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
 
 // Remove the given localities from the exclusion list.
 ACTOR Future<Void> includeLocalities(Database cx,
-                                     vector<std::string> localities,
+                                     std::vector<std::string> localities,
                                      bool failed = false,
                                      bool includeAll = false);
 
@@ -183,12 +171,12 @@ ACTOR Future<Void> includeLocalities(Database cx,
 ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
 
 // Get the current list of excluded servers
-ACTOR Future<vector<AddressExclusion>> getExcludedServers(Database cx);
-ACTOR Future<vector<AddressExclusion>> getExcludedServers(Transaction* tr);
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Database cx);
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr);
 
 // Get the current list of excluded localities
-ACTOR Future<vector<std::string>> getExcludedLocalities(Database cx);
-ACTOR Future<vector<std::string>> getExcludedLocalities(Transaction* tr);
+ACTOR Future<std::vector<std::string>> getExcludedLocalities(Database cx);
+ACTOR Future<std::vector<std::string>> getExcludedLocalities(Transaction* tr);
 
 std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers, const std::string& locality);
 
@@ -196,15 +184,15 @@ std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>
 // true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
 // unless any of them are explicitly included with includeServers()
 ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
-                                                                vector<AddressExclusion> servers,
+                                                                std::vector<AddressExclusion> servers,
                                                                 bool waitForAllExcluded);
 ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
                                                    std::set<AddressExclusion>* exclusions,
                                                    std::set<NetworkAddress>* inProgressExclusion);
 
 // Gets a list of all workers in the cluster (excluding testers)
-ACTOR Future<vector<ProcessData>> getWorkers(Database cx);
-ACTOR Future<vector<ProcessData>> getWorkers(Transaction* tr);
+ACTOR Future<std::vector<ProcessData>> getWorkers(Database cx);
+ACTOR Future<std::vector<ProcessData>> getWorkers(Transaction* tr);
 
 ACTOR Future<Void> timeKeeperSetDisable(Database cx);
 
@@ -322,6 +310,436 @@ Future<Void> removeCachedRange(Reference<DB> db, KeyRangeRef range) {
 	return changeCachedRange(db, range, false);
 }
 
+ACTOR template <class Tr>
+Future<std::vector<ProcessData>> getWorkers(Reference<Tr> tr,
+                                            typename Tr::template FutureT<RangeResult> processClassesF,
+                                            typename Tr::template FutureT<RangeResult> processDataF) {
+	// processClassesF and processDataF are used to hold standalone memory
+	processClassesF = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
+	processDataF = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
+	state Future<RangeResult> processClasses = safeThreadFutureToFuture(processClassesF);
+	state Future<RangeResult> processData = safeThreadFutureToFuture(processDataF);
+
+	wait(success(processClasses) && success(processData));
+	ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
+	ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
+
+	std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
+	for (int i = 0; i < processClasses.get().size(); i++) {
+		id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
+		    decodeProcessClassValue(processClasses.get()[i].value);
+	}
+
+	std::vector<ProcessData> results;
+
+	for (int i = 0; i < processData.get().size(); i++) {
+		ProcessData data = decodeWorkerListValue(processData.get()[i].value);
+		ProcessClass processClass = id_class[data.locality.processId()];
+
+		if (processClass.classSource() == ProcessClass::DBSource ||
+		    data.processClass.classType() == ProcessClass::UnsetClass)
+			data.processClass = processClass;
+
+		if (data.processClass.classType() != ProcessClass::TesterClass)
+			results.push_back(data);
+	}
+
+	return results;
+}
+
+// All versions of changeConfig apply the given set of configuration tokens to the database, and return a
+// ConfigurationResult (or error).
+
+// Accepts a full configuration in key/value format (from buildConfiguration)
+ACTOR template <class DB>
+Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string, std::string> m, bool force) {
+	state StringRef initIdKey = LiteralStringRef("\xff/init_id");
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+
+	if (!m.size()) {
+		return ConfigurationResult::NO_OPTIONS_PROVIDED;
+	}
+
+	// make sure we have essential configuration options
+	std::string initKey = configKeysPrefix.toString() + "initialized";
+	state bool creating = m.count(initKey) != 0;
+	state Optional<UID> locked;
+	{
+		auto iter = m.find(databaseLockedKey.toString());
+		if (iter != m.end()) {
+			if (!creating) {
+				return ConfigurationResult::LOCKED_NOT_NEW;
+			}
+			locked = UID::fromString(iter->second);
+			m.erase(iter);
+		}
+	}
+	if (creating) {
+		m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString();
+		if (!isCompleteConfiguration(m)) {
+			return ConfigurationResult::INCOMPLETE_CONFIGURATION;
+		}
+	}
+
+	state Future<Void> tooLong = delay(60);
+	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
+	state bool oldReplicationUsesDcId = false;
+	state bool warnPPWGradual = false;
+	state bool warnChangeStorageNoMigrate = false;
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
+
+			if (!creating && !force) {
+				state typename DB::TransactionT::template FutureT<RangeResult> fConfigF =
+				    tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
+				state Future<RangeResult> fConfig = safeThreadFutureToFuture(fConfigF);
+				state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
+				state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
+				state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr, processClassesF, processDataF);
+				wait(success(fConfig) || tooLong);
+
+				if (!fConfig.isReady()) {
+					return ConfigurationResult::DATABASE_UNAVAILABLE;
+				}
+
+				if (fConfig.isReady()) {
+					ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY);
+					state DatabaseConfiguration oldConfig;
+					oldConfig.fromKeyValues((VectorRef<KeyValueRef>)fConfig.get());
+					state DatabaseConfiguration newConfig = oldConfig;
+					for (auto kv : m) {
+						newConfig.set(kv.first, kv.second);
+					}
+					if (!newConfig.isValid()) {
+						return ConfigurationResult::INVALID_CONFIGURATION;
+					}
+
+					if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) {
+						return ConfigurationResult::REGION_REPLICATION_MISMATCH;
+					}
+
+					oldReplicationUsesDcId =
+					    oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
+
+					if (oldConfig.usableRegions != newConfig.usableRegions) {
+						// cannot change region configuration
+						std::map<Key, int32_t> dcId_priority;
+						for (auto& it : newConfig.regions) {
+							dcId_priority[it.dcId] = it.priority;
+						}
+						for (auto& it : oldConfig.regions) {
+							if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) {
+								return ConfigurationResult::REGIONS_CHANGED;
+							}
+						}
+
+						// must only have one region with priority >= 0
+						int activeRegionCount = 0;
+						for (auto& it : newConfig.regions) {
+							if (it.priority >= 0) {
+								activeRegionCount++;
+							}
+						}
+						if (activeRegionCount > 1) {
+							return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
+						}
+					}
+
+					state typename DB::TransactionT::template FutureT<RangeResult> fServerListF =
+					    tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+					state Future<RangeResult> fServerList =
+					    (newConfig.regions.size()) ? safeThreadFutureToFuture(fServerListF) : Future<RangeResult>();
+
+					if (newConfig.usableRegions == 2) {
+						if (oldReplicationUsesDcId) {
+							state typename DB::TransactionT::template FutureT<RangeResult> fLocalityListF =
+							    tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
+							state Future<RangeResult> fLocalityList = safeThreadFutureToFuture(fLocalityListF);
+							wait(success(fLocalityList) || tooLong);
+							if (!fLocalityList.isReady()) {
+								return ConfigurationResult::DATABASE_UNAVAILABLE;
+							}
+							RangeResult localityList = fLocalityList.get();
+							ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY);
+
+							std::set<Key> localityDcIds;
+							for (auto& s : localityList) {
+								auto dc = decodeTagLocalityListKey(s.key);
+								if (dc.present()) {
+									localityDcIds.insert(dc.get());
+								}
+							}
+
+							for (auto& it : newConfig.regions) {
+								if (localityDcIds.count(it.dcId) == 0) {
+									return ConfigurationResult::DCID_MISSING;
+								}
+							}
+						} else {
+							// all regions with priority >= 0 must be fully replicated
+							state std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>>
+							    replicasFuturesF;
+							state std::vector<Future<Optional<Value>>> replicasFutures;
+							for (auto& it : newConfig.regions) {
+								if (it.priority >= 0) {
+									replicasFuturesF.push_back(tr->get(datacenterReplicasKeyFor(it.dcId)));
+									replicasFutures.push_back(safeThreadFutureToFuture(replicasFuturesF.back()));
+								}
+							}
+							wait(waitForAll(replicasFutures) || tooLong);
+
+							for (auto& it : replicasFutures) {
+								if (!it.isReady()) {
+									return ConfigurationResult::DATABASE_UNAVAILABLE;
+								}
+								if (!it.get().present()) {
+									return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
+								}
+							}
+						}
+					}
+
+					if (newConfig.regions.size()) {
+						// all storage servers must be in one of the regions
+						wait(success(fServerList) || tooLong);
+						if (!fServerList.isReady()) {
+							return ConfigurationResult::DATABASE_UNAVAILABLE;
+						}
+						RangeResult serverList = fServerList.get();
+						ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
+
+						std::set<Key> newDcIds;
+						for (auto& it : newConfig.regions) {
+							newDcIds.insert(it.dcId);
+						}
+						std::set<Optional<Key>> missingDcIds;
+						for (auto& s : serverList) {
+							auto ssi = decodeServerListValue(s.value);
+							if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) {
+								missingDcIds.insert(ssi.locality.dcId());
+							}
+						}
+						if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) {
+							return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
+						}
+					}
+
+					wait(success(fWorkers) || tooLong);
+					if (!fWorkers.isReady()) {
+						return ConfigurationResult::DATABASE_UNAVAILABLE;
+					}
+
+					if (newConfig.regions.size()) {
+						std::map<Optional<Key>, std::set<Optional<Key>>> dcId_zoneIds;
+						for (auto& it : fWorkers.get()) {
+							if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
+								dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId());
+							}
+						}
+						for (auto& region : newConfig.regions) {
+							if (dcId_zoneIds[region.dcId].size() <
+							    std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
+								return ConfigurationResult::NOT_ENOUGH_WORKERS;
+							}
+							if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
+								int totalSatelliteProcesses = 0;
+								for (auto& sat : region.satellites) {
+									totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size();
+								}
+								if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) {
+									return ConfigurationResult::NOT_ENOUGH_WORKERS;
+								}
+							}
+						}
+					} else {
+						std::set<Optional<Key>> zoneIds;
+						for (auto& it : fWorkers.get()) {
+							if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
+								zoneIds.insert(it.locality.zoneId());
+							}
+						}
+						if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
+							return ConfigurationResult::NOT_ENOUGH_WORKERS;
+						}
+					}
+
+					if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
+					    newConfig.storageMigrationType == StorageMigrationType::DISABLED) {
+						return ConfigurationResult::STORAGE_MIGRATION_DISABLED;
+					} else if (newConfig.storageMigrationType == StorageMigrationType::GRADUAL &&
+					           newConfig.perpetualStorageWiggleSpeed == 0) {
+						warnPPWGradual = true;
+					}
+				}
+			}
+			if (creating) {
+				tr->setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
+				tr->addReadConflictRange(singleKeyRange(initIdKey));
+			} else if (m.size()) {
+				// might be used in an emergency transaction, so make sure it is retry-self-conflicting and
+				// CAUSAL_WRITE_RISKY
+				tr->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
+				tr->addReadConflictRange(singleKeyRange(m.begin()->first));
+			}
+
+			if (locked.present()) {
+				ASSERT(creating);
+				tr->atomicOp(databaseLockedKey,
+				             BinaryWriter::toValue(locked.get(), Unversioned())
+				                 .withPrefix(LiteralStringRef("0123456789"))
+				                 .withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
+				             MutationRef::SetVersionstampedValue);
+			}
+
+			for (auto i = m.begin(); i != m.end(); ++i) {
+				tr->set(StringRef(i->first), StringRef(i->second));
+			}
+
+			tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
+			tr->set(moveKeysLockOwnerKey, versionKey);
+
+			wait(safeThreadFutureToFuture(tr->commit()));
+			break;
+		} catch (Error& e) {
+			state Error e1(e);
+			if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
+				// The database now exists.  Determine whether we created it or it was already existing/created by
+				// someone else.  The latter is an error.
+				tr->reset();
+				loop {
+					try {
+						tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+						tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+						tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+						tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
+
+						state typename DB::TransactionT::template FutureT<Optional<Value>> vF = tr->get(initIdKey);
+						Optional<Value> v = wait(safeThreadFutureToFuture(vF));
+						if (v != m[initIdKey.toString()])
+							return ConfigurationResult::DATABASE_ALREADY_CREATED;
+						else
+							return ConfigurationResult::DATABASE_CREATED;
+					} catch (Error& e2) {
+						wait(safeThreadFutureToFuture(tr->onError(e2)));
+					}
+				}
+			}
+			wait(safeThreadFutureToFuture(tr->onError(e1)));
+		}
+	}
+
+	if (warnPPWGradual) {
+		return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
+	} else {
+		return ConfigurationResult::SUCCESS;
+	}
+}
+
+ACTOR template <class DB>
+Future<ConfigurationResult> autoConfig(Reference<DB> db, ConfigureAutoResult conf) {
+	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
+
+	if (!conf.address_class.size())
+		return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
+
+	loop {
+		try {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
+
+			state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
+			state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
+			std::vector<ProcessData> workers = wait(getWorkers(tr, processClassesF, processDataF));
+			std::map<NetworkAddress, Optional<Standalone<StringRef>>> address_processId;
+			for (auto& w : workers) {
+				address_processId[w.address] = w.locality.processId();
+			}
+
+			for (auto& it : conf.address_class) {
+				if (it.second.classSource() == ProcessClass::CommandLineSource) {
+					tr->clear(processClassKeyFor(address_processId[it.first].get()));
+				} else {
+					tr->set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second));
+				}
+			}
+
+			if (conf.address_class.size())
+				tr->set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString());
+
+			if (conf.auto_logs != conf.old_logs)
+				tr->set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs));
+
+			if (conf.auto_commit_proxies != conf.old_commit_proxies)
+				tr->set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies));
+
+			if (conf.auto_grv_proxies != conf.old_grv_proxies)
+				tr->set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies));
+
+			if (conf.auto_resolvers != conf.old_resolvers)
+				tr->set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
+
+			if (conf.auto_replication != conf.old_replication) {
+				std::vector<StringRef> modes;
+				modes.push_back(conf.auto_replication);
+				std::map<std::string, std::string> m;
+				auto r = buildConfiguration(modes, m);
+				if (r != ConfigurationResult::SUCCESS)
+					return r;
+
+				for (auto& kv : m)
+					tr->set(kv.first, kv.second);
+			}
+
+			tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
+			tr->set(moveKeysLockOwnerKey, versionKey);
+
+			wait(safeThreadFutureToFuture(tr->commit()));
+			return ConfigurationResult::SUCCESS;
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
+// Accepts tokens separated by spaces in a single string
+template <class DB>
+Future<ConfigurationResult> changeConfig(Reference<DB> db, std::string const& modes, bool force) {
+	TraceEvent("ChangeConfig").detail("Mode", modes);
+	std::map<std::string, std::string> m;
+	auto r = buildConfiguration(modes, m);
+	if (r != ConfigurationResult::SUCCESS)
+		return r;
+	return changeConfig(db, m, force);
+}
+
+// Accepts a vector of configuration tokens
+template <class DB>
+Future<ConfigurationResult> changeConfig(Reference<DB> db,
+                                         std::vector<StringRef> const& modes,
+                                         Optional<ConfigureAutoResult> const& conf,
+                                         bool force) {
+	if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
+		return autoConfig(db, conf.get());
+	}
+
+	std::map<std::string, std::string> m;
+	auto r = buildConfiguration(modes, m);
+	if (r != ConfigurationResult::SUCCESS)
+		return r;
+	return changeConfig(db, m, force);
+}
+
+// return the corresponding error message for the CoordinatorsResult
+// used by special keys and fdbcli
+std::string generateErrorMessage(const CoordinatorsResult& res);
+
 } // namespace ManagementAPI
 
 #include "flow/unactorcompiler.h"

fdbclient/MonitorLeader.actor.cpp

@@ -328,7 +328,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
 	return Void();
 }
 
-ClusterConnectionString::ClusterConnectionString(vector<NetworkAddress> servers, Key key) : coord(servers) {
+ClusterConnectionString::ClusterConnectionString(std::vector<NetworkAddress> servers, Key key) : coord(servers) {
 	parseKey(key.toString());
 }
 
@@ -383,9 +383,9 @@ ClientCoordinators::ClientCoordinators(Key clusterKey, std::vector<NetworkAddres
 }
 
 ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote)
-  : getLeader(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)),
-    openDatabase(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_OPENDATABASE)),
-    checkDescriptorMutable(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE)) {}
+  : getLeader(Endpoint::wellKnown({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)),
+    openDatabase(Endpoint::wellKnown({ remote }, WLTOKEN_CLIENTLEADERREG_OPENDATABASE)),
+    checkDescriptorMutable(Endpoint::wellKnown({ remote }, WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE)) {}
 
 ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
 	getLeader.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination);
@@ -394,9 +394,8 @@ ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
 	                                             TaskPriority::Coordination);
 }
 
-// Nominee is the worker among all workers that are considered as leader by a coordinator
-// This function contacts a coordinator coord to ask if the worker is considered as a leader (i.e., if the worker
-// is a nominee)
+// Nominee is the worker among all workers that are considered as leader by one coordinator
+// This function contacts a coordinator coord to ask who is its nominee.
 ACTOR Future<Void> monitorNominee(Key key,
                                   ClientLeaderRegInterface coord,
                                   AsyncTrigger* nomineeChange,
@@ -428,13 +427,13 @@ ACTOR Future<Void> monitorNominee(Key key,
 // Also used in fdbserver/LeaderElection.actor.cpp!
 // bool represents if the LeaderInfo is a majority answer or not.
 // This function also masks the first 7 bits of changeId of the nominees and returns the Leader with masked changeId
-Optional<std::pair<LeaderInfo, bool>> getLeader(const vector<Optional<LeaderInfo>>& nominees) {
+Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<LeaderInfo>>& nominees) {
 	// If any coordinator says that the quorum is forwarded, then it is
 	for (int i = 0; i < nominees.size(); i++)
 		if (nominees[i].present() && nominees[i].get().forward)
 			return std::pair<LeaderInfo, bool>(nominees[i].get(), true);
 
-	vector<std::pair<UID, int>> maskedNominees;
+	std::vector<std::pair<UID, int>> maskedNominees;
 	maskedNominees.reserve(nominees.size());
 	for (int i = 0; i < nominees.size(); i++) {
 		if (nominees[i].present()) {
@@ -529,18 +528,6 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConn
 	}
 }
 
-Future<Void> monitorLeaderRemotelyInternal(Reference<ClusterConnectionFile> const& connFile,
-                                           Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
-
-template <class LeaderInterface>
-Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connFile,
-                                   Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
-	LeaderDeserializer<LeaderInterface> deserializer;
-	auto serializedInfo = makeReference<AsyncVar<Value>>();
-	Future<Void> m = monitorLeaderRemotelyInternal(connFile, serializedInfo);
-	return m || deserializer(serializedInfo, outKnownLeader);
-}
-
 ACTOR Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> connFile,
                                          Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
 	state MonitorLeaderInfo info(connFile);
@@ -656,7 +643,7 @@ ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterCo
 		choose {
 			when(ClientDBInfo ni =
 			         wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
-				TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id())
+				TraceEvent("GetClientInfoFromLeaderGotClientInfo", knownLeader->get().get().clientInterface.id())
 				    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID())
 				    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID())
 				    .detail("ClientID", ni.id);
@@ -667,11 +654,11 @@ ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterCo
 	}
 }
 
-ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
-                                           vector<NetworkAddress> coordinators,
-                                           ClientData* clientData,
-                                           Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
-	state vector<ClientLeaderRegInterface> clientLeaderServers;
+ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
+                                                 std::vector<NetworkAddress> coordinators,
+                                                 ClientData* clientData,
+                                                 Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
+	state std::vector<ClientLeaderRegInterface> clientLeaderServers;
 	state AsyncTrigger nomineeChange;
 	state std::vector<Optional<LeaderInfo>> nominees;
 	state Future<Void> allActors;
@@ -695,7 +682,7 @@ ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
 
 	loop {
 		Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
-		TraceEvent("MonitorLeaderForProxiesChange")
+		TraceEvent("MonitorLeaderAndGetClientInfoLeaderChange")
 		    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1))
 		    .detail("Key", clusterKey.printable());
 		if (leader.present()) {
@@ -705,7 +692,7 @@ ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
 				outInfo.forward = leader.get().first.serializedInfo;
 				clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
 				leaderInfo->set(leader.get().first);
-				TraceEvent("MonitorLeaderForProxiesForwarding")
+				TraceEvent("MonitorLeaderAndGetClientInfoForwarding")
 				    .detail("NewConnStr", leader.get().first.serializedInfo.toString());
 				return Void();
 			}
@@ -762,7 +749,6 @@ void shrinkProxyList(ClientDBInfo& ni,
 	}
 }
 
-// Leader is the process that will be elected by coordinators as the cluster controller
 ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
     Reference<ClusterConnectionFile> connFile,
     Reference<AsyncVar<ClientDBInfo>> clientInfo,
@@ -771,9 +757,9 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup) {
 	state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
-	state vector<NetworkAddress> addrs = cs.coordinators();
+	state std::vector<NetworkAddress> addrs = cs.coordinators();
 	state int idx = 0;
-	state int successIdx = 0;
+	state int successIndex = 0;
 	state Optional<double> incorrectTime;
 	state std::vector<UID> lastCommitProxyUIDs;
 	state std::vector<CommitProxyInterface> lastCommitProxies;
@@ -840,11 +826,11 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			auto& ni = rep.get().mutate();
 			shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
 			clientInfo->set(ni);
-			successIdx = idx;
+			successIndex = idx;
 		} else {
 			TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller
 			idx = (idx + 1) % addrs.size();
-			if (idx == successIdx) {
+			if (idx == successIndex) {
 				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
 			}
 		}

fdbclient/MonitorLeader.h

@@ -61,17 +61,23 @@ struct MonitorLeaderInfo {
 	  : hasConnected(false), intermediateConnFile(intermediateConnFile) {}
 };
 
-// Monitors the given coordination group's leader election process and provides a best current guess
-// of the current leader.  If a leader is elected for long enough and communication with a quorum of
-// coordinators is possible, eventually outKnownLeader will be that leader's interface.
+Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<LeaderInfo>>& nominees);
+
+// This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect
+// nominees, the nominee with the most nomination is the leader. This function also monitors the change of the leader.
+// If a leader is elected for long enough and communication with a quorum of coordinators is possible, eventually
+// outKnownLeader will be that leader's interface.
 template <class LeaderInterface>
 Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
                            Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader);
 
-Future<Void> monitorLeaderForProxies(Value const& key,
-                                     vector<NetworkAddress> const& coordinators,
-                                     ClientData* const& clientData,
-                                     Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
+// This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect
+// nominees, the nominee with the most nomination is the leader, and collects client data from the leader. This function
+// also monitors the change of the leader.
+Future<Void> monitorLeaderAndGetClientInfo(Value const& key,
+                                           std::vector<NetworkAddress> const& coordinators,
+                                           ClientData* const& clientData,
+                                           Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
 
 Future<Void> monitorProxies(
     Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile,

fdbclient/MultiVersionTransaction.actor.cpp

@@ -606,7 +606,7 @@ void DLApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParame
 // MultiVersionTransaction
 MultiVersionTransaction::MultiVersionTransaction(Reference<MultiVersionDatabase> db,
                                                  UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions)
-  : db(db) {
+  : db(db), startTime(timer_monotonic()), timeoutTsav(new ThreadSingleAssignmentVar<Void>()) {
 	setDefaultOptions(defaultOptions);
 	updateTransaction();
 }
@@ -622,20 +622,23 @@ void MultiVersionTransaction::updateTransaction() {
 	TransactionInfo newTr;
 	if (currentDb.value) {
 		newTr.transaction = currentDb.value->createTransaction();
+	}
 
-		Optional<StringRef> timeout;
-		for (auto option : persistentOptions) {
-			if (option.first == FDBTransactionOptions::TIMEOUT) {
-				timeout = option.second.castTo<StringRef>();
-			} else {
-				newTr.transaction->setOption(option.first, option.second.castTo<StringRef>());
-			}
+	Optional<StringRef> timeout;
+	for (auto option : persistentOptions) {
+		if (option.first == FDBTransactionOptions::TIMEOUT) {
+			timeout = option.second.castTo<StringRef>();
+		} else if (currentDb.value) {
+			newTr.transaction->setOption(option.first, option.second.castTo<StringRef>());
 		}
+	}
 
-		// Setting a timeout can immediately cause a transaction to fail. The only timeout
-		// that matters is the one most recently set, so we ignore any earlier set timeouts
-		// that might inadvertently fail the transaction.
-		if (timeout.present()) {
+	// Setting a timeout can immediately cause a transaction to fail. The only timeout
+	// that matters is the one most recently set, so we ignore any earlier set timeouts
+	// that might inadvertently fail the transaction.
+	if (timeout.present()) {
+		setTimeout(timeout);
+		if (currentDb.value) {
 			newTr.transaction->setOption(FDBTransactionOptions::TIMEOUT, timeout);
 		}
 	}
@@ -670,19 +673,19 @@ void MultiVersionTransaction::setVersion(Version v) {
 }
 ThreadFuture<Version> MultiVersionTransaction::getReadVersion() {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getReadVersion() : ThreadFuture<Version>(Never());
+	auto f = tr.transaction ? tr.transaction->getReadVersion() : makeTimeout<Version>();
 	return abortableFuture(f, tr.onChange);
 }
 
 ThreadFuture<Optional<Value>> MultiVersionTransaction::get(const KeyRef& key, bool snapshot) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->get(key, snapshot) : ThreadFuture<Optional<Value>>(Never());
+	auto f = tr.transaction ? tr.transaction->get(key, snapshot) : makeTimeout<Optional<Value>>();
 	return abortableFuture(f, tr.onChange);
 }
 
 ThreadFuture<Key> MultiVersionTransaction::getKey(const KeySelectorRef& key, bool snapshot) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getKey(key, snapshot) : ThreadFuture<Key>(Never());
+	auto f = tr.transaction ? tr.transaction->getKey(key, snapshot) : makeTimeout<Key>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -692,8 +695,8 @@ ThreadFuture<RangeResult> MultiVersionTransaction::getRange(const KeySelectorRef
                                                             bool snapshot,
                                                             bool reverse) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse)
-	                        : ThreadFuture<RangeResult>(Never());
+	auto f =
+	    tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse) : makeTimeout<RangeResult>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -703,8 +706,8 @@ ThreadFuture<RangeResult> MultiVersionTransaction::getRange(const KeySelectorRef
                                                             bool snapshot,
                                                             bool reverse) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse)
-	                        : ThreadFuture<RangeResult>(Never());
+	auto f =
+	    tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse) : makeTimeout<RangeResult>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -713,8 +716,7 @@ ThreadFuture<RangeResult> MultiVersionTransaction::getRange(const KeyRangeRef& k
                                                             bool snapshot,
                                                             bool reverse) {
 	auto tr = getTransaction();
-	auto f =
-	    tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse) : ThreadFuture<RangeResult>(Never());
+	auto f = tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse) : makeTimeout<RangeResult>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -723,21 +725,20 @@ ThreadFuture<RangeResult> MultiVersionTransaction::getRange(const KeyRangeRef& k
                                                             bool snapshot,
                                                             bool reverse) {
 	auto tr = getTransaction();
-	auto f =
-	    tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse) : ThreadFuture<RangeResult>(Never());
+	auto f = tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse) : makeTimeout<RangeResult>();
 	return abortableFuture(f, tr.onChange);
 }
 
 ThreadFuture<Standalone<StringRef>> MultiVersionTransaction::getVersionstamp() {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getVersionstamp() : ThreadFuture<Standalone<StringRef>>(Never());
+	auto f = tr.transaction ? tr.transaction->getVersionstamp() : makeTimeout<Standalone<StringRef>>();
 	return abortableFuture(f, tr.onChange);
 }
 
 ThreadFuture<Standalone<VectorRef<const char*>>> MultiVersionTransaction::getAddressesForKey(const KeyRef& key) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getAddressesForKey(key)
-	                        : ThreadFuture<Standalone<VectorRef<const char*>>>(Never());
+	auto f =
+	    tr.transaction ? tr.transaction->getAddressesForKey(key) : makeTimeout<Standalone<VectorRef<const char*>>>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -750,7 +751,7 @@ void MultiVersionTransaction::addReadConflictRange(const KeyRangeRef& keys) {
 
 ThreadFuture<int64_t> MultiVersionTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : ThreadFuture<int64_t>(Never());
+	auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : makeTimeout<int64_t>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -758,7 +759,7 @@ ThreadFuture<Standalone<VectorRef<KeyRef>>> MultiVersionTransaction::getRangeSpl
                                                                                          int64_t chunkSize) {
 	auto tr = getTransaction();
 	auto f = tr.transaction ? tr.transaction->getRangeSplitPoints(range, chunkSize)
-	                        : ThreadFuture<Standalone<VectorRef<KeyRef>>>(Never());
+	                        : makeTimeout<Standalone<VectorRef<KeyRef>>>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -799,7 +800,7 @@ void MultiVersionTransaction::clear(const KeyRef& key) {
 
 ThreadFuture<Void> MultiVersionTransaction::watch(const KeyRef& key) {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->watch(key) : ThreadFuture<Void>(Never());
+	auto f = tr.transaction ? tr.transaction->watch(key) : makeTimeout<Void>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -812,7 +813,7 @@ void MultiVersionTransaction::addWriteConflictRange(const KeyRangeRef& keys) {
 
 ThreadFuture<Void> MultiVersionTransaction::commit() {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->commit() : ThreadFuture<Void>(Never());
+	auto f = tr.transaction ? tr.transaction->commit() : makeTimeout<Void>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -827,7 +828,7 @@ Version MultiVersionTransaction::getCommittedVersion() {
 
 ThreadFuture<int64_t> MultiVersionTransaction::getApproximateSize() {
 	auto tr = getTransaction();
-	auto f = tr.transaction ? tr.transaction->getApproximateSize() : ThreadFuture<int64_t>(Never());
+	auto f = tr.transaction ? tr.transaction->getApproximateSize() : makeTimeout<int64_t>();
 	return abortableFuture(f, tr.onChange);
 }
 
@@ -841,6 +842,11 @@ void MultiVersionTransaction::setOption(FDBTransactionOptions::Option option, Op
 	if (MultiVersionApi::apiVersionAtLeast(610) && itr->second.persistent) {
 		persistentOptions.emplace_back(option, value.castTo<Standalone<StringRef>>());
 	}
+
+	if (itr->first == FDBTransactionOptions::TIMEOUT) {
+		setTimeout(value);
+	}
+
 	auto tr = getTransaction();
 	if (tr.transaction) {
 		tr.transaction->setOption(option, value);
@@ -853,7 +859,7 @@ ThreadFuture<Void> MultiVersionTransaction::onError(Error const& e) {
 		return ThreadFuture<Void>(Void());
 	} else {
 		auto tr = getTransaction();
-		auto f = tr.transaction ? tr.transaction->onError(e) : ThreadFuture<Void>(Never());
+		auto f = tr.transaction ? tr.transaction->onError(e) : makeTimeout<Void>();
 		f = abortableFuture(f, tr.onChange);
 
 		return flatMapThreadFuture<Void, Void>(f, [this, e](ErrorOr<Void> ready) {
@@ -871,12 +877,95 @@ ThreadFuture<Void> MultiVersionTransaction::onError(Error const& e) {
 	}
 }
 
+// Waits for the specified duration and signals the assignment variable with a timed out error
+// This will be canceled if a new timeout is set, in which case the tsav will not be signaled.
+ACTOR Future<Void> timeoutImpl(Reference<ThreadSingleAssignmentVar<Void>> tsav, double duration) {
+	wait(delay(duration));
+
+	tsav->trySendError(transaction_timed_out());
+	return Void();
+}
+
+// Configure a timeout based on the options set for this transaction. This timeout only applies
+// if we don't have an underlying database object to connect with.
+void MultiVersionTransaction::setTimeout(Optional<StringRef> value) {
+	double timeoutDuration = extractIntOption(value, 0, std::numeric_limits<int>::max()) / 1000.0;
+
+	ThreadFuture<Void> prevTimeout;
+	double transactionStartTime = startTime;
+
+	{ // lock scope
+		ThreadSpinLockHolder holder(timeoutLock);
+
+		Reference<ThreadSingleAssignmentVar<Void>> tsav = timeoutTsav;
+		ThreadFuture<Void> newTimeout = onMainThread([transactionStartTime, tsav, timeoutDuration]() {
+			return timeoutImpl(tsav, timeoutDuration - std::max(0.0, now() - transactionStartTime));
+		});
+
+		prevTimeout = currentTimeout;
+		currentTimeout = newTimeout;
+	}
+
+	// Cancel the previous timeout now that we have a new one. This means that changing the timeout
+	// affects in-flight operations, which is consistent with the behavior in RYW.
+	if (prevTimeout.isValid()) {
+		prevTimeout.cancel();
+	}
+}
+
+// Creates a ThreadFuture<T> that will signal an error if the transaction times out.
+template <class T>
+ThreadFuture<T> MultiVersionTransaction::makeTimeout() {
+	ThreadFuture<Void> f;
+
+	{ // lock scope
+		ThreadSpinLockHolder holder(timeoutLock);
+
+		// Our ThreadFuture holds a reference to this TSAV,
+		// but the ThreadFuture does not increment the ref count
+		timeoutTsav->addref();
+		f = ThreadFuture<Void>(timeoutTsav.getPtr());
+	}
+
+	// When our timeoutTsav gets set, map it to the appropriate type
+	return mapThreadFuture<Void, T>(f, [](ErrorOr<Void> v) {
+		ASSERT(v.isError());
+		return ErrorOr<T>(v.getError());
+	});
+}
+
 void MultiVersionTransaction::reset() {
 	persistentOptions.clear();
+
+	// Reset the timeout state
+	Reference<ThreadSingleAssignmentVar<Void>> prevTimeoutTsav;
+	ThreadFuture<Void> prevTimeout;
+	startTime = timer_monotonic();
+
+	{ // lock scope
+		ThreadSpinLockHolder holder(timeoutLock);
+
+		prevTimeoutTsav = timeoutTsav;
+		timeoutTsav = makeReference<ThreadSingleAssignmentVar<Void>>();
+
+		prevTimeout = currentTimeout;
+		currentTimeout = ThreadFuture<Void>();
+	}
+
+	// Cancel any outstanding operations if they don't have an underlying transaction object to cancel them
+	prevTimeoutTsav->trySendError(transaction_cancelled());
+	if (prevTimeout.isValid()) {
+		prevTimeout.cancel();
+	}
+
 	setDefaultOptions(db->dbState->transactionDefaultOptions);
 	updateTransaction();
 }
 
+MultiVersionTransaction::~MultiVersionTransaction() {
+	timeoutTsav->trySendError(transaction_cancelled());
+}
+
 bool MultiVersionTransaction::isValid() {
 	auto tr = getTransaction();
 	return tr.transaction.isValid();
@@ -1896,8 +1985,28 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 			std::string valueStr;
 			try {
 				if (platform::getEnvironmentVar(("FDB_NETWORK_OPTION_" + option.second.name).c_str(), valueStr)) {
+					FDBOptionInfo::ParamType curParamType = option.second.paramType;
 					for (auto value : parseOptionValues(valueStr)) {
-						Standalone<StringRef> currentValue = StringRef(value);
+						Standalone<StringRef> currentValue;
+						int64_t intParamVal;
+						if (curParamType == FDBOptionInfo::ParamType::Int) {
+							try {
+								size_t nextIdx;
+								intParamVal = std::stoll(value, &nextIdx);
+								if (nextIdx != value.length()) {
+									throw invalid_option_value();
+								}
+							} catch (std::exception e) {
+								TraceEvent(SevError, "EnvironmentVariableParseIntegerFailed")
+								    .detail("Option", option.second.name)
+								    .detail("Value", valueStr)
+								    .detail("Error", e.what());
+								throw invalid_option_value();
+							}
+							currentValue = StringRef(reinterpret_cast<uint8_t*>(&intParamVal), 8);
+						} else {
+							currentValue = StringRef(value);
+						}
 						{ // lock scope
 							MutexHolder holder(lock);
 							if (setEnvOptions[option.first].count(currentValue) == 0) {

fdbclient/MultiVersionTransaction.h

@@ -334,6 +334,8 @@ public:
 	MultiVersionTransaction(Reference<MultiVersionDatabase> db,
 	                        UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
 
+	~MultiVersionTransaction() override;
+
 	void cancel() override;
 	void setVersion(Version v) override;
 	ThreadFuture<Version> getReadVersion() override;
@@ -400,6 +402,29 @@ private:
 		ThreadFuture<Void> onChange;
 	};
 
+	// Timeout related variables for MultiVersionTransaction objects that do not have an underlying ITransaction
+
+	// The time when the MultiVersionTransaction was last created or reset
+	std::atomic<double> startTime;
+
+	// A lock that needs to be held if using timeoutTsav or currentTimeout
+	ThreadSpinLock timeoutLock;
+
+	// A single assignment var (i.e. promise) that gets set with an error when the timeout elapses or the transaction
+	// is reset or destroyed.
+	Reference<ThreadSingleAssignmentVar<Void>> timeoutTsav;
+
+	// A reference to the current actor waiting for the timeout. This actor will set the timeoutTsav promise.
+	ThreadFuture<Void> currentTimeout;
+
+	// Configure a timeout based on the options set for this transaction. This timeout only applies
+	// if we don't have an underlying database object to connect with.
+	void setTimeout(Optional<StringRef> value);
+
+	// Creates a ThreadFuture<T> that will signal an error if the transaction times out.
+	template <class T>
+	ThreadFuture<T> makeTimeout();
+
 	TransactionInfo transaction;
 
 	TransactionInfo getTransaction();

fdbclient/NativeAPI.actor.cpp

@@ -55,6 +55,7 @@
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TransactionLineage.h"
 #include "fdbclient/versions.h"
+#include "fdbclient/WellKnownEndpoints.h"
 #include "fdbrpc/LoadBalance.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
@@ -85,10 +86,6 @@
 
 extern const char* getSourceVersion();
 
-using std::max;
-using std::min;
-using std::pair;
-
 namespace {
 
 TransactionLineageCollector transactionLineageCollector;
@@ -458,10 +455,10 @@ ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 			    .detail("TSSGetKeyLatency90", it.second->TSSgetKeyLatency.percentile(0.90))
 			    .detail("TSSGetKeyLatency99", it.second->TSSgetKeyLatency.percentile(0.99));
 
-			tssEv.detail("MeanSSGetKeyValuesLatency", it.second->SSgetKeyLatency.mean())
-			    .detail("MedianSSGetKeyValuesLatency", it.second->SSgetKeyLatency.median())
-			    .detail("SSGetKeyValuesLatency90", it.second->SSgetKeyLatency.percentile(0.90))
-			    .detail("SSGetKeyValuesLatency99", it.second->SSgetKeyLatency.percentile(0.99));
+			tssEv.detail("MeanSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.mean())
+			    .detail("MedianSSGetKeyValuesLatency", it.second->SSgetKeyValuesLatency.median())
+			    .detail("SSGetKeyValuesLatency90", it.second->SSgetKeyValuesLatency.percentile(0.90))
+			    .detail("SSGetKeyValuesLatency99", it.second->SSgetKeyValuesLatency.percentile(0.99));
 
 			tssEv.detail("MeanTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.mean())
 			    .detail("MedianTSSGetKeyValuesLatency", it.second->TSSgetKeyValuesLatency.median())
@@ -672,19 +669,82 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 	}
 }
 
-ACTOR static Future<Void> monitorProxiesChange(Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
+ACTOR Future<Void> assertFailure(GrvProxyInterface remote, Future<ErrorOr<GetReadVersionReply>> reply) {
+	try {
+		ErrorOr<GetReadVersionReply> res = wait(reply);
+		if (!res.isError()) {
+			TraceEvent(SevError, "GotStaleReadVersion")
+			    .detail("Remote", remote.getConsistentReadVersion.getEndpoint().addresses.address.toString())
+			    .detail("Provisional", remote.provisional)
+			    .detail("ReadVersion", res.get().version);
+			ASSERT_WE_THINK(false);
+		}
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw;
+		}
+		// we want this to fail -- so getting here is good, we'll just ignore the error.
+	}
+	return Void();
+}
+
+Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
+                                      std::vector<GrvProxyInterface> newProxies) {
+	Span span(deterministicRandom()->randomUniqueID(), "VerifyCausalReadRisky"_loc);
+	std::vector<Future<Void>> replies;
+	replies.reserve(oldProxies.size());
+	GetReadVersionRequest req(
+	    span.context, 1, TransactionPriority::IMMEDIATE, GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY);
+	TraceEvent evt("AttemptGRVFromOldProxies");
+	evt.detail("NumOldProxies", oldProxies.size()).detail("NumNewProxies", newProxies.size());
+	auto traceProxies = [&](std::vector<GrvProxyInterface>& proxies, std::string const& key) {
+		for (int i = 0; i < proxies.size(); ++i) {
+			auto k = key + std::to_string(i);
+			evt.detail(k.c_str(), proxies[i].id());
+		}
+	};
+	traceProxies(oldProxies, "OldProxy"s);
+	traceProxies(newProxies, "NewProxy"s);
+	evt.log();
+	for (auto& i : oldProxies) {
+		req.reply = ReplyPromise<GetReadVersionReply>();
+		replies.push_back(assertFailure(i, i.getConsistentReadVersion.tryGetReply(req)));
+	}
+	return waitForAll(replies);
+}
+
+ACTOR static Future<Void> monitorProxiesChange(DatabaseContext* cx,
+                                               Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
                                                AsyncTrigger* triggerVar) {
-	state vector<CommitProxyInterface> curCommitProxies;
-	state vector<GrvProxyInterface> curGrvProxies;
+	state std::vector<CommitProxyInterface> curCommitProxies;
+	state std::vector<GrvProxyInterface> curGrvProxies;
+	state ActorCollection actors(false);
 	curCommitProxies = clientDBInfo->get().commitProxies;
 	curGrvProxies = clientDBInfo->get().grvProxies;
 
 	loop {
-		wait(clientDBInfo->onChange());
-		if (clientDBInfo->get().commitProxies != curCommitProxies || clientDBInfo->get().grvProxies != curGrvProxies) {
-			curCommitProxies = clientDBInfo->get().commitProxies;
-			curGrvProxies = clientDBInfo->get().grvProxies;
-			triggerVar->trigger();
+		choose {
+			when(wait(clientDBInfo->onChange())) {
+				if (clientDBInfo->get().commitProxies != curCommitProxies ||
+				    clientDBInfo->get().grvProxies != curGrvProxies) {
+					// This condition is a bit complicated. Here we want to verify that we're unable to receive a read
+					// version from a proxy of an old generation after a successful recovery. The conditions are:
+					// 1. We only do this with a configured probability.
+					// 2. If the old set of Grv proxies is empty, there's nothing to do
+					// 3. If the new set of Grv proxies is empty, it means the recovery is not complete. So if an old
+					//    Grv proxy still gives out read versions, this would be correct behavior.
+					// 4. If we see a provisional proxy, it means the recovery didn't complete yet, so the same as (3)
+					//    applies.
+					if (deterministicRandom()->random01() < cx->verifyCausalReadsProp && !curGrvProxies.empty() &&
+					    !clientDBInfo->get().grvProxies.empty() && !clientDBInfo->get().grvProxies[0].provisional) {
+						actors.add(attemptGRVFromOldProxies(curGrvProxies, clientDBInfo->get().grvProxies));
+					}
+					curCommitProxies = clientDBInfo->get().commitProxies;
+					curGrvProxies = clientDBInfo->get().grvProxies;
+					triggerVar->trigger();
+				}
+			}
+			when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); }
 		}
 	}
 }
@@ -1145,11 +1205,13 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
     transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
     transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
-    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
-    commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), outstandingWatches(0),
-    transactionTracingEnabled(true), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
-    coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
-    detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
+    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
+    transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
+    latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
+    bytesPerCommit(1000), outstandingWatches(0), transactionTracingEnabled(true), taskID(taskID),
+    clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion),
+    mvCacheInsertLocation(0), healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
+    smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
     specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)) {
 	dbId = deterministicRandom()->randomUniqueID();
 	connected = (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size())
@@ -1168,7 +1230,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 	getValueSubmitted.init(LiteralStringRef("NativeAPI.GetValueSubmitted"));
 	getValueCompleted.init(LiteralStringRef("NativeAPI.GetValueCompleted"));
 
-	monitorProxiesInfoChange = monitorProxiesChange(clientInfo, &proxiesChangeTrigger);
+	monitorProxiesInfoChange = monitorProxiesChange(this, clientInfo, &proxiesChangeTrigger);
 	tssMismatchHandler = handleTssMismatches(this);
 	clientStatusUpdater.actor = clientStatusUpdateActor(this);
 	cacheListMonitor = monitorCacheList(this);
@@ -1397,9 +1459,10 @@ DatabaseContext::DatabaseContext(const Error& err)
     transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc),
     transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc),
     transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
-    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), latencies(1000), readLatencies(1000),
-    commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
-    transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT) {}
+    transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
+    transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
+    latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
+    bytesPerCommit(1000), transactionTracingEnabled(true), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT) {}
 
 // Static constructor used by server processes to create a DatabaseContext
 // For internal (fdbserver) use only
@@ -1435,7 +1498,7 @@ DatabaseContext::~DatabaseContext() {
 	locationCache.insert(allKeys, Reference<LocationInfo>());
 }
 
-pair<KeyRange, Reference<LocationInfo>> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) {
+std::pair<KeyRange, Reference<LocationInfo>> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) {
 	if (isBackward) {
 		auto range = locationCache.rangeContainingKeyBefore(key);
 		return std::make_pair(range->range(), range->value());
@@ -1446,7 +1509,7 @@ pair<KeyRange, Reference<LocationInfo>> DatabaseContext::getCachedLocation(const
 }
 
 bool DatabaseContext::getCachedLocations(const KeyRangeRef& range,
-                                         vector<std::pair<KeyRange, Reference<LocationInfo>>>& result,
+                                         std::vector<std::pair<KeyRange, Reference<LocationInfo>>>& result,
                                          int limit,
                                          Reverse reverse) {
 	result.clear();
@@ -1476,8 +1539,8 @@ bool DatabaseContext::getCachedLocations(const KeyRangeRef& range,
 }
 
 Reference<LocationInfo> DatabaseContext::setCachedLocation(const KeyRangeRef& keys,
-                                                           const vector<StorageServerInterface>& servers) {
-	vector<Reference<ReferencedInterface<StorageServerInterface>>> serverRefs;
+                                                           const std::vector<StorageServerInterface>& servers) {
+	std::vector<Reference<ReferencedInterface<StorageServerInterface>>> serverRefs;
 	serverRefs.reserve(servers.size());
 	for (const auto& interf : servers) {
 		serverRefs.push_back(StorageServerInfo::getInterface(this, interf, clientLocality));
@@ -1610,6 +1673,9 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional<Stri
 			validateOptionValueNotPresent(value);
 			useConfigDatabase = true;
 			break;
+		case FDBDatabaseOptions::TEST_CAUSAL_READ_RISKY:
+			verifyCausalReadsProp = double(extractIntOption(value, 0, 100)) / 100.0;
+			break;
 		default:
 			break;
 		}
@@ -2068,7 +2134,7 @@ void setupNetwork(uint64_t transportId, UseMetrics useMetrics) {
 	g_network = newNet2(tlsConfig, false, useMetrics || networkOptions.traceDirectory.present());
 	g_network->addStopCallback(Net2FileSystem::stop);
 	g_network->addStopCallback(TLS::DestroyOpenSSLGlobalState);
-	FlowTransport::createInstance(true, transportId);
+	FlowTransport::createInstance(true, transportId, WLTOKEN_RESERVED_COUNT);
 	Net2FileSystem::newFileSystem();
 
 	uncancellable(monitorNetworkBusyness());
@@ -2251,23 +2317,23 @@ ACTOR Future<Optional<StorageServerInterface>> fetchServerInterface(Database cx,
 	return decodeServerListValue(val.get());
 }
 
-ACTOR Future<Optional<vector<StorageServerInterface>>> transactionalGetServerInterfaces(Future<Version> ver,
-                                                                                        Database cx,
-                                                                                        TransactionInfo info,
-                                                                                        vector<UID> ids,
-                                                                                        TagSet tags) {
-	state vector<Future<Optional<StorageServerInterface>>> serverListEntries;
+ACTOR Future<Optional<std::vector<StorageServerInterface>>> transactionalGetServerInterfaces(Future<Version> ver,
+                                                                                             Database cx,
+                                                                                             TransactionInfo info,
+                                                                                             std::vector<UID> ids,
+                                                                                             TagSet tags) {
+	state std::vector<Future<Optional<StorageServerInterface>>> serverListEntries;
 	serverListEntries.reserve(ids.size());
 	for (int s = 0; s < ids.size(); s++) {
 		serverListEntries.push_back(fetchServerInterface(cx, info, ids[s], tags, ver));
 	}
 
-	vector<Optional<StorageServerInterface>> serverListValues = wait(getAll(serverListEntries));
-	vector<StorageServerInterface> serverInterfaces;
+	std::vector<Optional<StorageServerInterface>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<StorageServerInterface> serverInterfaces;
 	for (int s = 0; s < serverListValues.size(); s++) {
 		if (!serverListValues[s].present()) {
 			// A storage server has been removed from ServerList since we read keyServers
-			return Optional<vector<StorageServerInterface>>();
+			return Optional<std::vector<StorageServerInterface>>();
 		}
 		serverInterfaces.push_back(serverListValues[s].get());
 	}
@@ -2299,10 +2365,8 @@ void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) {
 
 // If isBackward == true, returns the shard containing the key before 'key' (an infinitely long, inexpressible key).
 // Otherwise returns the shard containing key
-ACTOR Future<pair<KeyRange, Reference<LocationInfo>>> getKeyLocation_internal(Database cx,
-                                                                              Key key,
-                                                                              TransactionInfo info,
-                                                                              Reverse isBackward = Reverse::False) {
+ACTOR Future<std::pair<KeyRange, Reference<LocationInfo>>>
+getKeyLocation_internal(Database cx, Key key, TransactionInfo info, Reverse isBackward = Reverse::False) {
 	state Span span("NAPI:getKeyLocation"_loc, info.spanID);
 	if (isBackward) {
 		ASSERT(key != allKeys.begin && key <= allKeys.end);
@@ -2337,11 +2401,11 @@ ACTOR Future<pair<KeyRange, Reference<LocationInfo>>> getKeyLocation_internal(Da
 }
 
 template <class F>
-Future<pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database const& cx,
-                                                               Key const& key,
-                                                               F StorageServerInterface::*member,
-                                                               TransactionInfo const& info,
-                                                               Reverse isBackward = Reverse::False) {
+Future<std::pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database const& cx,
+                                                                    Key const& key,
+                                                                    F StorageServerInterface::*member,
+                                                                    TransactionInfo const& info,
+                                                                    Reverse isBackward = Reverse::False) {
 	// we first check whether this range is cached
 	auto ssi = cx->getCachedLocation(key, isBackward);
 	if (!ssi.second) {
@@ -2359,11 +2423,8 @@ Future<pair<KeyRange, Reference<LocationInfo>>> getKeyLocation(Database const& c
 	return ssi;
 }
 
-ACTOR Future<vector<pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLocations_internal(Database cx,
-                                                                                            KeyRange keys,
-                                                                                            int limit,
-                                                                                            Reverse reverse,
-                                                                                            TransactionInfo info) {
+ACTOR Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>>
+getKeyRangeLocations_internal(Database cx, KeyRange keys, int limit, Reverse reverse, TransactionInfo info) {
 	state Span span("NAPI:getKeyRangeLocations"_loc, info.spanID);
 	if (info.debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.Before");
@@ -2384,7 +2445,7 @@ ACTOR Future<vector<pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLocatio
 					    "TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.After");
 				ASSERT(rep.results.size());
 
-				state vector<pair<KeyRange, Reference<LocationInfo>>> results;
+				state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> results;
 				state int shard = 0;
 				for (; shard < rep.results.size(); shard++) {
 					// FIXME: these shards are being inserted into the map sequentially, it would be much more CPU
@@ -2408,15 +2469,16 @@ ACTOR Future<vector<pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLocatio
 // Example: If query the function with  key range (b, d), the returned list of pairs could be something like:
 // [([a, b1), locationInfo), ([b1, c), locationInfo), ([c, d1), locationInfo)].
 template <class F>
-Future<vector<pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLocations(Database const& cx,
-                                                                             KeyRange const& keys,
-                                                                             int limit,
-                                                                             Reverse reverse,
-                                                                             F StorageServerInterface::*member,
-                                                                             TransactionInfo const& info) {
+Future<std::vector<std::pair<KeyRange, Reference<LocationInfo>>>> getKeyRangeLocations(
+    Database const& cx,
+    KeyRange const& keys,
+    int limit,
+    Reverse reverse,
+    F StorageServerInterface::*member,
+    TransactionInfo const& info) {
 	ASSERT(!keys.empty());
 
-	vector<pair<KeyRange, Reference<LocationInfo>>> locations;
+	std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations;
 	if (!cx->getCachedLocations(keys, locations, limit, reverse)) {
 		return getKeyRangeLocations_internal(cx, keys, limit, reverse, info);
 	}
@@ -2448,7 +2510,7 @@ ACTOR Future<Void> warmRange_impl(Transaction* self, Database cx, KeyRange keys)
 	state int totalRanges = 0;
 	state int totalRequests = 0;
 	loop {
-		vector<pair<KeyRange, Reference<LocationInfo>>> locations = wait(
+		std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations = wait(
 		    getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, Reverse::False, self->info));
 		totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT;
 		totalRequests++;
@@ -2493,7 +2555,7 @@ ACTOR Future<Optional<Value>> getValue(Future<Version> version,
 	cx->validateVersion(ver);
 
 	loop {
-		state pair<KeyRange, Reference<LocationInfo>> ssi =
+		state std::pair<KeyRange, Reference<LocationInfo>> ssi =
 		    wait(getKeyLocation(cx, key, &StorageServerInterface::getValue, info));
 		state Optional<UID> getValueID = Optional<UID>();
 		state uint64_t startTime;
@@ -2618,7 +2680,7 @@ ACTOR Future<Key> getKey(Database cx, KeySelector k, Future<Version> version, Tr
 		}
 
 		Key locationKey(k.getKey(), k.arena());
-		state pair<KeyRange, Reference<LocationInfo>> ssi =
+		state std::pair<KeyRange, Reference<LocationInfo>> ssi =
 		    wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, Reverse{ k.isBackward() }));
 
 		try {
@@ -2739,7 +2801,7 @@ ACTOR Future<Version> watchValue(Future<Version> version,
 	ASSERT(ver != latestVersion);
 
 	loop {
-		state pair<KeyRange, Reference<LocationInfo>> ssi =
+		state std::pair<KeyRange, Reference<LocationInfo>> ssi =
 		    wait(getKeyLocation(cx, key, &StorageServerInterface::watchValue, info));
 
 		try {
@@ -2992,7 +3054,7 @@ ACTOR Future<RangeResult> getExactRange(Database cx,
 
 	// printf("getExactRange( '%s', '%s' )\n", keys.begin.toString().c_str(), keys.end.toString().c_str());
 	loop {
-		state vector<pair<KeyRange, Reference<LocationInfo>>> locations = wait(getKeyRangeLocations(
+		state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations = wait(getKeyRangeLocations(
 		    cx, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValues, info));
 		ASSERT(locations.size());
 		state int shard = 0;
@@ -3315,7 +3377,7 @@ ACTOR Future<RangeResult> getRange(Database cx,
 
 			Key locationKey = reverse ? Key(end.getKey(), end.arena()) : Key(begin.getKey(), begin.arena());
 			Reverse locationBackward{ reverse ? (end - 1).isBackward() : begin.isBackward() };
-			state pair<KeyRange, Reference<LocationInfo>> beginServer =
+			state std::pair<KeyRange, Reference<LocationInfo>> beginServer =
 			    wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKeyValues, info, locationBackward));
 			state KeyRange shard = beginServer.first;
 			state bool modifiedSelectors = false;
@@ -3745,7 +3807,7 @@ ACTOR Future<Void> getRangeStreamFragment(ParallelStream<RangeResult>::Fragment*
                                           TagSet tags,
                                           SpanID spanContext) {
 	loop {
-		state vector<pair<KeyRange, Reference<LocationInfo>>> locations = wait(getKeyRangeLocations(
+		state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations = wait(getKeyRangeLocations(
 		    cx, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValuesStream, info));
 		ASSERT(locations.size());
 		state int shard = 0;
@@ -3806,7 +3868,7 @@ ACTOR Future<Void> getRangeStreamFragment(ParallelStream<RangeResult>::Fragment*
 						break;
 					}
 
-					vector<Future<Void>> ok(locations[shard].second->size());
+					std::vector<Future<Void>> ok(locations[shard].second->size());
 					for (int i = 0; i < ok.size(); i++) {
 						ok[i] = IFailureMonitor::failureMonitor().onStateEqual(
 						    locations[shard].second->get(i, &StorageServerInterface::getKeyValuesStream).getEndpoint(),
@@ -4048,7 +4110,7 @@ ACTOR Future<Void> getRangeStream(PromiseStream<RangeResult> _results,
 
 	state std::vector<Future<Void>> outstandingRequests;
 	while (b < e) {
-		state pair<KeyRange, Reference<LocationInfo>> ssi =
+		state std::pair<KeyRange, Reference<LocationInfo>> ssi =
 		    wait(getKeyLocation(cx, reverse ? e : b, &StorageServerInterface::getKeyValuesStream, info, reverse));
 		state KeyRange shardIntersection = intersect(ssi.first, KeyRangeRef(b, e));
 		state Standalone<VectorRef<KeyRef>> splitPoints =
@@ -4309,7 +4371,7 @@ ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Key key
                                                                          Database cx,
                                                                          TransactionInfo info,
                                                                          TransactionOptions options) {
-	state vector<StorageServerInterface> ssi;
+	state std::vector<StorageServerInterface> ssi;
 
 	// If key >= allKeys.end, then getRange will return a kv-pair with an empty value. This will result in our
 	// serverInterfaces vector being empty, which will cause us to return an empty addresses list.
@@ -4336,12 +4398,12 @@ ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Key key
 
 	ASSERT(serverUids.size()); // every shard needs to have a team
 
-	vector<UID> src;
-	vector<UID> ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in the
-	                    // case where this key is being relocated. But 'src' is the canonical location until the move is
-	                    // finished, because it could be cancelled at any time.
+	std::vector<UID> src;
+	std::vector<UID> ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in
+	                         // the case where this key is being relocated. But 'src' is the canonical location until
+	                         // the move is finished, because it could be cancelled at any time.
 	decodeKeyServersValue(serverTagResult, serverUids[0].value, src, ignore);
-	Optional<vector<StorageServerInterface>> serverInterfaces =
+	Optional<std::vector<StorageServerInterface>> serverInterfaces =
 	    wait(transactionalGetServerInterfaces(ver, cx, info, src, options.readTags));
 
 	ASSERT(serverInterfaces.present()); // since this is happening transactionally, /FF/keyServers and /FF/serverList
@@ -4709,8 +4771,9 @@ double Transaction::getBackoff(int errCode) {
 				auto tagItr = priorityItr->second.find(tag);
 				if (tagItr != priorityItr->second.end()) {
 					TEST(true); // Returning throttle backoff
-					returnedBackoff = std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL,
-					                           std::max(returnedBackoff, tagItr->second.throttleDuration()));
+					returnedBackoff = std::max(
+					    returnedBackoff,
+					    std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL, tagItr->second.throttleDuration()));
 					if (returnedBackoff == CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL) {
 						break;
 					}
@@ -4988,7 +5051,7 @@ ACTOR Future<Optional<ClientTrCommitCostEstimation>> estimateCommitCosts(Transac
 				++trCommitCosts.expensiveCostEstCount;
 				++self->getDatabase()->transactionsExpensiveClearCostEstCount;
 			} else {
-				std::vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+				std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 				    wait(getKeyRangeLocations(self->getDatabase(),
 				                              keyRange,
 				                              CLIENT_KNOBS->TOO_MANY,
@@ -5602,6 +5665,20 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 	state Future<Void> timeout;
 	state Optional<UID> debugID;
 	state bool send_batch;
+	state Reference<Histogram> batchSizeDist = Histogram::getHistogram(LiteralStringRef("GrvBatcher"),
+	                                                                   LiteralStringRef("ClientGrvBatchSize"),
+	                                                                   Histogram::Unit::countLinear,
+	                                                                   0,
+	                                                                   CLIENT_KNOBS->MAX_BATCH_SIZE * 2);
+	state Reference<Histogram> batchIntervalDist =
+	    Histogram::getHistogram(LiteralStringRef("GrvBatcher"),
+	                            LiteralStringRef("ClientGrvBatchInterval"),
+	                            Histogram::Unit::microseconds,
+	                            0,
+	                            CLIENT_KNOBS->GRV_BATCH_TIMEOUT * 1000000 * 2);
+	state Reference<Histogram> grvReplyLatencyDist = Histogram::getHistogram(
+	    LiteralStringRef("GrvBatcher"), LiteralStringRef("ClientGrvReplyLatency"), Histogram::Unit::microseconds);
+	state double lastRequestTime = now();
 
 	state TransactionTagMap<uint32_t> tags;
 
@@ -5626,22 +5703,34 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 					++tags[tag];
 				}
 
-				if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE)
+				if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE) {
 					send_batch = true;
-				else if (!timeout.isValid())
+					++cx->transactionGrvFullBatches;
+				} else if (!timeout.isValid()) {
 					timeout = delay(batchTime, TaskPriority::GetConsistentReadVersion);
+				}
+			}
+			when(wait(timeout.isValid() ? timeout : Never())) {
+				send_batch = true;
+				++cx->transactionGrvTimedOutBatches;
 			}
-			when(wait(timeout.isValid() ? timeout : Never())) { send_batch = true; }
 			// dynamic batching monitors reply latencies
 			when(double reply_latency = waitNext(replyTimes.getFuture())) {
 				double target_latency = reply_latency * 0.5;
-				batchTime = min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT);
+				batchTime = std::min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT);
+				grvReplyLatencyDist->sampleSeconds(reply_latency);
 			}
 			when(wait(collection)) {} // for errors
 		}
 		if (send_batch) {
 			int count = requests.size();
 			ASSERT(count);
+
+			batchSizeDist->sampleRecordCounter(count);
+			auto requestTime = now();
+			batchIntervalDist->sampleSeconds(requestTime - lastRequestTime);
+			lastRequestTime = requestTime;
+
 			// dynamic batching
 			Promise<GetReadVersionReply> GRVReply;
 			requests.push_back(GRVReply);
@@ -5827,7 +5916,8 @@ Future<Standalone<StringRef>> Transaction::getVersionstamp() {
 
 // Gets the protocol version reported by a coordinator via the protocol info interface
 ACTOR Future<ProtocolVersion> getCoordinatorProtocol(NetworkAddressList coordinatorAddresses) {
-	RequestStream<ProtocolInfoRequest> requestStream{ Endpoint{ { coordinatorAddresses }, WLTOKEN_PROTOCOL_INFO } };
+	RequestStream<ProtocolInfoRequest> requestStream{ Endpoint::wellKnown({ coordinatorAddresses },
+		                                                                  WLTOKEN_PROTOCOL_INFO) };
 	ProtocolInfoReply reply = wait(retryBrokenPromise(requestStream, ProtocolInfoRequest{}));
 
 	return reply.version;
@@ -5988,7 +6078,7 @@ ACTOR Future<StorageMetrics> doGetStorageMetrics(Database cx, KeyRange keys, Ref
 
 ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRange keys) {
 	state Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc);
-	vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+	std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 	    wait(getKeyRangeLocations(cx,
 	                              keys,
 	                              std::numeric_limits<int>::max(),
@@ -5996,7 +6086,7 @@ ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRang
 	                              &StorageServerInterface::waitMetrics,
 	                              TransactionInfo(TaskPriority::DataDistribution, span.context)));
 	state int nLocs = locations.size();
-	state vector<Future<StorageMetrics>> fx(nLocs);
+	state std::vector<Future<StorageMetrics>> fx(nLocs);
 	state StorageMetrics total;
 	KeyRef partBegin, partEnd;
 	for (int i = 0; i < nLocs; i++) {
@@ -6030,15 +6120,15 @@ ACTOR Future<Void> trackBoundedStorageMetrics(KeyRange keys,
 }
 
 ACTOR Future<StorageMetrics> waitStorageMetricsMultipleLocations(
-    vector<pair<KeyRange, Reference<LocationInfo>>> locations,
+    std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations,
     StorageMetrics min,
     StorageMetrics max,
     StorageMetrics permittedError) {
 	state int nLocs = locations.size();
-	state vector<Future<StorageMetrics>> fx(nLocs);
+	state std::vector<Future<StorageMetrics>> fx(nLocs);
 	state StorageMetrics total;
 	state PromiseStream<StorageMetrics> deltas;
-	state vector<Future<Void>> wx(fx.size());
+	state std::vector<Future<Void>> wx(fx.size());
 	state StorageMetrics halfErrorPerMachine = permittedError * (0.5 / nLocs);
 	state StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs - 1);
 	state StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs - 1);
@@ -6087,7 +6177,7 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 	loop {
 		int64_t shardLimit = 100; // Shard limit here does not really matter since this function is currently only used
 		                          // to find the read-hot sub ranges within a read-hot shard.
-		vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+		std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 		    wait(getKeyRangeLocations(cx,
 		                              keys,
 		                              shardLimit,
@@ -6106,7 +6196,7 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 			// 	    .detail("KeysBegin", keys.begin.printable().c_str())
 			// 	    .detail("KeysEnd", keys.end.printable().c_str());
 			// }
-			state vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
+			state std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
 			KeyRef partBegin, partEnd;
 			for (int i = 0; i < nLocs; i++) {
 				partBegin = (i == 0) ? keys.begin : locations[i].first.begin;
@@ -6155,7 +6245,7 @@ ACTOR Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(Databa
                                                                           int expectedShardCount) {
 	state Span span("NAPI:WaitStorageMetrics"_loc);
 	loop {
-		vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+		std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 		    wait(getKeyRangeLocations(cx,
 		                              keys,
 		                              shardLimit,
@@ -6247,7 +6337,7 @@ Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> Transaction::getReadHotRa
 ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Database cx, KeyRange keys, int64_t chunkSize) {
 	state Span span("NAPI:GetRangeSplitPoints"_loc);
 	loop {
-		state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+		state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 		    wait(getKeyRangeLocations(cx,
 		                              keys,
 		                              CLIENT_KNOBS->TOO_MANY,
@@ -6256,7 +6346,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Database cx, Key
 		                              TransactionInfo(TaskPriority::DataDistribution, span.context)));
 		try {
 			state int nLocs = locations.size();
-			state vector<Future<SplitRangeReply>> fReplies(nLocs);
+			state std::vector<Future<SplitRangeReply>> fReplies(nLocs);
 			KeyRef partBegin, partEnd;
 			for (int i = 0; i < nLocs; i++) {
 				partBegin = (i == 0) ? keys.begin : locations[i].first.begin;
@@ -6308,7 +6398,7 @@ ACTOR Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(Database cx,
                                                                 StorageMetrics estimated) {
 	state Span span("NAPI:SplitStorageMetrics"_loc);
 	loop {
-		state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+		state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 		    wait(getKeyRangeLocations(cx,
 		                              keys,
 		                              CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
@@ -6431,7 +6521,7 @@ ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID sn
 	}
 }
 
-ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exclusions) {
+ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions) {
 	TraceEvent("ExclusionSafetyCheckBegin")
 	    .detail("NumExclusion", exclusions.size())
 	    .detail("Exclusions", describe(exclusions));
@@ -6462,7 +6552,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exc
 	}
 	TraceEvent("ExclusionSafetyCheckCoordinators").log();
 	state ClientCoordinators coordinatorList(cx->getConnectionFile());
-	state vector<Future<Optional<LeaderInfo>>> leaderServers;
+	state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	leaderServers.reserve(coordinatorList.clientLeaderServers.size());
 	for (int i = 0; i < coordinatorList.clientLeaderServers.size(); i++) {
 		leaderServers.push_back(retryBrokenPromise(coordinatorList.clientLeaderServers[i].getLeader,
@@ -6597,7 +6687,7 @@ ACTOR Future<Standalone<VectorRef<MutationsAndVersionRef>>> getChangeFeedMutatio
 		throw unsupported_operation();
 	}
 	state KeyRange keys = std::get<0>(decodeChangeFeedValue(val.get())) & range;
-	state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+	state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 	    wait(getKeyRangeLocations(cx,
 	                              keys,
 	                              100,
@@ -6662,7 +6752,7 @@ ACTOR Future<Void> singleChangeFeedStream(StorageServerInterface interf,
 				           begin);
 				}*/
 				state int resultLoc = 0;
-				// TODO TELL EVAN about fix
+				// FIXME: fix better
 				while (resultLoc < rep.mutations.size()) {
 					if (rep.mutations[resultLoc].mutations.size() || rep.mutations[resultLoc].version + 1 == end ||
 					    (rep.mutations[resultLoc].mutations.empty() &&
@@ -6678,12 +6768,9 @@ ACTOR Future<Void> singleChangeFeedStream(StorageServerInterface interf,
 				}
 			}
 		} catch (Error& e) {
-			// FIXME shouldn't this also just send to the stream so it throws in mergeChangeFeedStream and retries in
-			/*if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
-			    e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed ||
-			    e.code() == error_code_actor_cancelled) {
-			    throw;
-			}*/
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
 			results.sendError(e);
 			return Void();
 		}
@@ -6759,18 +6846,17 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
                                             Version end,
                                             KeyRange range) {
 	state Database cx(db);
-	state Transaction tr(cx);
 	state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix);
 	state Span span("NAPI:GetChangeFeedStream"_loc);
 	state KeyRange keys;
 
 	loop {
+		state Transaction tr(cx);
 		loop {
 			try {
 				Version readVer = wait(tr.getReadVersion());
 				if (readVer < begin) {
 					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-					tr.reset();
 				} else {
 					Optional<Value> val = wait(tr.get(rangeIDKey));
 					if (!val.present()) {
@@ -6786,7 +6872,7 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 		}
 
 		try {
-			state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+			state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 			    wait(getKeyRangeLocations(cx,
 			                              keys,
 			                              1000,
@@ -6824,7 +6910,7 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 					continue;
 				}
 
-				vector<Future<Void>> ok(locations[loc].second->size());
+				std::vector<Future<Void>> ok(locations[loc].second->size());
 				for (int i = 0; i < ok.size(); i++) {
 					ok[i] = IFailureMonitor::failureMonitor().onStateEqual(
 					    locations[loc].second->get(i, &StorageServerInterface::changeFeedStream).getEndpoint(),
@@ -6876,10 +6962,10 @@ ACTOR Future<Void> getChangeFeedStreamActor(Reference<DatabaseContext> db,
 				throw;
 			}
 			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
-			    e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed) {
+			    e.code() == error_code_connection_failed || e.code() == error_code_unknown_change_feed ||
+			    e.code() == error_code_broken_promise) {
 				cx->invalidateCache(keys);
 				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY));
-				tr.reset();
 			} else {
 				results.sendError(e);
 				return Void();
@@ -6929,7 +7015,7 @@ ACTOR Future<std::vector<std::pair<Key, KeyRange>>> getOverlappingChangeFeedsAct
 
 	loop {
 		try {
-			state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+			state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 			    wait(getKeyRangeLocations(cx,
 			                              range,
 			                              1000,
@@ -6998,24 +7084,24 @@ ACTOR static Future<Void> popChangeFeedBackup(Database cx, Key rangeID, Version
 
 ACTOR Future<Void> popChangeFeedMutationsActor(Reference<DatabaseContext> db, Key rangeID, Version version) {
 	state Database cx(db);
-	state Transaction tr(cx);
 	state Key rangeIDKey = rangeID.withPrefix(changeFeedPrefix);
 	state Span span("NAPI:PopChangeFeedMutations"_loc);
-	state Optional<Value> val;
+
+	state Transaction tr(cx);
+	state KeyRange keys;
 	loop {
 		try {
-			Optional<Value> _val = wait(tr.get(rangeIDKey));
-			val = _val;
+			Optional<Value> val = wait(tr.get(rangeIDKey));
+			if (!val.present()) {
+				throw unsupported_operation();
+			}
+			keys = std::get<0>(decodeChangeFeedValue(val.get()));
 			break;
 		} catch (Error& e) {
 			wait(tr.onError(e));
 		}
 	}
-	if (!val.present()) {
-		throw unsupported_operation();
-	}
-	state KeyRange keys = std::get<0>(decodeChangeFeedValue(val.get()));
-	state vector<pair<KeyRange, Reference<LocationInfo>>> locations =
+	state std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 	    wait(getKeyRangeLocations(cx,
 	                              keys,
 	                              3,

fdbclient/NativeAPI.actor.h

@@ -415,7 +415,7 @@ public:
 	void setTransactionID(uint64_t id);
 	void setToken(uint64_t token);
 
-	const vector<Future<std::pair<Key, Key>>>& getExtraReadConflictRanges() const { return extraConflictRanges; }
+	const std::vector<Future<std::pair<Key, Key>>>& getExtraReadConflictRanges() const { return extraConflictRanges; }
 	Standalone<VectorRef<KeyRangeRef>> readConflictRanges() const {
 		return Standalone<VectorRef<KeyRangeRef>>(tr.transaction.read_conflict_ranges, tr.arena);
 	}
@@ -432,7 +432,7 @@ private:
 	CommitTransactionRequest tr;
 	Future<Version> readVersion;
 	Promise<Optional<Value>> metadataVersion;
-	vector<Future<std::pair<Key, Key>>> extraConflictRanges;
+	std::vector<Future<std::pair<Key, Key>>> extraConflictRanges;
 	Promise<Void> commitResult;
 	Future<Void> committing;
 };
@@ -453,7 +453,7 @@ int64_t extractIntOption(Optional<StringRef> value,
 ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
 
 // Checks with Data Distributor that it is safe to mark all servers in exclusions as failed
-ACTOR Future<bool> checkSafeExclusions(Database cx, vector<AddressExclusion> exclusions);
+ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions);
 
 inline uint64_t getWriteOperationCost(uint64_t bytes) {
 	return bytes / std::max(1, CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR) + 1;

fdbclient/ProcessInterface.h

@@ -21,8 +21,7 @@
 #include "fdbclient/AnnotateActor.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/fdbrpc.h"
-
-constexpr UID WLTOKEN_PROCESS(-1, 21);
+#include "fdbclient/WellKnownEndpoints.h"
 
 struct ProcessInterface {
 	constexpr static FileIdentifier file_identifier = 985636;

fdbclient/ReadYourWrites.actor.cpp

@@ -1342,7 +1342,7 @@ ACTOR Future<RangeResult> getWorkerInterfaces(Reference<ClusterConnectionFile> c
 
 	loop {
 		choose {
-			when(vector<ClientWorkerInterface> workers =
+			when(std::vector<ClientWorkerInterface> workers =
 			         wait(clusterInterface->get().present()
 			                  ? brokenPromiseToNever(
 			                        clusterInterface->get().get().getClientWorkers.getReply(GetClientWorkersRequest()))

fdbclient/Schemas.cpp

@@ -763,7 +763,14 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
          "grv_proxies":1,
          "proxies":6,
          "backup_worker_enabled":1,
-         "perpetual_storage_wiggle":0
+         "perpetual_storage_wiggle":0,
+         "perpetual_storage_wiggle_locality":"0",
+         "storage_migration_type": {
+             "$enum":[
+             "disabled",
+             "aggressive",
+             "gradual"
+         ]}
       },
       "data":{
          "least_operating_space_bytes_log_server":0,

fdbclient/ServerKnobs.cpp

@@ -102,7 +102,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PUSH_STATS_SLOW_RATIO,                                 0.5 );
 	init( TLOG_POP_BATCH_SIZE,                                  1000 ); if ( randomize && BUGGIFY ) TLOG_POP_BATCH_SIZE = 10;
 	init( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE,     250e6 );
-	init( ENABLE_DETAILED_TLOG_POP_TRACE,                       true );
+	init( ENABLE_DETAILED_TLOG_POP_TRACE,                      false ); if ( randomize && BUGGIFY ) ENABLE_DETAILED_TLOG_POP_TRACE = true;
 
 	// disk snapshot max timeout, to be put in TLog, storage and coordinator nodes
 	init( MAX_FORKED_PROCESS_OUTPUT,                            1024 );
@@ -444,7 +444,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BACKUP_TIMEOUT,                                        0.4 );
 	init( BACKUP_NOOP_POP_DELAY,                                 5.0 );
 	init( BACKUP_FILE_BLOCK_BYTES,                       1024 * 1024 );
-	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 1024;
+	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 15 * 1024;
 	init( BACKUP_UPLOAD_DELAY,                                  10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60;
 
 	//Cluster Controller
@@ -467,6 +467,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CHECK_OUTSTANDING_INTERVAL,                            0.5 ); if( randomize && BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001;
 	init( VERSION_LAG_METRIC_INTERVAL,                           0.5 ); if( randomize && BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0;
 	init( MAX_VERSION_DIFFERENCE,           20 * VERSIONS_PER_SECOND );
+	init( INITIAL_UPDATE_CROSS_DC_INFO_DELAY,                    300 );
+	init( CHECK_REMOTE_HEALTH_INTERVAL,                           60 );
 	init( FORCE_RECOVERY_CHECK_DELAY,                            5.0 );
 	init( RATEKEEPER_FAILURE_TIME,                               1.0 );
 	init( BLOB_MANAGER_FAILURE_TIME,                             1.0 );
@@ -482,7 +484,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CC_MAX_EXCLUSION_DUE_TO_HEALTH,                          2 );
 	init( CC_HEALTH_TRIGGER_RECOVERY,                          false );
 	init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL,               3600.0 );
-	init( CC_MAX_HEALTH_RECOVERY_COUNT,                            2 );
+	init( CC_MAX_HEALTH_RECOVERY_COUNT,                            5 );
+	init( CC_HEALTH_TRIGGER_FAILOVER,                          false );
+	init( CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION,               5 );
+	init( CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION,              10 );
 
 	init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL,                   600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
 	init( EXPECTED_MASTER_FITNESS,            ProcessClass::UnsetFit );

fdbclient/ServerKnobs.h

@@ -391,6 +391,10 @@ public:
 	double INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
 	double VERSION_LAG_METRIC_INTERVAL;
 	int64_t MAX_VERSION_DIFFERENCE;
+	double INITIAL_UPDATE_CROSS_DC_INFO_DELAY; // The intial delay in a new Cluster Controller just started to refresh
+	                                           // the info of remote DC, such as remote DC health, and whether we need
+	                                           // to take remote DC health info when making failover decision.
+	double CHECK_REMOTE_HEALTH_INTERVAL; // Remote DC health refresh interval.
 	double FORCE_RECOVERY_CHECK_DELAY;
 	double RATEKEEPER_FAILURE_TIME;
 	double BLOB_MANAGER_FAILURE_TIME;
@@ -414,7 +418,13 @@ public:
 	double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed
 	                                             // CC_MAX_HEALTH_RECOVERY_COUNT within
 	                                             // CC_TRACKING_HEALTH_RECOVERY_INTERVAL.
-	int CC_MAX_HEALTH_RECOVERY_COUNT;
+	int CC_MAX_HEALTH_RECOVERY_COUNT; // The max number of recoveries can be triggered due to worker health within
+	                                  // CC_TRACKING_HEALTH_RECOVERY_INTERVAL
+	bool CC_HEALTH_TRIGGER_FAILOVER; // Whether to enable health triggered failover in CC.
+	int CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION; // The minimum number of degraded servers that can trigger a
+	                                               // failover.
+	int CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION; // The maximum number of degraded servers that can trigger a
+	                                               // failover.
 
 	// Knobs used to select the best policy (via monte carlo)
 	int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)

fdbclient/SpecialKeySpace.actor.cpp

@@ -1145,7 +1145,7 @@ Future<RangeResult> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransac
 }
 
 ACTOR Future<RangeResult> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
-	vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
+	std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
 	auto workers = _workers; // strip const
 	// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
 	std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) {
@@ -1168,7 +1168,7 @@ ACTOR Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransa
 	ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-	vector<ProcessData> workers = wait(
+	std::vector<ProcessData> workers = wait(
 	    getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit()
 
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
@@ -1259,7 +1259,7 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef&
 }
 
 ACTOR Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
-	vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
+	std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
 	auto workers = _workers; // strip const
 	// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
 	std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) {
@@ -1322,7 +1322,7 @@ ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransa
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != uid) {
 		// check database not locked
 		// if locked already, throw error
-		msg = ManagementAPIError::toJsonString(false, "lock", "Database has already been locked");
+		throw database_locked();
 	} else if (!val.present()) {
 		// lock database
 		ryw->getTransaction().atomicOp(databaseLockedKey,
@@ -1623,7 +1623,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	state int index;
 	state bool parse_error = false;
 
-	// check update for cluster_description
+	// check update for coordinators
 	Key processes_key = LiteralStringRef("processes").withPrefix(kr.begin);
 	auto processes_entry = ryw->getSpecialKeySpaceWriteMap()[processes_key];
 	if (processes_entry.first) {
@@ -1691,29 +1691,14 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	    .detail("Result", r.present() ? static_cast<int>(r.get()) : -1); // -1 means success
 	if (r.present()) {
 		auto res = r.get();
-		std::string error_msg;
 		bool retriable = false;
-		if (res == CoordinatorsResult::INVALID_NETWORK_ADDRESSES) {
-			error_msg = "The specified network addresses are invalid";
-		} else if (res == CoordinatorsResult::SAME_NETWORK_ADDRESSES) {
-			error_msg = "No change (existing configuration satisfies request)";
-		} else if (res == CoordinatorsResult::NOT_COORDINATORS) {
-			error_msg = "Coordination servers are not running on the specified network addresses";
-		} else if (res == CoordinatorsResult::DATABASE_UNREACHABLE) {
-			error_msg = "Database unreachable";
-		} else if (res == CoordinatorsResult::BAD_DATABASE_STATE) {
-			error_msg = "The database is in an unexpected state from which changing coordinators might be unsafe";
-		} else if (res == CoordinatorsResult::COORDINATOR_UNREACHABLE) {
-			error_msg = "One of the specified coordinators is unreachable";
+		if (res == CoordinatorsResult::COORDINATOR_UNREACHABLE) {
 			retriable = true;
-		} else if (res == CoordinatorsResult::NOT_ENOUGH_MACHINES) {
-			error_msg = "Too few fdbserver machines to provide coordination at the current redundancy level";
 		} else if (res == CoordinatorsResult::SUCCESS) {
 			TraceEvent(SevError, "SpecialKeysForCoordinators").detail("UnexpectedSuccessfulResult", "");
-		} else {
 			ASSERT(false);
 		}
-		msg = ManagementAPIError::toJsonString(retriable, "coordinators", error_msg);
+		msg = ManagementAPIError::toJsonString(retriable, "coordinators", ManagementAPI::generateErrorMessage(res));
 	}
 	return msg;
 }
@@ -1758,7 +1743,9 @@ ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransac
 		// we could get not_enough_machines if we happen to see the database while the cluster controller is updating
 		// the worker list, so make sure it happens twice before returning a failure
 		ryw->setSpecialKeySpaceErrorMsg(ManagementAPIError::toJsonString(
-		    true, "auto_coordinators", "The auto change attempt did not get enough machines, please try again"));
+		    true,
+		    "auto_coordinators",
+		    "Too few fdbserver machines to provide coordination at the current redundancy level"));
 		throw special_keys_api_failure();
 	}
 
@@ -2105,7 +2092,7 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 	// Open endpoint to target process on each call. This can be optimized at
 	// some point...
 	state ProcessInterface process;
-	process.getInterface = RequestStream<GetProcessInterfaceRequest>(Endpoint({ host }, WLTOKEN_PROCESS));
+	process.getInterface = RequestStream<GetProcessInterfaceRequest>(Endpoint::wellKnown({ host }, WLTOKEN_PROCESS));
 	ProcessInterface p = wait(retryBrokenPromise(process.getInterface, GetProcessInterfaceRequest{}));
 	process = p;
 

fdbclient/StatusClient.actor.cpp

@@ -309,18 +309,18 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<C
 		state ClientCoordinators coord(f);
 		state StatusObject statusObj;
 
-		state vector<Future<Optional<LeaderInfo>>> leaderServers;
+		state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 		leaderServers.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++)
 			leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
 			                                           GetLeaderRequest(coord.clusterKey, UID()),
 			                                           TaskPriority::CoordinationReply));
 
-		state vector<Future<ProtocolInfoReply>> coordProtocols;
+		state std::vector<Future<ProtocolInfoReply>> coordProtocols;
 		coordProtocols.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
-			RequestStream<ProtocolInfoRequest> requestStream{ Endpoint{
-				{ coord.clientLeaderServers[i].getLeader.getEndpoint().addresses }, WLTOKEN_PROTOCOL_INFO } };
+			RequestStream<ProtocolInfoRequest> requestStream{ Endpoint::wellKnown(
+				{ coord.clientLeaderServers[i].getLeader.getEndpoint().addresses }, WLTOKEN_PROTOCOL_INFO) };
 			coordProtocols.push_back(retryBrokenPromise(requestStream, ProtocolInfoRequest{}));
 		}
 

fdbclient/StorageServerInterface.h

@@ -330,7 +330,14 @@ struct GetKeyValuesStreamReply : public ReplyPromiseStreamReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, data, version, more, cached, arena);
+		serializer(ar,
+		           ReplyPromiseStreamReply::acknowledgeToken,
+		           ReplyPromiseStreamReply::sequence,
+		           data,
+		           version,
+		           more,
+		           cached,
+		           arena);
 	}
 };
 
@@ -674,7 +681,7 @@ struct ChangeFeedStreamReply : public ReplyPromiseStreamReply {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, mutations, arena);
+		serializer(ar, ReplyPromiseStreamReply::acknowledgeToken, ReplyPromiseStreamReply::sequence, mutations, arena);
 	}
 };
 

fdbclient/SystemData.cpp

@@ -242,13 +242,13 @@ const Key storageCacheKey(const KeyRef& k) {
 	return k.withPrefix(storageCachePrefix);
 }
 
-const Value storageCacheValue(const vector<uint16_t>& serverIndices) {
+const Value storageCacheValue(const std::vector<uint16_t>& serverIndices) {
 	BinaryWriter wr((IncludeVersion(ProtocolVersion::withStorageCacheValue())));
 	wr << serverIndices;
 	return wr.toValue();
 }
 
-void decodeStorageCacheValue(const ValueRef& value, vector<uint16_t>& serverIndices) {
+void decodeStorageCacheValue(const ValueRef& value, std::vector<uint16_t>& serverIndices) {
 	serverIndices.clear();
 	if (value.size()) {
 		BinaryReader rd(value, IncludeVersion());
@@ -256,25 +256,26 @@ void decodeStorageCacheValue(const ValueRef& value, vector<uint16_t>& serverIndi
 	}
 }
 
-const Value logsValue(const vector<std::pair<UID, NetworkAddress>>& logs,
-                      const vector<std::pair<UID, NetworkAddress>>& oldLogs) {
+const Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
+                      const std::vector<std::pair<UID, NetworkAddress>>& oldLogs) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withLogsValue()));
 	wr << logs;
 	wr << oldLogs;
 	return wr.toValue();
 }
-std::pair<vector<std::pair<UID, NetworkAddress>>, vector<std::pair<UID, NetworkAddress>>> decodeLogsValue(
+std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID, NetworkAddress>>> decodeLogsValue(
     const ValueRef& value) {
-	vector<std::pair<UID, NetworkAddress>> logs;
-	vector<std::pair<UID, NetworkAddress>> oldLogs;
+	std::vector<std::pair<UID, NetworkAddress>> logs;
+	std::vector<std::pair<UID, NetworkAddress>> oldLogs;
 	BinaryReader reader(value, IncludeVersion());
 	reader >> logs;
 	reader >> oldLogs;
 	return std::make_pair(logs, oldLogs);
 }
 
-const KeyRef serverKeysPrefix = LiteralStringRef("\xff/serverKeys/");
-const ValueRef serverKeysTrue = LiteralStringRef("1"), // compatible with what was serverKeysTrue
+const KeyRef serverKeysPrefix = "\xff/serverKeys/"_sr;
+const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTrue
+    serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty.
     serverKeysFalse;
 
 const Key serverKeysKey(UID serverID, const KeyRef& key) {
@@ -299,7 +300,7 @@ UID serverKeysDecodeServer(const KeyRef& key) {
 	return server_id;
 }
 bool serverHasKey(ValueRef storedValue) {
-	return storedValue == serverKeysTrue;
+	return storedValue == serverKeysTrue || storedValue == serverKeysTrueEmptyRange;
 }
 
 const KeyRef cacheKeysPrefix = LiteralStringRef("\xff\x02/cacheKeys/");
@@ -629,6 +630,7 @@ const KeyRangeRef configKeys(LiteralStringRef("\xff/conf/"), LiteralStringRef("\
 const KeyRef configKeysPrefix = configKeys.begin;
 
 const KeyRef perpetualStorageWiggleKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle"));
+const KeyRef perpetualStorageWiggleLocalityKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle_locality"));
 const KeyRef wigglingStorageServerKey(LiteralStringRef("\xff/storageWigglePID"));
 
 const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint"));
@@ -1056,7 +1058,7 @@ std::tuple<KeyRange, Version, bool> decodeChangeFeedValue(ValueRef const& value)
 const KeyRangeRef changeFeedDurableKeys(LiteralStringRef("\xff\xff/cf/"), LiteralStringRef("\xff\xff/cf0"));
 const KeyRef changeFeedDurablePrefix = changeFeedDurableKeys.begin;
 
-const Value changeFeedDurableKey(Key const& feed, Version const& version) {
+const Value changeFeedDurableKey(Key const& feed, Version version) {
 	BinaryWriter wr(AssumeVersion(ProtocolVersion::withChangeFeed()));
 	wr.serializeBytes(changeFeedDurablePrefix);
 	wr << feed;
@@ -1071,16 +1073,19 @@ std::pair<Key, Version> decodeChangeFeedDurableKey(ValueRef const& key) {
 	reader >> version;
 	return std::make_pair(feed, bigEndian64(version));
 }
-const Value changeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations) {
+const Value changeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations, Version knownCommittedVersion) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withChangeFeed()));
 	wr << mutations;
+	wr << knownCommittedVersion;
 	return wr.toValue();
 }
-Standalone<VectorRef<MutationRef>> decodeChangeFeedDurableValue(ValueRef const& value) {
+std::pair<Standalone<VectorRef<MutationRef>>, Version> decodeChangeFeedDurableValue(ValueRef const& value) {
 	Standalone<VectorRef<MutationRef>> mutations;
+	Version knownCommittedVersion;
 	BinaryReader reader(value, IncludeVersion());
 	reader >> mutations;
-	return mutations;
+	reader >> knownCommittedVersion;
+	return std::make_pair(mutations, knownCommittedVersion);
 }
 
 const KeyRef configTransactionDescriptionKey = "\xff\xff/description"_sr;

fdbclient/SystemData.h

@@ -44,7 +44,7 @@ extern const KeyRangeRef specialKeys; // [FF][FF] to [FF][FF][FF], some client f
                                       // using these special keys, see pr#2662
 extern const KeyRef afterAllKeys;
 
-//    "\xff/keyServers/[[begin]]" := "[[vector<serverID>, vector<serverID>]|[vector<Tag>, vector<Tag>]]"
+//    "\xff/keyServers/[[begin]]" := "[[vector<serverID>, std::vector<serverID>]|[vector<Tag>, std::vector<Tag>]]"
 //	An internal mapping of where shards are located in the database. [[begin]] is the start of the shard range
 //	and the result is a list of serverIDs or Tags where these shards are located. These values can be changed
 //	as data movement occurs.
@@ -90,7 +90,7 @@ void decodeStorageCacheValue(const ValueRef& value, std::vector<uint16_t>& serve
 //	as the key, the value indicates whether the shard does or does not exist on the server.
 //	These values can be changed as data movement occurs.
 extern const KeyRef serverKeysPrefix;
-extern const ValueRef serverKeysTrue, serverKeysFalse;
+extern const ValueRef serverKeysTrue, serverKeysTrueEmptyRange, serverKeysFalse;
 const Key serverKeysKey(UID serverID, const KeyRef& keys);
 const Key serverKeysPrefixFor(UID serverID);
 UID serverKeysDecodeServer(const KeyRef& key);
@@ -211,6 +211,7 @@ extern const KeyRangeRef configKeys;
 extern const KeyRef configKeysPrefix;
 
 extern const KeyRef perpetualStorageWiggleKey;
+extern const KeyRef perpetualStorageWiggleLocalityKey;
 extern const KeyRef wigglingStorageServerKey;
 // Change the value of this key to anything and that will trigger detailed data distribution team info log.
 extern const KeyRef triggerDDTeamInfoPrintKey;
@@ -331,9 +332,9 @@ extern const KeyRef logsKey;
 //	Used during backup/recovery to restrict version requirements
 extern const KeyRef minRequiredCommitVersionKey;
 
-const Value logsValue(const vector<std::pair<UID, NetworkAddress>>& logs,
-                      const vector<std::pair<UID, NetworkAddress>>& oldLogs);
-std::pair<vector<std::pair<UID, NetworkAddress>>, vector<std::pair<UID, NetworkAddress>>> decodeLogsValue(
+const Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
+                      const std::vector<std::pair<UID, NetworkAddress>>& oldLogs);
+std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID, NetworkAddress>>> decodeLogsValue(
     const ValueRef& value);
 
 // The "global keys" are sent to each storage server any time they are changed
@@ -505,10 +506,10 @@ extern const KeyRef changeFeedPrivatePrefix;
 extern const KeyRangeRef changeFeedDurableKeys;
 extern const KeyRef changeFeedDurablePrefix;
 
-const Value changeFeedDurableKey(Key const& feed, Version const& version);
+const Value changeFeedDurableKey(Key const& feed, Version version);
 std::pair<Key, Version> decodeChangeFeedDurableKey(ValueRef const& key);
-const Value changeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations);
-Standalone<VectorRef<MutationRef>> decodeChangeFeedDurableValue(ValueRef const& value);
+const Value changeFeedDurableValue(Standalone<VectorRef<MutationRef>> const& mutations, Version knownCommittedVersion);
+std::pair<Standalone<VectorRef<MutationRef>>, Version> decodeChangeFeedDurableValue(ValueRef const& value);
 
 // Configuration database special keys
 extern const KeyRef configTransactionDescriptionKey;

fdbclient/TagThrottle.actor.h

@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "flow/Arena.h"
 #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TAG_THROTTLE_ACTOR_G_H)
 #define FDBCLIENT_TAG_THROTTLE_ACTOR_G_H
 #include "fdbclient/TagThrottle.actor.g.h"
@@ -247,7 +248,9 @@ ACTOR template <class Tr>
 Future<bool> getValidAutoEnabled(Reference<Tr> tr) {
 	state bool result;
 	loop {
-		Optional<Value> value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey)));
+		// hold the returned standalone object's memory
+		state typename Tr::template FutureT<Optional<Value>> valueF = tr->get(tagThrottleAutoEnabledKey);
+		Optional<Value> value = wait(safeThreadFutureToFuture(valueF));
 		if (!value.present()) {
 			tr->reset();
 			wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
@@ -466,10 +469,12 @@ Future<bool> unthrottleTags(Reference<DB> db,
 	loop {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		try {
+			state std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>> valueFutures;
 			state std::vector<Future<Optional<Value>>> values;
 			values.reserve(keys.size());
 			for (auto key : keys) {
-				values.push_back(safeThreadFutureToFuture(tr->get(key)));
+				valueFutures.push_back(tr->get(key));
+				values.push_back(safeThreadFutureToFuture(valueFutures.back()));
 			}
 
 			wait(waitForAll(values));
@@ -535,7 +540,9 @@ Future<Void> throttleTags(Reference<DB> db,
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		try {
 			if (throttleType == TagThrottleType::MANUAL) {
-				Optional<Value> oldThrottle = wait(safeThreadFutureToFuture(tr->get(key)));
+				// hold the returned standalone object's memory
+				state typename DB::TransactionT::template FutureT<Optional<Value>> oldThrottleF = tr->get(key);
+				Optional<Value> oldThrottle = wait(safeThreadFutureToFuture(oldThrottleF));
 				if (!oldThrottle.present()) {
 					wait(updateThrottleCount(tr, 1));
 				}
@@ -562,7 +569,10 @@ Future<Void> enableAuto(Reference<DB> db, bool enabled) {
 	loop {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		try {
-			Optional<Value> value = wait(safeThreadFutureToFuture(tr->get(tagThrottleAutoEnabledKey)));
+			// hold the returned standalone object's memory
+			state typename DB::TransactionT::template FutureT<Optional<Value>> valueF =
+			    tr->get(tagThrottleAutoEnabledKey);
+			Optional<Value> value = wait(safeThreadFutureToFuture(valueF));
 			if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) ||
 			    (!enabled && value.get() != LiteralStringRef("0"))) {
 				tr->set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0"));

fdbclient/WellKnownEndpoints.h

@@ -0,0 +1,54 @@
+/*
+ * WellKnownEndpoints.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_WELLKNOWNENDPOINTS_H
+#define FDBCLIENT_WELLKNOWNENDPOINTS_H
+#pragma once
+
+#include <fdbrpc/fdbrpc.h>
+
+/*
+ * All well-known endpoints of FDB must be listed here to guarantee their uniqueness
+ */
+enum WellKnownEndpoints {
+	WLTOKEN_CLIENTLEADERREG_GETLEADER = WLTOKEN_FIRST_AVAILABLE, // 2
+	WLTOKEN_CLIENTLEADERREG_OPENDATABASE, // 3
+	WLTOKEN_LEADERELECTIONREG_CANDIDACY, // 4
+	WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, // 5
+	WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, // 6
+	WLTOKEN_LEADERELECTIONREG_FORWARD, // 7
+	WLTOKEN_GENERATIONREG_READ, // 8
+	WLTOKEN_GENERATIONREG_WRITE, // 9
+	WLTOKEN_PROTOCOL_INFO, // 10 : the value of this endpoint should be stable and not change.
+	WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE, // 11
+	WLTOKEN_CONFIGTXN_GETGENERATION, // 12
+	WLTOKEN_CONFIGTXN_GET, // 13
+	WLTOKEN_CONFIGTXN_GETCLASSES, // 14
+	WLTOKEN_CONFIGTXN_GETKNOBS, // 15
+	WLTOKEN_CONFIGTXN_COMMIT, // 16
+	WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES, // 17
+	WLTOKEN_CONFIGFOLLOWER_GETCHANGES, // 18
+	WLTOKEN_CONFIGFOLLOWER_COMPACT, // 19
+	WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION, // 20
+	WLTOKEN_PROCESS, // 21
+	WLTOKEN_RESERVED_COUNT // 22
+};
+
+#endif

fdbclient/rapidjson/document.h

@@ -862,8 +862,7 @@ public:
 	    Helper function to enable support for common swap implementation pattern based on \c std::swap:
 	    \code
 	    void swap(MyClass& a, MyClass& b) {
-	        using std::swap;
-	        swap(a.value, b.value);
+	        std::swap(a.value, b.value);
 	        // ...
 	    }
 	    \endcode
@@ -2326,8 +2325,7 @@ public:
 	    Helper function to enable support for common swap implementation pattern based on \c std::swap:
 	    \code
 	    void swap(MyClass& a, MyClass& b) {
-	        using std::swap;
-	        swap(a.doc, b.doc);
+	        std::swap(a.doc, b.doc);
 	        // ...
 	    }
 	    \endcode

fdbclient/vexillographer/cpp.cs

@@ -47,8 +47,9 @@ namespace vexillographer
 
         private static string getCInfoLine(Option o, string indent, string structName)
         {
-            return String.Format("{0}ADD_OPTION_INFO({1}, {2}, \"{2}\", \"{3}\", \"{4}\", {5}, {6}, {7}, {8})",
-                indent, structName, o.name.ToUpper(), o.comment, o.getParameterComment(), (o.paramDesc != null).ToString().ToLower(), o.hidden.ToString().ToLower(), o.persistent.ToString().ToLower(), o.defaultFor);
+            return String.Format("{0}ADD_OPTION_INFO({1}, {2}, \"{2}\", \"{3}\", \"{4}\", {5}, {6}, {7}, {8}, FDBOptionInfo::ParamType::{9})",
+                indent, structName, o.name.ToUpper(), o.comment, o.getParameterComment(), (o.paramDesc != null).ToString().ToLower(), 
+                o.hidden.ToString().ToLower(), o.persistent.ToString().ToLower(), o.defaultFor, o.paramType);
         }
 
         private static void writeCppInfo(TextWriter outFile, Scope scope, IEnumerable<Option> options)

fdbclient/vexillographer/fdb.options

@@ -200,6 +200,8 @@ description is not currently required but encouraged.
             defaultFor="1100"/>
     <Option name="use_config_database" code="800"
             description="Use configuration database." />
+    <Option name="test_causal_read_risky" code="900"
+            description="An integer between 0 and 100 (default is 0) expressing the probability that a client will verify it can't read stale data whenever it detects a recovery." />
   </Scope>
   
   <Scope name="TransactionOption">

fdbmonitor/fdbmonitor.cpp

@@ -700,7 +700,12 @@ void start_process(Command* cmd, ProcessID id, uid_t uid, gid_t gid, int delay,
 			fflush(stdout);
 		}
 		execv(cmd->argv[0], (char* const*)cmd->argv);
-		fprintf(stderr, "Unable to launch %s for %s\n", cmd->argv[0], cmd->ssection.c_str());
+		fprintf(stderr,
+		        "Unable to launch %s for %s (execv error %d: %s)\n",
+		        cmd->argv[0],
+		        cmd->ssection.c_str(),
+		        errno,
+		        strerror(errno));
 		_exit(0);
 	}
 

fdbrpc/ActorFuzz.h

@@ -22,8 +22,6 @@
 #include <vector>
 #include "flow/actorcompiler.h"
 
-using std::vector;
-
 inline void throw_operation_failed() {
 	throw operation_failed();
 }
@@ -31,7 +29,7 @@ inline void throw_operation_failed() {
 // This is in dsltest.actor.cpp:
 bool testFuzzActor(Future<int> (*actor)(FutureStream<int> const&, PromiseStream<int> const&, Future<Void> const&),
                    const char* desc,
-                   vector<int> const& expectedOutput);
+                   std::vector<int> const& expectedOutput);
 
 // This is defined by ActorFuzz.actor.cpp (generated by actorFuzz.py)
 // Returns (tests passed, tests total)

fdbrpc/AsyncFileNonDurable.actor.h

@@ -415,7 +415,7 @@ private:
 	// results
 	ACTOR Future<int> onRead(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
 		wait(checkKilled(self, "Read"));
-		vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length);
+		std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length);
 		wait(waitForAll(priorModifications));
 		state Future<int> readFuture = self->file->read(data, length, offset);
 		wait(success(readFuture) || self->killed.getFuture());
@@ -513,7 +513,7 @@ private:
 		int diskPageLength = saveDurable ? length : 4096;
 		int diskSectorLength = saveDurable ? length : 512;
 
-		vector<Future<Void>> writeFutures;
+		std::vector<Future<Void>> writeFutures;
 		for (int writeOffset = 0; writeOffset < length;) {
 			// Number of bytes until the next diskPageLength file offset within the write or the end of the write.
 			int pageLength = diskPageLength;

fdbrpc/FailureMonitor.h

@@ -27,8 +27,6 @@
 #include <unordered_map>
 #include <unordered_set>
 
-using std::vector;
-
 /*
 
 IFailureMonitor is used by load balancing, data distribution and other components

fdbrpc/FlowTests.actor.cpp

@@ -32,8 +32,6 @@
 
 void forceLinkFlowTests() {}
 
-using std::vector;
-
 constexpr int firstLine = __LINE__;
 TEST_CASE("/flow/actorcompiler/lineNumbers") {
 	loop {
@@ -438,9 +436,9 @@ TEST_CASE("/flow/flow/networked futures") {
 }
 
 TEST_CASE("/flow/flow/quorum") {
-	vector<Promise<int>> ps(5);
-	vector<Future<int>> fs;
-	vector<Future<Void>> qs;
+	std::vector<Promise<int>> ps(5);
+	std::vector<Future<int>> fs;
+	std::vector<Future<Void>> qs;
 	for (auto& p : ps)
 		fs.push_back(p.getFuture());
 
@@ -774,7 +772,7 @@ TEST_CASE("/flow/perf/yieldedFuture") {
 
 	Promise<Void> p;
 	Future<Void> f = p.getFuture();
-	vector<Future<Void>> ys;
+	std::vector<Future<Void>> ys;
 
 	start = timer();
 	for (int i = 0; i < N; i++)
@@ -879,8 +877,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = oneWaitActor(pipe[i].getFuture());
@@ -893,8 +891,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = oneWaitActor(pipe[i].getFuture());
@@ -955,8 +953,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = chooseTwoActor(pipe[i].getFuture(), never);
@@ -969,8 +967,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = chooseTwoActor(pipe[i].getFuture(), pipe[i].getFuture());
@@ -983,8 +981,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = chooseTwoActor(chooseTwoActor(pipe[i].getFuture(), never), never);
@@ -1008,8 +1006,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			out[i] = oneWaitActor(chooseTwoActor(pipe[i].getFuture(), never));
@@ -1035,9 +1033,9 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out1(N);
-		vector<Future<Void>> out2(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out1(N);
+		std::vector<Future<Void>> out2(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			Future<Void> f = chooseTwoActor(pipe[i].getFuture(), never);
@@ -1052,9 +1050,9 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out1(N);
-		vector<Future<Void>> out2(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out1(N);
+		std::vector<Future<Void>> out2(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			Future<Void> f = chooseTwoActor(oneWaitActor(pipe[i].getFuture()), never);
@@ -1069,9 +1067,9 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 	}
 
 	{
-		vector<Promise<Void>> pipe(N);
-		vector<Future<Void>> out1(N);
-		vector<Future<Void>> out2(N);
+		std::vector<Promise<Void>> pipe(N);
+		std::vector<Future<Void>> out1(N);
+		std::vector<Future<Void>> out2(N);
 		start = timer();
 		for (int i = 0; i < N; i++) {
 			g_cheese = pipe[i].getFuture();
@@ -1101,8 +1099,8 @@ TEST_CASE("/flow/flow/perf/actor patterns") {
 
 	{
 		start = timer();
-		vector<Promise<Void>> ps(3);
-		vector<Future<Void>> fs(3);
+		std::vector<Promise<Void>> ps(3);
+		std::vector<Future<Void>> fs(3);
 
 		for (int i = 0; i < N; i++) {
 			ps.clear();

fdbrpc/FlowTransport.actor.cpp

@@ -45,13 +45,9 @@
 
 static NetworkAddressList g_currentDeliveryPeerAddress = NetworkAddressList();
 
-constexpr UID WLTOKEN_ENDPOINT_NOT_FOUND(-1, 0);
-constexpr UID WLTOKEN_PING_PACKET(-1, 1);
 constexpr int PACKET_LEN_WIDTH = sizeof(uint32_t);
 const uint64_t TOKEN_STREAM_FLAG = 1;
 
-static constexpr int WLTOKEN_COUNTS = 22; // number of wellKnownEndpoints
-
 class EndpointMap : NonCopyable {
 public:
 	// Reserve space for this many wellKnownEndpoints
@@ -97,7 +93,7 @@ void EndpointMap::realloc() {
 
 void EndpointMap::insertWellKnown(NetworkMessageReceiver* r, const Endpoint::Token& token, TaskPriority priority) {
 	int index = token.second();
-	ASSERT(index <= WLTOKEN_COUNTS);
+	ASSERT(index <= wellKnownEndpointCount);
 	ASSERT(data[index].receiver == nullptr);
 	data[index].receiver = r;
 	data[index].token() =
@@ -196,7 +192,8 @@ void EndpointMap::remove(Endpoint::Token const& token, NetworkMessageReceiver* r
 
 struct EndpointNotFoundReceiver final : NetworkMessageReceiver {
 	EndpointNotFoundReceiver(EndpointMap& endpoints) {
-		endpoints.insertWellKnown(this, WLTOKEN_ENDPOINT_NOT_FOUND, TaskPriority::DefaultEndpoint);
+		endpoints.insertWellKnown(
+		    this, Endpoint::wellKnownToken(WLTOKEN_ENDPOINT_NOT_FOUND), TaskPriority::DefaultEndpoint);
 	}
 
 	void receive(ArenaObjectReader& reader) override {
@@ -220,7 +217,7 @@ struct PingRequest {
 
 struct PingReceiver final : NetworkMessageReceiver {
 	PingReceiver(EndpointMap& endpoints) {
-		endpoints.insertWellKnown(this, WLTOKEN_PING_PACKET, TaskPriority::ReadSocket);
+		endpoints.insertWellKnown(this, Endpoint::wellKnownToken(WLTOKEN_PING_PACKET), TaskPriority::ReadSocket);
 	}
 	void receive(ArenaObjectReader& reader) override {
 		PingRequest req;
@@ -234,7 +231,7 @@ struct PingReceiver final : NetworkMessageReceiver {
 
 class TransportData {
 public:
-	TransportData(uint64_t transportId);
+	TransportData(uint64_t transportId, int maxWellKnownEndpoints);
 
 	~TransportData();
 
@@ -341,8 +338,8 @@ ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
 	}
 }
 
-TransportData::TransportData(uint64_t transportId)
-  : warnAlwaysForLargePacket(true), endpoints(WLTOKEN_COUNTS), endpointNotFoundReceiver(endpoints),
+TransportData::TransportData(uint64_t transportId, int maxWellKnownEndpoints)
+  : warnAlwaysForLargePacket(true), endpoints(maxWellKnownEndpoints), endpointNotFoundReceiver(endpoints),
     pingReceiver(endpoints), numIncompatibleConnections(0), lastIncompatibleMessage(0), transportId(transportId) {
 	degraded = makeReference<AsyncVar<bool>>(false);
 	pingLogger = pingLatencyLogger(this);
@@ -430,7 +427,7 @@ static ReliablePacket* sendPacket(TransportData* self,
                                   bool reliable);
 
 ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
-	state Endpoint remotePingEndpoint({ peer->destination }, WLTOKEN_PING_PACKET);
+	state Endpoint remotePingEndpoint({ peer->destination }, Endpoint::wellKnownToken(WLTOKEN_PING_PACKET));
 	loop {
 		if (!FlowTransport::isClient() && !peer->destination.isPublic() && peer->compatible) {
 			// Don't send ping messages to clients unless necessary. Instead monitor incoming client pings.
@@ -961,13 +958,13 @@ ACTOR static void deliver(TransportData* self,
 			if (self->isLocalAddress(destination.getPrimaryAddress())) {
 				sendLocal(self,
 				          SerializeSource<UID>(destination.token),
-				          Endpoint(destination.addresses, WLTOKEN_ENDPOINT_NOT_FOUND));
+				          Endpoint::wellKnown(destination.addresses, WLTOKEN_ENDPOINT_NOT_FOUND));
 			} else {
 				Reference<Peer> peer = self->getOrOpenPeer(destination.getPrimaryAddress());
 				sendPacket(self,
 				           peer,
 				           SerializeSource<UID>(destination.token),
-				           Endpoint(destination.addresses, WLTOKEN_ENDPOINT_NOT_FOUND),
+				           Endpoint::wellKnown(destination.addresses, WLTOKEN_ENDPOINT_NOT_FOUND),
 				           false);
 			}
 		}
@@ -1421,7 +1418,8 @@ ACTOR static Future<Void> multiVersionCleanupWorker(TransportData* self) {
 	}
 }
 
-FlowTransport::FlowTransport(uint64_t transportId) : self(new TransportData(transportId)) {
+FlowTransport::FlowTransport(uint64_t transportId, int maxWellKnownEndpoints)
+  : self(new TransportData(transportId, maxWellKnownEndpoints)) {
 	self->multiVersionCleanup = multiVersionCleanupWorker(self);
 }
 
@@ -1566,7 +1564,8 @@ static ReliablePacket* sendPacket(TransportData* self,
 
 	// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
 	if (!peer || (peer->outgoingConnectionIdle && !destination.getPrimaryAddress().isPublic()) ||
-	    (peer->incompatibleProtocolVersionNewer && destination.token != WLTOKEN_PING_PACKET)) {
+	    (peer->incompatibleProtocolVersionNewer &&
+	     destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
 		TEST(true); // Can't send to private address without a compatible open connection
 		return nullptr;
 	}
@@ -1651,7 +1650,7 @@ static ReliablePacket* sendPacket(TransportData* self,
 #endif
 
 	peer->send(pb, rp, firstUnsent);
-	if (destination.token != WLTOKEN_PING_PACKET) {
+	if (destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET)) {
 		peer->lastDataPacketSentTime = now();
 	}
 	return rp;
@@ -1716,8 +1715,9 @@ bool FlowTransport::incompatibleOutgoingConnectionsPresent() {
 	return self->numIncompatibleConnections > 0;
 }
 
-void FlowTransport::createInstance(bool isClient, uint64_t transportId) {
-	g_network->setGlobal(INetwork::enFlowTransport, (flowGlobalType) new FlowTransport(transportId));
+void FlowTransport::createInstance(bool isClient, uint64_t transportId, int maxWellKnownEndpoints) {
+	g_network->setGlobal(INetwork::enFlowTransport,
+	                     (flowGlobalType) new FlowTransport(transportId, maxWellKnownEndpoints));
 	g_network->setGlobal(INetwork::enNetworkAddressFunc, (flowGlobalType)&FlowTransport::getGlobalLocalAddress);
 	g_network->setGlobal(INetwork::enNetworkAddressesFunc, (flowGlobalType)&FlowTransport::getGlobalLocalAddresses);
 	g_network->setGlobal(INetwork::enFailureMonitor, (flowGlobalType) new SimpleFailureMonitor());

fdbrpc/FlowTransport.h

@@ -31,6 +31,8 @@
 #include "flow/Net2Packet.h"
 #include "fdbrpc/ContinuousSample.h"
 
+enum { WLTOKEN_ENDPOINT_NOT_FOUND = 0, WLTOKEN_PING_PACKET, WLTOKEN_FIRST_AVAILABLE };
+
 #pragma pack(push, 4)
 class Endpoint {
 public:
@@ -46,6 +48,12 @@ public:
 		choosePrimaryAddress();
 	}
 
+	static Token wellKnownToken(int wlTokenID) { return UID(-1, wlTokenID); }
+
+	static Endpoint wellKnown(const NetworkAddressList& addresses, int wlTokenID) {
+		return Endpoint(addresses, wellKnownToken(wlTokenID));
+	}
+
 	void choosePrimaryAddress() {
 		if (addresses.secondaryAddress.present() &&
 		    ((!g_network->getLocalAddresses().secondaryAddress.present() &&
@@ -175,12 +183,12 @@ struct Peer : public ReferenceCounted<Peer> {
 
 class FlowTransport {
 public:
-	FlowTransport(uint64_t transportId);
+	FlowTransport(uint64_t transportId, int maxWellKnownEndpoints);
 	~FlowTransport();
 
 	// Creates a new FlowTransport and makes FlowTransport::transport() return it.  This uses g_network->global()
 	// variables, so it will be private to a simulation.
-	static void createInstance(bool isClient, uint64_t transportId);
+	static void createInstance(bool isClient, uint64_t transportId, int maxWellKnownEndpoints);
 
 	static bool isClient() { return g_network->global(INetwork::enClientFailureMonitor) != nullptr; }
 

fdbrpc/LoadBalance.actor.h

@@ -41,8 +41,6 @@
 #include "fdbrpc/TSSComparison.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-using std::vector;
-
 ACTOR Future<Void> allAlternativesFailedDelay(Future<Void> okFuture);
 
 struct ModelHolder : NonCopyable, public ReferenceCounted<ModelHolder> {
@@ -609,7 +607,7 @@ Future<REPLY_TYPE(Request)> loadBalance(
 		if (!stream && !firstRequestData.isValid()) {
 			// Everything is down!  Wait for someone to be up.
 
-			vector<Future<Void>> ok(alternatives->size());
+			std::vector<Future<Void>> ok(alternatives->size());
 			for (int i = 0; i < ok.size(); i++) {
 				ok[i] = IFailureMonitor::failureMonitor().onStateEqual(alternatives->get(i, channel).getEndpoint(),
 				                                                       FailureStatus(false));
@@ -769,7 +767,7 @@ Future<REPLY_TYPE(Request)> basicLoadBalance(Reference<ModelInterface<Multi>> al
 		if (!stream) {
 			// Everything is down!  Wait for someone to be up.
 
-			vector<Future<Void>> ok(alternatives->size());
+			std::vector<Future<Void>> ok(alternatives->size());
 			for (int i = 0; i < ok.size(); i++) {
 				ok[i] = IFailureMonitor::failureMonitor().onStateEqual(alternatives->get(i, channel).getEndpoint(),
 				                                                       FailureStatus(false));

fdbrpc/MultiInterface.h

@@ -97,7 +97,7 @@ public:
 	// If balanceOnRequests is true, the client will load balance based on the number of GRVs released by each proxy
 	// If balanceOnRequests is false, the client will load balance based on the CPU usage of each proxy
 	// Only requests which take from the GRV budget on the proxy should set balanceOnRequests to true
-	ModelInterface(const vector<T>& v, bool balanceOnRequests) : balanceOnRequests(balanceOnRequests) {
+	ModelInterface(const std::vector<T>& v, bool balanceOnRequests) : balanceOnRequests(balanceOnRequests) {
 		for (int i = 0; i < v.size(); i++) {
 			alternatives.push_back(AlternativeInfo(v[i], 1.0 / v.size(), (i + 1.0) / v.size()));
 		}
@@ -174,14 +174,14 @@ public:
 	std::string description() { return describe(alternatives); }
 
 private:
-	vector<AlternativeInfo<T>> alternatives;
+	std::vector<AlternativeInfo<T>> alternatives;
 	Future<Void> updater;
 	bool balanceOnRequests;
 };
 
 template <class T>
 class MultiInterface : public ReferenceCounted<MultiInterface<T>> {
-	MultiInterface(const vector<T>& v, LocalityData const& locality = LocalityData()) {
+	MultiInterface(const std::vector<T>& v, LocalityData const& locality = LocalityData()) {
 		// This version of MultInterface is no longer used, but was kept around because of templating
 		ASSERT(false);
 	}

fdbrpc/PerfMetric.h

@@ -27,8 +27,6 @@
 #include "flow/BooleanParam.h"
 #include "flow/flow.h"
 
-using std::vector;
-
 FDB_DECLARE_BOOLEAN_PARAM(Averaged);
 
 struct PerfMetric {
@@ -62,7 +60,7 @@ private:
 
 struct PerfIntCounter {
 	PerfIntCounter(std::string name) : name(name), value(0) {}
-	PerfIntCounter(std::string name, vector<PerfIntCounter*>& v) : name(name), value(0) { v.push_back(this); }
+	PerfIntCounter(std::string name, std::vector<PerfIntCounter*>& v) : name(name), value(0) { v.push_back(this); }
 	void operator+=(int64_t delta) { value += delta; }
 	void operator++() { value += 1; }
 	PerfMetric getMetric() const { return PerfMetric(name, static_cast<double>(value), Averaged::False, "%.0lf"); }
@@ -76,7 +74,9 @@ private:
 
 struct PerfDoubleCounter {
 	PerfDoubleCounter(std::string name) : name(name), value(0) {}
-	PerfDoubleCounter(std::string name, vector<PerfDoubleCounter*>& v) : name(name), value(0) { v.push_back(this); }
+	PerfDoubleCounter(std::string name, std::vector<PerfDoubleCounter*>& v) : name(name), value(0) {
+		v.push_back(this);
+	}
 	void operator+=(double delta) { value += delta; }
 	void operator++() { value += 1.0; }
 	PerfMetric getMetric() const { return PerfMetric(name, value, Averaged::False); }

fdbrpc/Stats.actor.cpp

@@ -99,6 +99,11 @@ ACTOR Future<Void> traceCounters(std::string traceEventName,
 	for (ICounter* c : counters->counters)
 		c->resetInterval();
 
+	state Reference<EventCacheHolder> traceEventHolder;
+	if (!trackLatestName.empty()) {
+		traceEventHolder = makeReference<EventCacheHolder>(trackLatestName);
+	}
+
 	state double last_interval = now();
 
 	loop {
@@ -109,7 +114,7 @@ ACTOR Future<Void> traceCounters(std::string traceEventName,
 		decorator(te);
 
 		if (!trackLatestName.empty()) {
-			te.trackLatest(trackLatestName);
+			te.trackLatest(traceEventHolder->trackingKey);
 		}
 
 		last_interval = now();

fdbrpc/Stats.h

@@ -227,7 +227,8 @@ private:
 class LatencySample {
 public:
 	LatencySample(std::string name, UID id, double loggingInterval, int sampleSize)
-	  : name(name), id(id), sampleStart(now()), sample(sampleSize) {
+	  : name(name), id(id), sampleStart(now()), sample(sampleSize),
+	    latencySampleEventHolder(makeReference<EventCacheHolder>(id.toString() + "/" + name)) {
 		logger = recurring([this]() { logSample(); }, loggingInterval);
 	}
 
@@ -241,6 +242,8 @@ private:
 	ContinuousSample<double> sample;
 	Future<Void> logger;
 
+	Reference<EventCacheHolder> latencySampleEventHolder;
+
 	void logSample() {
 		TraceEvent(name.c_str(), id)
 		    .detail("Count", sample.getPopulationSize())
@@ -254,7 +257,7 @@ private:
 		    .detail("P95", sample.percentile(0.95))
 		    .detail("P99", sample.percentile(0.99))
 		    .detail("P99.9", sample.percentile(0.999))
-		    .trackLatest(id.toString() + "/" + name);
+		    .trackLatest(latencySampleEventHolder->trackingKey);
 
 		sample.clear();
 		sampleStart = now();

fdbrpc/dsltest.actor.cpp

@@ -28,16 +28,11 @@
 #include "flow/ThreadHelper.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-using std::cout;
-using std::endl;
-
-using std::vector;
-
 void* allocateLargePages(int total);
 
 bool testFuzzActor(Future<int> (*actor)(FutureStream<int> const&, PromiseStream<int> const&, Future<Void> const&),
                    const char* desc,
-                   vector<int> const& expectedOutput) {
+                   std::vector<int> const& expectedOutput) {
 	// Run the test 5 times with different "timing"
 	int i, outCount;
 	bool ok = true;
@@ -130,21 +125,21 @@ void memoryTest2() {
 	for(int threads=1; threads<=MaxThreads; threads++) {
 		double tstart = timer();
 
-		vector<ThreadFuture<Void>> done;
+		std::vector<ThreadFuture<Void>> done;
 		for(int t=0; t<threads; t++) {
 			char** r = random + Reads*t;
 			done.push_back(
 				inThread<Void>( [r,Reads] () -> Void {
 					for(int i=0; i<Reads; i++)
 						if ( *r[i] )
-							cout << "Does not happen" << endl;
+							std::cout << "Does not happen" << std::endl;
 					return Void();
 				}));
 		}
 		waitForAll(done).getBlocking();
 		double duration = timer() - tstart;
 
-		cout << format("%d threads: %f sec, %0.2fM/sec", threads, duration, Reads*threads/1e6/duration) << endl;
+		std::cout << format("%d threads: %f sec, %0.2fM/sec", threads, duration, Reads*threads/1e6/duration) << std::endl;
 	}
 }
 #endif
@@ -163,17 +158,17 @@ void memoryTest() {
 
 	const int N = 128<<20;	// 128 = 1GB
 	const int N2 = 8<<20;
-	cout << "Preparing memory test with " << N / 1e6 * sizeof(void*) << " MB" << endl;
+	std::cout << "Preparing memory test with " << N / 1e6 * sizeof(void*) << " MB" << std::endl;
 	void **x;
 	if (0) {
-		cout << "  NUMA large pages" << endl;
+		std::cout << "  NUMA large pages" << std::endl;
 		x = (void**)numaAllocate(size_t(N)*sizeof(void*));
 	} else if (1) {
-		cout << "  Normal pages" << endl;
+		std::cout << "  Normal pages" << std::endl;
 		x = new void*[ N ];
 		printf("  at %p\n", x);
 	} else {
-		cout << "  Large pages" << endl;
+		std::cout << "  Large pages" << std::endl;
 		x = (void**)allocate(N*sizeof(void*), true);
 	}
 	memset(x, 0, ((int64_t)N) * sizeof(void*));
@@ -181,7 +176,7 @@ void memoryTest() {
 	showNumaStatus();
 
 	if (1) {
-		cout <<"  Random permutation" << endl;
+		std::cout <<"  Random permutation" << std::endl;
 		// Random cyclic permutation
 		for(int i=0; i<N; i++)
 			x[i] = &x[i];
@@ -191,7 +186,7 @@ void memoryTest() {
 			std::swap( x[k], x[n] );
 		}
 	} else {
-		cout <<"  Sequential permutation" << endl;
+		std::cout <<"  Sequential permutation" << std::endl;
 		// Sequential
 		for(int i=0; i<N-1; i++)
 			x[i] = &x[i+1];
@@ -201,7 +196,7 @@ void memoryTest() {
 	for(int i=0; i<N; i++) {
 		p = (void**)*p;
 		if (p == x) {
-			cout << "Cycle " << i << endl;
+			std::cout << "Cycle " << i << std::endl;
 			if (i != N-1) terminate();
 		}
 	}
@@ -217,7 +212,7 @@ void memoryTest() {
 		}
 		for(int T=1; T<=MT; T+=T) {
 			double start = timer();
-			vector< Future<double> > done;
+			std::vector< Future<double> > done;
 			for(int t=0; t<T; t++) {
 				void*** start = starts + t*TraversalsPerThread;
 				done.push_back(
@@ -233,7 +228,7 @@ void memoryTest() {
 							}
 						for(int j=0; j<TraversalsPerThread; j++)
 							if (p[j] == p[(j+1)%TraversalsPerThread])
-								cout << "N";
+								std::cout << "N";
 						return timer();
 					}));
 			}
@@ -262,14 +257,14 @@ ACTOR template <int N, class X>
 ACTOR template <class A, class B>
 [[flow_allow_discard]] Future<Void> switchTest(FutureStream<A> as, Future<B> oneb) {
 	loop choose {
-		when(A a = waitNext(as)) { cout << "A " << a << endl; }
+		when(A a = waitNext(as)) { std::cout << "A " << a << std::endl; }
 		when(B b = wait(oneb)) {
-			cout << "B " << b << endl;
+			std::cout << "B " << b << std::endl;
 			break;
 		}
 	}
 	loop {
-		cout << "Done!" << endl;
+		std::cout << "Done!" << std::endl;
 		return Void();
 	}
 }
@@ -287,7 +282,7 @@ public:
 	}
 #if !defined(__INTEL_COMPILER)
 	void operator delete(void* buf) {
-		cout << "Freeing buffer" << endl;
+		std::cout << "Freeing buffer" << std::endl;
 		delete[](int*) buf;
 	}
 #endif
@@ -344,12 +339,12 @@ void fastAllocTest() {
 	}
 	std::sort(d.begin(), d.end());
 	if (std::unique(d.begin(), d.end()) != d.end())
-		cout << "Pointer returned twice!?" << endl;
+		std::cout << "Pointer returned twice!?" << std::endl;
 
 	for (int i = 0; i < 2; i++) {
 		void* p = FastAllocator<64>::allocate();
 		void* q = FastAllocator<64>::allocate();
-		cout << (intptr_t)p << " " << (intptr_t)q << endl;
+		std::cout << (intptr_t)p << " " << (intptr_t)q << std::endl;
 		FastAllocator<64>::release(p);
 		FastAllocator<64>::release(q);
 	}
@@ -358,13 +353,13 @@ void fastAllocTest() {
 	for (int i = 0; i < 1000000; i++)
 		(void)FastAllocator<64>::allocate();
 	t = timer() - t;
-	cout << "Allocations: " << (1 / t) << "M/sec" << endl;
+	std::cout << "Allocations: " << (1 / t) << "M/sec" << std::endl;
 
 	t = timer();
 	for (int i = 0; i < 1000000; i++)
 		FastAllocator<64>::release(FastAllocator<64>::allocate());
 	t = timer() - t;
-	cout << "Allocate/Release pairs: " << (1 / t) << "M/sec" << endl;
+	std::cout << "Allocate/Release pairs: " << (1 / t) << "M/sec" << std::endl;
 
 	t = timer();
 	void* pp[100];
@@ -375,13 +370,13 @@ void fastAllocTest() {
 			FastAllocator<64>::release(pp[j]);
 	}
 	t = timer() - t;
-	cout << "Allocate/Release interleaved(100): " << (1 / t) << "M/sec" << endl;
+	std::cout << "Allocate/Release interleaved(100): " << (1 / t) << "M/sec" << std::endl;
 
 	t = timer();
 	for (int i = 0; i < 1000000; i++)
 		delete new TestB;
 	t = timer() - t;
-	cout << "Allocate/Release TestB pairs: " << (1 / t) << "M/sec" << endl;
+	std::cout << "Allocate/Release TestB pairs: " << (1 / t) << "M/sec" << std::endl;
 
 #if FLOW_THREAD_SAFE
 	t = timer();
@@ -399,8 +394,8 @@ void fastAllocTest() {
 		}));
 	waitForAll(results).getBlocking();
 	t = timer() - t;
-	cout << "Threaded Allocate/Release TestB interleaved (100): " << results.size() << " x " << (1 / t) << "M/sec"
-	     << endl;
+	std::cout << "Threaded Allocate/Release TestB interleaved (100): " << results.size() << " x " << (1 / t) << "M/sec"
+	          << std::endl;
 #endif
 
 	volatile int32_t v = 0;
@@ -409,7 +404,7 @@ void fastAllocTest() {
 	for (int i = 0; i < 10000000; i++)
 		interlockedIncrement(&v);
 	t = timer() - t;
-	cout << "interlocked increment: " << 10.0 / t << "M/sec " << v << endl;
+	std::cout << "interlocked increment: " << 10.0 / t << "M/sec " << v << std::endl;
 
 	v = 5;
 	t = timer();
@@ -417,14 +412,14 @@ void fastAllocTest() {
 		interlockedCompareExchange(&v, 5, 5);
 	}
 	t = timer() - t;
-	cout << "1 state machine: " << 10.0 / t << "M/sec " << v << endl;
+	std::cout << "1 state machine: " << 10.0 / t << "M/sec " << v << std::endl;
 
 	v = 0;
 	t = timer();
 	for (int i = 0; i < 10000000; i++)
 		v++;
 	t = timer() - t;
-	cout << "volatile increment: " << 10.0 / t << "M/sec " << v << endl;
+	std::cout << "volatile increment: " << 10.0 / t << "M/sec " << v << std::endl;
 
 	{
 		Reference<TestBuffer> b(TestBuffer::create(1000));
@@ -436,14 +431,14 @@ void fastAllocTest() {
 			b = std::move(r);
 		}
 		t = timer() - t;
-		cout << "move Reference<Buffer>: " << 10.0 / t << "M/sec " << endl;
+		std::cout << "move Reference<Buffer>: " << 10.0 / t << "M/sec " << std::endl;
 
 		t = timer();
 		for (int i = 0; i < 10000000; i++) {
 			Reference<TestBuffer> r = b;
 		}
 		t = timer() - t;
-		cout << "copy (1) Reference<Buffer>: " << 10.0 / t << "M/sec " << endl;
+		std::cout << "copy (1) Reference<Buffer>: " << 10.0 / t << "M/sec " << std::endl;
 
 		Reference<TestBuffer> c = b;
 		t = timer();
@@ -451,27 +446,27 @@ void fastAllocTest() {
 			Reference<TestBuffer> r = b;
 		}
 		t = timer() - t;
-		cout << "copy (2) Reference<Buffer>: " << 10.0 / t << "M/sec " << endl;
+		std::cout << "copy (2) Reference<Buffer>: " << 10.0 / t << "M/sec " << std::endl;
 
-		cout << (const char*)b->begin() << endl;
+		std::cout << (const char*)b->begin() << std::endl;
 	}
 	t = timer();
 	for (int i = 0; i < 10000000; i++) {
 		delete new FastKey;
 	}
 	t = timer() - t;
-	cout << "delete new FastKey: " << 10.0 / t << "M/sec " << fastKeyCount << endl;
+	std::cout << "delete new FastKey: " << 10.0 / t << "M/sec " << fastKeyCount << std::endl;
 
 	t = timer();
 	for (int i = 0; i < 10000000; i++) {
 		Reference<FastKey> r(new FastKey);
 	}
 	t = timer() - t;
-	cout << "new Reference<FastKey>: " << 10.0 / t << "M/sec " << fastKeyCount << endl;
+	std::cout << "new Reference<FastKey>: " << 10.0 / t << "M/sec " << fastKeyCount << std::endl;
 }
 
 template <class PromiseT>
-Future<Void> threadSafetySender(vector<PromiseT>& v, Event& start, Event& ready, int iterations) {
+Future<Void> threadSafetySender(std::vector<PromiseT>& v, Event& start, Event& ready, int iterations) {
 	for (int i = 0; i < iterations; i++) {
 		start.block();
 		if (v.size() == 0)
@@ -500,7 +495,7 @@ void threadSafetyTest() {
 
 	int N = 10000, V = 100;
 
-	vector<Promise<Void>> v;
+	std::vector<Promise<Void>> v;
 	Event start, ready;
 	Future<Void> sender = inThread<Void>( [&] { return threadSafetySender( v, start, ready, N ); } );
 
@@ -508,7 +503,7 @@ void threadSafetyTest() {
 		v.clear();
 		for (int j = 0; j < V; j++)
 			v.push_back(Promise<Void>());
-		vector<Future<Void>> f( v.size() );
+		std::vector<Future<Void>> f( v.size() );
 		for(int i=0; i<v.size(); i++)
 			f[i] = v[i].getFuture();
 		std::random_shuffle( f.begin(), f.end() );
@@ -520,11 +515,11 @@ void threadSafetyTest() {
 		ready.block();
 
 		if (count != V)
-			cout << "Thread safety error: " << count << endl;
+			std::cout << "Thread safety error: " << count << std::endl;
 	}
 
 	t = timer()-t;
-	cout << "Thread safety test (2t): " << (V*N/1e6/t) << "M/sec" << endl;
+	std::cout << "Thread safety test (2t): " << (V*N/1e6/t) << "M/sec" << std::endl;
 }
 
 void threadSafetyTest2() {
@@ -532,16 +527,16 @@ void threadSafetyTest2() {
 
 	int N = 1000, V = 100;
 
-	// vector<PromiseStream<Void>> streams( 100 );
-	vector<PromiseStream<Void>> streams;
+	// std::vector<PromiseStream<Void>> streams( 100 );
+	std::vector<PromiseStream<Void>> streams;
 	for (int i = 0; i < 100; i++)
 		streams.push_back(PromiseStream<Void>());
-	vector<PromiseStream<Void>> v;
+	std::vector<PromiseStream<Void>> v;
 	Event start, ready;
 	Future<Void> sender = inThread<Void>( [&] { return threadSafetySender( v, start, ready, N ); } );
 
 	for(int i=0; i<N; i++) {
-		vector<int> counts( streams.size() );
+		std::vector<int> counts( streams.size() );
 		v.clear();
 		for(int k=0; k<V; k++) {
 			int i = deterministicRandom()->randomInt(0, (int)streams.size());
@@ -556,11 +551,11 @@ void threadSafetyTest2() {
 		ready.block();
 
 		if (count != V)
-			cout << "Thread safety error: " << count << endl;
+			std::cout << "Thread safety error: " << count << std::endl;
 	}
 
 	t = timer()-t;
-	cout << "Thread safety test 2 (2t): " << (V*N/1e6/t) << "M/sec" << endl;
+	std::cout << "Thread safety test 2 (2t): " << (V*N/1e6/t) << "M/sec" << std::endl;
 }
 
 volatile int32_t cancelled = 0, returned = 0;
@@ -579,8 +574,8 @@ ACTOR [[flow_allow_discard]] Future<Void> returnCancelRacer( Future<Void> f ) {
 void returnCancelRaceTest() {
 	int N = 100, M = 100;
 	for(int i=0; i<N; i++) {
-		vector< Promise<Void> > promises;
-		vector< Future<Void> > futures;
+		std::vector< Promise<Void> > promises;
+		std::vector< Future<Void> > futures;
 		for(int i=0; i < M; i++) {
 			promises.push_back( Promise<Void>() );
 			futures.push_back( returnCancelRacer( promises.back().getFuture() ) );
@@ -654,8 +649,8 @@ void arenaTest() {
 
 		for (auto i = test.begin(); i != test.end(); ++i)
 			for (auto j = i->begin(); j != i->end(); ++j)
-				cout << *j;
-		cout << endl;
+				std::cout << *j;
+		std::cout << std::endl;
 
 		wr << test;
 	}
@@ -667,8 +662,8 @@ void arenaTest() {
 
 		for (auto i = test2.begin(); i != test2.end(); ++i)
 			for (auto j = i->begin(); j != i->end(); ++j)
-				cout << *j;
-		cout << endl;
+				std::cout << *j;
+		std::cout << std::endl;
 	}
 
 	double t = timer();
@@ -686,7 +681,7 @@ void arenaTest() {
 ACTOR [[flow_allow_discard]] void testStream(FutureStream<int> xs) {
 	loop {
 		int x = waitNext(xs);
-		cout << x << endl;
+		std::cout << x << std::endl;
 	}
 }
 
@@ -947,10 +942,8 @@ ACTOR [[flow_allow_discard]] Future<Void> cancellable2() {
 	}
 }
 
-using std::string;
-
-ACTOR [[flow_allow_discard]] Future<int> introLoadValueFromDisk(Future<string> filename) {
-	string file = wait(filename);
+ACTOR [[flow_allow_discard]] Future<int> introLoadValueFromDisk(Future<std::string> filename) {
+	std::string file = wait(filename);
 
 	if (file == "/dev/threes")
 		return 3;
@@ -1075,7 +1068,7 @@ ACTOR [[flow_allow_discard]] void cycle(FutureStream<Void> in, PromiseStream<Voi
 }
 
 ACTOR [[flow_allow_discard]] Future<Void> cycleTime(int nodes, int times) {
-	state vector<PromiseStream<Void>> n(nodes);
+	state std::vector<PromiseStream<Void>> n(nodes);
 	state int total = 0;
 
 	// 1->2, 2->3, ..., n-1->0
@@ -1361,14 +1354,14 @@ void dsltest() {
 		Future<int> c = chooseTest(a.getFuture(), b.getFuture());
 		a.send(1);
 		b.send(2);
-		cout << "c=" << c.get() << endl;
+		std::cout << "c=" << c.get() << std::endl;
 	}
 
 	{
 		Promise<double> i;
 		Future<double> d = addN<20>(i.getFuture());
 		i.send(1.1);
-		cout << d.get() << endl;
+		std::cout << d.get() << std::endl;
 	}
 
 	{
@@ -1376,9 +1369,9 @@ void dsltest() {
 		i.sendError(operation_failed());
 		Future<double> d = addN<20>(i.getFuture());
 		if (d.isError() && d.getError().code() == error_code_operation_failed)
-			cout << "Error transmitted OK" << endl;
+			std::cout << "Error transmitted OK" << std::endl;
 		else
-			cout << "Error not transmitted!" << endl;
+			std::cout << "Error not transmitted!" << std::endl;
 	}
 
 	/*{
@@ -1386,10 +1379,10 @@ void dsltest() {
 	    PromiseStream<int> t;
 	    testStream(t.getFuture());
 	    if (Actor::allActors.size() != na+1)
-	        cout << "Actor not created!" << endl;
+	        std::cout << "Actor not created!" << std::endl;
 	    t = PromiseStream<int>();
 	    if (Actor::allActors.size() != na)
-	        cout << "Actor not cleaned up!" << endl;
+	        std::cout << "Actor not cleaned up!" << std::endl;
 	}*/
 
 	PromiseStream<int> as;
@@ -1439,7 +1432,7 @@ void pingtest() {
     Future<Void> pS = pingServer( serverInterface.getFuture(), 5000000 );
     Future<int> count = ping( serverInterface );
     double end = timer();
-    cout << count.get() << " pings completed in " << (end-start) << " sec" << endl;
+    std::cout << count.get() << " pings completed in " << (end-start) << " sec" << std::endl;
 }*/
 
 void copyTest() {
@@ -1495,12 +1488,12 @@ void copyTest() {
 
     loop choose {
         when( int j = waitNext( js.getFuture() ) ) {
-            cout << "J" << j << endl;
+            std::cout << "J" << j << std::endl;
         }
         when( int i = waitNext( is ) ) {
-            cout << "I" << i << endl;
+            std::cout << "I" << i << std::endl;
             js.send( i );
-            cout << "-I" << i << endl;
+            std::cout << "-I" << i << std::endl;
         }
     }
 }

fdbrpc/fdbrpc.h

@@ -255,6 +255,7 @@ void setReplyPriority(const ReplyPromise<Reply>& p, TaskPriority taskID) {
 
 struct ReplyPromiseStreamReply {
 	Optional<UID> acknowledgeToken;
+	uint16_t sequence;
 	ReplyPromiseStreamReply() {}
 };
 
@@ -277,15 +278,15 @@ struct AcknowledgementReceiver final : FlowReceiver, FastAllocated<Acknowledgeme
 	using FastAllocated<AcknowledgementReceiver>::operator new;
 	using FastAllocated<AcknowledgementReceiver>::operator delete;
 
-	int64_t bytesSent;
-	int64_t bytesAcknowledged;
-	int64_t bytesLimit;
+	uint16_t sequence = 0;
+	int64_t bytesSent = 0;
+	int64_t bytesAcknowledged = 0;
+	int64_t bytesLimit = 0;
 	Promise<Void> ready;
 	Future<Void> failures;
 
-	AcknowledgementReceiver() : bytesSent(0), bytesAcknowledged(0), bytesLimit(0), ready(nullptr) {}
-	AcknowledgementReceiver(const Endpoint& remoteEndpoint)
-	  : FlowReceiver(remoteEndpoint, false), bytesSent(0), bytesAcknowledged(0), bytesLimit(0), ready(nullptr) {}
+	AcknowledgementReceiver() : ready(nullptr) {}
+	AcknowledgementReceiver(const Endpoint& remoteEndpoint) : FlowReceiver(remoteEndpoint, false), ready(nullptr) {}
 
 	void receive(ArenaObjectReader& reader) override {
 		ErrorOr<AcknowledgementReply> message;
@@ -353,20 +354,29 @@ struct NetNotifiedQueueWithAcknowledgements final : NotifiedQueue<T>,
 				acknowledgements = AcknowledgementReceiver(
 				    FlowTransport::transport().loadedEndpoint(message.get().asUnderlyingType().acknowledgeToken.get()));
 			}
-			if (this->shouldFireImmediately()) {
-				// This message is going to be consumed by the client immediately (and therefore will not call pop()) so
-				// send an ack immediately
-				if (acknowledgements.getRawEndpoint().isValid()) {
-					acknowledgements.bytesAcknowledged += message.get().asUnderlyingType().expectedSize();
-					FlowTransport::transport().sendUnreliable(
-					    SerializeSource<ErrorOr<AcknowledgementReply>>(
-					        AcknowledgementReply(acknowledgements.bytesAcknowledged)),
-					    acknowledgements.getEndpoint(TaskPriority::ReadSocket),
-					    false);
+			if (acknowledgements.sequence != message.get().asUnderlyingType().sequence) {
+				TraceEvent(SevError, "StreamSequenceMismatch")
+				    .detail("Expected", acknowledgements.sequence)
+				    .detail("Actual", message.get().asUnderlyingType().sequence);
+				ASSERT_WE_THINK(false);
+				this->sendError(connection_failed());
+			} else {
+				acknowledgements.sequence++;
+				if (this->shouldFireImmediately()) {
+					// This message is going to be consumed by the client immediately (and therefore will not call
+					// pop()) so send an ack immediately
+					if (acknowledgements.getRawEndpoint().isValid()) {
+						acknowledgements.bytesAcknowledged += message.get().asUnderlyingType().expectedSize();
+						FlowTransport::transport().sendUnreliable(
+						    SerializeSource<ErrorOr<AcknowledgementReply>>(
+						        AcknowledgementReply(acknowledgements.bytesAcknowledged)),
+						    acknowledgements.getEndpoint(TaskPriority::ReadSocket),
+						    false);
+					}
 				}
-			}
 
-			this->send(std::move(message.get().asUnderlyingType()));
+				this->send(std::move(message.get().asUnderlyingType()));
+			}
 		}
 		this->delPromiseRef();
 	}
@@ -413,10 +423,14 @@ public:
 	template <class U>
 	void send(U&& value) const {
 		if (queue->isRemoteEndpoint()) {
+			if (queue->acknowledgements.failures.isError()) {
+				throw queue->acknowledgements.failures.getError();
+			}
 			if (!queue->acknowledgements.getRawEndpoint().isValid()) {
 				// register acknowledge receiver on sender and tell the receiver where to send acknowledge messages
 				value.acknowledgeToken = queue->acknowledgements.getEndpoint(TaskPriority::ReadSocket).token;
 			}
+			value.sequence = queue->acknowledgements.sequence++;
 			queue->acknowledgements.bytesSent += value.expectedSize();
 			FlowTransport::transport().sendUnreliable(
 			    SerializeSource<ErrorOr<EnsureTable<T>>>(value), getEndpoint(), false);
@@ -784,8 +798,9 @@ public:
 	const Endpoint& getEndpoint(TaskPriority taskID = TaskPriority::DefaultEndpoint) const {
 		return queue->getEndpoint(taskID);
 	}
-	void makeWellKnownEndpoint(Endpoint::Token token, TaskPriority taskID) {
-		queue->makeWellKnownEndpoint(token, taskID);
+
+	void makeWellKnownEndpoint(uint64_t wlTokenID, TaskPriority taskID) {
+		queue->makeWellKnownEndpoint(Endpoint::Token(-1, wlTokenID), taskID);
 	}
 
 	bool operator==(const RequestStream<T>& rhs) const { return queue == rhs.queue; }

fdbrpc/simulator.h

@@ -440,6 +440,8 @@ public:
 	bool hasDiffProtocolProcess; // true if simulator is testing a process with a different version
 	bool setDiffProtocol; // true if a process with a different protocol version has been started
 
+	bool allowStorageMigrationTypeChange = false;
+
 	flowGlobalType global(int id) const final { return getCurrentProcess()->global(id); };
 	void setGlobal(size_t id, flowGlobalType v) final { getCurrentProcess()->setGlobal(id, v); };
 

fdbserver/ApplyMetadataMutation.cpp

@@ -154,7 +154,7 @@ private:
 
 		KeyRef end = keyInfo->rangeContaining(k).end();
 		KeyRangeRef insertRange(k, end);
-		vector<UID> src, dest;
+		std::vector<UID> src, dest;
 		// txnStateStore is always an in-memory KVS, and must always be recovered before
 		// applyMetadataMutations is called, so a wait here should never be needed.
 		Future<RangeResult> fResult = txnStateStore->readRange(serverTagKeys);
@@ -261,7 +261,7 @@ private:
 			}
 			if (k != allKeys.end) {
 				KeyRef end = cacheInfo->rangeContaining(k).end();
-				vector<uint16_t> serverIndices;
+				std::vector<uint16_t> serverIndices;
 				decodeStorageCacheValue(m.param2, serverIndices);
 				cacheInfo->insert(KeyRangeRef(k, end), serverIndices.size() > 0);
 			}
@@ -930,7 +930,7 @@ private:
 
 		std::map<KeyRef, MutationRef>::iterator itr;
 		KeyRef keyBegin, keyEnd;
-		vector<uint16_t> serverIndices;
+		std::vector<uint16_t> serverIndices;
 		MutationRef mutationBegin, mutationEnd;
 
 		for (itr = cachedRangeInfo.begin(); itr != cachedRangeInfo.end(); ++itr) {

fdbserver/BackupProgress.actor.h

@@ -43,7 +43,7 @@ public:
 	// savedVersion is used.
 	void addBackupStatus(const WorkerBackupStatus& status);
 
-	// Returns a map of tuple<Epoch, endVersion, logRouterTags> : map<tag, savedVersion>, so that
+	// Returns a map of tuple<Epoch, endVersion, logRouterTags> : std::map<tag, savedVersion>, so that
 	// the backup range should be [savedVersion + 1, endVersion) for the "tag" of the "Epoch".
 	//
 	// Specifically, the backup ranges for each old epoch are:

fdbserver/BackupWorker.actor.cpp

@@ -839,6 +839,10 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			// make sure file is saved on version boundary
 			popVersion = lastVersion;
 			numMsg = lastVersionIndex;
+
+			// If we aren't able to process any messages and the lock is blocking us from
+			// queuing more, then we are stuck. This could suggest the lock capacity is too small.
+			ASSERT(numMsg > 0 || self->lock->waiters() == 0);
 		}
 		if (((numMsg > 0 || popVersion > lastPopVersion) && self->pulling) || self->pullFinished()) {
 			TraceEvent("BackupWorkerSave", self->myId)

fdbserver/CMakeLists.txt

@@ -53,7 +53,9 @@ set(FDBSERVER_SRCS
   LocalConfiguration.h
   LogProtocolMessage.h
   LogRouter.actor.cpp
+  LogSystem.cpp
   LogSystem.h
+  LogSystemConfig.cpp
   LogSystemConfig.h
   LogSystemDiskQueueAdapter.actor.cpp
   LogSystemDiskQueueAdapter.h
@@ -121,6 +123,7 @@ set(FDBSERVER_SRCS
   StorageMetrics.h
   storageserver.actor.cpp
   TagPartitionedLogSystem.actor.cpp
+  TagPartitionedLogSystem.actor.h
   template_fdb.h
   tester.actor.cpp
   TesterInterface.actor.h
@@ -234,6 +237,7 @@ set(FDBSERVER_SRCS
   workloads/SlowTaskWorkload.actor.cpp
   workloads/SnapTest.actor.cpp
   workloads/SpecialKeySpaceCorrectness.actor.cpp
+  workloads/StreamingRangeRead.actor.cpp
   workloads/StatusWorkload.actor.cpp
   workloads/Storefront.actor.cpp
   workloads/StreamingRead.actor.cpp

fdbserver/ClusterController.actor.cpp

@@ -26,6 +26,7 @@
 
 #include "fdbrpc/FailureMonitor.h"
 #include "flow/ActorCollection.h"
+#include "flow/SystemMonitor.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/BackupInterface.h"
 #include "fdbserver/CoordinationInterface.h"
@@ -324,7 +325,7 @@ public:
 	    DatabaseConfiguration const& conf,
 	    Reference<IReplicationPolicy> const& policy,
 	    Optional<Optional<Standalone<StringRef>>> const& dcId = Optional<Optional<Standalone<StringRef>>>()) {
-		std::map<ProcessClass::Fitness, vector<WorkerDetails>> fitness_workers;
+		std::map<ProcessClass::Fitness, std::vector<WorkerDetails>> fitness_workers;
 		std::vector<WorkerDetails> results;
 		Reference<LocalitySet> logServerSet = Reference<LocalitySet>(new LocalityMap<WorkerDetails>());
 		LocalityMap<WorkerDetails>* logServerMap = (LocalityMap<WorkerDetails>*)logServerSet.getPtr();
@@ -548,7 +549,7 @@ public:
 	                                                     bool checkStable,
 	                                                     const std::set<Optional<Key>>& dcIds,
 	                                                     const std::vector<UID>& exclusionWorkerIds) {
-		std::map<std::tuple<ProcessClass::Fitness, int, bool>, vector<WorkerDetails>> fitness_workers;
+		std::map<std::tuple<ProcessClass::Fitness, int, bool>, std::vector<WorkerDetails>> fitness_workers;
 
 		// Go through all the workers to list all the workers that can be recruited.
 		for (const auto& [worker_process_id, worker_info] : id_worker) {
@@ -793,7 +794,7 @@ public:
 	                                                    bool checkStable,
 	                                                    const std::set<Optional<Key>>& dcIds,
 	                                                    const std::vector<UID>& exclusionWorkerIds) {
-		std::map<std::tuple<ProcessClass::Fitness, int, bool, bool, bool>, vector<WorkerDetails>> fitness_workers;
+		std::map<std::tuple<ProcessClass::Fitness, int, bool, bool, bool>, std::vector<WorkerDetails>> fitness_workers;
 
 		// Go through all the workers to list all the workers that can be recruited.
 		for (const auto& [worker_process_id, worker_info] : id_worker) {
@@ -930,7 +931,7 @@ public:
 	    bool checkStable = false,
 	    const std::set<Optional<Key>>& dcIds = std::set<Optional<Key>>(),
 	    const std::vector<UID>& exclusionWorkerIds = {}) {
-		std::map<std::tuple<ProcessClass::Fitness, int, bool, bool>, vector<WorkerDetails>> fitness_workers;
+		std::map<std::tuple<ProcessClass::Fitness, int, bool, bool>, std::vector<WorkerDetails>> fitness_workers;
 		std::vector<WorkerDetails> results;
 		Reference<LocalitySet> logServerSet = Reference<LocalitySet>(new LocalityMap<WorkerDetails>());
 		LocalityMap<WorkerDetails>* logServerMap = (LocalityMap<WorkerDetails>*)logServerSet.getPtr();
@@ -1383,7 +1384,7 @@ public:
 	                                               std::map<Optional<Standalone<StringRef>>, int>& id_used,
 	                                               std::map<Optional<Standalone<StringRef>>, int> preferredSharing = {},
 	                                               bool checkStable = false) {
-		std::map<std::tuple<ProcessClass::Fitness, int, bool, int>, vector<WorkerDetails>> fitness_workers;
+		std::map<std::tuple<ProcessClass::Fitness, int, bool, int>, std::vector<WorkerDetails>> fitness_workers;
 
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(role);
@@ -1413,7 +1414,7 @@ public:
 		throw no_more_servers();
 	}
 
-	vector<WorkerDetails> getWorkersForRoleInDatacenter(
+	std::vector<WorkerDetails> getWorkersForRoleInDatacenter(
 	    Optional<Standalone<StringRef>> const& dcId,
 	    ProcessClass::ClusterRole role,
 	    int amount,
@@ -1422,8 +1423,8 @@ public:
 	    std::map<Optional<Standalone<StringRef>>, int> preferredSharing = {},
 	    Optional<WorkerFitnessInfo> minWorker = Optional<WorkerFitnessInfo>(),
 	    bool checkStable = false) {
-		std::map<std::tuple<ProcessClass::Fitness, int, bool, int>, vector<WorkerDetails>> fitness_workers;
-		vector<WorkerDetails> results;
+		std::map<std::tuple<ProcessClass::Fitness, int, bool, int>, std::vector<WorkerDetails>> fitness_workers;
+		std::vector<WorkerDetails> results;
 		if (minWorker.present()) {
 			results.push_back(minWorker.get().worker);
 		}
@@ -1486,7 +1487,7 @@ public:
 		  : bestFit(ProcessClass::NeverAssign), worstFit(ProcessClass::NeverAssign), role(ProcessClass::NoRole),
 		    count(0) {}
 
-		RoleFitness(const vector<WorkerDetails>& workers,
+		RoleFitness(const std::vector<WorkerDetails>& workers,
 		            ProcessClass::ClusterRole role,
 		            const std::map<Optional<Standalone<StringRef>>, int>& id_used)
 		  : role(role) {
@@ -1813,7 +1814,7 @@ public:
 			try {
 				auto reply = findWorkersForConfigurationFromDC(req, regions[0].dcId);
 				setPrimaryDesired = true;
-				vector<Optional<Key>> dcPriority;
+				std::vector<Optional<Key>> dcPriority;
 				dcPriority.push_back(regions[0].dcId);
 				dcPriority.push_back(regions[1].dcId);
 				desiredDcIds.set(dcPriority);
@@ -1840,7 +1841,7 @@ public:
 				    .error(e);
 				auto reply = findWorkersForConfigurationFromDC(req, regions[1].dcId);
 				if (!setPrimaryDesired) {
-					vector<Optional<Key>> dcPriority;
+					std::vector<Optional<Key>> dcPriority;
 					dcPriority.push_back(regions[1].dcId);
 					dcPriority.push_back(regions[0].dcId);
 					desiredDcIds.set(dcPriority);
@@ -1853,7 +1854,7 @@ public:
 				throw;
 			}
 		} else if (req.configuration.regions.size() == 1) {
-			vector<Optional<Key>> dcPriority;
+			std::vector<Optional<Key>> dcPriority;
 			dcPriority.push_back(req.configuration.regions[0].dcId);
 			desiredDcIds.set(dcPriority);
 			auto reply = findWorkersForConfigurationFromDC(req, req.configuration.regions[0].dcId);
@@ -2007,13 +2008,13 @@ public:
 
 			if (bestDC != clusterControllerDcId) {
 				TraceEvent("BestDCIsNotClusterDC").log();
-				vector<Optional<Key>> dcPriority;
+				std::vector<Optional<Key>> dcPriority;
 				dcPriority.push_back(bestDC);
 				desiredDcIds.set(dcPriority);
 				throw no_more_servers();
 			}
 			// If this cluster controller dies, do not prioritize recruiting the next one in the same DC
-			desiredDcIds.set(vector<Optional<Key>>());
+			desiredDcIds.set(std::vector<Optional<Key>>());
 			TraceEvent("FindWorkersForConfig")
 			    .detail("Replication", req.configuration.tLogReplicationFactor)
 			    .detail("DesiredLogs", req.configuration.getDesiredLogs())
@@ -2215,7 +2216,7 @@ public:
 			getWorkerForRoleInDatacenter(
 			    regions[0].dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, {}, true);
 
-			vector<Optional<Key>> dcPriority;
+			std::vector<Optional<Key>> dcPriority;
 			dcPriority.push_back(regions[0].dcId);
 			dcPriority.push_back(regions[1].dcId);
 			desiredDcIds.set(dcPriority);
@@ -2233,16 +2234,17 @@ public:
 		    db.recoveryStalled) {
 			if (db.config.regions.size() > 1) {
 				auto regions = db.config.regions;
-				if (clusterControllerDcId.get() == regions[0].dcId) {
+				if (clusterControllerDcId.get() == regions[0].dcId && regions[1].priority >= 0) {
 					std::swap(regions[0], regions[1]);
 				}
-				ASSERT(clusterControllerDcId.get() == regions[1].dcId);
+				ASSERT(regions[1].priority < 0 || clusterControllerDcId.get() == regions[1].dcId);
 				checkRegions(regions);
 			}
 		}
 	}
 
-	void updateIdUsed(const vector<WorkerDetails>& workers, std::map<Optional<Standalone<StringRef>>, int>& id_used) {
+	void updateIdUsed(const std::vector<WorkerDetails>& workers,
+	                  std::map<Optional<Standalone<StringRef>>, int>& id_used) {
 		for (auto& it : workers) {
 			id_used[it.interf.locality.processId()]++;
 		}
@@ -2268,7 +2270,7 @@ public:
 
 		if (db.config.regions.size() > 1 && db.config.regions[0].priority > db.config.regions[1].priority &&
 		    db.config.regions[0].dcId != clusterControllerDcId.get() && versionDifferenceUpdated &&
-		    datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE) {
+		    datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE && remoteDCIsHealthy()) {
 			checkRegions(db.config.regions);
 		}
 
@@ -2831,7 +2833,7 @@ public:
 	void updateWorkerHealth(const UpdateWorkerHealthRequest& req) {
 		std::string degradedPeersString;
 		for (int i = 0; i < req.degradedPeers.size(); ++i) {
-			degradedPeersString += i == 0 ? "" : " " + req.degradedPeers[i].toString();
+			degradedPeersString += (i == 0 ? "" : " ") + req.degradedPeers[i].toString();
 		}
 		TraceEvent("ClusterControllerUpdateWorkerHealth")
 		    .detail("WorkerAddress", req.address)
@@ -2965,24 +2967,9 @@ public:
 		return currentDegradedServersWithinLimit;
 	}
 
-	// Returns true when the cluster controller should trigger a recovery due to degraded servers are used in the
-	// transaction system in the primary data center.
-	bool shouldTriggerRecoveryDueToDegradedServers() {
-		if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) {
-			return false;
-		}
-
+	// Whether the transaction system (in primary DC if in HA setting) contains degraded servers.
+	bool transactionSystemContainsDegradedServers() {
 		const ServerDBInfo dbi = db.serverInfo->get();
-		if (dbi.recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-			return false;
-		}
-
-		// Do not trigger recovery if the cluster controller is excluded, since the master will change
-		// anyways once the cluster controller is moved
-		if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) {
-			return false;
-		}
-
 		for (const auto& excludedServer : degradedServers) {
 			if (dbi.master.addresses().contains(excludedServer)) {
 				return true;
@@ -3021,6 +3008,93 @@ public:
 		return false;
 	}
 
+	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded
+	// servers.
+	bool remoteTransactionSystemContainsDegradedServers() {
+		if (db.config.usableRegions <= 1) {
+			return false;
+		}
+
+		for (const auto& excludedServer : degradedServers) {
+			if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	// Returns true if remote DC is healthy and can failover to.
+	bool remoteDCIsHealthy() {
+		// When we just start, we ignore any remote DC health info since the current CC may be elected at wrong DC due
+		// to that all the processes are still starting.
+		if (machineStartTime() == 0) {
+			return true;
+		}
+
+		if (now() - machineStartTime() < SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
+			return true;
+		}
+
+		// When remote DC health is not monitored, we may not know whether the remote is healthy or not. So return false
+		// here to prevent failover.
+		if (!remoteDCMonitorStarted) {
+			return false;
+		}
+
+		return !remoteTransactionSystemContainsDegradedServers();
+	}
+
+	// Returns true when the cluster controller should trigger a recovery due to degraded servers used in the
+	// transaction system in the primary data center.
+	bool shouldTriggerRecoveryDueToDegradedServers() {
+		if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) {
+			return false;
+		}
+
+		if (db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			return false;
+		}
+
+		// Do not trigger recovery if the cluster controller is excluded, since the master will change
+		// anyways once the cluster controller is moved
+		if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) {
+			return false;
+		}
+
+		return transactionSystemContainsDegradedServers();
+	}
+
+	// Returns true when the cluster controller should trigger a failover due to degraded servers used in the
+	// transaction system in the primary data center, and no degradation in the remote data center.
+	bool shouldTriggerFailoverDueToDegradedServers() {
+		if (db.config.usableRegions <= 1) {
+			return false;
+		}
+
+		if (SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION >
+		    SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) {
+			TraceEvent(SevWarn, "TriggerFailoverDueToDegradedServersInvalidConfig")
+			    .suppressFor(1.0)
+			    .detail("Min", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION)
+			    .detail("Max", SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION);
+			return false;
+		}
+
+		if (degradedServers.size() < SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION ||
+		    degradedServers.size() > SERVER_KNOBS->CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION) {
+			return false;
+		}
+
+		// Do not trigger recovery if the cluster controller is excluded, since the master will change
+		// anyways once the cluster controller is moved
+		if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) {
+			return false;
+		}
+
+		return transactionSystemContainsDegradedServers() && !remoteTransactionSystemContainsDegradedServers();
+	}
+
 	int recentRecoveryCountDueToHealth() {
 		while (!recentHealthTriggeredRecoveryTime.empty() &&
 		       now() - recentHealthTriggeredRecoveryTime.front() > SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL) {
@@ -3046,10 +3120,10 @@ public:
 	Optional<Standalone<StringRef>> masterProcessId;
 	Optional<Standalone<StringRef>> clusterControllerProcessId;
 	Optional<Standalone<StringRef>> clusterControllerDcId;
-	AsyncVar<Optional<vector<Optional<Key>>>> desiredDcIds; // desired DC priorities
-	AsyncVar<std::pair<bool, Optional<vector<Optional<Key>>>>>
+	AsyncVar<Optional<std::vector<Optional<Key>>>> desiredDcIds; // desired DC priorities
+	AsyncVar<std::pair<bool, Optional<std::vector<Optional<Key>>>>>
 	    changingDcIds; // current DC priorities to change first, and whether that is the cluster controller
-	AsyncVar<std::pair<bool, Optional<vector<Optional<Key>>>>>
+	AsyncVar<std::pair<bool, Optional<std::vector<Optional<Key>>>>>
 	    changedDcIds; // current DC priorities to change second, and whether the cluster controller has been changed
 	UID id;
 	std::vector<RecruitFromConfigurationRequest> outstandingRecruitmentRequests;
@@ -3073,6 +3147,9 @@ public:
 	PromiseStream<Future<Void>> addActor;
 	bool versionDifferenceUpdated;
 
+	bool remoteDCMonitorStarted;
+	bool remoteTransactionSystemDegraded;
+
 	// recruitX is used to signal when role X needs to be (re)recruited.
 	// recruitingXID is used to track the ID of X's interface which is being recruited.
 	// We use AsyncVars to kill (i.e. halt) singletons that have been replaced.
@@ -3110,6 +3187,8 @@ public:
 	Counter registerMasterRequests;
 	Counter statusRequests;
 
+	Reference<EventCacheHolder> recruitedMasterWorkerEventHolder;
+
 	ClusterControllerData(ClusterControllerFullInterface const& ccInterface,
 	                      LocalityData const& locality,
 	                      ServerCoordinators const& coordinators)
@@ -3117,14 +3196,16 @@ public:
 	    clusterControllerDcId(locality.dcId()), id(ccInterface.id()), ac(false), outstandingRequestChecker(Void()),
 	    outstandingRemoteRequestChecker(Void()), startTime(now()), goodRecruitmentTime(Never()),
 	    goodRemoteRecruitmentTime(Never()), datacenterVersionDifference(0), versionDifferenceUpdated(false),
-	    recruitDistributor(false), recruitRatekeeper(false), recruitBlobManager(false),
+	    remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false), recruitDistributor(false),
+	    recruitRatekeeper(false), recruitBlobManager(false),
 	    clusterControllerMetrics("ClusterController", id.toString()),
 	    openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics),
 	    registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics),
 	    getWorkersRequests("GetWorkersRequests", clusterControllerMetrics),
 	    getClientWorkersRequests("GetClientWorkersRequests", clusterControllerMetrics),
 	    registerMasterRequests("RegisterMasterRequests", clusterControllerMetrics),
-	    statusRequests("StatusRequests", clusterControllerMetrics) {
+	    statusRequests("StatusRequests", clusterControllerMetrics),
+	    recruitedMasterWorkerEventHolder(makeReference<EventCacheHolder>("RecruitedMasterWorker")) {
 		auto serverInfo = ServerDBInfo();
 		serverInfo.id = deterministicRandom()->randomUniqueID();
 		serverInfo.infoGeneration = ++db.dbInfoCount;
@@ -3261,7 +3342,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster, ClusterC
 				// for status tool
 				TraceEvent("RecruitedMasterWorker", cluster->id)
 				    .detail("Address", fNewMaster.get().get().address())
-				    .trackLatest("RecruitedMasterWorker");
+				    .trackLatest(cluster->recruitedMasterWorkerEventHolder->trackingKey);
 
 				iMaster = fNewMaster.get().get();
 
@@ -3775,11 +3856,11 @@ struct FailureStatusInfo {
 	}
 };
 
-ACTOR Future<vector<TLogInterface>> requireAll(vector<Future<Optional<vector<TLogInterface>>>> in) {
-	state vector<TLogInterface> out;
+ACTOR Future<std::vector<TLogInterface>> requireAll(std::vector<Future<Optional<std::vector<TLogInterface>>>> in) {
+	state std::vector<TLogInterface> out;
 	state int i;
 	for (i = 0; i < in.size(); i++) {
-		Optional<vector<TLogInterface>> x = wait(in[i]);
+		Optional<std::vector<TLogInterface>> x = wait(in[i]);
 		if (!x.present())
 			throw recruitment_failed();
 		out.insert(out.end(), x.get().begin(), x.get().end());
@@ -4312,7 +4393,7 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 			}
 
 			// Get status but trap errors to send back to client.
-			vector<WorkerDetails> workers;
+			std::vector<WorkerDetails> workers;
 			std::vector<ProcessIssues> workerIssues;
 
 			for (auto& it : self->id_worker) {
@@ -4782,6 +4863,31 @@ ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self
 	}
 }
 
+// A background actor that periodically checks remote DC health, and `checkOutstandingRequests` if remote DC recovers.
+ACTOR Future<Void> updateRemoteDCHealth(ClusterControllerData* self) {
+	// The purpose of the initial delay is to wait for the cluster to achieve a steady state before checking remote DC
+	// health, since remote DC healthy may trigger a failover, and we don't want that to happen too frequently.
+	wait(delay(SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY));
+
+	self->remoteDCMonitorStarted = true;
+
+	// When the remote DC health just start, we may just recover from a health degradation. Check if we can failback if
+	// we are currently in the remote DC in the database configuration.
+	if (!self->remoteTransactionSystemDegraded) {
+		checkOutstandingRequests(self);
+	}
+
+	loop {
+		bool oldRemoteTransactionSystemDegraded = self->remoteTransactionSystemDegraded;
+		self->remoteTransactionSystemDegraded = self->remoteTransactionSystemContainsDegradedServers();
+
+		if (oldRemoteTransactionSystemDegraded && !self->remoteTransactionSystemDegraded) {
+			checkOutstandingRequests(self);
+		}
+		wait(delay(SERVER_KNOBS->CHECK_REMOTE_HEALTH_INTERVAL));
+	}
+}
+
 ACTOR Future<Void> doEmptyCommit(Database cx) {
 	state Transaction tr(cx);
 	loop {
@@ -4807,7 +4913,7 @@ ACTOR Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterCo
 		wait(fCommit || delay(SERVER_KNOBS->FORCE_RECOVERY_CHECK_DELAY));
 		if (!fCommit.isReady() || fCommit.isError()) {
 			if (self->clusterControllerDcId != req.dcId) {
-				vector<Optional<Key>> dcPriority;
+				std::vector<Optional<Key>> dcPriority;
 				dcPriority.push_back(req.dcId);
 				dcPriority.push_back(self->clusterControllerDcId);
 				self->desiredDcIds.set(dcPriority);
@@ -5201,6 +5307,24 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 						self->excludedDegradedServers.clear();
 						TraceEvent("DegradedServerDetectedAndSuggestRecovery").log();
 					}
+				} else if (self->shouldTriggerFailoverDueToDegradedServers()) {
+					double ccUpTime = now() - machineStartTime();
+					if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
+					    ccUpTime > SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
+						TraceEvent("DegradedServerDetectedAndTriggerFailover").log();
+						std::vector<Optional<Key>> dcPriority;
+						auto remoteDcId = self->db.config.regions[0].dcId == self->clusterControllerDcId.get()
+						                      ? self->db.config.regions[1].dcId
+						                      : self->db.config.regions[0].dcId;
+
+						// Switch the current primary DC and remote DC in desiredDcIds, so that the remote DC becomes
+						// the new primary, and the primary DC becomes the new remote.
+						dcPriority.push_back(remoteDcId);
+						dcPriority.push_back(self->clusterControllerDcId);
+						self->desiredDcIds.set(dcPriority);
+					} else {
+						TraceEvent("DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
+					}
 				}
 			}
 
@@ -5256,6 +5380,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 
 	if (SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) {
 		self.addActor.send(workerHealthMonitor(&self));
+		self.addActor.send(updateRemoteDCHealth(&self));
 	}
 
 	loop choose {
@@ -5292,7 +5417,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 		}
 		when(GetWorkersRequest req = waitNext(interf.getWorkers.getFuture())) {
 			++self.getWorkersRequests;
-			vector<WorkerDetails> workers;
+			std::vector<WorkerDetails> workers;
 
 			for (auto const& [id, worker] : self.id_worker) {
 				if ((req.flags & GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY) &&
@@ -5312,7 +5437,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 		}
 		when(GetClientWorkersRequest req = waitNext(interf.clientInterface.getClientWorkers.getFuture())) {
 			++self.getClientWorkersRequests;
-			vector<ClientWorkerInterface> workers;
+			std::vector<ClientWorkerInterface> workers;
 			for (auto& it : self.id_worker) {
 				if (it.second.details.processClass.classType() != ProcessClass::TesterClass) {
 					workers.push_back(it.second.details.interf.clientInterface);
@@ -5663,18 +5788,19 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	NetworkAddress backup(IPAddress(0x06060606), 1);
 	NetworkAddress proxy(IPAddress(0x07070707), 1);
 	NetworkAddress resolver(IPAddress(0x08080808), 1);
+	UID testUID(1, 2);
 
 	// Create a ServerDBInfo using above addresses.
 	ServerDBInfo testDbInfo;
 	testDbInfo.master.changeCoordinators =
-	    RequestStream<struct ChangeCoordinatorsRequest>(Endpoint({ master }, UID(1, 2)));
+	    RequestStream<struct ChangeCoordinatorsRequest>(Endpoint({ master }, testUID));
 
 	TLogInterface localTLogInterf;
-	localTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ tlog }, UID(1, 2)));
+	localTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ tlog }, testUID));
 	TLogInterface localLogRouterInterf;
-	localLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, UID(1, 2)));
+	localLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, testUID));
 	BackupInterface backupInterf;
-	backupInterf.waitFailure = RequestStream<ReplyPromise<Void>>(Endpoint({ backup }, UID(1, 2)));
+	backupInterf.waitFailure = RequestStream<ReplyPromise<Void>>(Endpoint({ backup }, testUID));
 	TLogSet localTLogSet;
 	localTLogSet.isLocal = true;
 	localTLogSet.tLogs.push_back(OptionalInterface(localTLogInterf));
@@ -5683,7 +5809,7 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	testDbInfo.logSystemConfig.tLogs.push_back(localTLogSet);
 
 	TLogInterface sateTLogInterf;
-	sateTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ satelliteTlog }, UID(1, 2)));
+	sateTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ satelliteTlog }, testUID));
 	TLogSet sateTLogSet;
 	sateTLogSet.isLocal = true;
 	sateTLogSet.locality = tagLocalitySatellite;
@@ -5691,18 +5817,18 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	testDbInfo.logSystemConfig.tLogs.push_back(sateTLogSet);
 
 	TLogInterface remoteTLogInterf;
-	remoteTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTlog }, UID(1, 2)));
+	remoteTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTlog }, testUID));
 	TLogSet remoteTLogSet;
 	remoteTLogSet.isLocal = false;
 	remoteTLogSet.tLogs.push_back(OptionalInterface(remoteTLogInterf));
 	testDbInfo.logSystemConfig.tLogs.push_back(remoteTLogSet);
 
 	GrvProxyInterface proxyInterf;
-	proxyInterf.getConsistentReadVersion = RequestStream<struct GetReadVersionRequest>(Endpoint({ proxy }, UID(1, 2)));
+	proxyInterf.getConsistentReadVersion = RequestStream<struct GetReadVersionRequest>(Endpoint({ proxy }, testUID));
 	testDbInfo.client.grvProxies.push_back(proxyInterf);
 
 	ResolverInterface resolverInterf;
-	resolverInterf.resolve = RequestStream<struct ResolveTransactionBatchRequest>(Endpoint({ resolver }, UID(1, 2)));
+	resolverInterf.resolve = RequestStream<struct ResolveTransactionBatchRequest>(Endpoint({ resolver }, testUID));
 	testDbInfo.resolvers.push_back(resolverInterf);
 
 	testDbInfo.recoveryState = RecoveryState::ACCEPTING_COMMITS;
@@ -5753,4 +5879,108 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	return Void();
 }
 
+TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServers") {
+	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	NetworkAddress master(IPAddress(0x01010101), 1);
+	NetworkAddress tlog(IPAddress(0x02020202), 1);
+	NetworkAddress satelliteTlog(IPAddress(0x03030303), 1);
+	NetworkAddress remoteTlog(IPAddress(0x04040404), 1);
+	NetworkAddress logRouter(IPAddress(0x05050505), 1);
+	NetworkAddress backup(IPAddress(0x06060606), 1);
+	NetworkAddress proxy(IPAddress(0x07070707), 1);
+	NetworkAddress proxy2(IPAddress(0x08080808), 1);
+	NetworkAddress resolver(IPAddress(0x09090909), 1);
+	UID testUID(1, 2);
+
+	data.db.config.usableRegions = 2;
+
+	// Create a ServerDBInfo using above addresses.
+	ServerDBInfo testDbInfo;
+	testDbInfo.master.changeCoordinators =
+	    RequestStream<struct ChangeCoordinatorsRequest>(Endpoint({ master }, testUID));
+
+	TLogInterface localTLogInterf;
+	localTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ tlog }, testUID));
+	TLogInterface localLogRouterInterf;
+	localLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, testUID));
+	BackupInterface backupInterf;
+	backupInterf.waitFailure = RequestStream<ReplyPromise<Void>>(Endpoint({ backup }, testUID));
+	TLogSet localTLogSet;
+	localTLogSet.isLocal = true;
+	localTLogSet.tLogs.push_back(OptionalInterface(localTLogInterf));
+	localTLogSet.logRouters.push_back(OptionalInterface(localLogRouterInterf));
+	localTLogSet.backupWorkers.push_back(OptionalInterface(backupInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(localTLogSet);
+
+	TLogInterface sateTLogInterf;
+	sateTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ satelliteTlog }, testUID));
+	TLogSet sateTLogSet;
+	sateTLogSet.isLocal = true;
+	sateTLogSet.locality = tagLocalitySatellite;
+	sateTLogSet.tLogs.push_back(OptionalInterface(sateTLogInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(sateTLogSet);
+
+	TLogInterface remoteTLogInterf;
+	remoteTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTlog }, testUID));
+	TLogSet remoteTLogSet;
+	remoteTLogSet.isLocal = false;
+	remoteTLogSet.tLogs.push_back(OptionalInterface(remoteTLogInterf));
+	testDbInfo.logSystemConfig.tLogs.push_back(remoteTLogSet);
+
+	GrvProxyInterface grvProxyInterf;
+	grvProxyInterf.getConsistentReadVersion = RequestStream<struct GetReadVersionRequest>(Endpoint({ proxy }, testUID));
+	testDbInfo.client.grvProxies.push_back(grvProxyInterf);
+
+	CommitProxyInterface commitProxyInterf;
+	commitProxyInterf.commit = RequestStream<struct CommitTransactionRequest>(Endpoint({ proxy2 }, testUID));
+	testDbInfo.client.commitProxies.push_back(commitProxyInterf);
+
+	ResolverInterface resolverInterf;
+	resolverInterf.resolve = RequestStream<struct ResolveTransactionBatchRequest>(Endpoint({ resolver }, testUID));
+	testDbInfo.resolvers.push_back(resolverInterf);
+
+	testDbInfo.recoveryState = RecoveryState::ACCEPTING_COMMITS;
+
+	// No failover when no degraded servers.
+	data.db.serverInfo->set(testDbInfo);
+	ASSERT(!data.shouldTriggerFailoverDueToDegradedServers());
+
+	// No failover when small number of degraded servers
+	data.degradedServers.insert(master);
+	ASSERT(!data.shouldTriggerFailoverDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// Trigger failover when enough servers in the txn system are degraded.
+	data.degradedServers.insert(master);
+	data.degradedServers.insert(tlog);
+	data.degradedServers.insert(proxy);
+	data.degradedServers.insert(proxy2);
+	data.degradedServers.insert(resolver);
+	ASSERT(data.shouldTriggerFailoverDueToDegradedServers());
+
+	// No failover when usable region is 1.
+	data.db.config.usableRegions = 1;
+	ASSERT(!data.shouldTriggerFailoverDueToDegradedServers());
+	data.db.config.usableRegions = 2;
+
+	// No failover when remote is also degraded.
+	data.degradedServers.insert(remoteTlog);
+	ASSERT(!data.shouldTriggerFailoverDueToDegradedServers());
+	data.degradedServers.clear();
+
+	// No failover when some are not from transaction system
+	data.degradedServers.insert(NetworkAddress(IPAddress(0x13131313), 1));
+	data.degradedServers.insert(NetworkAddress(IPAddress(0x13131313), 2));
+	data.degradedServers.insert(NetworkAddress(IPAddress(0x13131313), 3));
+	data.degradedServers.insert(NetworkAddress(IPAddress(0x13131313), 4));
+	data.degradedServers.insert(NetworkAddress(IPAddress(0x13131313), 5));
+	ASSERT(!data.shouldTriggerFailoverDueToDegradedServers());
+	data.degradedServers.clear();
+
+	return Void();
+}
+
 } // namespace

fdbserver/CommitProxyServer.actor.cpp

@@ -87,9 +87,9 @@ ACTOR void discardCommit(UID id, Future<LogSystemDiskQueueAdapter::CommitMessage
 
 struct ResolutionRequestBuilder {
 	ProxyCommitData* self;
-	vector<ResolveTransactionBatchRequest> requests;
-	vector<vector<int>> transactionResolverMap;
-	vector<CommitTransactionRef*> outTr;
+	std::vector<ResolveTransactionBatchRequest> requests;
+	std::vector<std::vector<int>> transactionResolverMap;
+	std::vector<CommitTransactionRef*> outTr;
 	std::vector<std::vector<std::vector<int>>>
 	    txReadConflictRangeIndexMap; // Used to report conflicting keys, the format is
 	                                 // [CommitTransactionRef_Index][Resolver_Index][Read_Conflict_Range_Index_on_Resolver]
@@ -186,7 +186,7 @@ struct ResolutionRequestBuilder {
 				requests[r].txnStateTransactions.push_back(requests[r].arena, transactionNumberInRequest);
 			}
 
-		vector<int> resolversUsed;
+		std::vector<int> resolversUsed;
 		for (int r = 0; r < outTr.size(); r++)
 			if (outTr[r]) {
 				resolversUsed.push_back(r);
@@ -277,7 +277,7 @@ ACTOR Future<Void> commitBatcher(ProxyCommitData* commitData,
 	}
 }
 
-void createWhitelistBinPathVec(const std::string& binPath, vector<Standalone<StringRef>>& binPathVec) {
+void createWhitelistBinPathVec(const std::string& binPath, std::vector<Standalone<StringRef>>& binPathVec) {
 	TraceEvent(SevDebug, "BinPathConverter").detail("Input", binPath);
 	StringRef input(binPath);
 	while (input != StringRef()) {
@@ -297,7 +297,7 @@ void createWhitelistBinPathVec(const std::string& binPath, vector<Standalone<Str
 	return;
 }
 
-bool isWhitelisted(const vector<Standalone<StringRef>>& binPathVec, StringRef binPath) {
+bool isWhitelisted(const std::vector<Standalone<StringRef>>& binPathVec, StringRef binPath) {
 	TraceEvent("BinPath").detail("Value", binPath);
 	for (const auto& item : binPathVec) {
 		TraceEvent("Element").detail("Value", item);
@@ -1407,7 +1407,7 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 
 // Commit one batch of transactions trs
 ACTOR Future<Void> commitBatch(ProxyCommitData* self,
-                               vector<CommitTransactionRequest>* trs,
+                               std::vector<CommitTransactionRequest>* trs,
                                int currentBatchMemBytesCount) {
 	// WARNING: this code is run at a high priority (until the first delay(0)), so it needs to do as little work as
 	// possible
@@ -1473,7 +1473,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	if (!req.end.present()) {
 		auto r = req.reverse ? commitData->keyInfo.rangeContainingKeyBefore(req.begin)
 		                     : commitData->keyInfo.rangeContaining(req.begin);
-		vector<StorageServerInterface> ssis;
+		std::vector<StorageServerInterface> ssis;
 		ssis.reserve(r.value().src_info.size());
 		for (auto& it : r.value().src_info) {
 			ssis.push_back(it->interf);
@@ -1485,7 +1485,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 		for (auto r = commitData->keyInfo.rangeContaining(req.begin);
 		     r != commitData->keyInfo.ranges().end() && count < req.limit && r.begin() < req.end.get();
 		     ++r) {
-			vector<StorageServerInterface> ssis;
+			std::vector<StorageServerInterface> ssis;
 			ssis.reserve(r.value().src_info.size());
 			for (auto& it : r.value().src_info) {
 				ssis.push_back(it->interf);
@@ -1498,7 +1498,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 		int count = 0;
 		auto r = commitData->keyInfo.rangeContainingKeyBefore(req.end.get());
 		while (count < req.limit && req.begin < r.end()) {
-			vector<StorageServerInterface> ssis;
+			std::vector<StorageServerInterface> ssis;
 			ssis.reserve(r.value().src_info.size());
 			for (auto& it : r.value().src_info) {
 				ssis.push_back(it->interf);
@@ -2008,7 +2008,7 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 	    proxy.id(), master, proxy.getConsistentReadVersion, recoveryTransactionVersion, proxy.commit, db, firstProxy);
 
 	state Future<Sequence> sequenceFuture = (Sequence)0;
-	state PromiseStream<std::pair<vector<CommitTransactionRequest>, int>> batchedCommits;
+	state PromiseStream<std::pair<std::vector<CommitTransactionRequest>, int>> batchedCommits;
 	state Future<Void> commitBatcherActor;
 	state Future<Void> lastCommitComplete = Void();
 
@@ -2100,9 +2100,10 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 			commitData.updateLatencyBandConfig(commitData.db->get().latencyBandConfig);
 		}
 		when(wait(onError)) {}
-		when(std::pair<vector<CommitTransactionRequest>, int> batchedRequests = waitNext(batchedCommits.getFuture())) {
+		when(std::pair<std::vector<CommitTransactionRequest>, int> batchedRequests =
+		         waitNext(batchedCommits.getFuture())) {
 			// WARNING: this code is run at a high priority, so it needs to do as little work as possible
-			const vector<CommitTransactionRequest>& trs = batchedRequests.first;
+			const std::vector<CommitTransactionRequest>& trs = batchedRequests.first;
 			int batchBytes = batchedRequests.second;
 			//TraceEvent("CommitProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount);
 			if (trs.size() || (commitData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS &&

fdbserver/ConfigDatabaseUnitTests.actor.cpp

@@ -108,6 +108,16 @@ public:
 
 	Future<Void> compact() { return cfi.compact.getReply(ConfigFollowerCompactRequest{ lastWrittenVersion }); }
 
+	Future<Void> rollback(Version version) { return cfi.rollback.getReply(ConfigFollowerRollbackRequest{ version }); }
+
+	Future<Void> rollforward(Version lastKnownCommitted,
+	                         Version target,
+	                         Standalone<VectorRef<VersionedConfigMutationRef>> mutations,
+	                         Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations) {
+		return cfi.rollforward.getReply(
+		    ConfigFollowerRollforwardRequest{ lastKnownCommitted, target, mutations, annotations });
+	}
+
 	void restartNode() {
 		cfiServer.cancel();
 		ctiServer.cancel();
@@ -119,6 +129,10 @@ public:
 
 	ConfigFollowerInterface getFollowerInterface() const { return cfi; }
 
+	void close() const { node->close(); }
+
+	Future<Void> onClosed() const { return node->onClosed(); }
+
 	Future<Void> getError() const { return cfiServer || ctiServer; }
 };
 
@@ -202,6 +216,9 @@ public:
 
 	LocalConfiguration& getMutableLocalConfiguration() { return *localConfiguration; }
 
+	void close() const { localConfiguration->close(); }
+	Future<Void> onClosed() const { return localConfiguration->onClosed(); }
+
 	Future<Void> getError() const { return consumer; }
 
 	Version lastSeenVersion() { return localConfiguration->lastSeenVersion(); }
@@ -307,6 +324,10 @@ public:
 
 	void compact() { broadcaster.compact(lastWrittenVersion); }
 
+	void close() const { readFrom.close(); }
+
+	Future<Void> onClosed() const { return readFrom.onClosed(); }
+
 	Future<Void> getError() const { return readFrom.getError() || broadcaster.getError(); }
 };
 
@@ -393,6 +414,13 @@ public:
 	}
 
 	Future<Void> compact() { return writeTo.compact(); }
+	Future<Void> rollback(Version version) { return writeTo.rollback(version); }
+	Future<Void> rollforward(Version lastKnownCommitted,
+	                         Version target,
+	                         Standalone<VectorRef<VersionedConfigMutationRef>> mutations,
+	                         Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations) {
+		return writeTo.rollforward(lastKnownCommitted, target, mutations, annotations);
+	}
 	Future<Void> getError() const { return writeTo.getError(); }
 };
 
@@ -427,8 +455,8 @@ public:
 		broadcastServer.cancel();
 		cbi->set(ConfigBroadcastInterface{});
 		readFrom.connectToBroadcaster(cbi);
-		broadcastServer =
-		    broadcaster.registerWorker(readFrom.lastSeenVersion(), readFrom.configClassSet(), Never(), cbi->get());
+		broadcastServer = broadcaster.registerWorker(
+		    readFrom.lastSeenVersion(), readFrom.configClassSet(), workerFailure.getFuture(), cbi->get());
 	}
 
 	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
@@ -456,6 +484,11 @@ public:
 	Future<Void> check(V T::*member, Optional<E> value) const {
 		return readFrom.checkEventually(member, value);
 	}
+	void close() const {
+		writeTo.close();
+		readFrom.close();
+	}
+	Future<Void> onClosed() const { return writeTo.onClosed() && readFrom.onClosed(); }
 	Future<Void> getError() const { return writeTo.getError() || readFrom.getError() || broadcaster.getError(); }
 };
 
@@ -495,6 +528,14 @@ Future<Void> compact(BroadcasterToLocalConfigEnvironment& env) {
 	env.compact();
 	return Void();
 }
+template <class Env, class... Args>
+Future<Void> rollback(Env& env, Args&&... args) {
+	return waitOrError(env.rollback(std::forward<Args>(args)...), env.getError());
+}
+template <class Env, class... Args>
+Future<Void> rollforward(Env& env, Args&&... args) {
+	return waitOrError(env.rollforward(std::forward<Args>(args)...), env.getError());
+}
 
 ACTOR template <class Env>
 Future<Void> testRestartLocalConfig(UnitTestParameters params) {
@@ -549,6 +590,9 @@ Future<Void> testKillWorker(UnitTestParameters params) {
 	env.killLocalConfig();
 	// Make sure broadcaster detects worker death in a timely manner.
 	wait(timeoutError(env.workerFailed(), 3));
+	Future<Void> closed = env.onClosed();
+	env.close();
+	wait(closed);
 	return Void();
 }
 
@@ -885,6 +929,101 @@ TEST_CASE("/fdbserver/ConfigDB/Transaction/CompactNode") {
 	return Void();
 }
 
+TEST_CASE("/fdbserver/ConfigDB/Transaction/Rollback") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 1 }));
+	// Rollback to version 0 should undo the set.
+	wait(rollback(env, 0));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{}));
+	// Make sure sets still work after rollback.
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 2 }));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{ 2 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/RollbackToCurrentVersion") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 1 }));
+	// Rollback to the latest written version shouldn't undo anything.
+	wait(rollback(env, 1));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{ 1 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/RollbackToNewerVersion") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 1 }));
+	// Rollback to the a version not yet committed should have no effect.
+	wait(rollback(env, 999));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{ 1 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/Rollforward") {
+	state TransactionEnvironment env(params.getDataDir());
+	Standalone<VectorRef<VersionedConfigMutationRef>> mutations;
+	appendVersionedMutation(
+	    mutations, 1, "class-A"_sr, "test_long_v1"_sr, KnobValueRef::create(int64_t{ 1 }).contents());
+	appendVersionedMutation(
+	    mutations, 2, "class-B"_sr, "test_long_v2"_sr, KnobValueRef::create(int64_t{ 2 }).contents());
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+	annotations.emplace_back_deep(annotations.arena(), 1, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	annotations.emplace_back_deep(annotations.arena(), 2, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	wait(rollforward(env, 0, 2, mutations, annotations));
+	wait(check(env, "class-A"_sr, "test_long_v1"_sr, Optional<int64_t>{ 1 }));
+	wait(check(env, "class-B"_sr, "test_long_v2"_sr, Optional<int64_t>{ 2 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/RollforwardWithExistingMutation") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 1 }));
+	Standalone<VectorRef<VersionedConfigMutationRef>> mutations;
+	appendVersionedMutation(
+	    mutations, 2, "class-A"_sr, "test_long_v2"_sr, KnobValueRef::create(int64_t{ 2 }).contents());
+	appendVersionedMutation(
+	    mutations, 3, "class-A"_sr, "test_long_v3"_sr, KnobValueRef::create(int64_t{ 3 }).contents());
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+	annotations.emplace_back_deep(annotations.arena(), 2, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	annotations.emplace_back_deep(annotations.arena(), 3, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	wait(rollforward(env, 1, 3, mutations, annotations));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{ 1 }));
+	wait(check(env, "class-A"_sr, "test_long_v2"_sr, Optional<int64_t>{ 2 }));
+	wait(check(env, "class-A"_sr, "test_long_v3"_sr, Optional<int64_t>{ 3 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/RollforwardWithInvalidMutation") {
+	state TransactionEnvironment env(params.getDataDir());
+	Standalone<VectorRef<VersionedConfigMutationRef>> mutations;
+	appendVersionedMutation(
+	    mutations, 1, "class-A"_sr, "test_long_v1"_sr, KnobValueRef::create(int64_t{ 1 }).contents());
+	appendVersionedMutation(
+	    mutations, 10, "class-A"_sr, "test_long_v10"_sr, KnobValueRef::create(int64_t{ 2 }).contents());
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+	annotations.emplace_back_deep(annotations.arena(), 1, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	wait(rollforward(env, 0, 5, mutations, annotations));
+	wait(check(env, "class-A"_sr, "test_long_v1"_sr, Optional<int64_t>{ 1 }));
+	wait(check(env, "class-A"_sr, "test_long_v10"_sr, Optional<int64_t>{}));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/RollbackThenRollforward") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, "test_long"_sr, int64_t{ 1 }));
+	wait(rollback(env, 0));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{}));
+	Standalone<VectorRef<VersionedConfigMutationRef>> mutations;
+	appendVersionedMutation(
+	    mutations, 1, "class-B"_sr, "test_long_v1"_sr, KnobValueRef::create(int64_t{ 2 }).contents());
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+	annotations.emplace_back_deep(annotations.arena(), 1, ConfigCommitAnnotationRef{ "unit_test"_sr, now() });
+	wait(rollforward(env, 0, 1, mutations, annotations));
+	wait(check(env, "class-A"_sr, "test_long"_sr, Optional<int64_t>{}));
+	wait(check(env, "class-B"_sr, "test_long_v1"_sr, Optional<int64_t>{ 2 }));
+	return Void();
+}
+
 TEST_CASE("/fdbserver/ConfigDB/Transaction/GetConfigClasses") {
 	wait(testGetConfigClasses(params, false));
 	return Void();

fdbserver/ConfigFollowerInterface.cpp

@@ -34,10 +34,10 @@ ConfigFollowerInterface::ConfigFollowerInterface() : _id(deterministicRandom()->
 
 ConfigFollowerInterface::ConfigFollowerInterface(NetworkAddress const& remote)
   : _id(deterministicRandom()->randomUniqueID()),
-    getSnapshotAndChanges(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES)),
-    getChanges(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCHANGES)),
-    compact(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_COMPACT)),
-    getCommittedVersion(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION)) {}
+    getSnapshotAndChanges(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES)),
+    getChanges(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCHANGES)),
+    compact(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_COMPACT)),
+    getCommittedVersion(Endpoint::wellKnown({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCOMMITTEDVERSION)) {}
 
 bool ConfigFollowerInterface::operator==(ConfigFollowerInterface const& rhs) const {
 	return _id == rhs._id;