Merge branch 'master' of github.com:apple/foundationdb into add-c-function-for-management-commands

CMakeLists.txt

@@ -164,6 +164,7 @@ endif()
 add_subdirectory(fdbbackup)
 add_subdirectory(contrib)
 add_subdirectory(tests)
+add_subdirectory(flowbench EXCLUDE_FROM_ALL)
 if(WITH_PYTHON)
   add_subdirectory(bindings)
 endif()
@@ -177,11 +178,6 @@ else()
   include(CPack)
 endif()
 
-set(BUILD_FLOWBENCH OFF CACHE BOOL "Build microbenchmark program (builds google microbenchmark dependency)")
-if(BUILD_FLOWBENCH)
-  add_subdirectory(flowbench)
-endif()
-
 if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
   add_link_options(-lexecinfo)
 endif()

bindings/bindingtester/tests/api.py

@@ -157,7 +157,7 @@ class ApiTest(Test):
         read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY']
         write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT']
         txn_sizes = ['GET_APPROXIMATE_SIZE']
-        storage_metrics = ['GET_ESTIMATED_RANGE_SIZE']
+        storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS']
 
         op_choices += reads
         op_choices += mutations
@@ -553,6 +553,23 @@ class ApiTest(Test):
                 instructions.push_args(key1, key2)
                 instructions.append(op)
                 self.add_strings(1)
+            elif op == 'GET_RANGE_SPLIT_POINTS':
+                # Protect against inverted range and identical keys
+                key1 = self.workspace.pack(self.random.random_tuple(1))
+                key2 = self.workspace.pack(self.random.random_tuple(1))
+
+                while key1 == key2:
+                    key1 = self.workspace.pack(self.random.random_tuple(1))
+                    key2 = self.workspace.pack(self.random.random_tuple(1))
+
+                if key1 > key2:
+                    key1, key2 = key2, key1
+
+                # TODO: randomize chunkSize but should not exceed 100M(shard limit)
+                chunkSize = 10000000 # 10M
+                instructions.push_args(key1, key2, chunkSize)
+                instructions.append(op)
+                self.add_strings(1)
 
             else:
                 assert False, 'Unknown operation: ' + op

bindings/c/CMakeLists.txt

@@ -71,17 +71,27 @@ if(NOT WIN32)
     test/mako/mako.h
     test/mako/utils.c
     test/mako/utils.h)
+  add_subdirectory(test/unit/third_party)
+  find_package(Threads REQUIRED)
+  set(UNIT_TEST_SRCS
+    test/unit/unit_tests.cpp
+    test/unit/fdb_api.cpp
+    test/unit/fdb_api.hpp)
 
   if(OPEN_FOR_IDE)
     add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
     add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h)
     add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)
     add_library(mako OBJECT ${MAKO_SRCS})
+    add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
+    add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
   else()
     add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
     add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
     add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)
     add_executable(mako ${MAKO_SRCS})
+    add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
+    add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
     strip_debug_symbols(fdb_c_performance_test)
     strip_debug_symbols(fdb_c_ryw_benchmark)
     strip_debug_symbols(fdb_c_txn_size_test)
@@ -89,9 +99,26 @@ if(NOT WIN32)
   target_link_libraries(fdb_c_performance_test PRIVATE fdb_c)
   target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c)
   target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c)
+
+  add_dependencies(fdb_c_setup_tests doctest)
+  add_dependencies(fdb_c_unit_tests doctest)
+  target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
+  target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
+  target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
+  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
+
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
   target_link_libraries(mako PRIVATE fdb_c)
+
+  add_fdbclient_test(
+    NAME fdb_c_setup_tests
+    COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
+  add_fdbclient_test(
+    NAME fdb_c_unit_tests
+    COMMAND $<TARGET_FILE:fdb_c_unit_tests>
+            @CLUSTER_FILE@
+            fdb)
 endif()
 
 set(c_workloads_srcs

bindings/c/fdb_c.cpp

@@ -153,7 +153,7 @@ void fdb_future_destroy( FDBFuture* f ) {
 
 extern "C" DLLEXPORT
 fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) {
-	CATCH_AND_RETURN( TSAVB(f)->blockUntilReady(); );
+	CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread(););
 }
 
 fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) {
@@ -171,12 +171,12 @@ public:
 				 void* userdata)
 		: callbackf(callbackf), f(f), userdata(userdata) {}
 
-	virtual bool canFire(int notMadeActive) { return true; }
-	virtual void fire(const Void& unused, int& userParam) {
+	bool canFire(int notMadeActive) const override { return true; }
+	void fire(const Void& unused, int& userParam) override {
 		(*callbackf)(f, userdata);
 		delete this;
 	}
-	virtual void error(const Error&, int& userParam) {
+	void error(const Error&, int& userParam) override {
 		(*callbackf)(f, userdata);
 		delete this;
 	}
@@ -281,6 +281,17 @@ fdb_error_t fdb_future_get_string_array(
 	);
 }
 
+extern "C" DLLEXPORT
+fdb_error_t fdb_future_get_key_array(
+	FDBFuture* f, FDBKey const** out_key_array, int* out_count)
+{
+	CATCH_AND_RETURN(
+		Standalone<VectorRef<KeyRef>> na = TSAV(Standalone<VectorRef<KeyRef>>, f)->get();
+		*out_key_array = (FDBKey*) na.begin();
+		*out_count = na.size();
+	);
+}
+
 FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
 	char *path;
 	if(cluster_file_path) {
@@ -601,7 +612,7 @@ fdb_error_t fdb_transaction_set_option_impl( FDBTransaction* tr,
 void fdb_transaction_set_option_v13( FDBTransaction* tr,
 									 FDBTransactionOption option )
 {
-	fdb_transaction_set_option_impl( tr, option, NULL, 0 );
+	fdb_transaction_set_option_impl( tr, option, nullptr, 0 );
 }
 
 extern "C" DLLEXPORT
@@ -631,13 +642,20 @@ fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const
 
 }
 
-extern "C" DLLEXPORT 
+extern "C" DLLEXPORT
 FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
         int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) {
 	KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
 	return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr());
 }
 
+extern "C" DLLEXPORT
+FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
+        int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) {
+	KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
+	return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr());
+}
+
 #include "fdb_c_function_pointers.g.h"
 
 #define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl);

bindings/c/foundationdb/fdb_c.h

@@ -91,6 +91,10 @@ extern "C" {
     DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter);
 
 #pragma pack(push, 4)
+    typedef struct key {
+        const uint8_t* key;
+        int key_length;
+    } FDBKey;
 #if FDB_API_VERSION >= 700
     typedef struct keyvalue {
         const uint8_t* key;
@@ -146,6 +150,9 @@ extern "C" {
     fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_kv,
                                    int* out_count, fdb_bool_t* out_more );
 #endif
+    DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
+    fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array,
+                                   int* out_count);
 
     DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f,
                             const char*** out_strings, int* out_count);
@@ -263,6 +270,10 @@ extern "C" {
     fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
         int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
 
+    DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
+    fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
+        int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size);
+
     #define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
     #define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0
     #define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1

bindings/c/test/mako/mako.c

@@ -96,6 +96,7 @@ int commit_transaction(FDBTransaction* transaction) {
 
 	f = fdb_transaction_commit(transaction);
 	fdb_wait_and_handle_error(commit_transaction, f, transaction);
+	fdb_future_destroy(f);
 
 	return FDB_SUCCESS;
 }

bindings/c/test/unit/fdb_api.cpp

@@ -0,0 +1,229 @@
+/*
+ * fdb_api.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdb_api.hpp"
+
+#include <iostream>
+
+namespace fdb {
+
+// Future
+
+Future::~Future() {
+  fdb_future_destroy(future_);
+}
+
+bool Future::is_ready() {
+  return fdb_future_is_ready(future_);
+}
+
+[[nodiscard]] fdb_error_t Future::block_until_ready() {
+  return fdb_future_block_until_ready(future_);
+}
+
+[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback,
+                                               void* callback_parameter) {
+  return fdb_future_set_callback(future_, callback, callback_parameter);
+}
+
+[[nodiscard]] fdb_error_t Future::get_error() {
+  return fdb_future_get_error(future_);
+}
+
+void Future::release_memory() {
+  fdb_future_release_memory(future_);
+}
+
+void Future::cancel() {
+  fdb_future_cancel(future_);
+}
+
+// Int64Future
+
+[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) {
+  return fdb_future_get_int64(future_, out);
+}
+
+// ValueFuture
+
+[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present,
+                                           const uint8_t** out_value,
+                                           int* out_value_length) {
+  return fdb_future_get_value(future_, out_present, out_value,
+                              out_value_length);
+}
+
+// KeyFuture
+
+[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key,
+                                         int* out_key_length) {
+  return fdb_future_get_key(future_, out_key, out_key_length);
+}
+
+// StringArrayFuture
+
+[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings,
+                                                 int* out_count) {
+  return fdb_future_get_string_array(future_, out_strings, out_count);
+}
+
+// KeyValueArrayFuture
+
+[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv,
+                                                   int* out_count,
+                                                   fdb_bool_t* out_more) {
+  return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
+}
+
+// Transaction
+
+Transaction::Transaction(FDBDatabase* db) {
+  if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
+    std::cerr << fdb_get_error(err) << std::endl;
+    std::abort();
+  }
+}
+
+Transaction::~Transaction() {
+  fdb_transaction_destroy(tr_);
+}
+
+void Transaction::reset() {
+  fdb_transaction_reset(tr_);
+}
+
+void Transaction::cancel() {
+  fdb_transaction_cancel(tr_);
+}
+
+[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option,
+                                                  const uint8_t* value,
+                                                  int value_length) {
+  return fdb_transaction_set_option(tr_, option, value, value_length);
+}
+
+void Transaction::set_read_version(int64_t version) {
+  fdb_transaction_set_read_version(tr_, version);
+}
+
+Int64Future Transaction::get_read_version() {
+  return Int64Future(fdb_transaction_get_read_version(tr_));
+}
+
+Int64Future Transaction::get_approximate_size() {
+  return Int64Future(fdb_transaction_get_approximate_size(tr_));
+}
+
+KeyFuture Transaction::get_versionstamp() {
+  return KeyFuture(fdb_transaction_get_versionstamp(tr_));
+}
+
+ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
+  return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(),
+                                         key.size(), snapshot));
+}
+
+KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length,
+                               fdb_bool_t or_equal, int offset,
+                               fdb_bool_t snapshot) {
+  return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length,
+                                           or_equal, offset, snapshot));
+}
+
+StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
+  return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_,
+                           (const uint8_t*)key.data(), key.size()));
+}
+
+KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
+                                           int begin_key_name_length,
+                                           fdb_bool_t begin_or_equal,
+                                           int begin_offset,
+                                           const uint8_t* end_key_name,
+                                           int end_key_name_length,
+                                           fdb_bool_t end_or_equal,
+                                           int end_offset, int limit,
+                                           int target_bytes,
+                                           FDBStreamingMode mode,
+                                           int iteration, fdb_bool_t snapshot,
+                                           fdb_bool_t reverse) {
+  return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name,
+                                                       begin_key_name_length,
+                                                       begin_or_equal,
+                                                       begin_offset,
+                                                       end_key_name,
+                                                       end_key_name_length,
+                                                       end_or_equal,
+                                                       end_offset,
+                                                       limit, target_bytes,
+                                                       mode, iteration,
+                                                       snapshot, reverse));
+}
+
+EmptyFuture Transaction::watch(std::string_view key) {
+  return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
+}
+
+EmptyFuture Transaction::commit() {
+  return EmptyFuture(fdb_transaction_commit(tr_));
+}
+
+EmptyFuture Transaction::on_error(fdb_error_t err) {
+  return EmptyFuture(fdb_transaction_on_error(tr_, err));
+}
+
+void Transaction::clear(std::string_view key) {
+  return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
+}
+
+void Transaction::clear_range(std::string_view begin_key,
+                              std::string_view end_key) {
+  fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(),
+                              begin_key.size(), (const uint8_t*)end_key.data(),
+                              end_key.size());
+}
+
+void Transaction::set(std::string_view key, std::string_view value) {
+  fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(),
+                      (const uint8_t*)value.data(), value.size());
+}
+
+void Transaction::atomic_op(std::string_view key, const uint8_t* param,
+                            int param_length, FDBMutationType operationType) {
+  return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(),
+                                   param, param_length, operationType);
+}
+
+[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
+  return fdb_transaction_get_committed_version(tr_, out_version);
+}
+
+fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
+                                            std::string_view end_key,
+                                            FDBConflictRangeType type) {
+  return fdb_transaction_add_conflict_range(tr_,
+                                            (const uint8_t*)begin_key.data(),
+                                            begin_key.size(),
+                                            (const uint8_t*)end_key.data(),
+                                            end_key.size(),
+                                            type);
+}
+
+}  // namespace fdb

bindings/c/test/unit/fdb_api.hpp

@@ -0,0 +1,243 @@
+/*
+ * fdb_api.hpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// A collection of C++ classes to wrap the C API to improve memory management
+// and add types to futures. Using the old C API may look something like:
+//
+//   FDBTransaction *tr;
+//   fdb_database_create_transaction(db, &tr);
+//   FDBFuture *f = fdb_transaction_get(tr, (const uint8_t*)"foo", 3, true);
+//   fdb_future_block_until_ready(f);
+//   fdb_future_get_value(f, ...);
+//   fdb_future_destroy(f);
+//   fdb_transaction_destroy(tr);
+//
+// Using the wrapper classes defined here, it will instead look like:
+//
+//   fdb::Transaction tr(db);
+//   fdb::ValueFuture f = tr.get((const uint8_t*)"foo", 3, true);
+//   f.block_until_ready();
+//   f.get_value(f, ...);
+//
+
+#pragma once
+
+#define FDB_API_VERSION 620
+#include <foundationdb/fdb_c.h>
+
+#include <string>
+#include <string_view>
+
+namespace fdb {
+
+// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
+// FDBFuture when this instance goes out of scope.
+class Future {
+ public:
+  virtual ~Future() = 0;
+
+  // Wrapper around fdb_future_is_ready.
+  bool is_ready();
+  // Wrapper around fdb_future_block_until_ready.
+  fdb_error_t block_until_ready();
+  // Wrapper around fdb_future_set_callback.
+  fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
+  // Wrapper around fdb_future_get_error.
+  fdb_error_t get_error();
+  // Wrapper around fdb_future_release_memory.
+  void release_memory();
+  // Wrapper around fdb_future_cancel.
+  void cancel();
+
+  // Conversion operator to allow Future instances to work interchangeably as
+  // an FDBFuture object.
+  // operator FDBFuture* () const {
+  //   return future_;
+  // }
+
+ protected:
+  Future(FDBFuture *f) : future_(f) {}
+  FDBFuture* future_;
+};
+
+
+class Int64Future : public Future {
+ public:
+   // Call this function instead of fdb_future_get_int64 when using the
+   // Int64Future type. It's behavior is identical to fdb_future_get_int64.
+  fdb_error_t get(int64_t* out);
+  
+ private:
+  friend class Transaction;
+  Int64Future(FDBFuture* f) : Future(f) {}
+};
+
+
+class KeyFuture : public Future {
+ public:
+   // Call this function instead of fdb_future_get_key when using the KeyFuture
+   // type. It's behavior is identical to fdb_future_get_key.
+  fdb_error_t get(const uint8_t** out_key, int* out_key_length);
+
+ private:
+  friend class Transaction;
+  KeyFuture(FDBFuture* f) : Future(f) {}
+};
+
+
+class ValueFuture : public Future {
+ public:
+   // Call this function instead of fdb_future_get_value when using the
+   // ValueFuture type. It's behavior is identical to fdb_future_get_value.
+  fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value,
+                  int* out_value_length);
+
+ private:
+  friend class Transaction;
+  ValueFuture(FDBFuture* f) : Future(f) {}
+};
+
+
+class StringArrayFuture : public Future {
+ public:
+   // Call this function instead of fdb_future_get_string_array when using the
+   // StringArrayFuture type. It's behavior is identical to
+   // fdb_future_get_string_array.
+  fdb_error_t get(const char*** out_strings, int* out_count);
+
+ private:
+  friend class Transaction;
+  StringArrayFuture(FDBFuture* f) : Future(f) {}
+};
+
+
+class KeyValueArrayFuture : public Future {
+ public:
+   // Call this function instead of fdb_future_get_keyvalue_array when using
+   // the KeyValueArrayFuture type. It's behavior is identical to
+   // fdb_future_get_keyvalue_array.
+  fdb_error_t get(const FDBKeyValue** out_kv, int* out_count,
+                  fdb_bool_t* out_more);
+
+ private:
+  friend class Transaction;
+  KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
+};
+
+
+class EmptyFuture : public Future {
+ private:
+  friend class Transaction;
+  EmptyFuture(FDBFuture* f) : Future(f) {}
+};
+
+
+// Wrapper around FDBTransaction, providing the same set of calls as the C API.
+// Handles cleanup of memory, removing the need to call
+// fdb_transaction_destroy.
+class Transaction final {
+ public:
+  // Given an FDBDatabase, initializes a new transaction.
+  Transaction(FDBDatabase* db);
+  ~Transaction();
+
+  // Wrapper around fdb_transaction_reset.
+  void reset();
+
+  // Wrapper around fdb_transaction_cancel.
+  void cancel();
+
+  // Wrapper around fdb_transaction_set_option.
+  fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value,
+                         int value_length);
+
+  // Wrapper around fdb_transaction_set_read_version.
+  void set_read_version(int64_t version);
+
+  // Returns a future which will be set to the transaction read version.
+  Int64Future get_read_version();
+
+  // Returns a future which will be set to the approximate transaction size so far.
+  Int64Future get_approximate_size();
+
+  // Returns a future which will be set to the versionstamp which was used by
+  // any versionstamp operations in the transaction.
+  KeyFuture get_versionstamp();
+
+  // Returns a future which will be set to the value of `key` in the database.
+  ValueFuture get(std::string_view key, fdb_bool_t snapshot);
+
+  // Returns a future which will be set to the key in the database matching the
+  // passed key selector.
+  KeyFuture get_key(const uint8_t* key_name, int key_name_length,
+                    fdb_bool_t or_equal, int offset, fdb_bool_t snapshot);
+
+  // Returns a future which will be set to an array of strings.
+  StringArrayFuture get_addresses_for_key(std::string_view key);
+
+  // Returns a future which will be set to an FDBKeyValue array.
+  KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
+                                int begin_key_name_length,
+                                fdb_bool_t begin_or_equal, int begin_offset,
+                                const uint8_t* end_key_name,
+                                int end_key_name_length,
+                                fdb_bool_t end_or_equal, int end_offset,
+                                int limit, int target_bytes,
+                                FDBStreamingMode mode, int iteration,
+                                fdb_bool_t snapshot, fdb_bool_t reverse);
+
+  // Wrapper around fdb_transaction_watch. Returns a future representing an
+  // empty value.
+  EmptyFuture watch(std::string_view key);
+
+  // Wrapper around fdb_transaction_commit. Returns a future representing an
+  // empty value.
+  EmptyFuture commit();
+
+  // Wrapper around fdb_transaction_on_error. Returns a future representing an
+  // empty value.
+  EmptyFuture on_error(fdb_error_t err);
+
+  // Wrapper around fdb_transaction_clear.
+  void clear(std::string_view key);
+
+  // Wrapper around fdb_transaction_clear_range.
+  void clear_range(std::string_view begin_key, std::string_view end_key);
+
+  // Wrapper around fdb_transaction_set.
+  void set(std::string_view key, std::string_view value);
+
+  // Wrapper around fdb_transaction_atomic_op.
+  void atomic_op(std::string_view key, const uint8_t* param, int param_length,
+                 FDBMutationType operationType);
+
+  // Wrapper around fdb_transaction_get_committed_version.
+  fdb_error_t get_committed_version(int64_t* out_version);
+
+  // Wrapper around fdb_transaction_add_conflict_range.
+  fdb_error_t add_conflict_range(std::string_view begin_key,
+                                 std::string_view end_key,
+                                 FDBConflictRangeType type);
+
+ private:
+  FDBTransaction* tr_;
+};
+
+}  // namespace fdb

bindings/c/test/unit/setup_tests.cpp

@@ -0,0 +1,75 @@
+/*
+ * setup_tests.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Unit tests for API setup, network initialization functions from the FDB C API.
+
+#define FDB_API_VERSION 620
+#include <foundationdb/fdb_c.h>
+#include <iostream>
+#include <thread>
+
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include "doctest.h"
+
+void fdb_check(fdb_error_t e) {
+  if (e) {
+    std::cerr << fdb_get_error(e) << std::endl;
+    std::abort();
+  }
+}
+
+TEST_CASE("setup") {
+  fdb_error_t err;
+  // Version passed here must be <= FDB_API_VERSION
+  err = fdb_select_api_version(9000);
+  CHECK(err);
+
+  // Select current API version
+  fdb_check(fdb_select_api_version(620));
+
+  // Error to call again after a successful return
+  err = fdb_select_api_version(620);
+  CHECK(err);
+
+  CHECK(fdb_get_max_api_version() >= 620);
+
+  fdb_check(fdb_setup_network());
+  // Calling a second time should fail
+  err = fdb_setup_network();
+  CHECK(err);
+
+  struct Context {
+    bool called = false;
+  };
+  Context context;
+  fdb_check(fdb_add_network_thread_completion_hook(
+      [](void *param) {
+        auto *context = static_cast<Context *>(param);
+        context->called = true;
+      },
+      &context));
+
+  std::thread network_thread{&fdb_run_network};
+
+  CHECK(!context.called);
+  fdb_check(fdb_stop_network());
+  network_thread.join();
+  CHECK(context.called);
+}

bindings/c/test/unit/third_party/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Download doctest repo.
+include(ExternalProject)
+find_package(Git REQUIRED)
+
+ExternalProject_Add(
+    doctest
+    PREFIX ${CMAKE_BINARY_DIR}/doctest
+    GIT_REPOSITORY https://github.com/onqtam/doctest.git
+    GIT_TAG 1c8da00c978c19e00a434b2b1f854fcffc9fba35 # v2.4.0
+    TIMEOUT 10
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND ""
+    LOG_DOWNLOAD ON
+)
+
+ExternalProject_Get_Property(doctest source_dir)
+set(DOCTEST_INCLUDE_DIR ${source_dir}/doctest CACHE INTERNAL "Path to include folder for doctest")

bindings/flow/fdb_flow.actor.cpp

@@ -134,6 +134,7 @@ namespace FDB {
 													   FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
 		
 		Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
+		Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
 
 		void addReadConflictRange(KeyRangeRef const& keys) override;
 		void addReadConflictKey(KeyRef const& key) override;
@@ -157,14 +158,14 @@ namespace FDB {
 		void cancel() override;
 		void reset() override;
 
-		TransactionImpl() : tr(NULL) {}
+		TransactionImpl() : tr(nullptr) {}
 	    TransactionImpl(TransactionImpl&& r) noexcept {
 		    tr = r.tr;
-		    r.tr = NULL;
+		    r.tr = nullptr;
 	    }
 	    TransactionImpl& operator=(TransactionImpl&& r) noexcept {
 		    tr = r.tr;
-		    r.tr = NULL;
+		    r.tr = nullptr;
 			return *this;
 	    }
 
@@ -207,10 +208,10 @@ namespace FDB {
 		if ( value.present() )
 			throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) );
 		else
-			throw_on_error( fdb_network_set_option( option, NULL, 0 ) );
+			throw_on_error( fdb_network_set_option( option, nullptr, 0 ) );
 	}
 
-	API* API::instance = NULL;
+	API* API::instance = nullptr;
 	API::API(int version) : version(version) {}
 
 	API* API::selectAPIVersion(int apiVersion) {
@@ -234,11 +235,11 @@ namespace FDB {
 	}
 
 	bool API::isAPIVersionSelected() {
-		return API::instance != NULL;
+		return API::instance != nullptr;
 	}
 
 	API* API::getInstance() {
-		if(API::instance == NULL) {
+		if(API::instance == nullptr) {
 			throw api_version_unset();
 		}
 		else {
@@ -280,7 +281,7 @@ namespace FDB {
 		if (value.present())
 			throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
 		else
-			throw_on_error(fdb_database_set_option(db, option, NULL, 0));
+			throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
 	}
 
 	TransactionImpl::TransactionImpl(FDBDatabase* db) {
@@ -356,6 +357,16 @@ namespace FDB {
 		});
 	}
 
+	Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) {
+		return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference<CFuture> f) {
+			FDBKey const* ks;
+			int count;
+			throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
+
+			return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
+		});
+	}
+
 	void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
 		throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
 	}
@@ -417,7 +428,7 @@ namespace FDB {
 		if ( value.present() ) {
 			throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) );
 		} else {
-			throw_on_error( fdb_transaction_set_option( tr, option, NULL, 0 ) );
+			throw_on_error( fdb_transaction_set_option( tr, option, nullptr, 0 ) );
 		}
 	}
 

bindings/flow/fdb_flow.h

@@ -31,7 +31,7 @@
 
 namespace FDB {
 	struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
-		CFuture() : f(NULL) {}
+		CFuture() : f(nullptr) {}
 		explicit CFuture(FDBFuture* f) : f(f) {}
 		~CFuture() {
 			if (f) {
@@ -90,6 +90,7 @@ namespace FDB {
 		}
 
 		virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
+		virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
 
 		virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
 		virtual void addReadConflictKey(KeyRef const& key) = 0;

bindings/flow/tester/Tester.actor.cpp

@@ -661,6 +661,33 @@ struct GetEstimatedRangeSize : InstructionFunc {
 const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE";
 REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize);
 
+struct GetRangeSplitPoints : InstructionFunc {
+	static const char* name;
+
+	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		state std::vector<StackItem> items = data->stack.pop(3);
+		if (items.size() != 3)
+			return Void();
+
+		Standalone<StringRef> s1 = wait(items[0].value);
+		state Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
+
+		Standalone<StringRef> s2 = wait(items[1].value);
+		state Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
+
+		Standalone<StringRef> s3 = wait(items[2].value);
+		state int64_t chunkSize = Tuple::unpack(s3).getInt(0);
+
+		Future<FDBStandalone<VectorRef<KeyRef>>> fsplitPoints = instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize);
+		FDBStandalone<VectorRef<KeyRef>> splitPoints = wait(fsplitPoints);
+		data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS"));
+
+		return Void();
+	}
+};
+const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS";
+REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints);
+
 struct GetKeyFunc : InstructionFunc {
 	static const char* name;
 

bindings/go/src/_stacktester/stacktester.go

@@ -579,6 +579,17 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) {
 		if e != nil {
 			panic(e)
 		}
+	case op == "GET_RANGE_SPLIT_POINTS":
+		r := sm.popKeyRange()
+		chunkSize := sm.waitAndPop().item.(int64)
+		_, e := rt.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
+			_ = rtr.GetRangeSplitPoints(r, chunkSize).MustGet()
+			sm.store(idx, []byte("GOT_RANGE_SPLIT_POINTS"))
+			return nil, nil
+		})
+		if e != nil {
+			panic(e)
+		}
 	case op == "COMMIT":
 		sm.store(idx, sm.currentTransaction().Commit())
 	case op == "RESET":

bindings/go/src/fdb/futures.go

@@ -306,6 +306,57 @@ func (f *futureKeyValueArray) Get() ([]KeyValue, bool, error) {
 	return ret, (more != 0), nil
 }
 
+// FutureKeyArray represents the asynchronous result of a function
+// that returns an array of keys. FutureKeyArray is a lightweight object
+// that may be efficiently copied, and is safe for concurrent use by multiple goroutines.
+type FutureKeyArray interface {
+
+	// Get returns an array of keys or an error if the asynchronous operation
+	// associated with this future did not successfully complete. The current
+	// goroutine will be blocked until the future is ready.
+	Get() ([]Key, error)
+
+	// MustGet returns an array of keys, or panics if the asynchronous operations
+	// associated with this future did not successfully complete. The current goroutine
+	// will be blocked until the future is ready.
+	MustGet() []Key
+}
+
+type futureKeyArray struct {
+	*future
+}
+
+func (f *futureKeyArray) Get() ([]Key, error) {
+	defer runtime.KeepAlive(f.future)
+
+	f.BlockUntilReady()
+
+	var ks *C.FDBKey
+	var count C.int
+
+	if err := C.fdb_future_get_key_array(f.ptr, &ks, &count); err != 0 {
+		return nil, Error{int(err)}
+	}
+
+	ret := make([]Key, int(count))
+
+	for i := 0; i < int(count); i++ {
+		kptr := unsafe.Pointer(uintptr(unsafe.Pointer(ks)) + uintptr(i*12))
+
+		ret[i] = stringRefToSlice(kptr)
+	}
+
+	return ret, nil
+}
+
+func (f *futureKeyArray) MustGet() []Key {
+	val, err := f.Get()
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
 // FutureInt64 represents the asynchronous result of a function that returns a
 // database version. FutureInt64 is a lightweight object that may be efficiently
 // copied, and is safe for concurrent use by multiple goroutines.

bindings/go/src/fdb/snapshot.go

@@ -87,6 +87,8 @@ func (s Snapshot) GetDatabase() Database {
 	return s.transaction.db
 }
 
+// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
+// stored in the given range.
 func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
 	beginKey, endKey := r.FDBRangeKeys()
 	return s.getEstimatedRangeSizeBytes(
@@ -94,3 +96,15 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
 		endKey.FDBKey(),
 	)
 }
+
+// GetRangeSplitPoints returns a list of keys that can split the given range
+// into (roughly) equally sized chunks based on chunkSize.
+// Note: the returned split points contain the start key and end key of the given range.
+func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
+	beginKey, endKey := r.FDBRangeKeys()
+	return s.getRangeSplitPoints(
+		beginKey.FDBKey(),
+		endKey.FDBKey(),
+		chunkSize,
+	)
+}

bindings/go/src/fdb/subspace/subspace.go

@@ -78,8 +78,9 @@ type Subspace interface {
 	// FoundationDB keys (corresponding to the prefix of this Subspace).
 	fdb.KeyConvertible
 
-	// All Subspaces implement fdb.ExactRange and fdb.Range, and describe all
-	// keys logically in this Subspace.
+	// All Subspaces implement fdb.ExactRange and fdb.Range, and describe all 
+	// keys strictly within the subspace that encode tuples. Specifically, 
+	// this will include all keys in [prefix + '\x00', prefix + '\xff').
 	fdb.ExactRange
 }
 

bindings/go/src/fdb/transaction.go

@@ -40,6 +40,7 @@ type ReadTransaction interface {
 	GetDatabase() Database
 	Snapshot() Snapshot
 	GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64
+	GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray
 
 	ReadTransactor
 }
@@ -318,7 +319,7 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur
 	}
 }
 
-// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
+// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
 // stored in the given range.
 // Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
 // algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
@@ -334,6 +335,31 @@ func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
 	)
 }
 
+func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize int64) FutureKeyArray {
+	return &futureKeyArray{
+		future: newFuture(C.fdb_transaction_get_range_split_points(
+			t.ptr,
+			byteSliceToPtr(beginKey),
+			C.int(len(beginKey)),
+			byteSliceToPtr(endKey),
+			C.int(len(endKey)),
+			C.int64_t(chunkSize),
+		)),
+	}
+}
+
+// GetRangeSplitPoints returns a list of keys that can split the given range
+// into (roughly) equally sized chunks based on chunkSize.
+// Note: the returned split points contain the start key and end key of the given range.
+func (t Transaction) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
+	beginKey, endKey := r.FDBRangeKeys()
+	return t.getRangeSplitPoints(
+		beginKey.FDBKey(),
+		endKey.FDBKey(),
+		chunkSize,
+	)
+}
+
 func (t *transaction) getReadVersion() FutureInt64 {
 	return &futureInt64{
 		future: newFuture(C.fdb_transaction_get_read_version(t.ptr)),

bindings/java/CMakeLists.txt

@@ -1,3 +1,6 @@
+set(RUN_JAVA_TESTS ON CACHE BOOL "Run Java unit tests")
+set(RUN_JUNIT_TESTS OFF CACHE BOOL "Compile and run junit tests")
+
 set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/async/AsyncIterable.java
   src/main/com/apple/foundationdb/async/AsyncIterator.java
@@ -29,6 +32,7 @@ set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/FDBTransaction.java
   src/main/com/apple/foundationdb/FutureInt64.java
   src/main/com/apple/foundationdb/FutureKey.java
+  src/main/com/apple/foundationdb/FutureKeyArray.java
   src/main/com/apple/foundationdb/FutureResult.java
   src/main/com/apple/foundationdb/FutureResults.java
   src/main/com/apple/foundationdb/FutureStrings.java
@@ -44,6 +48,7 @@ set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/package-info.java
   src/main/com/apple/foundationdb/Range.java
   src/main/com/apple/foundationdb/RangeQuery.java
+  src/main/com/apple/foundationdb/KeyArrayResult.java
   src/main/com/apple/foundationdb/RangeResult.java
   src/main/com/apple/foundationdb/RangeResultInfo.java
   src/main/com/apple/foundationdb/RangeResultSummary.java
@@ -102,6 +107,10 @@ set(JAVA_TESTS_SRCS
   src/test/com/apple/foundationdb/test/WatchTest.java
   src/test/com/apple/foundationdb/test/WhileTrueTest.java)
 
+set(JAVA_JUNIT_TESTS
+  src/junit/com/apple/foundationdb/tuple/AllTests.java
+  src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java)
+
 set(GENERATED_JAVA_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/main/com/apple/foundationdb)
 file(MAKE_DIRECTORY ${GENERATED_JAVA_DIR})
 
@@ -173,12 +182,6 @@ add_jar(fdb-java ${JAVA_BINDING_SRCS} ${GENERATED_JAVA_FILES} ${CMAKE_SOURCE_DIR
   OUTPUT_DIR ${PROJECT_BINARY_DIR}/lib VERSION ${CMAKE_PROJECT_VERSION} MANIFEST ${MANIFEST_FILE})
 add_dependencies(fdb-java fdb_java_options fdb_java)
 
-# TODO[mpilman]: The java RPM will require some more effort (mostly on debian). However,
-# most people will use the fat-jar, so it is not clear how high this priority is.
-
-#install_jar(fdb-java DESTINATION ${FDB_SHARE_DIR}/java COMPONENT java)
-#install(TARGETS fdb_java DESTINATION ${FDB_LIB_DIR} COMPONENT java)
-
 if(NOT OPEN_FOR_IDE)
   set(FAT_JAR_BINARIES "NOTFOUND" CACHE STRING
     "Path of a directory structure with libraries to include in fat jar (a lib directory)")
@@ -252,4 +255,30 @@ if(NOT OPEN_FOR_IDE)
   add_dependencies(fat-jar fdb-java)
   add_dependencies(fat-jar copy_lib)
   add_dependencies(packages fat-jar)
+
+  if(RUN_JAVA_TESTS)
+    set(enabled ENABLED)
+  else()
+    set(enabled DISABLED)
+  endif()
+  set(TEST_CP ${tests_jar} ${target_jar})
+  message(STATUS "TEST_CP ${TEST_CP}")
+  add_java_test(NAME DirectoryTest CLASS_PATH ${TEST_CP}
+    CLASS com.apple.foundationdb.test.DirectoryTest ${enabled})
+
+  if(RUN_JUNIT_TESTS)
+    file(DOWNLOAD "https://search.maven.org/remotecontent?filepath=junit/junit/4.13/junit-4.13.jar"
+      ${CMAKE_BINARY_DIR}/packages/junit-4.13.jar
+      EXPECTED_HASH SHA256=4b8532f63bdc0e0661507f947eb324a954d1dbac631ad19c8aa9a00feed1d863)
+    file(DOWNLOAD "https://repo1.maven.org/maven2/org/hamcrest/hamcrest-all/1.3/hamcrest-all-1.3.jar"
+      ${CMAKE_BINARY_DIR}/packages/hamcrest-all-1.3.jar
+      EXPECTED_HASH SHA256=4877670629ab96f34f5f90ab283125fcd9acb7e683e66319a68be6eb2cca60de)
+    add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} INCLUDE_JARS fdb-java ${CMAKE_BINARY_DIR}/packages/junit-4.13.jar)
+    get_property(junit_jar_path TARGET fdb-junit PROPERTY JAR_FILE)
+    add_test(NAME junit
+      COMMAND ${Java_JAVA_EXECUTABLE}
+              -cp "${target_jar}:${junit_jar_path}:${CMAKE_BINARY_DIR}/packages/junit-4.13.jar:${CMAKE_BINARY_DIR}/packages/hamcrest-all-1.3.jar"
+              -Djava.library.path=${CMAKE_BINARY_DIR}/lib
+              org.junit.runner.JUnitCore "com.apple.foundationdb.tuple.AllTests")
+  endif()
 endif()

bindings/java/fdbJNI.cpp

@@ -39,6 +39,8 @@ static thread_local bool is_external = false;
 static jclass range_result_summary_class;
 static jclass range_result_class;
 static jclass string_class;
+static jclass key_array_result_class;
+static jmethodID key_array_result_init;
 static jmethodID range_result_init;
 static jmethodID range_result_summary_init;
 
@@ -305,6 +307,77 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString
 	return arr;
 }
 
+
+JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyArray_1get(JNIEnv *jenv, jobject, jlong future) {
+	if( !future ) {
+		throwParamNotNull(jenv);
+		return JNI_NULL;
+	}
+ 
+	FDBFuture *f = (FDBFuture *)future;
+
+	const FDBKey *ks;
+	int count;
+	fdb_error_t err = fdb_future_get_key_array( f, &ks, &count );
+	if( err ) {
+		safeThrow( jenv, getThrowable( jenv, err ) );
+		return JNI_NULL;
+	}
+
+	int totalKeySize = 0;
+	for(int i = 0; i < count; i++) {
+		totalKeySize += ks[i].key_length;
+	}
+
+	jbyteArray keyArray = jenv->NewByteArray(totalKeySize);
+	if( !keyArray ) {
+		if( !jenv->ExceptionOccurred() )
+			throwOutOfMem(jenv);
+		return JNI_NULL;
+	}
+	uint8_t *keys_barr = (uint8_t *)jenv->GetByteArrayElements(keyArray, JNI_NULL); 
+	if (!keys_barr) {
+		throwRuntimeEx( jenv, "Error getting handle to native resources" );
+		return JNI_NULL;
+	}
+
+	jintArray lengthArray = jenv->NewIntArray(count);
+	if( !lengthArray ) {
+		if( !jenv->ExceptionOccurred() )
+			throwOutOfMem(jenv);
+
+		jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0);
+		return JNI_NULL;
+	}
+
+	jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); 
+	if( !length_barr ) {
+		if( !jenv->ExceptionOccurred() )
+			throwOutOfMem(jenv);
+
+		jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0);
+		return JNI_NULL;
+	}
+
+	int offset = 0;
+	for(int i = 0; i < count; i++) {
+		memcpy(keys_barr + offset, ks[i].key, ks[i].key_length);
+		length_barr[i] = ks[i].key_length;
+		offset += ks[i].key_length;
+	}
+
+	jenv->ReleaseByteArrayElements(keyArray, (jbyte *)keys_barr, 0);
+	jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0);
+
+	jobject result = jenv->NewObject(key_array_result_class, key_array_result_init, keyArray, lengthArray);
+	if( jenv->ExceptionOccurred() )
+		return JNI_NULL;
+
+	return result;
+
+}
+
+
 // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
 JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) {
 	if( !future ) {
@@ -695,6 +768,35 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 	return (jlong)f;
 }
 
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIEnv *jenv, jobject, jlong tPtr, 
+		jbyteArray beginKeyBytes, jbyteArray endKeyBytes, jlong chunkSize) {
+	if( !tPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBTransaction *tr = (FDBTransaction *)tPtr;
+
+	uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL );
+	if(!startKey) {
+		if( !jenv->ExceptionOccurred() )
+			throwRuntimeEx( jenv, "Error getting handle to native resources" );
+		return 0;
+	}
+
+	uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKey) {
+		jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT );
+		if( !jenv->ExceptionOccurred() )
+			throwRuntimeEx( jenv, "Error getting handle to native resources" );
+		return 0;
+	}
+
+	FDBFuture *f = fdb_transaction_get_range_split_points( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ), chunkSize );
+	jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT );
+	jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT );
+	return (jlong)f;
+}
+
 JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) {
 	if( !tPtr || !keyBytes || !valueBytes ) {
 		throwParamNotNull(jenv);
@@ -1071,6 +1173,10 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
 		range_result_init = env->GetMethodID(local_range_result_class, "<init>", "([B[IZ)V");
 		range_result_class = (jclass) (env)->NewGlobalRef(local_range_result_class);
 
+		jclass local_key_array_result_class = env->FindClass("com/apple/foundationdb/KeyArrayResult");
+		key_array_result_init = env->GetMethodID(local_key_array_result_class, "<init>", "([B[I)V");
+		key_array_result_class = (jclass) (env)->NewGlobalRef(local_key_array_result_class);
+
 		jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary");
 		range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "<init>", "([BIZ)V");
 		range_result_summary_class = (jclass) (env)->NewGlobalRef(local_range_result_summary_class);
@@ -1089,13 +1195,13 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) {
 		return;
 	} else {
 		// delete global references so the GC can collect them
-		if (range_result_summary_class != NULL) {
+		if (range_result_summary_class != JNI_NULL) {
 			env->DeleteGlobalRef(range_result_summary_class);
 		}
-		if (range_result_class != NULL) {
+		if (range_result_class != JNI_NULL) {
 			env->DeleteGlobalRef(range_result_class);
 		}
-		if (string_class != NULL) {
+		if (string_class != JNI_NULL) {
 			env->DeleteGlobalRef(string_class);
 		}
 	}

bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java

@@ -27,9 +27,14 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
+import org.junit.Ignore;
 
 /**
  * @author Ben
@@ -251,7 +256,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#bisectLeft(java.math.BigInteger[], java.math.BigInteger)}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testBisectLeft() {
 		fail("Not yet implemented");
 	}
@@ -259,7 +264,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#compareUnsigned(byte[], byte[])}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testCompare() {
 		fail("Not yet implemented");
 	}
@@ -267,7 +272,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#findNext(byte[], byte, int)}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testFindNext() {
 		fail("Not yet implemented");
 	}
@@ -275,7 +280,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#findTerminator(byte[], byte, byte, int)}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testFindTerminator() {
 		fail("Not yet implemented");
 	}
@@ -283,7 +288,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#copyOfRange(byte[], int, int)}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testCopyOfRange() {
 		fail("Not yet implemented");
 	}
@@ -291,7 +296,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#strinc(byte[])}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testStrinc() {
 		fail("Not yet implemented");
 	}
@@ -299,7 +304,7 @@ public class ArrayUtilTests {
 	/**
 	 * Test method for {@link ByteArrayUtil#printable(byte[])}.
 	 */
-	@Test
+	@Test @Ignore
 	public void testPrintable() {
 		fail("Not yet implemented");
 	}

bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java

@@ -80,6 +80,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 			return FDBTransaction.this.getEstimatedRangeSizeBytes(range);
 		}
 
+		@Override
+		public CompletableFuture<KeyArrayResult> getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) {
+			return FDBTransaction.this.getRangeSplitPoints(begin, end, chunkSize);
+		}
+
+		@Override
+		public CompletableFuture<KeyArrayResult> getRangeSplitPoints(Range range, long chunkSize) {
+			return FDBTransaction.this.getRangeSplitPoints(range, chunkSize);
+		}
+
 		///////////////////
 		//  getRange -> KeySelectors
 		///////////////////
@@ -282,6 +292,21 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 		return this.getEstimatedRangeSizeBytes(range.begin, range.end);
 	}
 
+	@Override
+	public CompletableFuture<KeyArrayResult> getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize) {
+		pointerReadLock.lock();
+		try {
+			return new FutureKeyArray(Transaction_getRangeSplitPoints(getPtr(), begin, end, chunkSize), executor);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<KeyArrayResult> getRangeSplitPoints(Range range, long chunkSize) {
+		return this.getRangeSplitPoints(range.begin, range.end, chunkSize);
+	}
+
 	///////////////////
 	//  getRange -> KeySelectors
 	///////////////////
@@ -686,4 +711,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
 	private native void Transaction_cancel(long cPtr);
 	private native long Transaction_getKeyLocations(long cPtr, byte[] key);
 	private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd);
+	private native long Transaction_getRangeSplitPoints(long cPtr, byte[] keyBegin, byte[] keyEnd, long chunkSize);
 }

bindings/java/src/main/com/apple/foundationdb/FutureKeyArray.java

@@ -0,0 +1,37 @@
+/*
+ * FutureKeyArray.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb;
+
+import java.util.concurrent.Executor;
+
+class FutureKeyArray extends NativeFuture<KeyArrayResult> {
+	FutureKeyArray(long cPtr, Executor executor) {
+		super(cPtr);
+		registerMarshalCallback(executor);
+	}
+
+	@Override
+	protected KeyArrayResult getIfDone_internal(long cPtr) throws FDBException {
+		return FutureKeyArray_get(cPtr);
+	}
+
+	private native KeyArrayResult FutureKeyArray_get(long cPtr) throws FDBException;
+}

bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java

@@ -0,0 +1,44 @@
+/*
+ * KeyArrayResult.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class KeyArrayResult {
+	final List<byte[]> keys;
+
+	KeyArrayResult(byte[] keyBytes, int[] keyLengths) {
+		int count = keyLengths.length;
+		keys = new ArrayList<byte[]>(count);
+
+		int offset = 0;
+		for(int i = 0; i < count; i++) {
+			int keyLength = keyLengths[i];
+
+			byte[] key = new byte[keyLength];
+			System.arraycopy(keyBytes, offset, key, 0, keyLength);
+
+			offset += keyLength;
+			keys.add(key);
+		}
+	}
+}

bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java

@@ -455,6 +455,28 @@ public interface ReadTransaction extends ReadTransactionContext {
 	 */
 	CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range);
 
+	/**
+	 * Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>.
+	 * Note: the returned split points contain the start key and end key of the given range.
+	 *
+	 * @param begin the beginning of the range (inclusive)
+	 * @param end the end of the range (exclusive)
+	 *
+	 * @return a handle to access the results of the asynchronous call
+	 */
+	CompletableFuture<KeyArrayResult> getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize);
+
+	/**
+	 * Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>
+	 * Note: the returned split points contain the start key and end key of the given range.
+	 *
+	 * @param range the range of the keys
+	 *
+	 * @return a handle to access the results of the asynchronous call
+	 */
+	CompletableFuture<KeyArrayResult> getRangeSplitPoints(Range range, long chunkSize);
+
+	
 	/**
 	 * Returns a set of options that can be set on a {@code Transaction}
 	 *

bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java

@@ -38,6 +38,7 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.FDBException;
 import com.apple.foundationdb.KeySelector;
 import com.apple.foundationdb.KeyValue;
+import com.apple.foundationdb.KeyArrayResult;
 import com.apple.foundationdb.MutationType;
 import com.apple.foundationdb.Range;
 import com.apple.foundationdb.StreamingMode;
@@ -229,6 +230,12 @@ public class AsyncStackTester {
 				inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes());
 			}, FDB.DEFAULT_EXECUTOR);
 		}
+		else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) {
+			List<Object> params = inst.popParams(3).join();
+			return inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).thenAcceptAsync(splitPoints -> {
+				inst.push("GOT_RANGE_SPLIT_POINTS".getBytes());
+			}, FDB.DEFAULT_EXECUTOR);
+		}
 		else if(op == StackOperation.GET_RANGE) {
 			return inst.popParams(5).thenComposeAsync(params -> {
 				int limit = StackUtils.getInt(params.get(2));

bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java

@@ -34,7 +34,7 @@ public class DirectoryTest {
 	public static void main(String[] args) throws Exception {
 		try {
 			FDB fdb = FDB.selectAPIVersion(700);
-			try(Database db = fdb.open()) {
+			try(Database db = args.length > 0 ? fdb.open(args[0]) : fdb.open()) {
 				runTests(db);
 			}
 		}

bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java

@@ -57,6 +57,7 @@ enum StackOperation {
 	GET_APPROXIMATE_SIZE,
 	GET_VERSIONSTAMP,
 	GET_ESTIMATED_RANGE_SIZE,
+	GET_RANGE_SPLIT_POINTS,
 	SET_READ_VERSION,
 	ON_ERROR,
 	SUB,

bindings/java/src/test/com/apple/foundationdb/test/StackTester.java

@@ -39,6 +39,7 @@ import com.apple.foundationdb.FDB;
 import com.apple.foundationdb.FDBException;
 import com.apple.foundationdb.KeySelector;
 import com.apple.foundationdb.KeyValue;
+import com.apple.foundationdb.KeyArrayResult;
 import com.apple.foundationdb.LocalityUtil;
 import com.apple.foundationdb.MutationType;
 import com.apple.foundationdb.Range;
@@ -211,6 +212,11 @@ public class StackTester {
 				Long size = inst.readTr.getEstimatedRangeSizeBytes((byte[])params.get(0), (byte[])params.get(1)).join();
 				inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes());
 			}
+			else if (op == StackOperation.GET_RANGE_SPLIT_POINTS) {
+				List<Object> params = inst.popParams(3).join();
+				KeyArrayResult splitPoints = inst.readTr.getRangeSplitPoints((byte[])params.get(0), (byte[])params.get(1), (long)params.get(2)).join();
+				inst.push("GOT_RANGE_SPLIT_POINTS".getBytes());
+			}
 			else if(op == StackOperation.GET_RANGE) {
 				List<Object> params = inst.popParams(5).join();
 

bindings/python/fdb/impl.py

@@ -462,16 +462,29 @@ class TransactionRead(_FDBBase):
         return self.get(key)
     
     def get_estimated_range_size_bytes(self, begin_key, end_key):
-        if begin_key is None:
-            begin_key = b''
-        if end_key is None:
-            end_key = b'\xff'
+        if begin_key is None or end_key is None:
+            if fdb.get_api_version() >= 700:
+                raise Exception('Invalid begin key or end key')
+            else:
+                if begin_key is None:
+                    begin_key = b''
+                if end_key is None:
+                    end_key = b'\xff'
         return FutureInt64(self.capi.fdb_transaction_get_estimated_range_size_bytes(
             self.tpointer,
             begin_key, len(begin_key),
             end_key, len(end_key)
             ))
-
+    
+    def get_range_split_points(self, begin_key, end_key, chunk_size):
+        if begin_key is None or end_key is None or chunk_size <=0:
+            raise Exception('Invalid begin key, end key or chunk size')
+        return FutureKeyArray(self.capi.fdb_transaction_get_range_split_points(
+            self.tpointer,
+            begin_key, len(begin_key),
+            end_key, len(end_key),
+            chunk_size
+            ))
 
 class Transaction(TransactionRead):
     """A modifiable snapshot of a Database.
@@ -736,6 +749,14 @@ class FutureKeyValueArray(Future):
         # the KVs on the python side and in most cases we are about to
         # destroy the future anyway
 
+class FutureKeyArray(Future):
+    def wait(self):
+        self.block_until_ready()
+        ks = ctypes.pointer(KeyStruct())
+        count = ctypes.c_int()
+        self.capi.fdb_future_get_key_array(self.fpointer, ctypes.byref(ks), ctypes.byref(count))
+        return [ctypes.string_at(x.key, x.key_length) for x in ks[0:count.value]]
+
 
 class FutureStringArray(Future):
     def wait(self):
@@ -1217,6 +1238,11 @@ class KeyValueStruct(ctypes.Structure):
                 ('value_length', ctypes.c_int)]
     _pack_ = 4
 
+class KeyStruct(ctypes.Structure):
+    _fields_ = [('key', ctypes.POINTER(ctypes.c_byte)),
+                ('key_length', ctypes.c_int)]
+    _pack_ = 4
+
 
 class KeyValue(object):
     def __init__(self, key, value):
@@ -1406,6 +1432,11 @@ def init_c_api():
     _capi.fdb_future_get_keyvalue_array.restype = int
     _capi.fdb_future_get_keyvalue_array.errcheck = check_error_code
 
+    _capi.fdb_future_get_key_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(
+        ctypes.POINTER(KeyStruct)), ctypes.POINTER(ctypes.c_int)]
+    _capi.fdb_future_get_key_array.restype = int
+    _capi.fdb_future_get_key_array.errcheck = check_error_code
+
     _capi.fdb_future_get_string_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)), ctypes.POINTER(ctypes.c_int)]
     _capi.fdb_future_get_string_array.restype = int
     _capi.fdb_future_get_string_array.errcheck = check_error_code
@@ -1451,6 +1482,9 @@ def init_c_api():
     _capi.fdb_transaction_get_estimated_range_size_bytes.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
     _capi.fdb_transaction_get_estimated_range_size_bytes.restype = ctypes.c_void_p
 
+    _capi.fdb_transaction_get_range_split_points.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
+    _capi.fdb_transaction_get_range_split_points.restype = ctypes.c_void_p
+
     _capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
     _capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int
     _capi.fdb_transaction_add_conflict_range.errcheck = check_error_code

bindings/python/tests/tester.py

@@ -393,6 +393,10 @@ class Tester:
                     begin, end = inst.pop(2)
                     estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait()
                     inst.push(b"GOT_ESTIMATED_RANGE_SIZE")
+                elif inst.op == six.u("GET_RANGE_SPLIT_POINTS"):
+                    begin, end, chunkSize = inst.pop(3)
+                    estimatedSize = obj.get_range_split_points(begin, end, chunkSize).wait()
+                    inst.push(b"GOT_RANGE_SPLIT_POINTS")
                 elif inst.op == six.u("GET_KEY"):
                     key, or_equal, offset, prefix = inst.pop(4)
                     result = obj.get_key(fdb.KeySelector(key, or_equal, offset))

bindings/ruby/lib/fdbimpl.rb

@@ -109,6 +109,7 @@ module FDB
       attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer
       attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer
       attach_function :fdb_transaction_get_estimated_range_size_bytes, [ :pointer, :pointer, :int, :pointer, :int ], :pointer
+      attach_function :fdb_transaction_get_range_split_points, [ :pointer, :pointer, :int, :pointer, :int, :int64 ], :pointer
       attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void
       attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void
       attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void
@@ -129,6 +130,12 @@ module FDB
              :value_length, :int
     end
 
+    class KeyStruct < FFI::Struct
+      pack 4
+      layout :key, :pointer,
+             :key_length, :int
+    end
+
     def self.check_error(code)
       raise Error.new(code) if code.nonzero?
       nil
@@ -472,6 +479,22 @@ module FDB
     end
   end
 
+  class FutureKeyArray < Future
+    def wait
+      block_until_ready
+
+      ks = FFI::MemoryPointer.new :pointer
+      count = FFI::MemoryPointer.new :int
+      FDBC.check_error FDBC.fdb_future_get_key_array(@fpointer, kvs, count)
+      ks = ks.read_pointer
+
+      (0..count.read_int-1).map{|i|
+        x = FDBC::KeyStruct.new(ks + (i * FDBC::KeyStruct.size))
+        x[:key].read_bytes(x[:key_length])
+       }
+    end
+  end
+
   class FutureStringArray < LazyFuture
     def getter
       strings = FFI::MemoryPointer.new :pointer
@@ -825,6 +848,15 @@ module FDB
       Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize))
     end
 
+    def get_range_split_points(begin_key, end_key, chunk_size)
+      if chunk_size <=0
+        raise ArgumentError, "Invalid chunk size"
+      end
+      bkey = FDB.key_to_bytes(begin_key)
+      ekey = FDB.key_to_bytes(end_key)
+      FutureKeyArray.new(FDBC.fdb_transaction_get_range_split_points(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize, chunk_size))
+    end
+
   end
 
   TransactionRead.class_variable_set("@@StreamingMode", @@StreamingMode)

bindings/ruby/tests/tester.rb

@@ -320,6 +320,9 @@ class Tester
         when "GET_ESTIMATED_RANGE_SIZE"
           inst.tr.get_estimated_range_size_bytes(inst.wait_and_pop, inst.wait_and_pop).to_i
           inst.push("GOT_ESTIMATED_RANGE_SIZE")
+        when "GET_RANGE_SPLIT_POINTS"
+          inst.tr.get_range_split_points(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop).length()
+          inst.push("GOT_RANGE_SPLIT_POINTS")
         when "GET_KEY"
           selector = FDB::KeySelector.new(inst.wait_and_pop, inst.wait_and_pop, inst.wait_and_pop)
           prefix = inst.wait_and_pop

build/Dockerfile

@@ -45,13 +45,13 @@ RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -
     cd .. && rm -rf ninja-1.9.0 ninja.zip
 
 # install openssl
-RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1d.tar.gz -o openssl.tar.gz &&\
-    echo "1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2  openssl.tar.gz" > openssl-sha.txt &&\
+RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz &&\
+    echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9  openssl.tar.gz" > openssl-sha.txt &&\
     sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\
-    cd openssl-1.1.1d && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
+    cd openssl-1.1.1h && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
     scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\
     ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\
-    cd /tmp/ && rm -rf /tmp/openssl-1.1.1d /tmp/openssl.tar.gz
+    cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz
 
 RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\
     echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee  rocksdb.tar.gz" > rocksdb-sha.txt &&\
@@ -61,8 +61,8 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.
 ARG TIMEZONEINFO=America/Los_Angeles
 RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime
 
-LABEL version=0.1.15
-ENV DOCKER_IMAGEVER=0.1.15
+LABEL version=0.1.17
+ENV DOCKER_IMAGEVER=0.1.17
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++

build/Dockerfile.c7.layered

@@ -0,0 +1,80 @@
+FROM centos:7
+
+RUN yum install -y centos-release-scl scl-utils
+RUN rpmkeys --import "http://pool.sks-keyservers.net/pks/lookup?op=get&search=0x3fa7e0328081bff6a14da29aa6a19b38d3d831ef"
+RUN	curl https://download.mono-project.com/repo/centos7-stable.repo | tee /etc/yum.repos.d/mono-centos7-stable.repo
+RUN yum install -y curl rpm-build wget git unzip devtoolset-8 devtoolset-8-libubsan-devel devtoolset-8-valgrind-devel \
+	rh-ruby26 go-toolset-7 rh-git218 rh-python36-devel java-11-openjdk-devel.x86_64 mono-devel dos2unix dpkg rh-python36
+
+# install Ninja
+RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\
+    unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\
+    cd .. && rm -rf ninja-1.9.0 ninja.zip
+
+# install cmake
+RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o /tmp/cmake.tar.gz &&\
+    echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76  /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\
+    sha256sum -c /tmp/cmake-sha.txt &&\
+    cd /tmp && tar xf cmake.tar.gz &&\
+    cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ &&\
+    rm -rf cmake.tar.gz cmake-3.13.4-Linux-x86_64 cmake-sha.txt
+
+# install LLVM
+RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz > /tmp/llvm.tar.xz
+RUN cd tmp &&\
+	echo "b7b639fc675fa1c86dd6d0bc32267be9eb34451748d2efd03f674b773000e92b  llvm.tar.xz" > llvm-sha.txt &&\
+	sha256sum -c llvm-sha.txt
+RUN cd /tmp && tar xf llvm.tar.xz --no-same-owner
+RUN mkdir /tmp/build && cd /tmp/build &&\
+	scl enable devtoolset-8 -- cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF \
+	-DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb"\
+	-DLLVM_STATIC_LINK_CXX_STDLIB=ON ../llvm-project-11.0.0/llvm
+RUN cd /tmp/build && cmake --build .
+RUN cd /tmp/build && cmake --build . --target install
+RUN rm -rf /tmp/*
+
+# install openssl
+RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz &&\
+    echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9  openssl.tar.gz" > openssl-sha.txt &&\
+    sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\
+    cd openssl-1.1.1h && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
+    scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\
+    ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\
+    cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz
+
+# install RocksDB
+RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\
+    echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee  rocksdb.tar.gz" > rocksdb-sha.txt &&\
+    sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt
+
+# install Boost
+# wget of bintray without forcing UTF-8 encoding results in 403 Forbidden
+RUN cd /opt/ &&\
+    curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 &&\
+    echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba  boost_1_67_0.tar.bz2" > boost-sha-67.txt &&\
+    sha256sum -c boost-sha-67.txt &&\
+    tar -xjf boost_1_67_0.tar.bz2 &&\
+    rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs &&\
+    curl -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 &&\
+    echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722  boost_1_72_0.tar.bz2" > boost-sha-72.txt &&\
+    sha256sum -c boost-sha-72.txt &&\
+    tar -xjf boost_1_72_0.tar.bz2 &&\
+    rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0/libs
+
+
+# Install CCACHE
+RUN cd /tmp && curl -L https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz > ccache.tar.gz &&\
+    echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45  ccache.tar.gz" > ccache-sha256.txt &&\
+    sha256sum -c ccache-sha256.txt &&\
+    tar xf ccache.tar.gz && rm -rf build && mkdir build && cd build && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache-4.0 &&\
+    cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/ccache-4.0
+
+# Install toml11
+RUN cd /tmp && curl -L https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz > toml.tar.gz &&\
+    echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d  toml.tar.gz" > toml-sha256.txt &&\
+    sha256sum -c toml-sha256.txt &&\
+    tar xf toml.tar.gz && rm -rf build && mkdir build && cd build && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml11-3.4.0 &&\
+    cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/toml11-3.4.0
+
+# do some cleanup
+RUN rm -rf /tmp/* && yum clean all && rm -rf /var/cache/yum

build/Dockerfile.devel

@@ -1,4 +1,4 @@
-FROM foundationdb/foundationdb-build:0.1.15
+FROM foundationdb/foundationdb-build:0.1.17
 
 USER root
 
@@ -7,7 +7,7 @@ ADD artifacts /mnt/artifacts
 
 # Install build tools for building via make
 RUN \
-	yum install -y distcc-server gperf rubygems python34 libmpc-devel npm cgdb
+	yum install -y distcc-server gperf rubygems python34 libmpc-devel npm cgdb jq
 
 # Download and install llvm-10.0.0
 RUN cd / &&\
@@ -50,8 +50,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\
 	ldconfig &&\
 	rm -rf /mnt/artifacts
 
-LABEL version=0.11.8
-ENV DOCKER_IMAGEVER=0.11.8
+LABEL version=0.11.9
+ENV DOCKER_IMAGEVER=0.11.9
 
 ENV CLANGCC=/usr/local/bin/clang.de8a65ef
 ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef

build/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   common: &common
-    image: foundationdb/foundationdb-build:0.1.15
+    image: foundationdb/foundationdb-build:0.1.17
 
   build-setup: &build-setup
     <<: *common
@@ -84,7 +84,7 @@ services:
 
   snapshot-ctest: &snapshot-ctest
     <<: *build-setup
-    command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure'
+    command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure'
 
   prb-ctest:
     <<: *snapshot-ctest

cmake/AddFdbTest.cmake

@@ -159,7 +159,7 @@ endif()
 # - OUT_DIR the directory where files will be staged
 # - CONTEXT the type of correctness package being built (e.g. 'valgrind correctness')
 function(stage_correctness_package)
-  set(oneValueArgs OUT_DIR CONTEXT)
+  set(oneValueArgs OUT_DIR CONTEXT OUT_FILES)
   cmake_parse_arguments(STAGE "" "${oneValueArgs}" "" "${ARGN}")
   file(MAKE_DIRECTORY ${STAGE_OUT_DIR}/bin)
   string(LENGTH "${CMAKE_SOURCE_DIR}/tests/" base_length)
@@ -202,6 +202,10 @@ function(stage_correctness_package)
     endforeach()
   endforeach()
   list(APPEND package_files ${STAGE_OUT_DIR}/bin/fdbserver
+                            ${STAGE_OUT_DIR}/bin/coverage.fdbserver.xml
+                            ${STAGE_OUT_DIR}/bin/coverage.fdbclient.xml
+                            ${STAGE_OUT_DIR}/bin/coverage.fdbrpc.xml
+                            ${STAGE_OUT_DIR}/bin/coverage.flow.xml
                             ${STAGE_OUT_DIR}/bin/TestHarness.exe
                             ${STAGE_OUT_DIR}/bin/TraceLogHelper.dll
                             ${STAGE_OUT_DIR}/CMakeCache.txt
@@ -210,17 +214,27 @@ function(stage_correctness_package)
     OUTPUT ${package_files}
     DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt
             ${CMAKE_BINARY_DIR}/packages/bin/fdbserver
+            ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml
+            ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml
+            ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml
+            ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml
             ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe
             ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll
     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/CMakeCache.txt ${STAGE_OUT_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbserver
+                                     ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml
+                                     ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml
+                                     ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml
+                                     ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml
                                      ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe
                                      ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll
                                      ${STAGE_OUT_DIR}/bin
     COMMENT "Copying files for ${STAGE_CONTEXT} package"
     )
   list(APPEND package_files ${test_files} ${external_files})
-  set(package_files ${package_files} PARENT_SCOPE)
+  if(STAGE_OUT_FILES)
+    set(${STAGE_OUT_FILES} ${package_files} PARENT_SCOPE)
+  endif()
 endfunction()
 
 function(create_correctness_package)
@@ -228,7 +242,7 @@ function(create_correctness_package)
     return()
   endif()
   set(out_dir "${CMAKE_BINARY_DIR}/correctness")
-  stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness")
+  stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness" OUT_FILES package_files)
   set(tar_file ${CMAKE_BINARY_DIR}/packages/correctness-${CMAKE_PROJECT_VERSION}.tar.gz)
   add_custom_command(
     OUTPUT ${tar_file}
@@ -255,7 +269,7 @@ function(create_valgrind_correctness_package)
   endif()
   if(USE_VALGRIND)
     set(out_dir "${CMAKE_BINARY_DIR}/valgrind_correctness")
-    stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness")
+    stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness" OUT_FILES package_files)
     set(tar_file ${CMAKE_BINARY_DIR}/packages/valgrind-${CMAKE_PROJECT_VERSION}.tar.gz)
     add_custom_command(
       OUTPUT ${tar_file}
@@ -363,3 +377,60 @@ function(package_bindingtester)
   add_custom_target(bindingtester ALL DEPENDS ${tar_file})
   add_dependencies(bindingtester copy_bindingtester_binaries)
 endfunction()
+
+function(add_fdbclient_test)
+  set(options DISABLED ENABLED)
+  set(oneValueArgs NAME)
+  set(multiValueArgs COMMAND)
+  cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
+  if(NOT T_ENABLED AND T_DISABLED)
+    return()
+  endif()
+  if(NOT T_NAME)
+    message(FATAL_ERROR "NAME is a required argument for add_fdbclient_test")
+  endif()
+  if(NOT T_COMMAND)
+    message(FATAL_ERROR "COMMAND is a required argument for add_fdbclient_test")
+  endif()
+  message(STATUS "Adding Client test ${T_NAME}")
+  add_test(NAME "${T_NAME}"
+    COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_cluster.py
+            --build-dir ${CMAKE_BINARY_DIR}
+            --
+            ${T_COMMAND})
+endfunction()
+
+function(add_java_test)
+  set(options DISABLED ENABLED)
+  set(oneValueArgs NAME CLASS)
+  set(multiValueArgs CLASS_PATH)
+  cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
+  if(NOT T_ENABLED AND T_DISABLED)
+    return()
+  endif()
+  if(NOT T_NAME)
+    message(FATAL_ERROR "NAME is a required argument for add_fdbclient_test")
+  endif()
+  if(NOT T_CLASS)
+    message(FATAL_ERROR "CLASS is a required argument for add_fdbclient_test")
+  endif()
+  set(cp "")
+  set(separator ":")
+  if (WIN32)
+    set(separator ";")
+  endif()
+  message(STATUS "CLASSPATH ${T_CLASS_PATH}")
+  foreach(path ${T_CLASS_PATH})
+    if(cp)
+      set(cp "${cp}${separator}${path}")
+    else()
+      set(cp "${path}")
+    endif()
+  endforeach()
+  add_fdbclient_test(
+    NAME ${T_NAME}
+    COMMAND ${Java_JAVA_EXECUTABLE}
+            -cp "${cp}"
+            -Djava.library.path=${CMAKE_BINARY_DIR}/lib
+            ${T_CLASS} "@CLUSTER_FILE@")
+endfunction()

cmake/ConfigureCompiler.cmake

@@ -4,12 +4,13 @@ env_set(USE_GPERFTOOLS OFF BOOL "Use gperfools for profiling")
 env_set(USE_DTRACE ON BOOL "Enable dtrace probes on supported platforms")
 env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage")
 env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest")
-env_set(VALGRIND_ARENA OFF BOOL "Inform valgrind about arena-allocated memory. Makes valgrind slower but more precise.")
 env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc")
 env_set(WITH_UNDODB OFF BOOL "Use rr or undodb")
 env_set(USE_ASAN OFF BOOL "Compile with address sanitizer")
+env_set(USE_GCOV OFF BOOL "Compile with gcov instrumentation")
+env_set(USE_MSAN OFF BOOL "Compile with memory sanitizer. To avoid false positives you need to dynamically link to a msan-instrumented libc++ and libc++abi, which you must compile separately. See https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo#instrumented-libc.")
+env_set(USE_TSAN OFF BOOL "Compile with thread sanitizer. It is recommended to dynamically link to a tsan-instrumented libc++ and libc++abi, which you can compile separately.")
 env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer")
-env_set(USE_TSAN OFF BOOL "Compile with thread sanitizer")
 env_set(FDB_RELEASE OFF BOOL "This is a building of a final release")
 env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available")
 env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info")
@@ -28,6 +29,9 @@ endif()
 if(STATIC_LINK_LIBCXX AND USE_TSAN)
   message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX doesn't work with tsan")
 endif()
+if(STATIC_LINK_LIBCXX AND USE_MSAN)
+  message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX doesn't work with msan")
+endif()
 
 set(rel_debug_paths OFF)
 if(RELATIVE_DEBUG_PATHS)
@@ -41,6 +45,7 @@ endif()
 add_compile_options(-DCMAKE_BUILD)
 add_compile_definitions(BOOST_ERROR_CODE_HEADER_ONLY BOOST_SYSTEM_NO_DEPRECATED)
 
+set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 if(ALLOC_INSTRUMENTATION)
   add_compile_options(-DALLOC_INSTRUMENTATION)
@@ -163,10 +168,28 @@ else()
   if(USE_ASAN)
     add_compile_options(
       -fsanitize=address
-      -DUSE_SANITIZER)
-    set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address")
-    set(CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fsanitize=address ${CMAKE_THREAD_LIBS_INIT}")
+      -DUSE_SANITIZER
+      -DADDRESS_SANITIZER
+      )
+    add_link_options(-fsanitize=address)
+  endif()
+
+  if(USE_MSAN)
+    if(NOT CLANG)
+      message(FATAL_ERROR "Unsupported configuration: USE_MSAN only works with Clang")
+    endif()
+    add_compile_options(
+      -fsanitize=memory
+      -fsanitize-memory-track-origins=2
+      -DUSE_SANITIZER
+      -DMEMORY_SANITIZER
+      )
+    add_link_options(-fsanitize=memory)
+  endif()
+
+  if(USE_GCOV)
+    add_compile_options(--coverage -DUSE_GCOV)
+    add_link_options(--coverage)
   endif()
 
   if(USE_UBSAN)
@@ -174,19 +197,20 @@ else()
       -fsanitize=undefined
       # TODO(atn34) Re-enable -fsanitize=alignment once https://github.com/apple/foundationdb/issues/1434 is resolved
       -fno-sanitize=alignment
-      -DUSE_SANITIZER)
-    set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=undefined")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=undefined")
-    set(CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fsanitize=undefined ${CMAKE_THREAD_LIBS_INIT}")
+      -DUSE_SANITIZER
+      -DUNDEFINED_BEHAVIOR_SANITIZER
+      )
+    add_link_options(-fsanitize=undefined)
   endif()
 
   if(USE_TSAN)
     add_compile_options(
       -fsanitize=thread
-      -DUSE_SANITIZER)
-    set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=thread")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=thread")
-    set(CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fsanitize=thread ${CMAKE_THREAD_LIBS_INIT}")
+      -DUSE_SANITIZER
+      -DTHREAD_SANITIZER
+      -DDYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1
+      )
+    add_link_options(-fsanitize=thread)
   endif()
 
   if(PORTABLE_BINARY)
@@ -248,9 +272,6 @@ else()
   if (USE_VALGRIND)
     add_compile_options(-DVALGRIND=1 -DUSE_VALGRIND=1)
   endif()
-  if (VALGRIND_ARENA)
-    add_compile_options(-DVALGRIND_ARENA=1)
-  endif()
   if (CLANG)
     add_compile_options()
     # Clang has link errors unless `atomic` is specifically requested.
@@ -363,4 +384,3 @@ else()
     endif()
   endif()
 endif()
-

cmake/FDBComponents.cmake

@@ -121,7 +121,24 @@ endif()
 
 # TOML can download and install itself into the binary directory, so it should
 # always be available.
-find_package(TOML11)
+find_package(toml11 QUIET)
+if(toml11_FOUND)
+  add_library(toml11_target INTERFACE)
+  add_dependencies(toml11_target INTERFACE toml11::toml11)
+else()
+  include(ExternalProject)
+
+  ExternalProject_add(toml11Project
+    URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz"
+    URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d
+    CMAKE_CACHE_ARGS
+      -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11
+      -Dtoml11_BUILD_TEST:BOOL=OFF
+    BUILD_ALWAYS ON)
+  add_library(toml11_target INTERFACE)
+  add_dependencies(toml11_target toml11Project)
+  target_include_directories(toml11_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/toml11/include)
+endif()
 
 ################################################################################
 

cmake/FindGperftools.cmake

@@ -52,6 +52,7 @@ mark_as_advanced(
 
 if (GPERFTOOLS_FOUND)
   add_library(gperftools UNKNOWN IMPORTED)
+  target_compile_definitions(gperftools PUBLIC USE_GPERFTOOLS)
   set_target_properties(gperftools PROPERTIES
     IMPORTED_LOCATION ${GPERFTOOLS_TCMALLOC_AND_PROFILER}
     INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}")

cmake/FindTOML11.cmake

@@ -1,28 +0,0 @@
-find_path(TOML11_INCLUDE_DIR
-  NAMES
-    toml.hpp
-  PATH_SUFFIXES
-    include
-    toml11
-    include/toml11
-   HINTS
-    "${_TOML11_HINTS}"
-)
-
-if (NOT TOML11_INCLUDE_DIR)
-  include(ExternalProject)
-
-  ExternalProject_add(toml11
-    URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz"
-    URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d
-    CMAKE_CACHE_ARGS
-      -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11
-      -Dtoml11_BUILD_TEST:BOOL=OFF)
-
-  set(TOML11_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/toml11/include")
-endif()
-
-find_package_handle_standard_args(TOML11
-  REQUIRED_VARS
-    TOML11_INCLUDE_DIR
-)

contrib/Joshua/scripts/bindingTest.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-pkill fdbserver
 ulimit -S -c unlimited
 
 unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
@@ -8,4 +7,4 @@ WORKDIR="$(pwd)/tmp/$$"
 if [ ! -d "${WORKDIR}" ] ; then
     mkdir -p "${WORKDIR}"
 fi
-DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" FDBSERVERPORT="${PORT_FDBSERVER:-4500}" ${SCRIPTDIR}/bindingTestScript.sh 1
+DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" ${SCRIPTDIR}/bindingTestScript.sh 1

contrib/Joshua/scripts/bindingTestScript.sh

@@ -7,7 +7,7 @@ SCRIPTID="${$}"
 SAVEONERROR="${SAVEONERROR:-1}"
 PYTHONDIR="${BINDIR}/tests/python"
 testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh"
-VERSION="1.6"
+VERSION="1.9"
 
 source ${SCRIPTDIR}/localClusterStart.sh
 
@@ -23,19 +23,22 @@ cycles="${1}"
 
 if [ "${DEBUGLEVEL}" -gt 0 ]
 then
-	echo "Work dir:    ${WORKDIR}"
-	echo "Bin dir:     ${BINDIR}"
-	echo "Log dir:     ${LOGDIR}"
-	echo "Python path: ${PYTHONDIR}"
-	echo "Lib dir:     ${LIBDIR}"
-	echo "Server port: ${FDBSERVERPORT}"
-	echo "Script Id:   ${SCRIPTID}"
-	echo "Version:     ${VERSION}"
+	echo "Work dir:       ${WORKDIR}"
+	echo "Bin dir:        ${BINDIR}"
+	echo "Log dir:        ${LOGDIR}"
+	echo "Python path:    ${PYTHONDIR}"
+	echo "Lib dir:        ${LIBDIR}"
+	echo "Cluster String: ${FDBCLUSTERTEXT}"
+	echo "Script Id:      ${SCRIPTID}"
+	echo "Version:        ${VERSION}"
 fi
 
 # Begin the cluster using the logic in localClusterStart.sh.
 startCluster
 
+# Stop the cluster on exit
+trap "stopCluster" EXIT
+
 # Display user message
 if [ "${status}" -ne 0 ]; then
 	:
@@ -58,8 +61,8 @@ fi
 # Display directory and log information, if an error occurred
 if [ "${status}" -ne 0 ]
 then
-	ls "${WORKDIR}" > "${LOGDIR}/dir.log"
-	ps -eafw > "${LOGDIR}/process-preclean.log"
+	ls "${WORKDIR}" &> "${LOGDIR}/dir.log"
+	ps -eafwH &> "${LOGDIR}/process-preclean.log"
 	if [ -f "${FDBCONF}" ]; then
 		cp -f "${FDBCONF}" "${LOGDIR}/"
 	fi
@@ -71,10 +74,15 @@ fi
 
 # Save debug information files, environment, and log information, if an error occurred
 if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then
-	ps -eafw > "${LOGDIR}/process-exit.log"
-	netstat -na > "${LOGDIR}/netstat.log"
-	df -h > "${LOGDIR}/disk.log"
-	env > "${LOGDIR}/env.log"
+	ps -eafwH &> "${LOGDIR}/process-exit.log"
+	netstat -na &> "${LOGDIR}/netstat.log"
+	df -h &> "${LOGDIR}/disk.log"
+	env &> "${LOGDIR}/env.log"
+fi
+
+# Stop the cluster
+if stopCluster; then
+	unset FDBSERVERID
 fi
 
 exit "${status}"

contrib/Joshua/scripts/localClusterStart.sh

@@ -5,311 +5,398 @@ WORKDIR="${WORKDIR:-${SCRIPTDIR}/tmp/fdb.work}"
 LOGDIR="${WORKDIR}/log"
 ETCDIR="${WORKDIR}/etc"
 BINDIR="${BINDIR:-${SCRIPTDIR}}"
-FDBSERVERPORT="${FDBSERVERPORT:-4500}"
+FDBPORTSTART="${FDBPORTSTART:-4000}"
+FDBPORTTOTAL="${FDBPORTTOTAL:-1000}"
+SERVERCHECKS="${SERVERCHECKS:-10}"
+CONFIGUREWAIT="${CONFIGUREWAIT:-240}"
 FDBCONF="${ETCDIR}/fdb.cluster"
 LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}"
+AUDITCLUSTER="${AUDITCLUSTER:-0}"
+AUDITLOG="${AUDITLOG:-/tmp/audit-cluster.log}"
 
 # Initialize the variables
 status=0
 messagetime=0
 messagecount=0
 
+# Do nothing, if cluster string is already defined
+if [ -n "${FDBCLUSTERTEXT}" ]
+then
+  :
+# Otherwise, define the cluster text
+else
+  # Define a random ip address and port on localhost
+  if [ -z "${IPADDRESS}" ]; then
+      let index2="${RANDOM} % 256"
+      let index3="${RANDOM} % 256"
+      let index4="(${RANDOM} % 255) + 1"
+      IPADDRESS="127.${index2}.${index3}.${index4}"
+  fi
+  if [ -z "${FDBPORT}" ]; then
+      let FDBPORT="(${RANDOM} % ${FDBPORTTOTAL}) + ${FDBPORTSTART}"
+  fi
+  FDBCLUSTERTEXT="${IPADDRESS}:${FDBPORT}"
+fi
+
 function log
 {
-	local status=0
-	if [ "$#" -lt 1 ]
-	then
-		echo "Usage: log <message> [echo]"
-		echo
-		echo "Logs the message and timestamp to LOGFILE (${LOGFILE}) and, if the"
-		echo "second argument is either not present or is set to 1, stdout."
-		let status="${status} + 1"
-	else
-		# Log to stdout.
-		if [ "$#" -lt 2 ] || [ "${2}" -ge 1 ]
-		then
-			echo "${1}"
-		fi
+  local status=0
+  if [ "$#" -lt 1 ]
+  then
+    echo "Usage: log <message> [echo]"
+    echo
+    echo "Logs the message and timestamp to LOGFILE (${LOGFILE}) and, if the"
+    echo "second argument is either not present or is set to 1, stdout."
+    let status="${status} + 1"
+  else
+    # Log to stdout.
+    if [ "$#" -lt 2 ] || [ "${2}" -ge 1 ]
+    then
+      echo "${1}"
+    fi
 
-		# Log to file.
-		datestr=$(date +"%Y-%m-%d %H:%M:%S (%s)")
-		dir=$(dirname "${LOGFILE}")
-		if ! [ -d "${dir}" ] && ! mkdir -p "${dir}"
-		then
-			echo "Could not create directory to log output."
-			let status="${status} + 1"
-		elif ! [ -f "${LOGFILE}" ] && ! touch "${LOGFILE}"
-		then
-			echo "Could not create file ${LOGFILE} to log output."
-			let status="${status} + 1"
-		elif ! echo "[ ${datestr} ] ${1}" >> "${LOGFILE}"
-		then
-			echo "Could not log output to ${LOGFILE}."
-			let status="${status} + 1"
-		fi
-	fi
+    # Log to file.
+    datestr=$(date +"%Y-%m-%d %H:%M:%S (%s)")
+    dir=$(dirname "${LOGFILE}")
+    if ! [ -d "${dir}" ] && ! mkdir -p "${dir}"
+    then
+      echo "Could not create directory to log output."
+      let status="${status} + 1"
+    elif ! [ -f "${LOGFILE}" ] && ! touch "${LOGFILE}"
+    then
+      echo "Could not create file ${LOGFILE} to log output."
+      let status="${status} + 1"
+    elif ! echo "[ ${datestr} ] ${1}" >> "${LOGFILE}"
+    then
+      echo "Could not log output to ${LOGFILE}."
+      let status="${status} + 1"
+    fi
+  fi
 
-	return "${status}"
+  return "${status}"
 }
 
 # Display a message for the user.
 function displayMessage
 {
-	local status=0
+  local status=0
 
-	if [ "$#" -lt 1 ]
-	then
-		echo "displayMessage <message>"
-		let status="${status} + 1"
-	elif ! log "${1}" 0
-	then
-		log "Could not write message to file."
-	else
-		# Increment the message counter
-		let messagecount="${messagecount} + 1"
+  if [ "$#" -lt 1 ]
+  then
+    echo "displayMessage <message>"
+    let status="${status} + 1"
+  elif ! log "${1}" 0
+  then
+    log "Could not write message to file."
+  else
+    # Increment the message counter
+    let messagecount="${messagecount} + 1"
 
-		# Display successful message, if previous message
-		if [ "${messagecount}" -gt 1 ]
-		then
-			# Determine the amount of transpired time
-			let timespent="${SECONDS}-${messagetime}"
+    # Display successful message, if previous message
+    if [ "${messagecount}" -gt 1 ]
+    then
+      # Determine the amount of transpired time
+      let timespent="${SECONDS}-${messagetime}"
 
-			if [ "${DEBUGLEVEL}" -gt 0 ]; then
-				printf "... done in %3d seconds\n" "${timespent}"
-			fi
-		fi
+      if [ "${DEBUGLEVEL}" -gt 0 ]; then
+        printf "... done in %3d seconds\n" "${timespent}"
+      fi
+    fi
 
-		# Display message
-		if [ "${DEBUGLEVEL}" -gt 0 ]; then
-			printf "%-16s	  %-35s " "$(date "+%F %H-%M-%S")" "$1"
-		fi
+    # Display message
+    if [ "${DEBUGLEVEL}" -gt 0 ]; then
+      printf "%-16s    %-35s " "$(date "+%F %H-%M-%S")" "$1"
+    fi
 
-		# Update the variables
-		messagetime="${SECONDS}"
-	fi
+    # Update the variables
+    messagetime="${SECONDS}"
+  fi
 
-	return "${status}"
+  return "${status}"
 }
 
 # Create the directories used by the server.
-function createDirectories {
-	# Display user message
-	if ! displayMessage "Creating directories"
-	then
-		echo 'Failed to display user message'
-		let status="${status} + 1"
-	
-	elif ! mkdir -p "${LOGDIR}" "${ETCDIR}"
-	then
-		log "Failed to create directories"
-		let status="${status} + 1"
-	
-	# Display user message
-	elif ! displayMessage "Setting file permissions"
-	then
-		log 'Failed to display user message'
-		let status="${status} + 1"
-	
-	elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli"
-	then
-		log "Failed to set file permissions"
-		let status="${status} + 1"
-	
-	else
-		while read filepath
-		do
-				if [ -f "${filepath}" ] && [ ! -x "${filepath}" ]
-				then
-					# if [ "${DEBUGLEVEL}" -gt 1 ]; then
-					# 	log "   Enable executable: ${filepath}"
-					# fi
-					log "   Enable executable: ${filepath}" "${DEBUGLEVEL}"
-					if ! chmod 755 "${filepath}"
-					then
-						log "Failed to set executable for file: ${filepath}"
-						let status="${status} + 1"
-					fi
-				fi
-		done < <(find "${BINDIR}" -iname '*.py' -o -iname '*.rb' -o -iname 'fdb_flow_tester' -o -iname '_stacktester' -o -iname '*.js' -o -iname '*.sh' -o -iname '*.ksh')
-	fi
+function createDirectories
+{
+  local status=0
 
-	return ${status}
+  # Display user message
+  if ! displayMessage "Creating directories"
+  then
+    echo 'Failed to display user message'
+    let status="${status} + 1"
+
+  elif ! mkdir -p "${LOGDIR}" "${ETCDIR}"
+  then
+    log "Failed to create directories"
+    let status="${status} + 1"
+
+  # Display user message
+  elif ! displayMessage "Setting file permissions"
+  then
+    log 'Failed to display user message'
+    let status="${status} + 1"
+
+  elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli"
+  then
+    log "Failed to set file permissions"
+    let status="${status} + 1"
+
+  else
+    while read filepath
+    do
+        if [ -f "${filepath}" ] && [ ! -x "${filepath}" ]
+        then
+          # if [ "${DEBUGLEVEL}" -gt 1 ]; then
+          #   log "   Enable executable: ${filepath}"
+          # fi
+          log "   Enable executable: ${filepath}" "${DEBUGLEVEL}"
+          if ! chmod 755 "${filepath}"
+          then
+            log "Failed to set executable for file: ${filepath}"
+            let status="${status} + 1"
+          fi
+        fi
+    done < <(find "${BINDIR}" -iname '*.py' -o -iname '*.rb' -o -iname 'fdb_flow_tester' -o -iname '_stacktester' -o -iname '*.js' -o -iname '*.sh' -o -iname '*.ksh')
+  fi
+
+  return ${status}
 }
 
 # Create a cluster file for the local cluster.
-function createClusterFile {
-	if [ "${status}" -ne 0 ]; then
-		:
-	# Display user message
-	elif ! displayMessage "Creating Fdb Cluster file"
-	then
-		log 'Failed to display user message'
-		let status="${status} + 1"
-	else
-		description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8)
-		random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8)
-		echo "$description:$random_str@127.0.0.1:${FDBSERVERPORT}" > "${FDBCONF}"
-	fi
+function createClusterFile
+{
+  local status=0
 
-	if [ "${status}" -ne 0 ]; then
-		:
-	elif ! chmod 0664 "${FDBCONF}"; then
-		log "Failed to set permissions on fdbconf: ${FDBCONF}"
-		let status="${status} + 1"
-	fi
+  if [ "${status}" -ne 0 ]; then
+    :
+  # Display user message
+  elif ! displayMessage "Creating Fdb Cluster file"
+  then
+    log 'Failed to display user message'
+    let status="${status} + 1"
+  else
+    description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8)
+    random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8)
+    echo "${description}:${random_str}@${FDBCLUSTERTEXT}" > "${FDBCONF}"
+  fi
 
-	return ${status}
+  if [ "${status}" -ne 0 ]; then
+    :
+  elif ! chmod 0664 "${FDBCONF}"; then
+    log "Failed to set permissions on fdbconf: ${FDBCONF}"
+    let status="${status} + 1"
+  fi
+
+  return ${status}
+}
+
+# Stop the Cluster from running.
+function stopCluster
+{
+  local status=0
+
+  # Add an audit entry, if enabled
+  if [ "${AUDITCLUSTER}" -gt 0 ]; then
+    printf '%-15s (%6s)  Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBCLUSTERTEXT}" "${FDBSERVERID}" >> "${AUDITLOG}"
+  fi
+  if [ -z "${FDBSERVERID}" ]; then
+    log 'FDB Server process is not defined'
+    let status="${status} + 1"
+  elif ! kill -0 "${FDBSERVERID}"; then
+    log "Failed to locate FDB Server process (${FDBSERVERID})"
+    let status="${status} + 1"
+  elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${FDBCLUSTERTEXT}; sleep 3" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log"
+  then
+    # Ensure that process is dead
+    if ! kill -0 "${FDBSERVERID}" 2> /dev/null; then
+      log "Killed cluster (${FDBSERVERID}) via cli"
+    elif ! kill -9 "${FDBSERVERID}"; then
+      log "Failed to kill FDB Server process (${FDBSERVERID}) via cli or kill command"
+      let status="${status} + 1"
+    else
+      log "Forcibly killed FDB Server process (${FDBSERVERID}) since cli failed"
+    fi
+  elif ! kill -9 "${FDBSERVERID}"; then
+    log "Failed to forcibly kill FDB Server process (${FDBSERVERID})"
+    let status="${status} + 1"
+  else
+    log "Forcibly killed FDB Server process (${FDBSERVERID})"
+  fi
+  return "${status}"
 }
 
 # Start the server running.
-function startFdbServer {
-	if [ "${status}" -ne 0 ]; then
-		:
-	elif ! displayMessage "Starting Fdb Server"
-	then
-		log 'Failed to display user message'
-		let status="${status} + 1"
+function startFdbServer
+{
+  local status=0
 
-	elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "auto:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/$$" &> "${LOGDIR}/fdbserver.log" &
-	then
-		log "Failed to start FDB Server"
-		# Maybe the server is already running
-		FDBSERVERID="$(pidof fdbserver)"
-		let status="${status} + 1"
-	else
-		FDBSERVERID="${!}"
-	fi
+  # Add an audit entry, if enabled
+  if [ "${AUDITCLUSTER}" -gt 0 ]; then
+    printf '%-15s (%6s)  Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${FDBCLUSTERTEXT}" >> "${AUDITLOG}"
+  fi
 
-	if ! kill -0 ${FDBSERVERID} ; then
-		log "FDB Server start failed."
-		let status="${status} + 1"
-	fi
+  if ! displayMessage "Starting Fdb Server"
+  then
+    log 'Failed to display user message'
+    let status="${status} + 1"
 
-	return ${status}
+  else
+    "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${FDBCLUSTERTEXT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" &
+    if [ "${?}" -ne 0 ]
+    then
+        log "Failed to start FDB Server"
+        let status="${status} + 1"
+    else
+        FDBSERVERID="${!}"
+    fi
+  fi
+
+  if [ -z "${FDBSERVERID}" ]; then
+    log "FDB Server start failed because no process"
+    let status="${status} + 1"
+  elif ! kill -0 "${FDBSERVERID}" ; then
+    log "FDB Server start failed because process terminated unexpectedly"
+    let status="${status} + 1"
+  fi
+
+  return ${status}
 }
 
-function getStatus {
-	if [ "${status}" -ne 0 ]; then
-		:
-	elif ! date &>> "${LOGDIR}/fdbclient.log"
-	then
-		log 'Failed to get date'
-		let status="${status} + 1"
-	elif ! "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 120 &>> "${LOGDIR}/fdbclient.log"
-	then
-		log 'Failed to get status from fdbcli'
-		let status="${status} + 1"
-	elif !  date &>> "${LOGDIR}/fdbclient.log"
-	then
-		log 'Failed to get date'
-		let status="${status} + 1"
-	fi
+function getStatus
+{
+  local status=0
 
-	return ${status}
+  if [ "${status}" -ne 0 ]; then
+    :
+  elif ! date &>> "${LOGDIR}/fdbclient.log"
+  then
+    log 'Failed to get date'
+    let status="${status} + 1"
+  elif ! "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 120 &>> "${LOGDIR}/fdbclient.log"
+  then
+    log 'Failed to get status from fdbcli'
+    let status="${status} + 1"
+  elif !  date &>> "${LOGDIR}/fdbclient.log"
+  then
+    log 'Failed to get date'
+    let status="${status} + 1"
+  fi
+
+  return ${status}
 }
 
 # Verify that the cluster is available.
-function verifyAvailable {
-	# Verify that the server is running.
-	if ! kill -0 "${FDBSERVERID}"
-	then
-		log "FDB server process (${FDBSERVERID}) is not running"
-		let status="${status} + 1"
-		return 1
+function verifyAvailable
+{
+  local status=0
 
-	# Display user message.
-	elif ! displayMessage "Checking cluster availability"
-	then
-		log 'Failed to display user message'
-		let status="${status} + 1"
-		return 1
-
-	# Determine if status json says the database is available.
-	else
-		avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 10 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'`
-		log "Avail value: ${avail}" "${DEBUGLEVEL}"
-		if [[ -n "${avail}" ]] ; then
-			return 0
-		else
-			return 1
-		fi
-	fi
+  if [ -z "${FDBSERVERID}" ]; then
+    log "FDB Server process is not defined."
+    let status="${status} + 1"
+  # Verify that the server is running.
+  elif ! kill -0 "${FDBSERVERID}"
+  then
+    log "FDB server process (${FDBSERVERID}) is not running"
+    let status="${status} + 1"
+  # Display user message.
+  elif ! displayMessage "Checking cluster availability"
+  then
+    log 'Failed to display user message'
+    let status="${status} + 1"
+  # Determine if status json says the database is available.
+  else
+    avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout "${SERVERCHECKS}" 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'`
+    log "Avail value: ${avail}" "${DEBUGLEVEL}"
+    if [[ -n "${avail}" ]] ; then
+      :
+    else
+      let status="${status} + 1"
+    fi
+  fi
+  return "${status}"
 }
 
 # Configure the database on the server.
-function createDatabase {
-	if [ "${status}" -ne 0 ]; then
-		:
-	# Ensure that the server is running
-	elif ! kill -0 "${FDBSERVERID}"
-	then
-		log "FDB server process: (${FDBSERVERID}) is not running"
-		let status="${status} + 1"
+function createDatabase
+{
+  local status=0
 
-	# Display user message
-	elif ! displayMessage "Creating database"
-	then
-		log 'Failed to display user message'
-		let status="${status} + 1"
-	elif ! echo "Client log:" &> "${LOGDIR}/fdbclient.log"
-	then
-		log 'Failed to create fdbclient.log'
-		let status="${status} + 1"
-	elif ! getStatus
-	then
-		log 'Failed to get status'
-		let status="${status} + 1"
+  if [ "${status}" -ne 0 ]; then
+    :
+  # Ensure that the server is running
+  elif ! kill -0 "${FDBSERVERID}"
+  then
+    log "FDB server process: (${FDBSERVERID}) is not running"
+    let status="${status} + 1"
 
-	# Configure the database.
-	else
-		"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout 240 --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log"
+  # Display user message
+  elif ! displayMessage "Creating database"
+  then
+    log 'Failed to display user message'
+    let status="${status} + 1"
+  elif ! echo "Client log:" &> "${LOGDIR}/fdbclient.log"
+  then
+    log 'Failed to create fdbclient.log'
+    let status="${status} + 1"
+  elif ! getStatus
+  then
+    log 'Failed to get status'
+    let status="${status} + 1"
 
-		if ! displayMessage "Checking if config succeeded"
-		then
-			log 'Failed to display user message.'
-		fi
+  # Configure the database.
+  else
+    "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout "${CONFIGUREWAIT}" --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log"
 
-		iteration=0
-		while [[ "${iteration}" -lt 10 ]] && ! verifyAvailable
-		do
-			log "Database not created (iteration ${iteration})."
-			let iteration="${iteration} + 1"
-		done
+    if ! displayMessage "Checking if config succeeded"
+    then
+      log 'Failed to display user message.'
+    fi
 
-		if ! verifyAvailable
-		then
-			log "Failed to create database via cli"
-			getStatus
-			cat "${LOGDIR}/fdbclient.log"
-			log "Ignoring -- moving on"
-			#let status="${status} + 1"
-		fi
-	fi
+    iteration=0
+    while [[ "${iteration}" -lt "${SERVERCHECKS}" ]] && ! verifyAvailable
+    do
+      log "Database not created (iteration ${iteration})."
+      let iteration="${iteration} + 1"
+    done
 
-	return ${status}
+    if ! verifyAvailable
+    then
+      log "Failed to create database via cli"
+      getStatus
+      cat "${LOGDIR}/fdbclient.log"
+      log "Ignoring -- moving on"
+      #let status="${status} + 1"
+    fi
+  fi
+
+  return ${status}
 }
 
 # Begin the local cluster from scratch.
-function startCluster {
-	if [ "${status}" -ne 0 ]; then
-		:
-	elif ! createDirectories
-	then
-		log "Could not create directories."
-		let status="${status} + 1"
-	elif ! createClusterFile
-	then
-		log "Could not create cluster file."
-		let status="${status} + 1"
-	elif ! startFdbServer
-	then
-		log "Could not start FDB server."
-		let status="${status} + 1"
-	elif ! createDatabase
-	then
-		log "Could not create database."
-		let status="${status} + 1"
-	fi
+function startCluster
+{
+  local status=0
 
-	return ${status}
+  if [ "${status}" -ne 0 ]; then
+    :
+  elif ! createDirectories
+  then
+    log "Could not create directories."
+    let status="${status} + 1"
+  elif ! createClusterFile
+  then
+    log "Could not create cluster file."
+    let status="${status} + 1"
+  elif ! startFdbServer
+  then
+    log "Could not start FDB server."
+    let status="${status} + 1"
+  elif ! createDatabase
+  then
+    log "Could not create database."
+    let status="${status} + 1"
+  fi
+
+  return ${status}
 }

contrib/TestHarness/Program.cs.cmake

@@ -307,10 +307,13 @@ namespace SummarizeTest
                     int unseed;
                     string uid = Guid.NewGuid().ToString();
                     bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
-                    result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout);
+                    bool useToml = File.Exists(testFile + "-1.toml");
+                    string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt";
+                    result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout);
                     if (result == 0)
                     {
-                        result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout);
+                        string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt";
+                        result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout);
                     }
                 }
                 else

contrib/commit_debug.py

@@ -24,22 +24,22 @@ def parse_args():
 # (e)nd of a span with a better given name
 locationToPhase = {
         "NativeAPI.commit.Before": [],
-        "MasterProxyServer.batcher": [("b", "Commit")],
-        "MasterProxyServer.commitBatch.Before": [],
-        "MasterProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")],
-        "MasterProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")],
+        "CommitProxyServer.batcher": [("b", "Commit")],
+        "CommitProxyServer.commitBatch.Before": [],
+        "CommitProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")],
+        "CommitProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")],
         "Resolver.resolveBatch.Before": [("b", "Resolver.PipelineWait")],
         "Resolver.resolveBatch.AfterQueueSizeCheck": [],
         "Resolver.resolveBatch.AfterOrderer": [("e", "Resolver.PipelineWait"), ("b", "Resolver.Conflicts")],
         "Resolver.resolveBatch.After": [("e", "Resolver.Conflicts")],
-        "MasterProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")],
-        "MasterProxyServer.commitBatch.ProcessingMutations": [],
-        "MasterProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")],
+        "CommitProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")],
+        "CommitProxyServer.commitBatch.ProcessingMutations": [],
+        "CommitProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")],
         "TLog.tLogCommit.BeforeWaitForVersion": [("b", "TLog.PipelineWait")],
         "TLog.tLogCommit.Before": [("e", "TLog.PipelineWait")],
         "TLog.tLogCommit.AfterTLogCommit": [("b", "TLog.FSync")],
         "TLog.tLogCommit.After": [("e", "TLog.FSync")],
-        "MasterProxyServer.commitBatch.AfterLogPush": [("e", "Commit")],
+        "CommitProxyServer.commitBatch.AfterLogPush": [("e", "Commit")],
         "NativeAPI.commit.After": [],
 }
 

design/backup_v2_partitioned_logs.md

@@ -16,7 +16,7 @@ As an essential component of a database system, backup and restore is commonly u
 
 ## Background
 
-FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at Proxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of Proxy CPU time.
+FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at CommitProxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of CommitProxy CPU time.
 
 The design of old backup system is [here](https://github.com/apple/foundationdb/blob/master/design/backup.md), and the data format of range files and mutations files is [here](https://github.com/apple/foundationdb/blob/master/design/backup-dataFormat.md). The technical overview of FDB is [here](https://github.com/apple/foundationdb/wiki/Technical-Overview-of-the-Database). The FDB recovery is described in this [doc](https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md).
 
@@ -37,7 +37,7 @@ The design of old backup system is [here](https://github.com/apple/foundationdb/
 
 Feature priorities: Feature 1, 2, 3, 4, 5 are must-have; Feature 6 is better to have.
 
-1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the Proxy, thus reduce TLog write bandwidth usage by half and significantly improve Proxy CPU usage;
+1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the CommitProxy, thus reduce TLog write bandwidth usage by half and significantly improve CommitProxy CPU usage;
 2. **Correctness**: The restored database must be consistent: each *restored* state (i.e., key-value pair) at a version `v` must match the original state at version `v`.
 3. **Performance**: The backup system should be performant, mostly measured as a small CPU overhead on transaction logs and backup workers. The version lag on backup workers is an indicator of performance.
 4. **Fault-tolerant**: The backup system should be fault-tolerant to node failures in the FDB cluster.
@@ -153,9 +153,9 @@ The requirement of the new backup system raises several design challenges:
 
 **Master**: The master is responsible for coordinating the transition of the FDB transaction sub-system from one generation to the next. In particular, the master recruits backup workers during the recovery.
 
-**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs.
+**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the commit proxy in version order, and only respond to the commit proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs.
 
-**Proxy**: The proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, Proxies are responsible to group mutations into backup mutations and write them to the database.
+**CommitProxy**: The commit proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, commit proxies are responsible to group mutations into backup mutations and write them to the database.
 
 **GrvProxy**: The GRV proxies are responsible for providing read versions.
 ## System overview
@@ -229,7 +229,7 @@ The operator’s backup request can indicate if an old backup or a new backup is
 2. All backup workers monitor the key `\xff\x02/backupStarted`, see the change, and start logging mutations.
 3. After all backup workers have started, the `fdbbackup` tool initiates the backup of all or specified key ranges by issuing a transaction `Ts`.
 
-Compared to the old backup system, the above step 1 and 2 are new and is only triggered if client requests for a new type of backup. The purpose is to allow backup workers to function as no-op if there are no ongoing backups. However, the backup workers should still continuously pop their corresponding tags, otherwise mutations will be kept in the TLog. In order to know the version to pop, backup workers can obtain the read version from any proxy. Because the read version must be a committed version, so popping to this version is safe.
+Compared to the old backup system, the above step 1 and 2 are new and is only triggered if client requests for a new type of backup. The purpose is to allow backup workers to function as no-op if there are no ongoing backups. However, the backup workers should still continuously pop their corresponding tags, otherwise mutations will be kept in the TLog. In order to know the version to pop, backup workers can obtain the read version from any GRV proxy. Because the read version must be a committed version, so popping to this version is safe.
 
 **Backup Submission Protocol**
 Protocol for `submitBackup()` to ensure that all backup workers of the current epoch have started logging mutations:

design/data-distributor-internals.md

@@ -22,7 +22,7 @@ Data distribution manages the lifetime of storage servers, decides which storage
 
 **Data distribution queue (`struct DDQueueData`)**: It receives shards to be relocated (i.e., RelocateShards), decides which shard should be moved to which server team, prioritizes the data movement based on relocate shard’s priority, and controls the progress of data movement based on servers’ workload.
 
-**Special keys in the system keyspace**: DD saves its state in the system keyspace to recover from failure and to ensure every process (e.g., proxies, tLogs and storage servers) has a consistent view of which storage server is responsible for which key range.
+**Special keys in the system keyspace**: DD saves its state in the system keyspace to recover from failure and to ensure every process (e.g., commit proxies, tLogs and storage servers) has a consistent view of which storage server is responsible for which key range.
 
 *serverKeys* sub-space (`\xff/serverKeys/`): It records the start key of each shard a server is responsible for. The format is *\xff/serverKeys/[serverID]/[start_key]*. To get start keys of all shards for a server, DD can read the key range with prefix *\xff/serverKeys/[serverID]/* and decode the value of [start_key].
 
@@ -32,9 +32,9 @@ Data distribution manages the lifetime of storage servers, decides which storage
 
 When a new DD is initialized, it will set itself as the owner by setting its random UID to the `moveKeysLockOwnerKey`. Since the owner key has only one value, at most one DD can own the DD-related system subspace. This avoids the potential race condition between multiple DDs which may co-exit during DD recruitment.
 
-**Transaction State Store (txnStateStore)**: It is a replica of the special keyspace that stores the cluster’s states, such as which SS is responsible for which shard. Because proxies use txnStateStore to decide which tLog and SS should receive a mutation, proxies must have a consistent view of txnStateStore. Therefore, changes to txnStateStore must be populated to all proxies in total order. To achieve that, we use the special transaction (`applyMetaMutations`) to update txnStateStore and use resolvers to ensure the total ordering (serializable snapshot isolation).
+**Transaction State Store (txnStateStore)**: It is a replica of the special keyspace that stores the cluster’s states, such as which SS is responsible for which shard. Because commit proxies use txnStateStore to decide which tLog and SS should receive a mutation, commit proxies must have a consistent view of txnStateStore. Therefore, changes to txnStateStore must be populated to all commit proxies in total order. To achieve that, we use the special transaction (`applyMetaMutations`) to update txnStateStore and use resolvers to ensure the total ordering (serializable snapshot isolation).
 
-**Private mutation**: A private mutation is a mutation updating a special system key, such as keyServersKey (`\xff/keyServers/`) and serverKeysKey (`\xff/serverKeys/`). Like a normal mutation, a private mutation will be processed by the transaction systems (i.e., proxy, resolver and tLog) and be routed to a set of storage servers, based on the mutation’s tag, to update the key-value in the storage engine. Private mutations also keep the serializable snapshot isolation and consensus: The results of committed concurrent private mutations can be reproduced by sequentially executing the mutations, and all components in FDB have the same view of the mutations.
+**Private mutation**: A private mutation is a mutation updating a special system key, such as keyServersKey (`\xff/keyServers/`) and serverKeysKey (`\xff/serverKeys/`). Like a normal mutation, a private mutation will be processed by the transaction systems (i.e., commit proxy, resolver and tLog) and be routed to a set of storage servers, based on the mutation’s tag, to update the key-value in the storage engine. Private mutations also keep the serializable snapshot isolation and consensus: The results of committed concurrent private mutations can be reproduced by sequentially executing the mutations, and all components in FDB have the same view of the mutations.
 
 
 ## Operations
@@ -51,7 +51,7 @@ Whenever the team builder is invoked, it aims to build the desired number of ser
 
 **Data distribution queue server (`dataDistributionQueue` actor)**: It is created when DD is initialized. It behaves as a server to handle RelocateShard related requests. For example, it waits on the stream of RelocateShard. When a new RelocateShard is sent by teamTracker, it enqueues the new shard, and cancels the inflight shards that overlap with the new relocate shard.
 
-**`applyMetaMutations`**: This is special logic to handle *private transactions* that modify txnStateStore and special system keys. Transaction systems (i.e., proxy, resolver and tLogs) and storage servers perform extra operations for the special transactions. For any update, it will be executed on all proxies in order so that all proxies have a consistent view of the txnStateStore. It will also send special keys to storage servers so that storage servers know the new keyspace they are now responsible for.
+**`applyMetaMutations`**: This is special logic to handle *private transactions* that modify txnStateStore and special system keys. Transaction systems (i.e., commit proxy, resolver and tLogs) and storage servers perform extra operations for the special transactions. For any update, it will be executed on all commit proxies in order so that all commit proxies have a consistent view of the txnStateStore. It will also send special keys to storage servers so that storage servers know the new keyspace they are now responsible for.
 
 A storage server (SS) processes all requests sent to the server in its `storageServerCore` actor. When a (private) mutation request is sent to a SS, the server will call the `update()` function. Eventually, the `StorageUpdater` class will be invoked to apply the mutation in `applyMutation()` function, which handles private mutations `applyPrivateData()` function.
 
@@ -84,9 +84,9 @@ Actors are created to monitor the reasons of key movement:
 
 A key range is a shard. A shard is the minimum unit of moving data. The storage server’s ownership of a shard -- which SS owns which shard -- is stored in the system keyspace *serverKeys* (`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). To simplify the explanation, we refer to the storage server’s ownership of a shard as a shard’s ownership.
 
-A shard’s ownership is used in transaction systems (proxy and tLogs) to route mutations to tLogs and storage servers. When a proxy receives a mutation,dd it uses the shard’s ownership to decide which *k* tLogs receive the mutation, assuming *k* is the replias factor. When a storage server pulls mutations from tLogs, it uses the shard’s ownership to decide which shards the SS is responsible for and which tLog the SS should pull the data from.
+A shard’s ownership is used in transaction systems (commit proxy and tLogs) to route mutations to tLogs and storage servers. When a commit proxy receives a mutation, it uses the shard’s ownership to decide which *k* tLogs receive the mutation, assuming *k* is the replias factor. When a storage server pulls mutations from tLogs, it uses the shard’s ownership to decide which shards the SS is responsible for and which tLog the SS should pull the data from.
 
-A shard’s ownership must be consistent across transaction systems and SSes, so that mutations can be correctly routed to SSes. Moving keys from a SS to another requires changing the shard’s ownership under ACID property. The ACID property is achieved by using FDB transactions to change the *serverKeys *(`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). The mutation on the *serverKeys *and* keyServers *will be categorized as private mutations in transaction system. Compared to normal mutation, the private mutations will change the transaction state store (txnStateStore) that maintains the *serverKeys* and *keyServers* for transaction systems (proxy and tLog) when it arrives on each transaction component (e.g., tLog). Because mutations are processed in total order with the ACID guarantees, the change to the txnStateStore will be executed in total order on each node and the change on the shard’s ownership will also be consistent.
+A shard’s ownership must be consistent across transaction systems and SSes, so that mutations can be correctly routed to SSes. Moving keys from a SS to another requires changing the shard’s ownership under ACID property. The ACID property is achieved by using FDB transactions to change the *serverKeys *(`\xff/serverKeys/`) and *keyServers* (`\xff/keyServers/`). The mutation on the *serverKeys *and* keyServers *will be categorized as private mutations in transaction system. Compared to normal mutation, the private mutations will change the transaction state store (txnStateStore) that maintains the *serverKeys* and *keyServers* for transaction systems (commit proxy and tLog) when it arrives on each transaction component (e.g., tLog). Because mutations are processed in total order with the ACID guarantees, the change to the txnStateStore will be executed in total order on each node and the change on the shard’s ownership will also be consistent.
 
 The data movement from one server (called source server) to another (called destination server) has four steps:
 (1) DD adds the destination server as the shard’s new owner;

design/recovery-internals.md

@@ -8,12 +8,12 @@ This document explains at the high level how the recovery works in a single clus
 
 ## `ServerDBInfo` data structure
 
-This data structure contains transient information which is broadcast to all workers for a database, permitting them to communicate with each other. It contains, for example, the interfaces for cluster controller (CC), master, ratekeeper, and resolver, and holds the log system's configuration. Only part of the data structure, such as `ClientDBInfo` that contains the list of proxies, is  available to the client.
+This data structure contains transient information which is broadcast to all workers for a database, permitting them to communicate with each other. It contains, for example, the interfaces for cluster controller (CC), master, ratekeeper, and resolver, and holds the log system's configuration. Only part of the data structure, such as `ClientDBInfo` that contains the list of GRV proxies and commit proxies, is available to the client.
 
 Whenever a field of the `ServerDBInfo`is changed, the new value of the field, say new master's interface, will be sent to the CC and CC will propagate the new `ServerDBInfo` to all workers in the cluster.
 
 ## When will recovery happen?
-Failure of certain roles in FDB can cause recovery. Those roles are cluster controller, master, proxy, transaction logs (tLog), resolvers, and log router.
+Failure of certain roles in FDB can cause recovery. Those roles are cluster controller, master, GRV proxy, commit proxy, transaction logs (tLog), resolvers, log router, and backup workers.
 
 Network partition or failures can make CC unable to reach some roles, treating those roles as dead and causing recovery. If CC cannot connect to a majority of coordinators, it will be treated as dead by coordinators and recovery will happen.
 
@@ -97,7 +97,7 @@ Master interface is stored in `serverDBInfo`. Once the CC recruits the master, i
 Once the master locks the cstate, it will recruit the still-alive tLogs from the previous generation for the benefit of faster recovery. The master gets the old tLogs’ interfaces from the READING_CSTATE phase and uses those interfaces to track which old tLog are still alive, the implementation of which is in `trackRejoins()`.
 
 
-Once the master gets enough tLogs, it calculates the known committed version (i.e., `knownCommittedVersion` in code). `knownCommittedVersion` is the highest version that a proxy tells a given tLog that it had durably committed on *all* tLogs. The master's is the maximum of all of that. `knownCommittedVersion` is  important, because it defines the lower bound of what version range of mutations need to be copied to the new generation. That is, any versions larger than the master's `knownCommittedVersion` is not guaranteed to persist on all replicas. The master chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy.
+Once the master gets enough tLogs, it calculates the known committed version (i.e., `knownCommittedVersion` in code). `knownCommittedVersion` is the highest version that a commit proxy tells a given tLog that it had durably committed on *all* tLogs. The master's is the maximum of all of that. `knownCommittedVersion` is  important, because it defines the lower bound of what version range of mutations need to be copied to the new generation. That is, any versions larger than the master's `knownCommittedVersion` is not guaranteed to persist on all replicas. The master chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy.
 
 Later, the master will use the recruited tLogs to create a new `TagPartitionedLogSystem` for the new generation.
 
@@ -121,9 +121,9 @@ Consider an old generation with three TLogs: `A, B, C`. Their durable versions a
 
 Once we have a `knownCommittedVersion`, the master will reconstruct the transaction state store (txnStateStore) by peeking the txnStateTag in oldLogSystem.
 Recall that the txnStateStore includes the transaction system’s configuration, such as the assignment of shards to SS and to tLogs and that the txnStateStore was durable on disk in the oldLogSystem.
-Once we get the txnStateStore, we know the configuration of the transaction system, such as the number of proxies. The master then can ask the CC to recruit roles for the new generation in the `recruitEverything()` function. Those recruited roles includes proxies, tLogs and seed SSes, which are the storage servers created for an empty database in the first generation to host the first shard and serve as the starting point of the bootstrap process to recruit more SSes. Once all roles are recruited, the master starts a new epoch in `newEpoch()`.
+Once we get the txnStateStore, we know the configuration of the transaction system, such as the number of GRV proxies and commit proxies. The master then can ask the CC to recruit roles for the new generation in the `recruitEverything()` function. Those recruited roles includes GRV proxies, commit proxies, tLogs and seed SSes, which are the storage servers created for an empty database in the first generation to host the first shard and serve as the starting point of the bootstrap process to recruit more SSes. Once all roles are recruited, the master starts a new epoch in `newEpoch()`.
 
-At this point, we have recovered the txnStateStore, recruited new proxies and tLogs, and copied data from old tLogs to new tLogs. We have a working transaction system in the new generation now.
+At this point, we have recovered the txnStateStore, recruited new GRV proxies, commit proxies and tLogs, and copied data from old tLogs to new tLogs. We have a working transaction system in the new generation now.
 
 ### Where can the recovery get stuck in this phase?
 
@@ -151,7 +151,7 @@ Not every FDB role participates in the recovery phases 1-3. This phase tells the
 Storage servers (SSes) are not involved in the recovery phase 1 - 3. To notify SSes about the recovery, the master commits a recovery transaction, the first transaction in the new generation, which contains the txnStateStore information. Once storage servers receive the recovery transaction, it will compare its latest data version and the recovery version, and rollback to the recovery version if its data version is newer. Note that storage servers may have newer data than the recovery version because they pre-fetch mutations from tLogs before the mutations are durable to reduce the latency to read newly written data.
 
 
-Proxies haven’t recovered the transaction system state and cannot accept transactions yet. The master recovers proxies’ states by sending the txnStateStore to proxies through proxies’ (`txnState`) interfaces in `sendIntialCommitToResolvers()` function. Once proxies have recovered their states, they can start processing transactions. The recovery transaction that was waiting on proxies will be processed.
+Commit proxies haven’t recovered the transaction system state and cannot accept transactions yet. The master recovers proxies’ states by sending the txnStateStore to commit proxies through commit proxies’ (`txnState`) interfaces in `sendIntialCommitToResolvers()` function. Once commit proxies have recovered their states, they can start processing transactions. The recovery transaction that was waiting on commit proxies will be processed.
 
 
 The resolvers haven’t known the recovery version either. The master needs to send the lastEpochEnd version (i.e., last commit of the previous generation) to resolvers via resolvers’ (`resolve`) interface.
@@ -162,7 +162,7 @@ At the end of this phase, every role should be aware of the recovery and start r
 
 ## Phase 5: WRITING_CSTATE
 
-Coordinators store the transaction systems’ information. The master needs to write the new tLogs into coordinators’ states to achieve consensus and fault tolerance. Only when the coordinators’ states are updated with the new transaction system’s configuration will the cluster controller tell clients about the new transaction system (such as the new proxies).
+Coordinators store the transaction systems’ information. The master needs to write the new tLogs into coordinators’ states to achieve consensus and fault tolerance. Only when the coordinators’ states are updated with the new transaction system’s configuration will the cluster controller tell clients about the new transaction system (such as the new GRV proxies and commit proxies).
 
 The master only needs to write the new tLogs to a quorum of coordinators for a running cluster. The only time the master has to write all coordinators is when creating a brand new database.
 

design/tlog-spilling.md.html

@@ -7,17 +7,17 @@
 (This assumes a basic familiarity with [FoundationDB's architecture](https://www.youtu.be/EMwhsGsxfPU).)
 
 Transaction logs are a distributed Write-Ahead-Log for FoundationDB.  They
-receive commits from proxies, and are responsible for durably storing those
-commits, and making them available to storage servers for reading.
+receive commits from commit proxies, and are responsible for durably storing 
+those commits, and making them available to storage servers for reading.
 
 Clients send *mutations*, the list of their set, clears, atomic operations,
-etc., to proxies.  Proxies collect mutations into a *batch*, which is the list
-of all changes that need to be applied to the database to bring it from version
-`N-1` to `N`.  Proxies then walk through their in-memory mapping of shard
-boundaries to associate one or more *tags*, a small integer uniquely
-identifying a destination storage server, with each mutation.  They then send a
-*commit*, the full list of `(tags, mutation)` for each mutation in a batch, to
-the transaction logs.
+etc., to commit proxies. Commit proxies collect mutations into a *batch*, which
+is the list of all changes that need to be applied to the database to bring it
+from version `N-1` to `N`. Commit proxies then walk through their in-memory 
+mapping of shard boundaries to associate one or more *tags*, a small integer 
+uniquely identifying a destination storage server, with each mutation. They 
+then send a *commit*, the full list of `(tags, mutation)` for each mutation in
+a batch, to the transaction logs.
 
 The transaction log has two responsibilities: it must persist the commits to
 disk and notify the proxy when a commit is durably stored, and it must make the

documentation/sphinx/source/administration.rst

@@ -28,8 +28,6 @@ Starting and stopping
 
 After installation, FoundationDB is set to start automatically. You can manually start and stop the database with the commands shown below.
 
-These commands start and stop the master ``fdbmonitor`` process, which in turn starts ``fdbserver`` and ``backup-agent`` processes.  See :ref:`administration_fdbmonitor` for details.
-
 Linux
 -----
 
@@ -58,6 +56,15 @@ It can be stopped and prevented from starting at boot as follows::
 
     host:~ user$ sudo launchctl unload -w /Library/LaunchDaemons/com.foundationdb.fdbmonitor.plist
 
+Start, stop and restart behavior
+=================================
+
+These commands above start and stop the master ``fdbmonitor`` process, which in turn starts ``fdbserver`` and ``backup-agent`` processes.  See :ref:`administration_fdbmonitor` for details.
+
+After any child process has terminated by any reason, ``fdbmonitor`` tries to restart it. See :ref:`restarting parameters <configuration-restarting>`.
+
+When ``fdbmonitor`` itself is killed unexpectedly (for example, by the ``out-of-memory killer``), all the child processes are also terminated. Then the operating system is responsible for restarting it. See :ref:`Configuring autorestart of fdbmonitor <configuration-restart-fdbmonitor>`.
+
 .. _foundationdb-cluster-file:
 
 Cluster files
@@ -259,7 +266,8 @@ Use the ``status`` command of ``fdbcli`` to determine if the cluster is up and r
       Redundancy mode        - triple
       Storage engine         - ssd-2
       Coordinators           - 5
-      Desired Proxies        - 5
+      Desired GRV Proxies    - 1
+      Desired Commit Proxies - 4
       Desired Logs           - 8
 
     Cluster:
@@ -299,7 +307,8 @@ The summary fields are interpreted as follows:
 Redundancy mode         The currently configured redundancy mode (see the section :ref:`configuration-choosing-redundancy-mode`)
 Storage engine          The currently configured storage engine (see the section :ref:`configuration-configuring-storage-subsystem`)
 Coordinators            The number of FoundationDB coordination servers
-Desired Proxies         Number of proxies desired. If replication mode is 3 then default number of proxies is 3
+Desired GRV Proxies     Number of GRV proxies desired. (default 1)
+Desired Commit Proxies  Number of commit proxies desired. If replication mode is 3 then default number of commit proxies is 3
 Desired Logs            Number of logs desired. If replication mode is 3 then default number of logs is 3
 FoundationDB processes  Number of FoundationDB processes participating in the cluster
 Machines                Number of physical machines running at least one FoundationDB process that is participating in the cluster
@@ -565,7 +574,7 @@ When configured, the ``status json`` output will include additional fields to re
     filtered: 1
   }
 
-The ``grv_latency_bands`` and ``commit_latency_bands`` objects will only be logged for ``proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. 
+The ``grv_latency_bands`` objects will only be logged for ``grv_proxy`` roles, ``commit_latency_bands`` objects will only be logged for ``commit_proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. 
 
 For example, ``0.1: 1`` in ``read_latency_bands`` indicates that there has been 1 read request with a latency in the range ``[0.01, 0.1)``. For the smallest specified threshold, the lower bound is 0 (e.g. ``[0, 0.01)`` in the example above). Requests that took longer than any defined latency band will be reported in the ``inf`` (infinity) band. Requests that were filtered by the configuration (e.g. using ``max_read_bytes``) are reported in the ``filtered`` category.
 

documentation/sphinx/source/api-c.rst

@@ -263,9 +263,9 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
 .. function:: fdb_error_t fdb_future_block_until_ready(FDBFuture* future)
 
-   Blocks the calling thread until the given Future is ready. It will return success even if the Future is set to an error -- you must call :func:`fdb_future_get_error()` to determine that. :func:`fdb_future_block_until_ready()` will return an error only in exceptional conditions (e.g. out of memory or other operating system resources).
+   Blocks the calling thread until the given Future is ready. It will return success even if the Future is set to an error -- you must call :func:`fdb_future_get_error()` to determine that. :func:`fdb_future_block_until_ready()` will return an error only in exceptional conditions (e.g. deadlock detected, out of memory or other operating system resources).
 
-   .. warning:: Never call this function from a callback passed to :func:`fdb_future_set_callback()`. This may block the thread on which :func:`fdb_run_network()` was invoked, resulting in a deadlock.
+   .. warning:: Never call this function from a callback passed to :func:`fdb_future_set_callback()`. This may block the thread on which :func:`fdb_run_network()` was invoked, resulting in a deadlock. In some cases the client can detect the deadlock and throw a ``blocked_from_network_thread`` error.
 
 .. function:: fdb_bool_t fdb_future_is_ready(FDBFuture* future)
 
@@ -301,6 +301,12 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
    |future-get-return1| |future-get-return2|.
 
+.. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count)
+
+   Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
+
+   |future-get-return1| |future-get-return2|.
+
 .. function:: fdb_error_t fdb_future_get_key(FDBFuture* future, uint8_t const** out_key, int* out_key_length)
 
    Extracts a key from an :type:`FDBFuture` into caller-provided variables of type ``uint8_t*`` (a pointer to the beginning of the key) and ``int`` (the length of the key). |future-warning|
@@ -480,6 +486,12 @@ Applications must provide error handling and an appropriate retry loop around th
 
    |future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2|
 
+.. function:: FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size)
+   Returns a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``.
+   .. note:: The returned split points contain the start key and end key of the given range
+
+   |future-return0| the list of split points. |future-return1| call :func:`fdb_future_get_key_array()` to extract the array, |future-return2|
+
 .. function:: FDBFuture* fdb_transaction_get_key(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t or_equal, int offset, fdb_bool_t snapshot)
 
    Resolves a :ref:`key selector <key-selectors>` against the keys in the database snapshot represented by ``transaction``.

documentation/sphinx/source/api-error-codes.rst

@@ -40,7 +40,7 @@ FoundationDB may return the following error codes from API functions. If you nee
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | external_client_already_loaded                | 1040| External client has already been loaded                                        |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
-| proxy_memory_limit_exceeded                   | 1042| Proxy commit memory limit exceeded                                             |
+| proxy_memory_limit_exceeded                   | 1042| CommitProxy commit memory limit exceeded                                       |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | batch_transaction_throttled                   | 1051| Batch GRV request rate limit exceeded                                          |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
@@ -114,8 +114,12 @@ FoundationDB may return the following error codes from API functions. If you nee
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | transaction_read_only                         | 2023| Attempted to commit a transaction specified as read-only                       |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| invalid_cache_eviction_policy                 | 2024| Invalid cache eviction policy, only random and lru are supported               |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | network_cannot_be_restarted                   | 2025| Network can only be started once                                               |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
+| blocked_from_network_thread                   | 2026| Detected a deadlock in a callback called from the network thread               |
++-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | incompatible_protocol_version                 | 2100| Incompatible protocol version                                                  |
 +-----------------------------------------------+-----+--------------------------------------------------------------------------------+
 | transaction_too_large                         | 2101| Transaction exceeds byte limit                                                 |

documentation/sphinx/source/api-python.rst

@@ -799,9 +799,15 @@ Transaction misc functions
 
 .. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
 
-    Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
+    Gets the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
     .. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
 
+.. method:: Transaction.get_range_split_points(self, begin_key, end_key, chunk_size)
+
+    Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
+    .. note:: The returned split points contain the start key and end key of the given range
+
+
 .. _api-python-transaction-options:
 
 Transaction misc functions

documentation/sphinx/source/api-ruby.rst

@@ -741,11 +741,16 @@ Most applications should use the read version that FoundationDB determines autom
 Transaction misc functions
 --------------------------
 
-.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
+.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) -> Int64Future
 
-    Get the estimated byte size of the given key range. Returns a :class:`Int64Future`.
+    Gets the estimated byte size of the given key range. Returns a :class:`Int64Future`.
     .. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
 
+.. method:: Transaction.get_range_split_points(begin_key, end_key, chunk_size) -> FutureKeyArray
+
+    Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
+    .. note:: The returned split points contain the start key and end key of the given range
+
 .. method:: Transaction.get_approximate_size() -> Int64Future
 
     |transaction-get-approximate-size-blurb|. Returns a :class:`Int64Future`.

documentation/sphinx/source/api-version-upgrade-guide.rst

@@ -9,6 +9,19 @@ This document provides an overview of changes that an application developer may
 
 For more details about API versions, see :ref:`api-versions`.
 
+.. _api-version-upgrade-guide-700:
+
+API version 700
+===============
+
+General
+-------
+
+Python bindings
+---------------
+
+* The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``.
+
 .. _api-version-upgrade-guide-630:
 
 API version 630

documentation/sphinx/source/architecture.rst

@@ -26,7 +26,7 @@ and servers use the coordinators to connect with the cluster controller.
 The servers will attempt to become the cluster controller if one does
 not exist, and register with the cluster controller once one has been
 elected. Clients use the cluster controller to keep an up-to-date list
-of proxies.
+of GRV proxies and commit proxies.
 
 Cluster Controller
 ~~~~~~~~~~~~~~~~~~
@@ -42,10 +42,11 @@ Master
 
 The master is responsible for coordinating the transition of the write
 sub-system from one generation to the next. The write sub-system
-includes the master, proxies, resolvers, and transaction logs. The three
-roles are treated as a unit, and if any of them fail, we will recruit a
-replacement for all three roles. The master provides the commit versions
-for batches of the mutations to the proxies.
+includes the master, GRV proxies, commit proxies, resolvers, and 
+transaction logs. The three roles are treated as a unit, and if any of 
+them fail, we will recruit a replacement for all three roles. The master 
+provides the commit versions for batches of the mutations to the commit 
+proxies.
 
 Historically, Ratekeeper and Data Distributor are coupled with Master on
 the same process. Since 6.2, both have become a singleton in the
@@ -53,16 +54,22 @@ cluster. The life time is no longer tied with Master.
 
 |image1|
 
-Proxies
-~~~~~~~
+GRV Proxies
+~~~~~~~~~~~
 
-The proxies are responsible for providing read versions, committing
-transactions, and tracking the storage servers responsible for each
-range of keys. To provide a read version, a proxy will ask all other
-proxies to see the largest committed version at this point in time,
-while simultaneously checking that the transaction logs have not been
-stopped. Ratekeeper will artificially slow down the rate at which the
-proxy provides read versions.
+The GRV proxies are responsible for providing read versions, communicating
+with ratekeeper to control the rate providing read versions. To provide a 
+read version, a GRV proxy will ask all master to see the largest committed
+version at this point in time, while simultaneously checking that the 
+transaction logs have not been stopped. Ratekeeper will artificially slow 
+down the rate at which the GRV proxy provides read versions.
+
+Commit Proxies
+~~~~~~~~~~~~~~
+
+The proxies are responsible for committing transactions, report committed
+versions to master and tracking the storage servers responsible for each 
+range of keys. 
 
 Commits are accomplished by:
 
@@ -73,20 +80,20 @@ Commits are accomplished by:
 
 The key space starting with the ``\xff`` byte is reserved for system
 metadata. All mutations committed into this key space are distributed to
-all of the proxies through the resolvers. This metadata includes a
+all of the commit proxies through the resolvers. This metadata includes a
 mapping between key ranges and the storage servers which have the data
-for that range of keys. The proxies provides this information to clients
-on-demand. The clients cache this mapping; if they ask a storage server
-for a key it does not have, they will clear their cache and get a more
-up-to-date list of servers from the proxies.
+for that range of keys. The commit proxies provides this information to 
+clients on-demand. The clients cache this mapping; if they ask a storage
+server for a key it does not have, they will clear their cache and get a
+more up-to-date list of servers from the commit proxies.
 
 Transaction Logs
 ~~~~~~~~~~~~~~~~
 
 The transaction logs make mutations durable to disk for fast commit
-latencies. The logs receive commits from the proxy in version order, and
-only respond to the proxy once the data has been written and fsync’ed to
-an append only mutation log on disk. Before the data is even written to
+latencies. The logs receive commits from the commit proxy in version order, 
+and only respond to the commit proxy once the data has been written and fsync’ed
+to an append only mutation log on disk. Before the data is even written to
 disk we forward it to the storage servers responsible for that mutation.
 Once the storage servers have made the mutation durable, they pop it
 from the log. This generally happens roughly 6 seconds after the
@@ -153,7 +160,7 @@ Transaction Processing
 ----------------------
 
 A database transaction in FoundationDB starts by a client contacting one
-of the Proxies to obtain a read version, which is guaranteed to be
+of the GRV proxies to obtain a read version, which is guaranteed to be
 larger than any of commit version that client may know about (even
 through side channels outside the FoundationDB cluster). This is needed
 so that a client will see the result of previous commits that have
@@ -165,64 +172,51 @@ memory without contacting the cluster. By default, reading a key that
 was written in the same transaction will return the newly written value.
 
 At commit time, the client sends the transaction data (all reads and
-writes) to one of the Proxies and waits for commit or abort response
-from the proxy. If the transaction conflicts with another one and cannot
-commit, the client may choose to retry the transaction from the
-beginning again. If the transaction commits, the proxy also returns the
-commit version back to the client. Note this commit version is larger
-than the read version and is chosen by the master.
+writes) to one of the commit proxies and waits for commit or abort response
+from the commit proxy. If the transaction conflicts with another one and 
+cannot commit, the client may choose to retry the transaction from the
+beginning again. If the transaction commits, the commit proxy also returns
+the commit version back to the client and to master so that GRV proxies can
+get access to the latest committed version. Note this commit version is 
+larger than the read version and is chosen by the master.
 
 The FoundationDB architecture separates the scaling of client reads and
 writes (i.e., transaction commits). Because clients directly issue reads
 to sharded storage servers, reads scale linearly to the number of
 storage servers. Similarly, writes are scaled by adding more processes
-to Proxies, Resolvers, and Log Servers in the transaction system.
+to Commit Proxies, Resolvers, and Log Servers in the transaction system.
 
 Determine Read Version
 ~~~~~~~~~~~~~~~~~~~~~~
 
-When a client requests a read version from a proxy, the proxy asks all
-other proxies for their last commit versions, and checks a set of
-transaction logs satisfying replication policy are live. Then the proxy
-returns the maximum commit version as the read version to the client.
+When a client requests a read version from a GRV proxy, the GRV proxy asks 
+master for the latest committed version, and checks a set of transaction 
+logs satisfying replication policy are live. Then the GRV proxy returns 
+the maximum committed version as the read version to the client.
 
 |image2|
 
-The reason for the proxy to contact all other proxies for commit
-versions is to ensure the read version is larger than any previously
-committed version. Consider that if proxy ``A`` commits a transaction,
-and then the client asks proxy ``B`` for a read version. The read
-version from proxy ``B`` must be larger than the version committed by
-proxy ``A``. The only way to get this information is by asking proxy
-``A`` for its largest committed version.
+The reason for the GRV proxy to contact master for the latest committed
+versions is to because master is a central place to keep the largest of 
+all commit proxies' committed version.
 
 The reason for checking a set of transaction logs satisfying replication
-policy are live is to ensure the proxy is not replaced with newer
-generation of proxies. This is because proxy is a stateless role
-recruited in each generation. If a recovery has happened and the old
-proxy is still live, this old proxy could still give out read versions.
+policy are live is to ensure the GRV proxy is not replaced with newer
+generation of GRV proxies. This is because GRV proxy is a stateless role
+recruited in each generation. If a recovery has happened and the old GRV
+proxy is still live, this old GRV proxy could still give out read versions.
 As a result, a *read-only* transaction may see stale results (a
 read-write transaction will be aborted). By checking a set of
-transaction logs satisfying replication policy are live, the proxy makes
+transaction logs satisfying replication policy are live, the GRV proxy makes
 sure no recovery has happened, thus the *read-only* transaction sees the
 latest data.
 
-Note that the client cannot simply ask the master for read versions. The
-master gives out versions to proxies to be committed, but the master
-does not know when the versions it gives out are durable on the
-transaction logs. Therefore it is not safe to do reads at the largest
-version the master has provided because that version might be rolled
-back in the event of a failure, so the client could end up reading data
-that was never committed. In order for the client to use versions from
-the master, the client needs to wait until all in-flight
-transaction-batches (a write version is used for a batch of
-transactions) to commit. This can take a long time and thus is
-inefficient. Another drawback of this approach is putting more work
-towards the master, because the master role can’t be scaled. Even though
-giving out read-versions isn’t very expensive, it still requires the
-master to get a transaction budget from the Ratekeeper, batches
-requests, and potentially maintains thousands of network connections
-from clients.
+Note that the client cannot simply ask the master for read versions because
+this approach is putting more work towards the master, because the master 
+role can’t be scaled. Even though giving out read-versions isn’t very 
+expensive, it still requires the master to get a transaction budget from the
+Ratekeeper, batches requests, and potentially maintains thousands of network
+connections from clients.
 
 |image3|
 
@@ -231,27 +225,27 @@ Transaction Commit
 
 A client transaction commits in the following steps:
 
-1. A client sends a transaction to a proxy.
-2. The proxy asks the master for a commit version.
+1. A client sends a transaction to a commit proxy.
+2. The commit proxy asks the master for a commit version.
 3. The master sends back a commit version that is higher than any commit
    version seen before.
-4. The proxy sends the read and write conflict ranges to the resolver(s)
+4. The commit proxy sends the read and write conflict ranges to the resolver(s)
    with the commit version included.
 5. The resolver responds back with whether the transaction has any
    conflicts with previous transactions by sorting transactions
    according to their commit versions and computing if such a serial
    execution order is conflict-free.
 
-   -  If there are conflicts, the proxy responds back to the client with
+   -  If there are conflicts, the commit proxy responds back to the client with
       a not_committed error.
-   -  If there are no conflicts, the proxy sends the mutations and
+   -  If there are no conflicts, the commit proxy sends the mutations and
       commit version of this transaction to the transaction logs.
 
-6. Once the mutations are durable on the logs, the proxy responds back
+6. Once the mutations are durable on the logs, the commit proxy responds back
    success to the user.
 
-Note the proxy sends each resolver their respective key ranges, if any
-one of the resolvers detects a conflict then the transaction is not
+Note the commit proxy sends each resolver their respective key ranges, if 
+any one of the resolvers detects a conflict then the transaction is not
 committed. This has the flaw that if only one of the resolvers detects a
 conflict, the other resolver will still think the transaction has
 succeeded and may fail future transactions with overlapping write
@@ -273,8 +267,8 @@ Background Work
 There are a number of background work happening besides the transaction
 processing:
 
--  **Ratekeeper** collects statistic information from proxies,
-   transaction logs, and storage servers and compute the target
+-  **Ratekeeper** collects statistic information from GRV proxies, Commit
+   proxies, transaction logs, and storage servers and compute the target
    transaction rate for the cluster.
 
 -  **Data distribution** monitors all storage servers and perform load
@@ -284,7 +278,7 @@ processing:
 -  **Storage servers** pull mutations from transaction logs, write them
    into storage engine to persist on disks.
 
--  **Proxies** periodically send empty commits to transaction logs to
+-  **Commit proxies** periodically send empty commits to transaction logs to
    keep commit versions increasing, in case there is no client generated
    transactions.
 
@@ -299,9 +293,9 @@ latency. A typical recovery takes about a few hundred milliseconds, but
 longer recovery time (usually a few seconds) can happen. Whenever there
 is a failure in the transaction system, a recovery process is performed
 to restore the transaction system to a new configuration, i.e., a clean
-state. Specifically, the Master process monitors the health of Proxies,
-Resolvers, and Transaction Logs. If any one of the monitored process
-failed, the Master process terminates. The Cluster Controller will
+state. Specifically, the Master process monitors the health of GRV Proxies,
+Commit Proxies,  Resolvers, and Transaction Logs. If any one of the monitored 
+process failed, the Master process terminates. The Cluster Controller will
 detect this event, and then recruits a new Master, which coordinates the
 recovery and recruits a new transaction system instance. In this way,
 the transaction processing is divided into a number of epochs, where
@@ -314,20 +308,20 @@ Coordinators and lock the coordinated states to prevent another Master
 process from recovering at the same time. Then the Master recovers
 previous transaction system states, including all Log Servers’
 Information, stops these Log Servers from accepting transactions, and
-recruits a new set of Proxies, Resolvers, and Transaction Logs. After
-previous Log Servers are stopped and new transaction system is
-recruited, the Master writes the coordinated states with current
+recruits a new set of GRV Proxies, Commit Proxies, Resolvers, and 
+Transaction Logs. After previous Log Servers are stopped and new transaction
+system is recruited, the Master writes the coordinated states with current
 transaction system information. Finally, the Master accepts new
 transaction commits. See details in this
 `documentation <https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md>`__.
 
-Because Proxies and Resolvers are stateless, their recoveries have no
-extra work. In contrast, Transaction Logs save the logs of committed
-transactions, and we need to ensure all previously committed
-transactions are durable and retrievable by storage servers. That is,
-for any transactions that the Proxies may have sent back commit
-response, their logs are persisted in multiple Log Servers (e.g., three
-servers if replication degree is 3).
+Because GRV Proxies, Commit Proxies and Resolvers are stateless, their 
+recoveries have no extra work. In contrast, Transaction Logs save the 
+logs of committed transactions, and we need to ensure all previously 
+committed transactions are durable and retrievable by storage servers. 
+That is, for any transactions that the Commit Proxies may have sent back
+commit response, their logs are persisted in multiple Log Servers (e.g., 
+three servers if replication degree is 3).
 
 Finally, a recovery will *fast forward* time by 90 seconds, which would
 abort any in-progress client transactions with ``transaction_too_old``
@@ -335,7 +329,7 @@ error. During retry, these client transactions will find the new
 generation of transaction system and commit.
 
 **``commit_result_unknown`` error:** If a recovery happened while a
-transaction is committing (i.e., a proxy has sent mutations to
+transaction is committing (i.e., a commit proxy has sent mutations to
 transaction logs). A client would have received
 ``commit_result_unknown``, and then retried the transaction. It’s
 completely permissible for FDB to commit both the first attempt, and the

documentation/sphinx/source/command-line-interface.rst

@@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu
 configure
 ---------
 
-The ``configure`` command changes the database configuration. Its syntax is ``configure [new] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [proxies=<N>] [resolvers=<N>] [logs=<N>]``.
+The ``configure`` command changes the database configuration. Its syntax is ``configure [new] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>]``.
 
 The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified.
 
@@ -98,11 +98,12 @@ A FoundationDB cluster employs server processes of different types. It automatic
 
 For large clusters, you can manually set the allocated number of processes of a given type. Valid process types are:
 
-* ``proxies``
+* ``grv_proxies``
+* ``commit_proxies``
 * ``resolvers``
 * ``logs``
 
-Set the process using ``configure [proxies|resolvers|logs]=<N>``, where ``<N>`` is an integer greater than 0, or -1 to reset the value to its default.
+Set the process using ``configure [grv_proxies|commit_proxies|resolvers|logs]=<N>``, where ``<N>`` is an integer greater than 0, or -1 to reset the value to its default.
 
 For recommendations on appropriate values for process types in large clusters, see :ref:`guidelines-process-class-config`.
 
@@ -357,7 +358,7 @@ setclass
 
 The ``setclass`` command can be used to change the :ref:`process class <guidelines-process-class-config>` for a given process. Its syntax is ``setclass [<ADDRESS> <CLASS>]``. If no arguments are specified, then the process classes of all processes are listed. Setting the class to ``default`` to revert to the process class specified on the command line.
 
-The available process classes are ``unset``, ``storage``, ``transaction``, ``resolution``, ``proxy``, ``master``, ``test``, ``unset``, ``stateless``, ``log``, ``router``, ``cluster_controller``, ``fast_restore``, ``data_distributor``, ``coordinator``, ``ratekeeper``, ``storage_cache``, ``backup``, and ``default``.
+The available process classes are ``unset``, ``storage``, ``transaction``, ``resolution``, ``grv_proxy``, ``commit_proxy``, ``master``, ``test``, ``unset``, ``stateless``, ``log``, ``router``, ``cluster_controller``, ``fast_restore``, ``data_distributor``, ``coordinator``, ``ratekeeper``, ``storage_cache``, ``backup``, and ``default``.
 
 sleep
 -----

documentation/sphinx/source/configuration.rst

@@ -229,6 +229,8 @@ Contains settings applicable to all processes (e.g. fdbserver, backup_agent).
 * ``kill_on_configuration_change``: If ``true``, affected processes will be restarted whenever the configuration file changes. Defaults to ``true``.
 * ``disable_lifecycle_logging``: If ``true``, ``fdbmonitor`` will not write log events when processes start or terminate. Defaults to ``false``.
 
+.. _configuration-restarting:
+
 The ``[general]`` section also contains some parameters to control how processes are restarted when they die. ``fdbmonitor`` uses backoff logic to prevent a process that dies repeatedly from cycling too quickly, and it also introduces up to +/-10% random jitter into the delay to avoid multiple processes all restarting simultaneously. ``fdbmonitor`` tracks separate backoff state for each process, so the restarting of one process will have no effect on the backoff behavior of another.
 
 * ``restart_delay``: The maximum number of seconds (subject to jitter) that fdbmonitor will delay before restarting a failed process.
@@ -236,6 +238,8 @@ The ``[general]`` section also contains some parameters to control how processes
 * ``restart_backoff``: Controls how quickly ``fdbmonitor`` backs off when a process dies repeatedly. The previous delay (or 1, if the previous delay is 0) is multiplied by ``restart_backoff`` to get the next delay, maxing out at the value of ``restart_delay``. Defaults to the value of ``restart_delay``, meaning that the second and subsequent failures will all delay ``restart_delay`` between restarts.
 * ``restart_delay_reset_interval``: The number of seconds a process must be running before resetting the backoff back to the value of ``initial_restart_delay``. Defaults to the value of ``restart_delay``.
 
+ These ``restart_`` parameters are not applicable to the ``fdbmonitor`` process itself. See :ref:`Configuring autorestart of fdbmonitor <configuration-restart-fdbmonitor>` for details.
+
 As an example, let's say the following parameters have been set:
 
 .. code-block:: ini
@@ -322,6 +326,24 @@ Backup agent sections
 
 These sections run and configure the backup agent process used for :doc:`point-in-time backups <backups>` of FoundationDB. These don't usually need to be modified. The structure and functionality is similar to the ``[fdbserver]`` and ``[fdbserver.<ID>]`` sections.
 
+.. _configuration-restart-fdbmonitor:
+
+Configuring autorestart of fdbmonitor
+=====================================
+
+Configuring the restart parameters for ``fdbmonitor`` is operating system-specific.
+
+Linux (RHEL/CentOS)
+-------------------
+
+ ``systemd`` controls the ``foundationdb`` service. When ``fdbmonitor`` is killed unexpectedly, by default, systemd restarts it in 60 seconds. To adjust this value you have to create a file ``/etc/systemd/system/foundationdb.service.d/override.conf`` with the overriding values. For example:
+
+.. code-block:: ini
+
+    [Service]
+    RestartSec=20s
+
+To disable auto-restart of ``fdbmonitor``, put ``Restart=no`` in the same section.
 
 .. _configuration-choosing-redundancy-mode:
 
@@ -777,16 +799,17 @@ The 6.2 release still has a number of rough edges related to region configuratio
 Guidelines for setting process class
 ====================================
 
-In a FoundationDB cluster, each of the ``fdbserver`` processes perform different tasks. Each process is recruited to do a particular task based on its process ``class``. For example, processes with ``class=storage`` are given preference to be recruited for doing storage server tasks, ``class=transaction`` are for log server processes and ``class=stateless`` are for stateless processes like proxies, resolvers, etc.,
+In a FoundationDB cluster, each of the ``fdbserver`` processes perform different tasks. Each process is recruited to do a particular task based on its process ``class``. For example, processes with ``class=storage`` are given preference to be recruited for doing storage server tasks, ``class=transaction`` are for log server processes and ``class=stateless`` are for stateless processes like commit proxies, resolvers, etc.,
 
-The recommended minimum number of ``class=transaction`` (log server) processes is 8 (active) + 2 (standby) and the recommended minimum number for ``class=stateless`` processes is 4 (proxy) + 1 (resolver) + 1 (cluster controller) + 1 (master) + 2 (standby). It is better to spread the transaction and stateless processes across as many machines as possible.
+The recommended minimum number of ``class=transaction`` (log server) processes is 8 (active) + 2 (standby) and the recommended minimum number for ``class=stateless`` processes is 1 (GRV proxy) + 3 (commit proxy) + 1 (resolver) + 1 (cluster controller) + 1 (master) + 2 (standby). It is better to spread the transaction and stateless processes across as many machines as possible.
 
 ``fdbcli`` is used to set the desired number of processes of a particular process type. To do so, you would issue the ``fdbcli`` commands::
 
-  fdb> configure proxies=5
+  fdb> configure grv_proxies=1
+  fdb> configure grv_proxies=4
   fdb> configure logs=8
 
-.. note:: In the present release, the default value for proxies and log servers is 3 and for resolvers is 1. You should not set the value of a process type to less than its default.
+.. note:: In the present release, the default value for commit proxies and log servers is 3 and for GRV proxies and resolvers is 1. You should not set the value of a process type to less than its default.
 
 .. warning:: The conflict-resolution algorithm used by FoundationDB is conservative: it guarantees that no conflicting transactions will be committed, but it may fail to commit some transactions that theoretically could have been. The effects of this conservatism may increase as you increase the number of resolvers. It is therefore important to employ the recommended techniques for :ref:`minimizing conflicts <developer-guide-transaction-conflicts>` when increasing the number of resolvers.
 

documentation/sphinx/source/developer-guide.rst

@@ -838,7 +838,7 @@ Caveats
 
 #. ``\xff\xff/transaction/read_conflict_range/`` The conflict range for a read is sometimes not known until that read completes (e.g. range reads with limits, key selectors). When you read from these special keys, the returned future first blocks until all pending reads are complete so it can give an accurate response.
 #. ``\xff\xff/transaction/write_conflict_range/`` The conflict range range for a ``set_versionstamped_key`` atomic op is not known until commit time. You'll get an approximate range (the actual range will be a subset of the approximate range) until the precise range is known.
-#. ``\xff\xff/transaction/conflicting_keys/`` Since using this feature costs server (i.e., proxy and resolver) resources, it's disabled by default. You must opt in by setting the ``report_conflicting_keys`` transaction option.
+#. ``\xff\xff/transaction/conflicting_keys/`` Since using this feature costs server (i.e., commit proxy and resolver) resources, it's disabled by default. You must opt in by setting the ``report_conflicting_keys`` transaction option.
 
 Metrics module
 --------------
@@ -942,7 +942,7 @@ that process, and wait for necessary data to be moved away.
 
 #. ``\xff\xff/management/excluded/<exclusion>`` Read/write. Indicates that the cluster should move data away from processes matching ``<exclusion>``, so that they can be safely removed. See :ref:`removing machines from a cluster <removing-machines-from-a-cluster>` for documentation for the corresponding fdbcli command.
 #. ``\xff\xff/management/failed/<exclusion>`` Read/write. Indicates that the cluster should consider matching processes as permanently failed. This allows the cluster to avoid maintaining extra state and doing extra work in the hope that these processes come back. See :ref:`removing machines from a cluster <removing-machines-from-a-cluster>` for documentation for the corresponding fdbcli command.
-#. ``\xff\xff/management/inProgressExclusion/<address>`` Read-only. Indicates that the process matching ``<address>`` matches an exclusion, but still has necessary data and can't yet be safely removed.
+#. ``\xff\xff/management/in_progress_exclusion/<address>`` Read-only. Indicates that the process matching ``<address>`` matches an exclusion, but still has necessary data and can't yet be safely removed.
 #. ``\xff\xff/management/options/excluded/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/excluded/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 #. ``\xff\xff/management/options/failed/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/failed/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
 
@@ -1059,22 +1059,21 @@ How Versions are Generated and Assigned
 
 Versions are generated by the process that runs the *master* role. FoundationDB guarantees that no version will be generated twice and that the versions are monotonically increasing.
 
-In order to assign read and commit versions to transactions, a client will never talk to the master. Instead it will get both from a proxy. Getting a read version is more complex than a commit version. Let's first look at commit versions:
+In order to assign read and commit versions to transactions, a client will never talk to the master. Instead it will get them from a GRV proxy and a commit proxy. Getting a read version is more complex than a commit version. Let's first look at commit versions:
 
-#. The client will send a commit message to a proxy.
-#. The proxy will put this commit message in a queue in order to build a batch.
-#. In parallel, the proxy will ask for a new version from the master (note that this means that only proxies will ever ask for new versions - which scales much better as it puts less stress on the network).
-#. The proxy will then resolve all transactions within that batch (discussed later) and assign the version it got from the master to *all* transactions within that batch. It will then write the transactions to the transaction log system to make it durable.
+#. The client will send a commit message to a commit proxy.
+#. The commit proxy will put this commit message in a queue in order to build a batch.
+#. In parallel, the commit proxy will ask for a new version from the master (note that this means that only commit proxies will ever ask for new versions - which scales much better as it puts less stress on the network).
+#. The commit proxy will then resolve all transactions within that batch (discussed later) and assign the version it got from the master to *all* transactions within that batch. It will then write the transactions to the transaction log system to make it durable.
 #. If the transaction succeeded, it will send back the version as commit version to the client. Otherwise it will send back an error.
 
-As mentioned before, the algorithm to assign read versions is a bit more complex. At the start of a transaction, a client will ask a proxy server for a read version. The proxy will reply with the last committed version as of the time it received the request - this is important to guarantee external consistency. This is how this is achieved:
+As mentioned before, the algorithm to assign read versions is a bit more complex. At the start of a transaction, a client will ask a GRV proxy server for a read version. The GRV proxy will reply with the last committed version as of the time it received the request - this is important to guarantee external consistency. This is how this is achieved:
 
-#. The client will send a GRV (get read version) request to a proxy.
-#. The proxy will batch GRV requests for a short amount of time (it depends on load and configuartion how big these batches will be).
+#. The client will send a GRV (get read version) request to a GRV proxy.
+#. The GRV proxy will batch GRV requests for a short amount of time (it depends on load and configuartion how big these batches will be).
 #. The proxy will do the following steps in parallel:
-   * Ask all other proxies for their most recent committed version (the largest version they received from the master for which it successfully wrote the transactions to the transaction log system).
-   * Send a message to the transaction log system to verify that it is still writable. This is to prevent that we fetch read versions from a proxy that has been declared to be dead.
-#. It will then take the largest committed version from all proxies (including its own) and send it back to the clients.
+   * Ask master for their most recent committed version (the largest version of proxies' committed version for which the transactions are successfully written to the transaction log system).
+   * Send a message to the transaction log system to verify that it is still writable. This is to prevent that we fetch read versions from a GRV proxy that has been declared to be dead.
 
 Checking whether the log-system is still writeable can be especially expensive if a clusters runs in a multi-region configuration. If a user is fine to sacrifice strict serializability they can use :ref:`option-causal-read-risky <api-python-option-set-causal-read-risky>`.
 
@@ -1148,8 +1147,8 @@ The ``commit_unknown_result`` Error
 
 ``commit_unknown_result`` can be thrown during a commit. This error is difficult to handle as you won't know whether your transaction was committed or not. There are mostly two reasons why you might see this error:
 
-#. The client lost the connection to the proxy to which it did send the commit. So it never got a reply and therefore can't know whether the commit was successful or not.
-#. There was a FoundationDB failure - for example a proxy failed during the commit. In that case there is no way for the client know whether the transaction succeeded or not.
+#. The client lost the connection to the commit proxy to which it did send the commit. So it never got a reply and therefore can't know whether the commit was successful or not.
+#. There was a FoundationDB failure - for example a commit proxy failed during the commit. In that case there is no way for the client know whether the transaction succeeded or not.
 
 However, there is one guarantee FoundationDB gives to the caller: at the point of time where you receive this error, the transaction either committed or not and if it didn't commit, it will never commit in the future. Or: it is guaranteed that the transaction is not in-flight anymore. This is an important guarantee as it means that if your transaction is idempotent you can simply retry. For more explanations see developer-guide-unknown-results_.
 

documentation/sphinx/source/disk-snapshot-backup.rst

@@ -104,7 +104,7 @@ Field Name                        Description
 ``Name for the snapshot file``    recommended name for the disk snapshot                     cluster-name:ip-addr:port:UID
 ================================  ========================================================   ========================================================
 
-``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or MasterProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally.
+``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or CommitProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally.
 
 Management of disk snapshots
 ----------------------------

documentation/sphinx/source/downloads.rst

@@ -10,38 +10,38 @@ macOS
 
 The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
 
-* `FoundationDB-6.3.5.pkg <https://www.foundationdb.org/downloads/6.3.5/macOS/installers/FoundationDB-6.3.5.pkg>`_
+* `FoundationDB-6.3.9.pkg <https://www.foundationdb.org/downloads/6.3.9/macOS/installers/FoundationDB-6.3.9.pkg>`_
 
 Ubuntu
 ------
 
 The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
 
-* `foundationdb-clients-6.3.5-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.5/ubuntu/installers/foundationdb-clients_6.3.5-1_amd64.deb>`_
-* `foundationdb-server-6.3.5-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.5/ubuntu/installers/foundationdb-server_6.3.5-1_amd64.deb>`_ (depends on the clients package)
+* `foundationdb-clients-6.3.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.9/ubuntu/installers/foundationdb-clients_6.3.9-1_amd64.deb>`_
+* `foundationdb-server-6.3.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.9/ubuntu/installers/foundationdb-server_6.3.9-1_amd64.deb>`_ (depends on the clients package)
 
 RHEL/CentOS EL6
 ---------------
 
 The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
 
-* `foundationdb-clients-6.3.5-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.5/rhel6/installers/foundationdb-clients-6.3.5-1.el6.x86_64.rpm>`_
-* `foundationdb-server-6.3.5-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.5/rhel6/installers/foundationdb-server-6.3.5-1.el6.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.3.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-clients-6.3.9-1.el6.x86_64.rpm>`_
+* `foundationdb-server-6.3.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-server-6.3.9-1.el6.x86_64.rpm>`_ (depends on the clients package)
 
 RHEL/CentOS EL7
 ---------------
 
 The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
 
-* `foundationdb-clients-6.3.5-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.5/rhel7/installers/foundationdb-clients-6.3.5-1.el7.x86_64.rpm>`_
-* `foundationdb-server-6.3.5-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.5/rhel7/installers/foundationdb-server-6.3.5-1.el7.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.3.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-clients-6.3.9-1.el7.x86_64.rpm>`_
+* `foundationdb-server-6.3.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-server-6.3.9-1.el7.x86_64.rpm>`_ (depends on the clients package)
 
 Windows
 -------
 
 The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
 
-* `foundationdb-6.3.5-x64.msi <https://www.foundationdb.org/downloads/6.3.5/windows/installers/foundationdb-6.3.5-x64.msi>`_
+* `foundationdb-6.3.9-x64.msi <https://www.foundationdb.org/downloads/6.3.9/windows/installers/foundationdb-6.3.9-x64.msi>`_
 
 API Language Bindings
 =====================
@@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
 
 If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package:
 
-* `foundationdb-6.3.5.tar.gz <https://www.foundationdb.org/downloads/6.3.5/bindings/python/foundationdb-6.3.5.tar.gz>`_
+* `foundationdb-6.3.9.tar.gz <https://www.foundationdb.org/downloads/6.3.9/bindings/python/foundationdb-6.3.9.tar.gz>`_
 
 Ruby 1.9.3/2.0.0+
 -----------------
 
-* `fdb-6.3.5.gem <https://www.foundationdb.org/downloads/6.3.5/bindings/ruby/fdb-6.3.5.gem>`_
+* `fdb-6.3.9.gem <https://www.foundationdb.org/downloads/6.3.9/bindings/ruby/fdb-6.3.9.gem>`_
 
 Java 8+
 -------
 
-* `fdb-java-6.3.5.jar <https://www.foundationdb.org/downloads/6.3.5/bindings/java/fdb-java-6.3.5.jar>`_
-* `fdb-java-6.3.5-javadoc.jar <https://www.foundationdb.org/downloads/6.3.5/bindings/java/fdb-java-6.3.5-javadoc.jar>`_
+* `fdb-java-6.3.9.jar <https://www.foundationdb.org/downloads/6.3.9/bindings/java/fdb-java-6.3.9.jar>`_
+* `fdb-java-6.3.9-javadoc.jar <https://www.foundationdb.org/downloads/6.3.9/bindings/java/fdb-java-6.3.9-javadoc.jar>`_
 
 Go 1.11+
 --------

documentation/sphinx/source/kv-architecture.rst

@@ -5,7 +5,7 @@ FoundationDB Architecture
 Coordinators
 ============
 
-All clients and servers connect to a FoundationDB cluster with a cluster file, which contains the IP:PORT of the coordinators. Both the clients and servers use the coordinators to connect with the cluster controller. The servers will attempt to become the cluster controller if one does not exist, and register with the cluster controller once one has been elected. Clients use the cluster controller to keep an up-to-date list of proxies.
+All clients and servers connect to a FoundationDB cluster with a cluster file, which contains the IP:PORT of the coordinators. Both the clients and servers use the coordinators to connect with the cluster controller. The servers will attempt to become the cluster controller if one does not exist, and register with the cluster controller once one has been elected. Clients use the cluster controller to keep an up-to-date list of GRV proxies and commit proxies.
 
 Cluster Controller
 ==================
@@ -15,12 +15,12 @@ The cluster controller is a singleton elected by a majority of coordinators. It
 Master
 ======
 
-The master is responsible for coordinating the transition of the write sub-system from one generation to the next. The write sub-system includes the master, proxies, resolvers, and transaction logs. The three roles are treated as a unit, and if any of them fail, we will recruit a replacement for all three roles. The master provides the commit versions for batches of the mutations to the proxies, runs data distribution algorithm, and runs ratekeeper.
+The master is responsible for coordinating the transition of the write sub-system from one generation to the next. The write sub-system includes the master, GRV proxies, commit proxies, resolvers, and transaction logs. The three roles are treated as a unit, and if any of them fail, we will recruit a replacement for all three roles. The master keeps commit proxies' committed version, provides read version for GRV proxies, provides the commit versions for batches of the mutations to the commit proxies, runs data distribution algorithm, and runs ratekeeper.
 
-Proxies
-=======
+GRV Proxies and Commit Proxies
+==============================
 
-The proxies are responsible for providing read versions, committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a proxy will ask all other proxies to see the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the proxy provides read versions.
+The GRV proxies are responsible for providing read versions. The commit proxies are responsible for committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a GRV proxy will ask master the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the GRV proxy provides read versions.
 
 Commits are accomplished by:
 
@@ -33,7 +33,7 @@ The key space starting with the '\xff' byte is reserved for system metadata. All
 Transaction Logs
 ================
 
-The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Before the data is even written to disk we forward it to the storage servers responsible for that mutation. Once the storage servers have made the mutation durable, they pop it from the log. This generally happens roughly 6 seconds after the mutation was originally committed to the log. We only read from the log's disk when the process has been rebooted. If a storage server has failed, mutations bound for that storage server will build up on the logs. Once data distribution makes a different storage server responsible for all of the missing storage server's data we will discard the log data bound for the failed server.
+The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the commit proxy in version order, and only respond to the commit proxy once the data has been written and fsync'ed to an append only mutation log on disk. Before the data is even written to disk we forward it to the storage servers responsible for that mutation. Once the storage servers have made the mutation durable, they pop it from the log. This generally happens roughly 6 seconds after the mutation was originally committed to the log. We only read from the log's disk when the process has been rebooted. If a storage server has failed, mutations bound for that storage server will build up on the logs. Once data distribution makes a different storage server responsible for all of the missing storage server's data we will discard the log data bound for the failed server.
 
 Resolvers
 =========
@@ -48,4 +48,4 @@ The vast majority of processes in a cluster are storage servers. Storage servers
 Clients
 =======
 
-Clients must get a read version at the start of every transaction. During the transaction all of the reads are done at that read version, and write are kept in memory until transaction is committed. When the transaction is committed, all of the reads and writes are sent to the proxy. If the transaction conflicts with another transaction the client is responsible for retrying the transaction. By default, reading a key that was written in the same transaction will return the newly written value.
+Clients must get a read version at the start of every transaction. During the transaction all of the reads are done at that read version, and write are kept in memory until transaction is committed. When the transaction is committed, all of the reads and writes are sent to the commit proxy. If the transaction conflicts with another transaction the client is responsible for retrying the transaction. By default, reading a key that was written in the same transaction will return the newly written value.

documentation/sphinx/source/mr-status-json-schemas.rst.inc

@@ -27,7 +27,7 @@
                   "storage",
                   "transaction",
                   "resolution",
-                  "proxy",
+                  "commit_proxy",
                   "grv_proxy",
                   "master",
                   "test",
@@ -61,7 +61,7 @@
                   "role":{
                      "$enum":[
                         "master",
-                        "proxy",
+                        "commit_proxy",
                         "grv_proxy",
                         "log",
                         "storage",
@@ -446,8 +446,9 @@
          }
       ],
       "recovery_state":{
+         "seconds_since_last_recovered":1,
          "required_resolvers":1,
-         "required_proxies":1,
+         "required_commit_proxies":1,
          "required_grv_proxies":1,
          "name":{ // "fully_recovered" is the healthy state; other states are normal to transition through but not to persist in
             "$enum":[
@@ -633,11 +634,11 @@
                "address":"10.0.4.1"
             }
          ],
-         "auto_proxies":3,
+         "auto_commit_proxies":3,
          "auto_resolvers":1,
          "auto_logs":3,
          "backup_worker_enabled":1,
-         "proxies":5 // this field will be absent if a value has not been explicitly set
+         "commit_proxies":5 // this field will be absent if a value has not been explicitly set
       },
       "data":{
          "least_operating_space_bytes_log_server":0,

documentation/sphinx/source/release-notes/release-notes-620.rst

@@ -2,6 +2,18 @@
 Release Notes
 #############
 
+6.2.27
+======
+* For clusters with a large number of shards, avoid slow tasks in the data distributor by adding yields to the shard map destruction. `(PR #3834) <https://github.com/apple/foundationdb/pull/3834>`_
+* Reset the network connection between a proxy and master or resolvers if the proxy is too far behind in processing transactions. `(PR #3891) <https://github.com/apple/foundationdb/pull/3891>`_
+
+6.2.26
+======
+
+* Fixed undefined behavior in configuring supported FoundationDB versions while starting up a client. `(PR #3849) <https://github.com/apple/foundationdb/pull/3849>`_
+* Updated OpenSSL to version 1.1.1h. `(PR #3809) <https://github.com/apple/foundationdb/pull/3809>`_
+* Attempt to detect when calling :func:`fdb_future_block_until_ready` would cause a deadlock, and throw ``blocked_from_network_thread`` if it would definitely cause a deadlock. `(PR #3786) <https://github.com/apple/foundationdb/pull/3786>`_
+
 6.2.25
 ======
 
@@ -11,50 +23,31 @@ Release Notes
 6.2.24
 ======
 
-Features
---------
-
 * Added the ``suspend`` command to ``fdbcli`` which kills a process and prevents it from rejoining the cluster for a specified duration. `(PR #3550) <https://github.com/apple/foundationdb/pull/3550>`_
 
 6.2.23
 ======
 
-Fixes
------
-
 * When configured with ``usable_regions=2`` data distribution could temporarily lower the replication of a shard when moving it. `(PR #3487) <https://github.com/apple/foundationdb/pull/3487>`_
 * Prevent data distribution from running out of memory by fetching the source servers for too many shards in parallel. `(PR #3487) <https://github.com/apple/foundationdb/pull/3487>`_
 * Reset network connections between log routers and satellite tlogs if the latencies are larger than 500ms. `(PR #3487) <https://github.com/apple/foundationdb/pull/3487>`_
-
-Status
-------
-
 * Added per-process server request latency statistics reported in the role section of relevant processes. These are named ``grv_latency_statistics`` and ``commit_latency_statistics`` on proxy roles and ``read_latency_statistics`` on storage roles. `(PR #3480) <https://github.com/apple/foundationdb/pull/3480>`_
 * Added ``cluster.active_primary_dc`` that indicates which datacenter is serving as the primary datacenter in multi-region setups. `(PR #3320) <https://github.com/apple/foundationdb/pull/3320>`_
 
 6.2.22
 ======
 
-Fixes
------
-
 * Coordinator class processes could be recruited as the cluster controller. `(PR #3282) <https://github.com/apple/foundationdb/pull/3282>`_
 * HTTPS requests made by backup would fail (introduced in 6.2.21). `(PR #3284) <https://github.com/apple/foundationdb/pull/3284>`_
 
 6.2.21
 ======
 
-Fixes
------
-
 * HTTPS requests made by backup could hang indefinitely. `(PR #3027) <https://github.com/apple/foundationdb/pull/3027>`_
 * ``fdbrestore`` prefix options required exactly a single hyphen instead of the standard two. `(PR #3056) <https://github.com/apple/foundationdb/pull/3056>`_
 * Commits could stall on a newly elected proxy because of inaccurate compute estimates. `(PR #3123) <https://github.com/apple/foundationdb/pull/3123>`_
 * A transaction class process with a bad disk could be repeatedly recruited as a transaction log. `(PR #3268) <https://github.com/apple/foundationdb/pull/3268>`_
 * Fix a potential race condition that could lead to undefined behavior when connecting to a database using the multi-version client API. `(PR #3265) <https://github.com/apple/foundationdb/pull/3265>`_
-
-Features
---------
 * Added the ``getversion`` command to ``fdbcli`` which returns the current read version of the cluster.  `(PR #2882) <https://github.com/apple/foundationdb/pull/2882>`_
 * Added the ``advanceversion`` command to ``fdbcli`` which increases the current version of a cluster.  `(PR #2965) <https://github.com/apple/foundationdb/pull/2965>`_
 * Added the ``lock`` and ``unlock`` commands to ``fdbcli`` which lock or unlock a cluster. `(PR #2890) <https://github.com/apple/foundationdb/pull/2890>`_
@@ -62,9 +55,6 @@ Features
 6.2.20
 ======
 
-Fixes
------
-
 * In rare scenarios, clients could send corrupted data to the server. `(PR #2976) <https://github.com/apple/foundationdb/pull/2976>`_
 * Internal tools like ``fdbbackup`` are no longer tracked as clients in status (introduced in 6.2.18) `(PR #2849) <https://github.com/apple/foundationdb/pull/2849>`_
 * Changed TLS error handling to match the behavior of 6.2.15. `(PR #2993) <https://github.com/apple/foundationdb/pull/2993>`_ `(PR #2977) <https://github.com/apple/foundationdb/pull/2977>`_
@@ -72,9 +62,6 @@ Fixes
 6.2.19
 ======
 
-Fixes
------
-
 * Protect the proxies from running out of memory when bombarded with requests from clients. `(PR #2812) <https://github.com/apple/foundationdb/pull/2812>`_.
 * One process with a ``proxy`` class would not become the first proxy when put with other ``stateless`` class processes. `(PR #2819) <https://github.com/apple/foundationdb/pull/2819>`_.
 * If a transaction log stalled on a disk operation during recruitment the cluster would become unavailable until the process died. `(PR #2815) <https://github.com/apple/foundationdb/pull/2815>`_.
@@ -82,70 +69,37 @@ Fixes
 * Prevent the cluster from having too many active generations as a safety measure against repeated failures. `(PR #2814) <https://github.com/apple/foundationdb/pull/2814>`_.
 * ``fdbcli`` status JSON could become truncated because of unprintable characters. `(PR #2807) <https://github.com/apple/foundationdb/pull/2807>`_.
 * The data distributor used too much CPU in large clusters (broken in 6.2.16). `(PR #2806) <https://github.com/apple/foundationdb/pull/2806>`_.
-
-Status
-------
-
 * Added ``cluster.workload.operations.memory_errors`` to measure the number of requests rejected by the proxies because the memory limit has been exceeded. `(PR #2812) <https://github.com/apple/foundationdb/pull/2812>`_.
 * Added ``cluster.workload.operations.location_requests`` to measure the number of outgoing key server location responses from the proxies. `(PR #2812) <https://github.com/apple/foundationdb/pull/2812>`_.
 * Added ``cluster.recovery_state.active_generations`` to track the number of generations for which the cluster still requires transaction logs. `(PR #2814) <https://github.com/apple/foundationdb/pull/2814>`_.
 * Added ``network.tls_policy_failures`` to the ``processes`` section to record the number of TLS policy failures each process has observed. `(PR #2811) <https://github.com/apple/foundationdb/pull/2811>`_.
-
-Features
---------
-
 * Added ``--debug-tls`` as a command line argument to ``fdbcli`` to help diagnose TLS issues. `(PR #2810) <https://github.com/apple/foundationdb/pull/2810>`_.
 
 6.2.18
 ======
 
-Fixes
------
-
 * When configuring a cluster to usable_regions=2, data distribution would not react to machine failures while copying data to the remote region. `(PR #2774) <https://github.com/apple/foundationdb/pull/2774>`_.
 * When a cluster is configured with usable_regions=2, data distribution could push a cluster into saturation by relocating too many shards simulatenously. `(PR #2776) <https://github.com/apple/foundationdb/pull/2776>`_.
 * Do not allow the cluster controller to mark any process as failed within 30 seconds of startup. `(PR #2780) <https://github.com/apple/foundationdb/pull/2780>`_.
 * Backup could not establish TLS connections (broken in 6.2.16). `(PR #2775) <https://github.com/apple/foundationdb/pull/2775>`_.
 * Certificates were not refreshed automatically (broken in 6.2.16). `(PR #2781) <https://github.com/apple/foundationdb/pull/2781>`_.
-
-Performance
------------
-
 * Improved the efficiency of establishing large numbers of network connections. `(PR #2777) <https://github.com/apple/foundationdb/pull/2777>`_.
-
-Features
---------
-
 * Add support for setting knobs to modify the behavior of ``fdbcli``. `(PR #2773) <https://github.com/apple/foundationdb/pull/2773>`_.
-
-Other Changes
--------------
- 
 * Setting invalid knobs in backup and DR binaries is now a warning instead of an error and will not result in the application being terminated. `(PR #2773) <https://github.com/apple/foundationdb/pull/2773>`_.
 
 6.2.17
 ======
 
-Fixes
------
-
 * Restored the ability to set TLS configuration using environment variables (broken in 6.2.16). `(PR #2755) <https://github.com/apple/foundationdb/pull/2755>`_.
 
 6.2.16
 ======
 
-Performance
------------
-
 * Reduced tail commit latencies by improving commit pipelining on the proxies. `(PR #2589) <https://github.com/apple/foundationdb/pull/2589>`_.
 * Data distribution does a better job balancing data when disks are more than 70% full. `(PR #2722) <https://github.com/apple/foundationdb/pull/2722>`_.
 * Reverse range reads could read too much data from disk, resulting in poor performance relative to forward range reads. `(PR #2650) <https://github.com/apple/foundationdb/pull/2650>`_.
 * Switched from LibreSSL to OpenSSL to improve the speed of establishing connections. `(PR #2646) <https://github.com/apple/foundationdb/pull/2646>`_.
 * The cluster controller does a better job avoiding multiple recoveries when first recruited. `(PR #2698) <https://github.com/apple/foundationdb/pull/2698>`_.
-
-Fixes
------
-
 * Storage servers could fail to advance their version correctly in response to empty commits. `(PR #2617) <https://github.com/apple/foundationdb/pull/2617>`_.
 * Status could not label more than 5 processes as proxies. `(PR #2653) <https://github.com/apple/foundationdb/pull/2653>`_.
 * The ``TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER``, ``TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS``, ``TR_FLAG_DISABLE_SERVER_TEAM_REMOVER``, and ``BUGGIFY_ALL_COORDINATION`` knobs could not be set at runtime. `(PR #2661) <https://github.com/apple/foundationdb/pull/2661>`_.
@@ -157,17 +111,11 @@ Fixes
 6.2.15
 ======
 
-Fixes
------
-
 * TLS throttling could block legitimate connections. `(PR #2575) <https://github.com/apple/foundationdb/pull/2575>`_.
 
 6.2.14
 ======
 
-Fixes
------
-
 * Data distribution was prioritizing shard merges too highly. `(PR #2562) <https://github.com/apple/foundationdb/pull/2562>`_.
 * Status would incorrectly mark clusters as having no fault tolerance. `(PR #2562) <https://github.com/apple/foundationdb/pull/2562>`_.
 * A proxy could run out of memory if disconnected from the cluster for too long. `(PR #2562) <https://github.com/apple/foundationdb/pull/2562>`_.
@@ -175,26 +123,16 @@ Fixes
 6.2.13
 ======
 
-Performance
------------
-
 * Optimized the commit path the proxies to significantly reduce commit latencies in large clusters. `(PR #2536) <https://github.com/apple/foundationdb/pull/2536>`_.
 * Data distribution could create temporarily untrackable shards which could not be split if they became hot. `(PR #2546) <https://github.com/apple/foundationdb/pull/2546>`_.
 
 6.2.12
 ======
 
-Performance
------------
-
 * Throttle TLS connect attempts from misconfigured clients. `(PR #2529) <https://github.com/apple/foundationdb/pull/2529>`_.
 * Reduced master recovery times in large clusters. `(PR #2430) <https://github.com/apple/foundationdb/pull/2430>`_.
 * Improved performance while a remote region is catching up. `(PR #2527) <https://github.com/apple/foundationdb/pull/2527>`_.
 * The data distribution algorithm does a better job preventing hot shards while recovering from machine failures. `(PR #2526) <https://github.com/apple/foundationdb/pull/2526>`_.
-
-Fixes
------
-
 * Improve the reliability of a ``kill`` command from ``fdbcli``. `(PR #2512) <https://github.com/apple/foundationdb/pull/2512>`_.
 * The ``--traceclock`` parameter to fdbserver incorrectly had no effect. `(PR #2420) <https://github.com/apple/foundationdb/pull/2420>`_.
 * Clients could throw an internal error during ``commit`` if client buggification was enabled. `(PR #2427) <https://github.com/apple/foundationdb/pull/2427>`_.
@@ -204,9 +142,6 @@ Fixes
 6.2.11
 ======
 
-Fixes
------
-
 * Clients could hang indefinitely on reads if all storage servers holding a keyrange were removed from a cluster since the last time the client read a key in the range. `(PR #2377) <https://github.com/apple/foundationdb/pull/2377>`_.
 * In rare scenarios, status could falsely report no replicas remain of some data. `(PR #2380) <https://github.com/apple/foundationdb/pull/2380>`_.
 * Latency band tracking could fail to configure correctly after a recovery or upon process startup. `(PR #2371) <https://github.com/apple/foundationdb/pull/2371>`_.
@@ -214,17 +149,11 @@ Fixes
 6.2.10
 ======
 
-Fixes
------
-
 * ``backup_agent`` crashed on startup. `(PR #2356) <https://github.com/apple/foundationdb/pull/2356>`_.
 
 6.2.9
 =====
 
-Fixes
------
-
 * Small clusters using specific sets of process classes could cause the data distributor to be continuously killed and re-recruited. `(PR #2344) <https://github.com/apple/foundationdb/pull/2344>`_.
 * The data distributor and ratekeeper could be recruited on non-optimal processes. `(PR #2344) <https://github.com/apple/foundationdb/pull/2344>`_.
 * A ``kill`` command from ``fdbcli`` could take a long time before being executed by a busy process. `(PR #2339) <https://github.com/apple/foundationdb/pull/2339>`_.
@@ -234,9 +163,6 @@ Fixes
 6.2.8
 =====
 
-Fixes
------
-
 * Significantly improved the rate at which the transaction logs in a remote region can pull data from the primary region. `(PR #2307) <https://github.com/apple/foundationdb/pull/2307>`_ `(PR #2323) <https://github.com/apple/foundationdb/pull/2323>`_.
 * The ``system_kv_size_bytes`` status field could report a size much larger than the actual size of the system keyspace. `(PR #2305) <https://github.com/apple/foundationdb/pull/2305>`_.
 

documentation/sphinx/source/release-notes/release-notes-630.rst

@@ -2,7 +2,7 @@
 Release Notes
 #############
 
-6.3.5
+6.3.9
 =====
 
 Features
@@ -61,6 +61,8 @@ Fixes
 * In very rare scenarios, the data distributor process would crash when being shutdown. `(PR #3530) <https://github.com/apple/foundationdb/pull/3530>`_
 * The master would die immediately if it did not have the correct cluster controller interface when recruited. [6.3.4] `(PR #3537) <https://github.com/apple/foundationdb/pull/3537>`_
 * Fix an issue where ``fdbcli --exec 'exclude no_wait ...'`` would incorrectly report that processes can safely be removed from the cluster. [6.3.5] `(PR #3566) <https://github.com/apple/foundationdb/pull/3566>`_
+* Commit latencies could become large because of inaccurate compute estimates. [6.3.9] `(PR #3845) <https://github.com/apple/foundationdb/pull/3845>`_
+* Added a timeout on TLS handshakes to prevent them from hanging indefinitely. [6.3.9] `(PR #3850) <https://github.com/apple/foundationdb/pull/3850>`_
 
 Status
 ------
@@ -108,6 +110,10 @@ Other Changes
 * Updated boost to 1.72.  `(PR #2684) <https://github.com/apple/foundationdb/pull/2684>`_
 * Calling ``fdb_run_network`` multiple times in a single run of a client program now returns an error instead of causing undefined behavior. [6.3.1] `(PR #3229) <https://github.com/apple/foundationdb/pull/3229>`_
 * Blob backup URL parameter ``request_timeout`` changed to ``request_timeout_min``, with prior name still supported. `(PR #3533) <https://github.com/apple/foundationdb/pull/3533>`_
+* Support query command in backup CLI that allows users to query restorable files by key ranges. [6.3.6] `(PR #3703) <https://github.com/apple/foundationdb/pull/3703>`_
+* Report missing old tlogs information when in recovery before storage servers are fully recovered. [6.3.6] `(PR #3706) <https://github.com/apple/foundationdb/pull/3706>`_
+* Updated OpenSSL to version 1.1.1h. [6.3.7] `(PR #3809) <https://github.com/apple/foundationdb/pull/3809>`_
+* Lowered the amount of time a watch will remain registered on a storage server from 900 seconds to 30 seconds. [6.3.8] `(PR #3833) <https://github.com/apple/foundationdb/pull/3833>`_
 
 Fixes from previous versions
 ----------------------------
@@ -124,6 +130,9 @@ Fixes only impacting 6.3.0+
 * Refreshing TLS certificates could cause crashes. [6.3.2] `(PR #3352) <https://github.com/apple/foundationdb/pull/3352>`_
 * All storage class processes attempted to connect to the same coordinator. [6.3.2] `(PR #3361) <https://github.com/apple/foundationdb/pull/3361>`_
 * Adjusted the proxy load balancing algorithm to be based on the CPU usage of the process instead of the number of requests processed. [6.3.5] `(PR #3653) <https://github.com/apple/foundationdb/pull/3653>`_
+* Only return the error code ``batch_transaction_throttled`` for API versions greater than or equal to 630. [6.3.6] `(PR #3799) <https://github.com/apple/foundationdb/pull/3799>`_
+* The fault tolerance calculation in status did not take into account region configurations. [6.3.8] `(PR #3836) <https://github.com/apple/foundationdb/pull/3836>`_
+* Get read version tail latencies were high because some proxies were serving more read versions than other proxies. [6.3.9] `(PR #3845) <https://github.com/apple/foundationdb/pull/3845>`_
 
 Earlier release notes
 ---------------------

documentation/sphinx/source/release-notes/release-notes-700.rst

@@ -9,7 +9,7 @@ Release Notes
 
 Features
 --------
-
+* Added a new API in all bindings that can be used to get a list of split points that will split the given range into (roughly) equally sized chunks. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
 
 
 Performance
@@ -34,7 +34,7 @@ Status
 
 Bindings
 --------
-
+* Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
 
 
 Other Changes

fdbbackup/backup.actor.cpp

@@ -18,6 +18,10 @@
  * limitations under the License.
  */
 
+#include "fdbclient/JsonBuilder.h"
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/Trace.h"
 #define BOOST_DATE_TIME_NO_LIB
 #include <boost/interprocess/managed_shared_memory.hpp>
 
@@ -81,7 +85,22 @@ enum enumProgramExe {
 };
 
 enum enumBackupType {
-	BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP
+	BACKUP_UNDEFINED = 0,
+	BACKUP_START,
+	BACKUP_MODIFY,
+	BACKUP_STATUS,
+	BACKUP_ABORT,
+	BACKUP_WAIT,
+	BACKUP_DISCONTINUE,
+	BACKUP_PAUSE,
+	BACKUP_RESUME,
+	BACKUP_EXPIRE,
+	BACKUP_DELETE,
+	BACKUP_DESCRIBE,
+	BACKUP_LIST,
+	BACKUP_QUERY,
+	BACKUP_DUMP,
+	BACKUP_CLEANUP
 };
 
 enum enumDBType {
@@ -121,6 +140,7 @@ enum {
 	OPT_TAGNAME,
 	OPT_BACKUPKEYS,
 	OPT_WAITFORDONE,
+	OPT_BACKUPKEYS_FILTER,
 	OPT_INCREMENTALONLY,
 
 	// Backup Modify
@@ -624,6 +644,40 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
 	SO_END_OF_OPTIONS
 };
 
+CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
+#ifdef _WIN32
+	{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
+#endif
+	{ OPT_RESTORE_TIMESTAMP, "--query_restore_timestamp", SO_REQ_SEP },
+	{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
+	{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
+	{ OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP },
+	{ OPT_RESTORE_VERSION, "--query_restore_version", SO_REQ_SEP },
+	{ OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP },
+	{ OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP },
+	{ OPT_TRACE, "--log", SO_NONE },
+	{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
+	{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
+	{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
+	{ OPT_QUIET, "-q", SO_NONE },
+	{ OPT_QUIET, "--quiet", SO_NONE },
+	{ OPT_VERSION, "-v", SO_NONE },
+	{ OPT_VERSION, "--version", SO_NONE },
+	{ OPT_CRASHONERROR, "--crash", SO_NONE },
+	{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
+	{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
+	{ OPT_HELP, "-?", SO_NONE },
+	{ OPT_HELP, "-h", SO_NONE },
+	{ OPT_HELP, "--help", SO_NONE },
+	{ OPT_DEVHELP, "--dev-help", SO_NONE },
+	{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
+	{ OPT_KNOB, "--knob_", SO_REQ_SEP },
+#ifndef TLS_DISABLED
+	TLS_OPTION_FLAGS
+#endif
+	    SO_END_OF_OPTIONS
+};
+
 // g_rgRestoreOptions is used by fdbrestore and fastrestore_tool
 CSimpleOpt::SOption g_rgRestoreOptions[] = {
 #ifdef _WIN32
@@ -959,13 +1013,16 @@ void printBackupContainerInfo() {
 
 static void printBackupUsage(bool devhelp) {
 	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
-	printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str());
+	printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | "
+	       "list | query | cleanup) [OPTIONS]\n\n",
+	       exeBackup.toString().c_str());
 	printf("  -C CONNFILE    The path of a file containing the connection string for the\n"
 		   "                 FoundationDB cluster. The default is first the value of the\n"
 		   "                 FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
 		   "                 then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
 	printf("  -d, --destcontainer URL\n"
-	       "                 The Backup container URL for start, modify, describe, expire, and delete operations.\n");
+	       "                 The Backup container URL for start, modify, describe, query, expire, and delete "
+	       "operations.\n");
 	printBackupContainerInfo();
 	printf("  -b, --base_url BASEURL\n"
 		   "                 Base backup URL for list operations.  This looks like a Backup URL but without a backup name.\n");
@@ -979,6 +1036,12 @@ static void printBackupUsage(bool devhelp) {
 	printf("  --delete_before_days NUM_DAYS\n"
 		   "                 Another way to specify version cutoff for expire operations.  Deletes data files containing no data at or after a\n"
 		   "                 version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n");
+	printf("  -qrv --query_restore_version VERSION\n"
+	       "                 For query operations, set target version for restoring a backup. Set -1 for maximum\n"
+	       "                 restorable version (default) and -2 for minimum restorable version.\n");
+	printf("  --query_restore_timestamp DATETIME\n"
+	       "                 For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str());
+	printf("                 and it will be converted to a version from that time using metadata in the cluster file.\n");
 	printf("  --restorable_after_timestamp DATETIME\n"
 		   "                 For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n");
 	printf("  --restorable_after_version VERSION\n"
@@ -997,8 +1060,8 @@ static void printBackupUsage(bool devhelp) {
 	       "                 Specifies a UID to verify against the BackupUID of the running backup.  If provided, the UID is verified in the same transaction\n"
 	       "                 which sets the new backup parameters (if the UID matches).\n");
 	printf("  -e ERRORLIMIT  The maximum number of errors printed by status (default is 10).\n");
-	printf("  -k KEYS        List of key ranges to backup.\n"
-		   "                 If not specified, the entire database will be backed up.\n");
+	printf("  -k KEYS        List of key ranges to backup or to filter the backup in query operations.\n"
+	       "                 If not specified, the entire database will be backed up or no filter will be applied.\n");
 	printf("  --partitioned_log_experimental  Starts with new type of backup system using partitioned logs.\n");
 	printf("  -n, --dryrun   For backup start or restore start, performs a trial run with no actual changes made.\n");
 	printf("  --log          Enables trace file logging for the CLI session.\n"
@@ -1320,6 +1383,7 @@ enumBackupType	getBackupType(std::string backupType)
 		values["delete"] = BACKUP_DELETE;
 		values["describe"] = BACKUP_DESCRIBE;
 		values["list"] = BACKUP_LIST;
+		values["query"] = BACKUP_QUERY;
 		values["dump"] = BACKUP_DUMP;
 		values["modify"] = BACKUP_MODIFY;
 	}
@@ -1449,7 +1513,7 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		for (KeyBackedTag eachTag : backupTags) {
 			Version last_restorable_version = tagLastRestorableVersions[j].get();
 			double last_restorable_seconds_behind = ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
-			BackupAgentBase::enumState status = (BackupAgentBase::enumState)tagStates[j].get();
+			EBackupState status = tagStates[j].get();
 			const char *statusText = fba.getStateText(status);
 
 			// The object for this backup tag inside this instance's subdocument
@@ -1458,8 +1522,9 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 			tagRoot.create("current_status") = statusText;
 			tagRoot.create("last_restorable_version") = tagLastRestorableVersions[j].get();
 			tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind;
-			tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING);
-			tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
+			tagRoot.create("running_backup") =
+			    (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING);
+			tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL);
 			tagRoot.create("range_bytes_written") = tagRangeBytes[j].get();
 			tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get();
 			tagRoot.create("mutation_stream_id") = backupTagUids[j].toString();
@@ -1474,7 +1539,7 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		tr2->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Standalone<RangeResultRef> tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000, snapshot));
 		state std::vector<Future<Optional<Key>>> backupVersion;
-		state std::vector<Future<int>> backupStatus;
+		state std::vector<Future<EBackupState>> backupStatus;
 		state std::vector<Future<int64_t>> tagRangeBytesDR;
 		state std::vector<Future<int64_t>> tagLogBytesDR;
 		state Future<Optional<Value>> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot);
@@ -1499,11 +1564,12 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		for (int i = 0; i < tagNames.size(); i++) {
 			std::string tagName = dba.sourceTagNames.unpack(tagNames[i].key).getString(0).toString();
 
-			BackupAgentBase::enumState status = (BackupAgentBase::enumState)backupStatus[i].get();
+			auto status = backupStatus[i].get();
 
 			JSONDoc tagRoot = tagsRoot.create(tagName);
-			tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING);
-			tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
+			tagRoot.create("running_backup") =
+			    (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING);
+			tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL);
 			tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get();
 			tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get();
 			tagRoot.create("mutation_stream_id") = drTagUids[i].toString();
@@ -1787,7 +1853,7 @@ ACTOR Future<Void> submitBackup(Database db, std::string url, int snapshotInterv
 				EBackupState backupStatus = wait(config.stateEnum().getOrThrow(db));
 
 				// Throw error if a backup is currently running until we support parallel backups
-				if (BackupAgentBase::isRunnable((BackupAgentBase::enumState)backupStatus)) {
+				if (BackupAgentBase::isRunnable(backupStatus)) {
 					throw backup_duplicate();
 				}
 			}
@@ -2012,10 +2078,10 @@ ACTOR Future<Void> waitBackup(Database db, std::string tagName, bool stopWhenDon
 	{
 		state FileBackupAgent backupAgent;
 
-		int status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone));
+		EBackupState status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone));
 
 		printf("The backup on tag `%s' %s.\n", printable(StringRef(tagName)).c_str(),
-			BackupAgentBase::getStateText((BackupAgentBase::enumState) status));
+		       BackupAgentBase::getStateText(status));
 	}
 	catch (Error& e) {
 		if(e.code() == error_code_actor_cancelled)
@@ -2456,6 +2522,135 @@ ACTOR Future<Void> describeBackup(const char *name, std::string destinationConta
 	return Void();
 }
 
+static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) {
+	result["error"] = errorMessage;
+	printf("%s\n", result.getJson().c_str());
+	TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage);
+}
+
+// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for
+// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version
+// resolved to that timestamp.
+ACTOR Future<Void> queryBackup(const char* name, std::string destinationContainer,
+                               Standalone<VectorRef<KeyRangeRef>> keyRangesFilter, Version restoreVersion,
+                               std::string originalClusterFile, std::string restoreTimestamp, bool verbose) {
+	state UID operationId = deterministicRandom()->randomUniqueID();
+	state JsonBuilderObject result;
+	state std::string errorMessage;
+	result["key_ranges_filter"] = printable(keyRangesFilter);
+	result["destination_container"] = destinationContainer;
+
+	TraceEvent("BackupQueryStart")
+	    .detail("OperationId", operationId)
+	    .detail("DestinationContainer", destinationContainer)
+	    .detail("KeyRangesFilter", printable(keyRangesFilter))
+	    .detail("SpecifiedRestoreVersion", restoreVersion)
+	    .detail("RestoreTimestamp", restoreTimestamp)
+	    .detail("BackupClusterFile", originalClusterFile);
+
+	// Resolve restoreTimestamp if given
+	if (!restoreTimestamp.empty()) {
+		if (originalClusterFile.empty()) {
+			reportBackupQueryError(
+			    operationId, result,
+			    format("an original cluster file must be given in order to resolve restore target timestamp '%s'",
+			           restoreTimestamp.c_str()));
+			return Void();
+		}
+
+		if (!fileExists(originalClusterFile)) {
+			reportBackupQueryError(operationId, result,
+			                       format("The specified original source database cluster file '%s' does not exist\n",
+			                              originalClusterFile.c_str()));
+			return Void();
+		}
+
+		Database origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST);
+		Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb));
+		result["restore_timestamp"] = restoreTimestamp;
+		result["restore_timestamp_resolved_version"] = v;
+		restoreVersion = v;
+	}
+
+	try {
+		state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer);
+		if (restoreVersion == invalidVersion) {
+			BackupDescription desc = wait(bc->describeBackup());
+			if (desc.maxRestorableVersion.present()) {
+				restoreVersion = desc.maxRestorableVersion.get();
+				// Use continuous log end version for the maximum restorable version for the key ranges.
+			} else if (keyRangesFilter.size() && desc.contiguousLogEnd.present()) {
+				restoreVersion = desc.contiguousLogEnd.get();
+			} else {
+				reportBackupQueryError(
+				    operationId, result,
+				    errorMessage = format("the backup for the specified key ranges is not restorable to any version"));
+			}
+		}
+
+		if (restoreVersion < 0 && restoreVersion != latestVersion) {
+			reportBackupQueryError(operationId, result,
+			                       errorMessage =
+			                           format("the specified restorable version %ld is not valid", restoreVersion));
+			return Void();
+		}
+		Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter));
+		if (fileSet.present()) {
+			int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0;
+			result["restore_version"] = fileSet.get().targetVersion;
+			JsonBuilderArray rangeFilesJson;
+			JsonBuilderArray logFilesJson;
+			for (const auto& rangeFile : fileSet.get().ranges) {
+				JsonBuilderObject object;
+				object["file_name"] = rangeFile.fileName;
+				object["file_size"] = rangeFile.fileSize;
+				object["version"] = rangeFile.version;
+				object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0
+				                          ? "none"
+				                          : fileSet.get().keyRanges.at(rangeFile.fileName).toString();
+				rangeFilesJson.push_back(object);
+				totalRangeFilesSize += rangeFile.fileSize;
+			}
+			for (const auto& log : fileSet.get().logs) {
+				JsonBuilderObject object;
+				object["file_name"] = log.fileName;
+				object["file_size"] = log.fileSize;
+				object["begin_version"] = log.beginVersion;
+				object["end_version"] = log.endVersion;
+				logFilesJson.push_back(object);
+				totalLogFilesSize += log.fileSize;
+			}
+
+			result["total_range_files_size"] = totalRangeFilesSize;
+			result["total_log_files_size"] = totalLogFilesSize;
+
+			if (verbose) {
+				result["ranges"] = rangeFilesJson;
+				result["logs"] = logFilesJson;
+			}
+
+			TraceEvent("BackupQueryReceivedRestorableFilesSet")
+			    .detail("DestinationContainer", destinationContainer)
+			    .detail("KeyRangesFilter", printable(keyRangesFilter))
+			    .detail("ActualRestoreVersion", fileSet.get().targetVersion)
+			    .detail("NumRangeFiles", fileSet.get().ranges.size())
+			    .detail("NumLogFiles", fileSet.get().logs.size())
+			    .detail("RangeFilesBytes", totalRangeFilesSize)
+			    .detail("LogFilesBytes", totalLogFilesSize);
+		} else {
+			reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges");
+			return Void();
+		}
+
+	} catch (Error& e) {
+		reportBackupQueryError(operationId, result, e.what());
+		return Void();
+	}
+
+	printf("%s\n", result.getJson().c_str());
+	return Void();
+}
+
 ACTOR Future<Void> listBackup(std::string baseUrl) {
 	try {
 		std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl));
@@ -2825,6 +3020,9 @@ int main(int argc, char* argv[]) {
 				case BACKUP_LIST:
 					args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT);
 					break;
+				case BACKUP_QUERY:
+					args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT);
+					break;
 				case BACKUP_MODIFY:
 					args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT);
 					break;
@@ -2964,6 +3162,7 @@ int main(int argc, char* argv[]) {
 		std::string addPrefix;
 		std::string removePrefix;
 		Standalone<VectorRef<KeyRangeRef>> backupKeys;
+		Standalone<VectorRef<KeyRangeRef>> backupKeysFilter;
 		int maxErrors = 20;
 		Version beginVersion = invalidVersion;
 		Version restoreVersion = invalidVersion;
@@ -3186,6 +3385,15 @@ int main(int argc, char* argv[]) {
 						return FDB_EXIT_ERROR;
 					}
 					break;
+				case OPT_BACKUPKEYS_FILTER:
+					try {
+						addKeyRange(args->OptionArg(), backupKeysFilter);
+					}
+					catch (Error &) {
+						printHelpTeaser(argv[0]);
+						return FDB_EXIT_ERROR;
+					}
+					break;
 				case OPT_DESTCONTAINER:
 					destinationContainer = args->OptionArg();
 					// If the url starts with '/' then prepend "file://" for backwards compatibility
@@ -3725,6 +3933,12 @@ int main(int argc, char* argv[]) {
 				f = stopAfter( listBackup(baseUrl) );
 				break;
 
+			case BACKUP_QUERY:
+				initTraceFile();
+				f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion,
+				                          restoreClusterFileOrig, restoreTimestamp, !quietDisplay));
+				break;
+
 			case BACKUP_DUMP:
 				initTraceFile();
 				f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) );

fdbcli/FlowLineNoise.actor.cpp

@@ -36,18 +36,18 @@
 #endif
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
-struct LineNoiseReader : IThreadPoolReceiver {
-    virtual void init() {}
+struct LineNoiseReader final : IThreadPoolReceiver {
+	void init() override {}
 
-    struct Read : TypedAction<LineNoiseReader, Read> {
-        std::string prompt;
+	struct Read final : TypedAction<LineNoiseReader, Read> {
+		std::string prompt;
         ThreadReturnPromise<Optional<std::string>> result;
 
-        virtual double getTimeEstimate() { return 0.0; }
-        explicit Read(std::string const& prompt) : prompt(prompt) {}
-    };
+		double getTimeEstimate() const override { return 0.0; }
+		explicit Read(std::string const& prompt) : prompt(prompt) {}
+	};
 
-    void action(Read& r) {
+	void action(Read& r) {
         try {
             r.result.send( read(r.prompt) );
         } catch (Error& e) {
@@ -117,7 +117,7 @@ LineNoise::LineNoise(
             Hint h = onMainThread( [line]() -> Future<Hint> {
                 return hint_callback(line);
             }).getBlocking();
-            if (!h.valid) return NULL;
+            if (!h.valid) return nullptr;
             *color = h.color;
             *bold = h.bold;
             return strdup( h.text.c_str() );

fdbcli/fdbcli.actor.cpp

@@ -20,6 +20,7 @@
 
 #include "boost/lexical_cast.hpp"
 #include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/FDBTypes.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/DatabaseContext.h"
@@ -102,7 +103,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
 void printAtCol(const char* text, int col) {
 	const char* iter = text;
 	const char* start = text;
-	const char* space = NULL;
+	const char* space = nullptr;
 
 	do {
 		iter++;
@@ -112,7 +113,7 @@ void printAtCol(const char* text, int col) {
 			printf("%.*s\n", (int)(space - start), start);
 			start = space;
 			if (*start == ' ' || *start == '\n') start++;
-			space = NULL;
+			space = nullptr;
 		}
 	} while (*iter);
 }
@@ -120,7 +121,7 @@ void printAtCol(const char* text, int col) {
 std::string lineWrap(const char* text, int col) {
 	const char* iter = text;
 	const char* start = text;
-	const char* space = NULL;
+	const char* space = nullptr;
 	std::string out = "";
 	do {
 		iter++;
@@ -130,7 +131,7 @@ std::string lineWrap(const char* text, int col) {
 			out += format("%.*s\n", (int)(space - start), start);
 			start = space;
 			if (*start == ' '/* || *start == '\n'*/) start++;
-			space = NULL;
+			space = nullptr;
 		}
 	} while (*iter);
 	return out;
@@ -470,8 +471,8 @@ void initHelp() {
 		"All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK);
 	helpMap["configure"] = CommandHelp(
 	    "configure [new] "
-	    "<single|double|triple|three_data_hall|three_datacenter|ssd|memory|memory-radixtree-beta|proxies=<PROXIES>|grv_"
-	    "proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
+	    "<single|double|triple|three_data_hall|three_datacenter|ssd|memory|memory-radixtree-beta|proxies=<PROXIES>|"
+	    "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
 	    "change the database configuration",
 	    "The `new' option, if present, initializes a new database with the given configuration rather than changing "
 	    "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be "
@@ -479,13 +480,19 @@ void initHelp() {
 	    "of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - "
 	    "See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage "
 	    "engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small "
-	    "datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set "
-	    "to -1 which restores the number of proxies to the default value.\n\ngrv_proxies=<GRV_PROXIES>: Sets the "
-	    "desired number of GRV proxies in the cluster. Must be at least 1, or set to -1 which restores the number of "
-	    "proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be "
-	    "at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: "
-	    "Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the "
-	    "number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
+	    "datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. The proxy role is being "
+		"deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' "
+		"separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and 'grv_proxies' "
+		"should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and 'commit_proxies'. "
+		"Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the "
+		"default value.\n\ncommit_proxies=<COMMIT_PROXIES>: Sets the desired number of commit proxies in the cluster. "
+	    "Must be at least 1, or set to -1 which restores the number of commit proxies to the default "
+	    "value.\n\ngrv_proxies=<GRV_PROXIES>: Sets the desired number of GRV proxies in the cluster. Must be at least "
+	    "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=<LOGS>: Sets the "
+	    "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of "
+	    "logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. "
+	    "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the "
+	    "FoundationDB Administration Guide for more information.");
 	helpMap["fileconfigure"] = CommandHelp(
 		"fileconfigure [new] <FILENAME>",
 		"change the database configuration from a file",
@@ -871,12 +878,13 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
 							fatalRecoveryState = true;
 
 							if (name == "recruiting_transaction_servers") {
-								description += format("\nNeed at least %d log servers across unique zones, %d proxies, "
-								                      "%d GRV proxies and %d resolvers.",
-								                      recoveryState["required_logs"].get_int(),
-								                      recoveryState["required_proxies"].get_int(),
-								                      recoveryState["required_grv_proxies"].get_int(),
-								                      recoveryState["required_resolvers"].get_int());
+								description +=
+								    format("\nNeed at least %d log servers across unique zones, %d commit proxies, "
+								           "%d GRV proxies and %d resolvers.",
+								           recoveryState["required_logs"].get_int(),
+								           recoveryState["required_commit_proxies"].get_int(),
+								           recoveryState["required_grv_proxies"].get_int(),
+								           recoveryState["required_resolvers"].get_int());
 								if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) {
 									auto numOfNonExcludedProcessesAndZones = getNumOfNonExcludedProcessAndZones(statusObjCluster);
 									description += format("\nHave %d non-excluded processes on %d machines across %d zones.", numOfNonExcludedProcessesAndZones.first, getNumofNonExcludedMachines(statusObjCluster), numOfNonExcludedProcessesAndZones.second);
@@ -1026,8 +1034,8 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
 					outputString += format("\n  Exclusions             - %d (type `exclude' for details)", excludedServersArr.size());
 				}
 
-				if (statusObjConfig.get("proxies", intVal))
-					outputString += format("\n  Desired Proxies        - %d", intVal);
+				if (statusObjConfig.get("commit_proxies", intVal))
+					outputString += format("\n  Desired Commit Proxies - %d", intVal);
 
 				if (statusObjConfig.get("grv_proxies", intVal))
 					outputString += format("\n  Desired GRV Proxies    - %d", intVal);
@@ -1055,10 +1063,10 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
 				if (statusObjConfig.has("regions")) {
 					outputString += "\n  Regions: ";
 					regions = statusObjConfig["regions"].get_array();
-					bool isPrimary = false;
-					std::vector<std::string> regionSatelliteDCs;
-					std::string regionDC;
 					for (StatusObjectReader region : regions) {
+						bool isPrimary = false;
+						std::vector<std::string> regionSatelliteDCs;
+						std::string regionDC;
 						for (StatusObjectReader dc : region["datacenters"].get_array()) {
 							if (!dc.has("satellite")) {
 								regionDC = dc["id"].get_str();
@@ -1233,14 +1241,54 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
 
 						int minLoss = std::min(availLoss, dataLoss);
 						const char *faultDomain = machinesAreZones ? "machine" : "zone";
-						if (minLoss == 1)
-							outputString += format("1 %s", faultDomain);
-						else
-							outputString += format("%d %ss", minLoss, faultDomain);
+						outputString += format("%d %ss", minLoss, faultDomain);
 
 						if (dataLoss > availLoss){
 							outputString += format(" (%d without data loss)", dataLoss);
 						}
+
+						if (dataLoss == -1) {
+							ASSERT_WE_THINK(availLoss == -1);
+							outputString += format(
+							    "\n\n  Warning: the database may have data loss and availability loss. Please restart "
+							    "following tlog interfaces, otherwise storage servers may never be able to catch "
+							    "up.\n");
+							StatusObjectReader logs;
+							if (statusObjCluster.has("logs")) {
+								for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) {
+									bool possiblyLosingData;
+									if (logEpoch.get("possibly_losing_data", possiblyLosingData) &&
+									    !possiblyLosingData) {
+										continue;
+									}
+									// Current epoch doesn't have an end version.
+									int64_t epoch, beginVersion, endVersion = invalidVersion;
+									bool current;
+									logEpoch.get("epoch", epoch);
+									logEpoch.get("begin_version", beginVersion);
+									logEpoch.get("end_version", endVersion);
+									logEpoch.get("current", current);
+									std::string missing_log_interfaces;
+									if (logEpoch.has("log_interfaces")) {
+										for (StatusObjectReader logInterface : logEpoch.last().get_array()) {
+											bool healthy;
+											std::string address, id;
+											if (logInterface.get("healthy", healthy) && !healthy) {
+												logInterface.get("id", id);
+												logInterface.get("address", address);
+												missing_log_interfaces += format("%s,%s ", id.c_str(), address.c_str());
+											}
+										}
+									}
+									outputString += format(
+									    "  %s log epoch: %ld begin: %ld end: %s, missing "
+									    "log interfaces(id,address): %s\n",
+									    current ? "Current" : "Old", epoch, beginVersion,
+									    endVersion == invalidVersion ? "(unknown)" : format("%ld", endVersion).c_str(),
+									    missing_log_interfaces.c_str());
+								}
+							}
+						}
 					}
 				}
 
@@ -1764,7 +1812,7 @@ ACTOR Future<Void> commitTransaction( Reference<ReadYourWritesTransaction> tr )
 }
 
 ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Reference<ClusterConnectionFile> ccf, LineNoise* linenoise, Future<Void> warn ) {
-	state ConfigurationResult::Type result;
+	state ConfigurationResult result;
 	state int startToken = 1;
 	state bool force = false;
 	if (tokens.size() < 2)
@@ -1790,14 +1838,14 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 
 			bool noChanges = conf.get().old_replication == conf.get().auto_replication &&
 			                 conf.get().old_logs == conf.get().auto_logs &&
-			                 conf.get().old_proxies == conf.get().auto_proxies &&
+			                 conf.get().old_commit_proxies == conf.get().auto_commit_proxies &&
 			                 conf.get().old_grv_proxies == conf.get().auto_grv_proxies &&
 			                 conf.get().old_resolvers == conf.get().auto_resolvers &&
 			                 conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction &&
 			                 conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction;
 
 			bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs &&
-			                        conf.get().old_proxies == conf.get().desired_proxies &&
+			                        conf.get().old_commit_proxies == conf.get().desired_commit_proxies &&
 			                        conf.get().old_grv_proxies == conf.get().desired_grv_proxies &&
 			                        conf.get().old_resolvers == conf.get().desired_resolvers;
 
@@ -1816,8 +1864,11 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 			outputString += format("| replication                 | %16s | %16s |\n", conf.get().old_replication.c_str(), conf.get().auto_replication.c_str());
 			outputString += format("| logs                        | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs);
 			outputString += conf.get().auto_logs != conf.get().desired_logs ? format(" (manually set; would be %d)\n", conf.get().desired_logs) : "\n";
-			outputString += format("| proxies                     | %16d | %16d |", conf.get().old_proxies, conf.get().auto_proxies);
-			outputString += conf.get().auto_proxies != conf.get().desired_proxies ? format(" (manually set; would be %d)\n", conf.get().desired_proxies) : "\n";
+			outputString += format("| commit_proxies              | %16d | %16d |", conf.get().old_commit_proxies,
+			                       conf.get().auto_commit_proxies);
+			outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies
+			                    ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies)
+			                    : "\n";
 			outputString += format("| grv_proxies                 | %16d | %16d |", conf.get().old_grv_proxies,
 			                       conf.get().auto_grv_proxies);
 			outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies
@@ -1842,7 +1893,8 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 			}
 		}
 
-		ConfigurationResult::Type r  = wait( makeInterruptable( changeConfig( db, std::vector<StringRef>(tokens.begin()+startToken,tokens.end()), conf, force) ) );
+		ConfigurationResult r = wait(makeInterruptable(
+		    changeConfig(db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force)));
 		result = r;
 	}
 
@@ -1968,7 +2020,7 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDa
 			return true;
 		}
 	}
-	ConfigurationResult::Type result = wait( makeInterruptable( changeConfig(db, configString, force) ) );
+	ConfigurationResult result = wait(makeInterruptable(changeConfig(db, configString, force)));
 	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
 	// there are various results specific to changeConfig() that we need to report:
 	bool ret;
@@ -2099,7 +2151,7 @@ ACTOR Future<bool> coordinators( Database db, std::vector<StringRef> tokens, boo
 	}
 	if(setName.size()) change = nameQuorumChange( setName.toString(), change );
 
-	CoordinatorsResult::Type r = wait( makeInterruptable( changeQuorum( db, change ) ) );
+	CoordinatorsResult r = wait(makeInterruptable(changeQuorum(db, change)));
 
 	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
 	// there are various results specific to changeConfig() that we need to report:
@@ -2472,7 +2524,7 @@ void compGenerator(const char* text, bool help, std::vector<std::string>& lc) {
 	std::map<std::string, CommandHelp>::const_iterator iter;
 	int len = strlen(text);
 
-	const char* helpExtra[] = {"escaping", "options", NULL};
+	const char* helpExtra[] = {"escaping", "options", nullptr};
 
 	const char** he = helpExtra;
 
@@ -2531,11 +2583,24 @@ void onOffGenerator(const char* text, const char *line, std::vector<std::string>
 }
 
 void configureGenerator(const char* text, const char *line, std::vector<std::string>& lc) {
-	const char* opts[] = {
-		"new",          "single", "double",     "triple",   "three_data_hall", "three_datacenter",      "ssd",
-		"ssd-1",        "ssd-2",  "memory",     "memory-1", "memory-2",        "memory-radixtree-beta", "proxies=",
-		"grv_proxies=", "logs=",  "resolvers=", nullptr
-	};
+	const char* opts[] = { "new",
+		                   "single",
+		                   "double",
+		                   "triple",
+		                   "three_data_hall",
+		                   "three_datacenter",
+		                   "ssd",
+		                   "ssd-1",
+		                   "ssd-2",
+		                   "memory",
+		                   "memory-1",
+		                   "memory-2",
+		                   "memory-radixtree-beta",
+		                   "commit_proxies=",
+		                   "grv_proxies=",
+		                   "logs=",
+		                   "resolvers=",
+		                   nullptr };
 	arrayGenerator(text, line, opts, lc);
 }
 
@@ -2973,7 +3038,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 			.detail("SourceVersion", getSourceVersion())
 			.detail("Version", FDB_VT_VERSION)
 			.detail("PackageName", FDB_VT_PACKAGE_NAME)
-			.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
+			.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 			.detail("ClusterFile", ccf->getFilename().c_str())
 			.detail("ConnectionString", ccf->getConnectionString().toString())
 			.setMaxFieldLength(10000)
@@ -4548,7 +4613,7 @@ int main(int argc, char **argv) {
 	sigemptyset( &act.sa_mask );
 	act.sa_flags = 0;
 	act.sa_handler = SIG_IGN;
-	sigaction(SIGINT, &act, NULL);
+	sigaction(SIGINT, &act, nullptr);
 #endif
 
 	CLIOptions opt(argc, argv);

fdbclient/AsyncFileBlobStore.actor.h

@@ -59,7 +59,7 @@ public:
 	virtual void delref() { ReferenceCounted<AsyncFileBlobStoreWrite>::delref(); }
 
 	struct Part : ReferenceCounted<Part> {
-		Part(int n, int minSize) : number(n), writer(content.getWriteBuffer(minSize), NULL, Unversioned()), length(0) {
+		Part(int n, int minSize) : number(n), writer(content.getWriteBuffer(minSize), nullptr, Unversioned()), length(0) {
 			etag = std::string();
 			::MD5_Init(&content_md5_buf);
 		}

fdbclient/BackupAgent.actor.h

@@ -46,13 +46,15 @@ public:
 		return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
 	}
 
-	// Type of program being executed
-	enum enumActionResult {
-		RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3
-	};
-
-	enum enumState {
-		STATE_ERRORED = 0, STATE_SUBMITTED = 1, STATE_RUNNING = 2, STATE_RUNNING_DIFFERENTIAL = 3, STATE_COMPLETED = 4, STATE_NEVERRAN = 5, STATE_ABORTED = 6, STATE_PARTIALLY_ABORTED = 7
+	enum class EnumState {
+		STATE_ERRORED = 0,
+		STATE_SUBMITTED = 1,
+		STATE_RUNNING = 2,
+		STATE_RUNNING_DIFFERENTIAL = 3,
+		STATE_COMPLETED = 4,
+		STATE_NEVERRAN = 5,
+		STATE_ABORTED = 6,
+		STATE_PARTIALLY_ABORTED = 7
 	};
 
 	static const Key keyFolderId;
@@ -85,70 +87,68 @@ public:
 	static const int logHeaderSize;
 
 	// Convert the status text to an enumerated value
-	static enumState getState(std::string stateText)
-	{
-		enumState enState = STATE_ERRORED;
+	static EnumState getState(std::string stateText) {
+		auto enState = EnumState::STATE_ERRORED;
 
 		if (stateText.empty()) {
-			enState = STATE_NEVERRAN;
+			enState = EnumState::STATE_NEVERRAN;
 		}
 
 		else if (!stateText.compare("has been submitted")) {
-			enState = STATE_SUBMITTED;
+			enState = EnumState::STATE_SUBMITTED;
 		}
 
 		else if (!stateText.compare("has been started")) {
-			enState = STATE_RUNNING;
+			enState = EnumState::STATE_RUNNING;
 		}
 
 		else if (!stateText.compare("is differential")) {
-			enState = STATE_RUNNING_DIFFERENTIAL;
+			enState = EnumState::STATE_RUNNING_DIFFERENTIAL;
 		}
 
 		else if (!stateText.compare("has been completed")) {
-			enState = STATE_COMPLETED;
+			enState = EnumState::STATE_COMPLETED;
 		}
 
 		else if (!stateText.compare("has been aborted")) {
-			enState = STATE_ABORTED;
+			enState = EnumState::STATE_ABORTED;
 		}
 
 		else if (!stateText.compare("has been partially aborted")) {
-			enState = STATE_PARTIALLY_ABORTED;
+			enState = EnumState::STATE_PARTIALLY_ABORTED;
 		}
 
 		return enState;
 	}
 
 	// Convert the status enum to a text description
-	static const char* getStateText(enumState enState)
-	{
+	static const char* getStateText(EnumState enState) {
 		const char* stateText;
 
 		switch (enState)
 		{
-		case STATE_ERRORED:
+		case EnumState::STATE_ERRORED:
 			stateText = "has errored";
 			break;
-		case STATE_NEVERRAN:
+		case EnumState::STATE_NEVERRAN:
 			stateText = "has never been started";
 			break;
-		case STATE_SUBMITTED:
+		case EnumState::STATE_SUBMITTED:
 			stateText = "has been submitted";
 			break;
-		case STATE_RUNNING:
+		case EnumState::STATE_RUNNING:
 			stateText = "has been started";
 			break;
-		case STATE_RUNNING_DIFFERENTIAL:
+		case EnumState::STATE_RUNNING_DIFFERENTIAL:
 			stateText = "is differential";
 			break;
-		case STATE_COMPLETED:
+		case EnumState::STATE_COMPLETED:
 			stateText = "has been completed";
 			break;
-		case STATE_ABORTED:
+		case EnumState::STATE_ABORTED:
 			stateText = "has been aborted";
 			break;
-		case STATE_PARTIALLY_ABORTED:
+		case EnumState::STATE_PARTIALLY_ABORTED:
 			stateText = "has been partially aborted";
 			break;
 		default:
@@ -160,34 +160,33 @@ public:
 	}
 
 	// Convert the status enum to a name
-	static const char* getStateName(enumState enState)
-	{
+	static const char* getStateName(EnumState enState) {
 		const char* s;
 
 		switch (enState)
 		{
-		case STATE_ERRORED:
+		case EnumState::STATE_ERRORED:
 			s = "Errored";
 			break;
-		case STATE_NEVERRAN:
+		case EnumState::STATE_NEVERRAN:
 			s = "NeverRan";
 			break;
-		case STATE_SUBMITTED:
+		case EnumState::STATE_SUBMITTED:
 			s = "Submitted";
 			break;
-		case STATE_RUNNING:
+		case EnumState::STATE_RUNNING:
 			s = "Running";
 			break;
-		case STATE_RUNNING_DIFFERENTIAL:
+		case EnumState::STATE_RUNNING_DIFFERENTIAL:
 			s = "RunningDifferentially";
 			break;
-		case STATE_COMPLETED:
+		case EnumState::STATE_COMPLETED:
 			s = "Completed";
 			break;
-		case STATE_ABORTED:
+		case EnumState::STATE_ABORTED:
 			s = "Aborted";
 			break;
-		case STATE_PARTIALLY_ABORTED:
+		case EnumState::STATE_PARTIALLY_ABORTED:
 			s = "Aborting";
 			break;
 		default:
@@ -199,16 +198,15 @@ public:
 	}
 
 	// Determine if the specified state is runnable
-	static bool isRunnable(enumState enState)
-	{
+	static bool isRunnable(EnumState enState) {
 		bool isRunnable = false;
 
 		switch (enState)
 		{
-		case STATE_SUBMITTED:
-		case STATE_RUNNING:
-		case STATE_RUNNING_DIFFERENTIAL:
-		case STATE_PARTIALLY_ABORTED:
+		case EnumState::STATE_SUBMITTED:
+		case EnumState::STATE_RUNNING:
+		case EnumState::STATE_RUNNING_DIFFERENTIAL:
+		case EnumState::STATE_PARTIALLY_ABORTED:
 			isRunnable = true;
 			break;
 		default:
@@ -359,7 +357,8 @@ public:
 
 	// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
 	// will return when the backup directory is restorable.
-	Future<int> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr);
+	Future<EnumState> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true,
+	                             Reference<IBackupContainer>* pContainer = nullptr, UID* pUID = nullptr);
 
 	static const Key keyLastRestorable;
 
@@ -432,8 +431,8 @@ public:
 
 	Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
 
-	Future<int> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
-	Future<int> getStateValue(Database cx, UID logUid) {
+	Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
+	Future<EnumState> getStateValue(Database cx, UID logUid) {
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
 	}
 
@@ -452,8 +451,8 @@ public:
 
 	// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
 	// will return when the backup directory is restorable.
-	Future<int> waitBackup(Database cx, Key tagName, bool stopWhenDone = true);
-	Future<int> waitSubmitted(Database cx, Key tagName);
+	Future<EnumState> waitBackup(Database cx, Key tagName, bool stopWhenDone = true);
+	Future<EnumState> waitSubmitted(Database cx, Key tagName);
 	Future<Void> waitUpgradeToLatestDrVersion(Database cx, Key tagName);
 
 	static const Key keyAddPrefix;
@@ -522,9 +521,15 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
                                   NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
 ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
 
-typedef BackupAgentBase::enumState EBackupState;
-template<> inline Tuple Codec<EBackupState>::pack(EBackupState const &val) { return Tuple().append(val); }
-template<> inline EBackupState Codec<EBackupState>::unpack(Tuple const &val) { return (EBackupState)val.getInt(0); }
+using EBackupState = BackupAgentBase::EnumState;
+template <>
+inline Tuple Codec<EBackupState>::pack(EBackupState const& val) {
+	return Tuple().append(static_cast<int>(val));
+}
+template <>
+inline EBackupState Codec<EBackupState>::unpack(Tuple const& val) {
+	return static_cast<EBackupState>(val.getInt(0));
+}
 
 // Key backed tags are a single-key slice of the TagUidMap, defined below.
 // The Value type of the key is a UidAndAbortedFlagT which is a pair of {UID, aborted_flag}

fdbclient/BackupContainer.actor.cpp

@@ -23,6 +23,7 @@
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/JsonBuilder.h"
+#include "flow/Arena.h"
 #include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include "flow/Hash3.h"
@@ -245,7 +246,7 @@ std::string BackupDescription::toJSON() const {
  *     file written will be after the start version of the snapshot's execution.
  *
  *   Log files are at file paths like
- *       /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize
+ *       /plogs/.../log,startVersion,endVersion,UID,tagID-of-N,blocksize
  *       /logs/.../log,startVersion,endVersion,UID,blockSize
  *     where ... is a multi level path which sorts lexically into version order and results in approximately 1
  *     unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs"
@@ -1403,8 +1404,15 @@ public:
 	}
 
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc,
-	                                                                    Version targetVersion, bool logsOnly,
-	                                                                    Version beginVersion) {
+	                                                                    Version targetVersion,
+	                                                                    VectorRef<KeyRangeRef> keyRangesFilter, bool logsOnly = false,
+	                                                                    Version beginVersion = invalidVersion) {
+		// Does not support use keyRangesFilter for logsOnly yet
+		if (logsOnly && !keyRangesFilter.empty()) {
+			TraceEvent(SevError, "BackupContainerRestoreSetUnsupportedAPI").detail("KeyRangesFilter", keyRangesFilter.size());
+			return Optional<RestorableFileSet>();
+		}
+
 		if (logsOnly) {
 			state RestorableFileSet restorableSet;
 			state std::vector<LogFile> logFiles;
@@ -1416,23 +1424,55 @@ public:
 				return getRestoreSetFromLogs(logFiles, targetVersion, restorableSet);
 			}
 		}
-		// Find the most recent keyrange snapshot to end at or before targetVersion
-		state Optional<KeyspaceSnapshotFile> snapshot;
-		std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
-		for(auto const &s : snapshots) {
-			if(s.endVersion <= targetVersion)
-				snapshot = s;
-		}
 
-		if(snapshot.present()) {
+		// Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion.
+		state std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
+		state int i = snapshots.size() - 1;
+		for (; i >= 0; i--) {
+			// The smallest version of filtered range files >= snapshot beginVersion > targetVersion
+			if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) {
+				continue;
+			}
+
 			state RestorableFileSet restorable;
-			restorable.snapshot = snapshot.get();
-			restorable.targetVersion = targetVersion;
+			state Version minKeyRangeVersion = MAX_VERSION;
+			state Version maxKeyRangeVersion = -1;
 
 			std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>> results =
-			    wait(bc->readKeyspaceSnapshot(snapshot.get()));
-			restorable.ranges = std::move(results.first);
-			restorable.keyRanges = std::move(results.second);
+			    wait(bc->readKeyspaceSnapshot(snapshots[i]));
+
+			// Old backup does not have metadata about key ranges and can not be filtered with key ranges.
+			if (keyRangesFilter.size() && results.second.empty() && !results.first.empty()) {
+				throw backup_not_filterable_with_key_ranges();
+			}
+
+			// Filter by keyRangesFilter.
+			if (keyRangesFilter.empty()) {
+				restorable.ranges = std::move(results.first);
+				restorable.keyRanges = std::move(results.second);
+				minKeyRangeVersion = snapshots[i].beginVersion;
+				maxKeyRangeVersion = snapshots[i].endVersion;
+			} else {
+				for (const auto& rangeFile : results.first) {
+					const auto& keyRange = results.second.at(rangeFile.fileName);
+					if (keyRange.intersects(keyRangesFilter)) {
+						restorable.ranges.push_back(rangeFile);
+						restorable.keyRanges[rangeFile.fileName] = keyRange;
+						minKeyRangeVersion = std::min(minKeyRangeVersion, rangeFile.version);
+						maxKeyRangeVersion = std::max(maxKeyRangeVersion, rangeFile.version);
+					}
+				}
+				// No range file matches 'keyRangesFilter'.
+				if (restorable.ranges.empty()) {
+					throw backup_not_overlapped_with_keys_filter();
+				}
+			}
+			// 'latestVersion' represents using the minimum restorable version in a snapshot.
+			restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
+			// Any version < maxKeyRangeVersion is not restorable.
+			if (restorable.targetVersion < maxKeyRangeVersion) continue;
+
+			restorable.snapshot = snapshots[i];
 			// TODO: Reenable the sanity check after TooManyFiles error is resolved
 			if (false && g_network->isSimulated()) {
 				// Sanity check key ranges
@@ -1446,18 +1486,21 @@ public:
 				}
 			}
 
-			// No logs needed if there is a complete key space snapshot at the target version.
-			if (snapshot.get().beginVersion == snapshot.get().endVersion &&
-			    snapshot.get().endVersion == targetVersion) {
+			// No logs needed if there is a complete filtered key space snapshot at the target version.
+			if (minKeyRangeVersion == maxKeyRangeVersion && maxKeyRangeVersion == restorable.targetVersion) {
 				restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion;
+				TraceEvent("BackupContainerGetRestorableFilesWithoutLogs")
+				    .detail("KeyRangeVersion", restorable.targetVersion)
+				    .detail("NumberOfRangeFiles", restorable.ranges.size())
+				    .detail("KeyRangesFilter", printable(keyRangesFilter));
 				return Optional<RestorableFileSet>(restorable);
 			}
 
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
 			state std::vector<LogFile> logs;
 			state std::vector<LogFile> plogs;
-			wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) &&
-			     store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true)));
+			wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) &&
+			     store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true)));
 
 			if (plogs.size() > 0) {
 				logs.swap(plogs);
@@ -1469,13 +1512,12 @@ public:
 
 				// Remove duplicated log files that can happen for old epochs.
 				std::vector<LogFile> filtered = filterDuplicates(logs);
-
 				restorable.logs.swap(filtered);
 				// sort by version order again for continuous analysis
 				std::sort(restorable.logs.begin(), restorable.logs.end());
-				if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) {
-					restorable.continuousBeginVersion = snapshot.get().beginVersion;
-					restorable.continuousEndVersion = targetVersion + 1; // not inclusive
+				if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) {
+					restorable.continuousBeginVersion = minKeyRangeVersion;
+					restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive
 					return Optional<RestorableFileSet>(restorable);
 				}
 				return Optional<RestorableFileSet>();
@@ -1483,20 +1525,19 @@ public:
 
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logs.begin(), logs.end());
-
-			// If there are logs and the first one starts at or before the snapshot begin version then proceed
-			if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
+			// If there are logs and the first one starts at or before the keyrange's snapshot begin version, then
+			// it is valid restore set and proceed
+			if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) {
 				return getRestoreSetFromLogs(logs, targetVersion, restorable);
 			}
 		}
-
 		return Optional<RestorableFileSet>();
 	}
 
-	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly,
-	                                                  Version beginVersion) final {
-		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, logsOnly,
-		                          beginVersion);
+	Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, VectorRef<KeyRangeRef> keyRangesFilter,
+	                                                  bool logsOnly, Version beginVersion) final {
+		return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter,
+		                          logsOnly, beginVersion);
 	}
 
 private:
@@ -1796,7 +1837,7 @@ private:
 	std::string m_path;
 };
 
-class BackupContainerBlobStore : public BackupContainerFileSystem, ReferenceCounted<BackupContainerBlobStore> {
+class BackupContainerBlobStore final : public BackupContainerFileSystem, ReferenceCounted<BackupContainerBlobStore> {
 private:
 	// Backup files to under a single folder prefix with subfolders for each named backup
 	static const std::string DATAFOLDER;
@@ -1836,15 +1877,13 @@ public:
 		}
 	}
 
-	void addref() final { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
-	void delref() final { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
+	void addref() override { return ReferenceCounted<BackupContainerBlobStore>::addref(); }
+	void delref() override { return ReferenceCounted<BackupContainerBlobStore>::delref(); }
 
 	static std::string getURLFormat() {
 		return BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)";
 	}
 
-	virtual ~BackupContainerBlobStore() {}
-
 	Future<Reference<IAsyncFile>> readFile(std::string path) final {
 		return Reference<IAsyncFile>(
 			new AsyncFileReadAheadCache(

fdbclient/BackupContainer.h

@@ -280,10 +280,13 @@ public:
 
 	virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
 
-	// Get exactly the files necessary to restore to targetVersion.  Returns non-present if
-	// restore to given version is not possible.
-	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly = false,
-	                                                          Version beginVersion = -1) = 0;
+	// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
+	// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
+	// If logsOnly is set, only use log files in [beginVersion, targetVervions) in restore set.
+	// Returns non-present if restoring to the given version is not possible.
+	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
+	                                                          VectorRef<KeyRangeRef> keyRangesFilter = {},
+	                                                          bool logsOnly = false, Version beginVersion = -1) = 0;
 
 	// Get an IBackupContainer based on a container spec string
 	static Reference<IBackupContainer> openContainer(std::string url);

fdbclient/BlobStore.actor.cpp

@@ -277,7 +277,7 @@ ACTOR Future<bool> bucketExists_impl(Reference<BlobStoreEndpoint> b, std::string
 	std::string resource = std::string("/") + bucket;
 	HTTP::Headers headers;
 
-	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404}));
+	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404}));
 	return r->code == 200;
 }
 
@@ -291,7 +291,7 @@ ACTOR Future<bool> objectExists_impl(Reference<BlobStoreEndpoint> b, std::string
 	std::string resource = std::string("/") + bucket + "/" + object;
 	HTTP::Headers headers;
 
-	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404}));
+	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404}));
 	return r->code == 200;
 }
 
@@ -305,7 +305,7 @@ ACTOR Future<Void> deleteObject_impl(Reference<BlobStoreEndpoint> b, std::string
 	std::string resource = std::string("/") + bucket + "/" + object;
 	HTTP::Headers headers;
 	// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered successful
-	Reference<HTTP::Response> r = wait(b->doRequest("DELETE", resource, headers, NULL, 0, {200, 204, 404}));
+	Reference<HTTP::Response> r = wait(b->doRequest("DELETE", resource, headers, nullptr, 0, {200, 204, 404}));
 
 	// But if the object already did not exist then the 'delete' is assumed to be successful but a warning is logged.
 	if(r->code == 404) {
@@ -386,7 +386,7 @@ ACTOR Future<Void> createBucket_impl(Reference<BlobStoreEndpoint> b, std::string
 	if(!exists) {
 		std::string resource = std::string("/") + bucket;
 		HTTP::Headers headers;
-		Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, NULL, 0, {200, 409}));
+		Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, {200, 409}));
 	}
 	return Void();
 }
@@ -401,7 +401,7 @@ ACTOR Future<int64_t> objectSize_impl(Reference<BlobStoreEndpoint> b, std::strin
 	std::string resource = std::string("/") + bucket + "/" + object;
 	HTTP::Headers headers;
 
-	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, NULL, 0, {200, 404}));
+	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, {200, 404}));
 	if(r->code == 404)
 		throw file_not_found();
 	return r->contentLen;
@@ -737,7 +737,7 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
 		HTTP::Headers headers;
 		state std::string fullResource = resource + HTTP::urlEncode(lastFile);
 		lastFile.clear();
-		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, NULL, 0, {200}));
+		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, {200}));
 		listReleaser.release();
 
 		try {
@@ -782,7 +782,7 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
 					if(size == nullptr) {
 						throw http_bad_response();
 					}
-					object.size = strtoull(size->value(), NULL, 10);
+					object.size = strtoull(size->value(), nullptr, 10);
 
 					listResult.objects.push_back(object);
 				}
@@ -893,7 +893,7 @@ ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<BlobStoreEndpo
 
 		HTTP::Headers headers;
 		state std::string fullResource = resource + HTTP::urlEncode(lastName);
-		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, NULL, 0, {200}));
+		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, {200}));
 		listReleaser.release();
 
 		try {
@@ -1024,7 +1024,7 @@ ACTOR Future<std::string> readEntireFile_impl(Reference<BlobStoreEndpoint> bstor
 
 	std::string resource = std::string("/") + bucket + "/" + object;
 	HTTP::Headers headers;
-	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, NULL, 0, {200, 404}));
+	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, {200, 404}));
 	if(r->code == 404)
 		throw file_not_found();
 	return r->content;
@@ -1057,7 +1057,7 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> b
 
 ACTOR Future<Void> writeEntireFile_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, std::string content) {
 	state UnsentPacketQueue packets;
-	PacketWriter pw(packets.getWriteBuffer(content.size()), NULL, Unversioned());
+	PacketWriter pw(packets.getWriteBuffer(content.size()), nullptr, Unversioned());
 	pw.serializeBytes(content);
 	if(content.size() > bstore->knobs.multipart_max_part_size)
 		throw file_too_large();
@@ -1095,7 +1095,7 @@ ACTOR Future<int> readObject_impl(Reference<BlobStoreEndpoint> bstore, std::stri
 	std::string resource = std::string("/") + bucket + "/" + object;
 	HTTP::Headers headers;
 	headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1);
-	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, NULL, 0, {200, 206, 404}));
+	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, {200, 206, 404}));
 	if(r->code == 404)
 		throw file_not_found();
 	if(r->contentLen != r->content.size())  // Double check that this wasn't a header-only response, probably unnecessary
@@ -1114,7 +1114,7 @@ ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<BlobStoreEn
 
 	std::string resource = std::string("/") + bucket + "/" + object + "?uploads";
 	HTTP::Headers headers;
-	Reference<HTTP::Response> r = wait(bstore->doRequest("POST", resource, headers, NULL, 0, {200}));
+	Reference<HTTP::Response> r = wait(bstore->doRequest("POST", resource, headers, nullptr, 0, {200}));
 
 	try {
 		xml_document<> doc;
@@ -1180,7 +1180,7 @@ ACTOR Future<Void> finishMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstor
 
 	std::string resource = format("/%s/%s?uploadId=%s", bucket.c_str(), object.c_str(), uploadID.c_str());
 	HTTP::Headers headers;
-	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), NULL, Unversioned());
+	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned());
 	pw.serializeBytes(manifest);
 	Reference<HTTP::Response> r = wait(bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), {200}));
 	// TODO:  In the event that the client times out just before the request completes (so the client is unaware) then the next retry

fdbclient/CMakeLists.txt

@@ -33,7 +33,7 @@ set(FDBCLIENT_SRCS
   Knobs.h
   ManagementAPI.actor.cpp
   ManagementAPI.actor.h
-  MasterProxyInterface.h
+  CommitProxyInterface.h
   MetricLogger.actor.cpp
   MetricLogger.h
   MonitorLeader.actor.cpp

fdbclient/ClientWorkerInterface.h

@@ -25,7 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbclient/Status.h"
-#include "fdbclient/MasterProxyInterface.h"
+#include "fdbclient/CommitProxyInterface.h"
 
 // Streams from WorkerInterface that are safe and useful to call from a client.
 // A ClientWorkerInterface is embedded as the first element of a WorkerInterface.

fdbclient/ClusterInterface.h

@@ -25,7 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbclient/Status.h"
-#include "fdbclient/MasterProxyInterface.h"
+#include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/ClientWorkerInterface.h"
 
 struct ClusterInterface {

fdbclient/CommitProxyInterface.h

@@ -1,6 +1,6 @@
 
 /*
- * MasterProxyInterface.h
+ * CommitProxyInterface.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,8 +19,8 @@
  * limitations under the License.
  */
 
-#ifndef FDBCLIENT_MASTERPROXYINTERFACE_H
-#define FDBCLIENT_MASTERPROXYINTERFACE_H
+#ifndef FDBCLIENT_COMMITPROXYINTERFACE_H
+#define FDBCLIENT_COMMITPROXYINTERFACE_H
 #pragma once
 
 #include <utility>
@@ -36,7 +36,7 @@
 #include "fdbrpc/TimedRequest.h"
 #include "GrvProxyInterface.h"
 
-struct MasterProxyInterface {
+struct CommitProxyInterface {
 	constexpr static FileIdentifier file_identifier = 8954922;
 	enum { LocationAwareLoadBalance = 1 };
 	enum { AlwaysFresh = 1 };
@@ -59,8 +59,8 @@ struct MasterProxyInterface {
 
 	UID id() const { return commit.getEndpoint().token; }
 	std::string toString() const { return id().shortString(); }
-	bool operator == (MasterProxyInterface const& r) const { return id() == r.id(); }
-	bool operator != (MasterProxyInterface const& r) const { return id() != r.id(); }
+	bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); }
+	bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); }
 	NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); }
 
 	template <class Archive>
@@ -100,9 +100,10 @@ struct MasterProxyInterface {
 struct ClientDBInfo {
 	constexpr static FileIdentifier file_identifier = 5355080;
 	UID id;  // Changes each time anything else changes
-	vector< GrvProxyInterface > grvProxies;
-	vector< MasterProxyInterface > masterProxies;
-	Optional<MasterProxyInterface> firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
+	vector<GrvProxyInterface> grvProxies;
+	vector<CommitProxyInterface> commitProxies;
+	Optional<CommitProxyInterface>
+	    firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk
 	double clientTxnInfoSampleRate;
 	int64_t clientTxnInfoSizeLimit;
 	Optional<Value> forward;
@@ -122,7 +123,7 @@ struct ClientDBInfo {
 		if constexpr (!is_fb_function<Archive>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, grvProxies, masterProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward,
+		serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward,
 		           transactionTagSampleRate, transactionTagSampleCost);
 	}
 };
@@ -165,7 +166,8 @@ struct CommitTransactionRequest : TimedRequest {
 	Optional<ClientTrCommitCostEstimation> commitCostEstimation;
 	Optional<TagSet> tagSet;
 
-	CommitTransactionRequest() : flags(0) {}
+	CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {}
+	CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {}
 
 	template <class Ar> 
 	void serialize(Ar& ar) { 

fdbclient/CommitTransaction.h

@@ -49,6 +49,7 @@ static const char* typeString[] = { "SetValue",
 	                                "MinV2",
 	                                "AndV2",
 	                                "CompareAndClear",
+	                                "Reserved_For_SpanContextMessage",
 	                                "MAX_ATOMIC_OP" };
 
 struct MutationRef {
@@ -75,6 +76,7 @@ struct MutationRef {
 		MinV2,
 		AndV2,
 		CompareAndClear,
+		Reserved_For_SpanContextMessage /* See fdbserver/SpanContextMessage.h */,
 		MAX_ATOMIC_OP
 	};
 	// This is stored this way for serialization purposes.

fdbclient/CoordinationInterface.h

@@ -25,7 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/Locality.h"
-#include "fdbclient/MasterProxyInterface.h"
+#include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/ClusterInterface.h"
 
 const int MAX_CLUSTER_FILE_BYTES = 60000;

fdbclient/DatabaseBackupAgent.actor.cpp

@@ -129,9 +129,9 @@ namespace dbBackup {
 
 	struct BackupRangeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -421,16 +421,15 @@ namespace dbBackup {
 
 	};
 	StringRef BackupRangeTaskFunc::name = LiteralStringRef("dr_backup_range");
-	const uint32_t BackupRangeTaskFunc::version = 1;
 	const Key BackupRangeTaskFunc::keyAddBackupRangeTasks = LiteralStringRef("addBackupRangeTasks");
 	const Key BackupRangeTaskFunc::keyBackupRangeBeginKey = LiteralStringRef("backupRangeBeginKey");
 	REGISTER_TASKFUNC(BackupRangeTaskFunc);
 
 	struct FinishFullBackupTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			wait(checkTaskVersion(tr, task, FinishFullBackupTaskFunc::name, FinishFullBackupTaskFunc::version));
 
@@ -467,14 +466,13 @@ namespace dbBackup {
 
 	};
 	StringRef FinishFullBackupTaskFunc::name = LiteralStringRef("dr_finish_full_backup");
-	const uint32_t FinishFullBackupTaskFunc::version = 1;
 	REGISTER_TASKFUNC(FinishFullBackupTaskFunc);
 
 	struct EraseLogRangeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		StringRef getName() const { return name; };
+	    StringRef getName() const { return name; };
 
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
@@ -523,14 +521,13 @@ namespace dbBackup {
 		}
 	};
 	StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("dr_erase_log_range");
-	const uint32_t EraseLogRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
 
 	struct CopyLogRangeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -773,15 +770,14 @@ namespace dbBackup {
 		}
 	};
 	StringRef CopyLogRangeTaskFunc::name = LiteralStringRef("dr_copy_log_range");
-	const uint32_t CopyLogRangeTaskFunc::version = 1;
 	const Key CopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion");
 	REGISTER_TASKFUNC(CopyLogRangeTaskFunc);
 
 	struct CopyLogsTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Subspace conf = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			wait(checkTaskVersion(tr, task, CopyLogsTaskFunc::name, CopyLogsTaskFunc::version));
@@ -876,13 +872,12 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyLogsTaskFunc::name = LiteralStringRef("dr_copy_logs");
-	const uint32_t CopyLogsTaskFunc::version = 1;
 	REGISTER_TASKFUNC(CopyLogsTaskFunc);
 
 	struct FinishedFullBackupTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
-		static const Key keyInsertTask;
+	    static constexpr uint32_t version = 1;
+	    static const Key keyInsertTask;
 
 		StringRef getName() const { return name; };
 
@@ -965,9 +960,10 @@ namespace dbBackup {
 			tr->clear(KeyRangeRef(logsPath, strinc(logsPath)));
 
 			tr->clear(conf.range());
-			tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_COMPLETED)));
+		    tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
+		            StringRef(BackupAgentBase::getStateText(EBackupState::STATE_COMPLETED)));
 
-			wait(taskBucket->finish(tr, task));
+		    wait(taskBucket->finish(tr, task));
 			return Void();
 		}
 
@@ -975,15 +971,14 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("dr_finished_full_backup");
-	const uint32_t FinishedFullBackupTaskFunc::version = 1;
 	const Key FinishedFullBackupTaskFunc::keyInsertTask = LiteralStringRef("insertTask");
 	REGISTER_TASKFUNC(FinishedFullBackupTaskFunc);
 
 	struct CopyDiffLogsTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Subspace conf = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			state Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			wait(checkTaskVersion(tr, task, CopyDiffLogsTaskFunc::name, CopyDiffLogsTaskFunc::version));
@@ -1058,15 +1053,14 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyDiffLogsTaskFunc::name = LiteralStringRef("dr_copy_diff_logs");
-	const uint32_t CopyDiffLogsTaskFunc::version = 1;
 	REGISTER_TASKFUNC(CopyDiffLogsTaskFunc);
 
 	// Skip unneeded EraseLogRangeTaskFunc in 5.1
 	struct SkipOldEraseLogRangeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 			wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 			return Void();
@@ -1078,16 +1072,15 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef SkipOldEraseLogRangeTaskFunc::name = LiteralStringRef("dr_skip_legacy_task");
-	const uint32_t SkipOldEraseLogRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(SkipOldEraseLogRangeTaskFunc);
 	REGISTER_TASKFUNC_ALIAS(SkipOldEraseLogRangeTaskFunc, db_erase_log_range);
 
 	// This is almost the same as CopyLogRangeTaskFunc in 5.1. The only purpose is to support DR upgrade
 	struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<int64_t> bytesWritten() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -1254,15 +1247,14 @@ namespace dbBackup {
 		}
 	};
 	StringRef OldCopyLogRangeTaskFunc::name = LiteralStringRef("db_copy_log_range");
-	const uint32_t OldCopyLogRangeTaskFunc::version = 1;
 	const Key OldCopyLogRangeTaskFunc::keyNextBeginVersion = LiteralStringRef("nextBeginVersion");
 	REGISTER_TASKFUNC(OldCopyLogRangeTaskFunc);
 
 	struct AbortOldBackupTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state DatabaseBackupAgent srcDrAgent(taskBucket->src);
 			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 			state Key tagNameKey;
@@ -1315,7 +1307,6 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortOldBackupTaskFunc::name = LiteralStringRef("dr_abort_legacy_backup");
-	const uint32_t AbortOldBackupTaskFunc::version = 1;
 	REGISTER_TASKFUNC(AbortOldBackupTaskFunc);
 	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_backup_range);
 	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTaskFunc, db_finish_full_backup);
@@ -1327,9 +1318,9 @@ namespace dbBackup {
 	//Upgrade DR from 5.1
 	struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
 			state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
 			state Subspace config = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
@@ -1434,14 +1425,13 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef CopyDiffLogsUpgradeTaskFunc::name = LiteralStringRef("db_copy_diff_logs");
-	const uint32_t CopyDiffLogsUpgradeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(CopyDiffLogsUpgradeTaskFunc);
 
 	struct BackupRestorableTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
 			wait(checkTaskVersion(cx, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version));
 			state Transaction tr(taskBucket->src);
@@ -1449,9 +1439,10 @@ namespace dbBackup {
 				try {
 					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 					tr.addReadConflictRange(singleKeyRange(sourceStates.pack(DatabaseBackupAgent::keyStateStatus)));
-					tr.set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING_DIFFERENTIAL)));
+				    tr.set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus),
+				           StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING_DIFFERENTIAL)));
 
-					Key versionKey = task->params[DatabaseBackupAgent::keyConfigLogUid].withPrefix(task->params[BackupAgentBase::destUid]).withPrefix(backupLatestVersionsPrefix);
+				    Key versionKey = task->params[DatabaseBackupAgent::keyConfigLogUid].withPrefix(task->params[BackupAgentBase::destUid]).withPrefix(backupLatestVersionsPrefix);
 					Optional<Key> prevBeginVersion = wait(tr.get(versionKey));
 					if (!prevBeginVersion.present()) {
 						return Void();
@@ -1489,9 +1480,10 @@ namespace dbBackup {
 				wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
 			}
 			else { // Start the writing of logs, if differential
-				tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING_DIFFERENTIAL)));
+			    tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
+			            StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING_DIFFERENTIAL)));
 
-				allPartsDone = futureBucket->future(tr);
+			    allPartsDone = futureBucket->future(tr);
 
 				Version prevBeginVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned());
 				wait(success(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone))));
@@ -1524,14 +1516,13 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupRestorableTaskFunc::name = LiteralStringRef("dr_backup_restorable");
-	const uint32_t BackupRestorableTaskFunc::version = 1;
 	REGISTER_TASKFUNC(BackupRestorableTaskFunc);
 
 	struct StartFullBackupTaskFunc : TaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
 			state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
 			wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version));
@@ -1623,9 +1614,10 @@ namespace dbBackup {
 
 					srcTr2->set( Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceTagName).pack(task->params[BackupAgentBase::keyTagName]), logUidValue );
 					srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyFolderId), task->params[DatabaseBackupAgent::keyFolderId] );
-					srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING)));
+				    srcTr2->set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus),
+				                StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING)));
 
-					state Key destPath = destUidValue.withPrefix(backupLogKeys.begin);
+				    state Key destPath = destUidValue.withPrefix(backupLogKeys.begin);
 					// Start logging the mutations for the specified ranges of the tag
 					for (auto &backupRange : backupRanges) {
 						srcTr2->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, destPath));
@@ -1666,9 +1658,10 @@ namespace dbBackup {
 
 			tr->set(logUidValue.withPrefix(applyMutationsBeginRange.begin), BinaryWriter::toValue(beginVersion, Unversioned()));
 			tr->set(logUidValue.withPrefix(applyMutationsEndRange.begin), BinaryWriter::toValue(beginVersion, Unversioned()));
-			tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING)));
+		    tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
+		            StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING)));
 
-			state Reference<TaskFuture>	kvBackupRangeComplete = futureBucket->future(tr);
+		    state Reference<TaskFuture>	kvBackupRangeComplete = futureBucket->future(tr);
 			state Reference<TaskFuture>	kvBackupComplete = futureBucket->future(tr);
 			state int rangeCount = 0;
 
@@ -1721,7 +1714,6 @@ namespace dbBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("dr_start_full_backup");
-	const uint32_t StartFullBackupTaskFunc::version = 1;
 	REGISTER_TASKFUNC(StartFullBackupTaskFunc);
 }
 
@@ -1817,7 +1809,7 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade
 
 class DatabaseBackupAgentImpl {
 public:
-	static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
+	static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
 
 	ACTOR static Future<Void> waitUpgradeToLatestDrVersion(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
 		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
@@ -1851,7 +1843,8 @@ public:
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<int> waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, bool stopWhenDone) {
+	ACTOR static Future<EBackupState> waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName,
+	                                             bool stopWhenDone) {
 		state std::string backTrace;
 		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
 		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus);
@@ -1862,15 +1855,15 @@ public:
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			try {
-				state int status = wait(backupAgent->getStateValue(tr, logUid));
+				state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
 
 				// Break, if no longer runnable
-				if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status) || BackupAgentBase::STATE_PARTIALLY_ABORTED == status) {
+				if (!DatabaseBackupAgent::isRunnable(status) || EBackupState::STATE_PARTIALLY_ABORTED == status) {
 					return status;
 				}
 
 				// Break, if in differential mode (restorable) and stopWhenDone is not enabled
-				if ((!stopWhenDone) && (BackupAgentBase::STATE_RUNNING_DIFFERENTIAL == status)) {
+				if ((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status)) {
 					return status;
 				}
 
@@ -1885,7 +1878,7 @@ public:
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<int> waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
+	ACTOR static Future<EBackupState> waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
 		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
 		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus);
 
@@ -1895,10 +1888,10 @@ public:
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			try {
-				state int status = wait(backupAgent->getStateValue(tr, logUid));
+				state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
 
 				// Break, if no longer runnable
-				if( BackupAgentBase::STATE_SUBMITTED != status) {
+				if (EBackupState::STATE_SUBMITTED != status) {
 					return status;
 				}
 
@@ -1924,9 +1917,9 @@ public:
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
 		// We will use the global status for now to ensure that multiple backups do not start place with different tags
-		state int status = wait(backupAgent->getStateValue(tr, logUidCurrent));
+		state EBackupState status = wait(backupAgent->getStateValue(tr, logUidCurrent));
 
-		if (DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) {
+		if (DatabaseBackupAgent::isRunnable(status)) {
 			throw backup_duplicate();
 		}
 
@@ -1987,7 +1980,8 @@ public:
 		tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid);
 		tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid); // written to config and states because it's also used by abort
 		tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyConfigBackupRanges), BinaryWriter::toValue(backupRanges, IncludeVersion(ProtocolVersion::withDRBackupRanges())));
-		tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_SUBMITTED)));
+		tr->set(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus),
+		        StringRef(BackupAgentBase::getStateText(EBackupState::STATE_SUBMITTED)));
 		if (stopWhenDone) {
 			tr->set(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey), StringRef());
 		}
@@ -2033,10 +2027,10 @@ public:
 	ACTOR static Future<Void> atomicSwitchover(DatabaseBackupAgent* backupAgent, Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction) {
 		state DatabaseBackupAgent drAgent(dest);
 		state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName));
-		state int status = wait(backupAgent->getStateValue(dest, destlogUid));
+		state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid));
 
 		TraceEvent("DBA_SwitchoverStart").detail("Status", status);
-		if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL && status != BackupAgentBase::STATE_COMPLETED) {
+		if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL && status != EBackupState::STATE_COMPLETED) {
 			throw backup_duplicate();
 		}
 
@@ -2153,10 +2147,10 @@ public:
 	ACTOR static Future<Void> discontinueBackup(DatabaseBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key tagName) {
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state UID logUid = wait(backupAgent->getLogUid(tr, tagName));
-		state int status = wait(backupAgent->getStateValue(tr, logUid));
+		state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
 
 		TraceEvent("DBA_Discontinue").detail("Status", status);
-		if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) {
+		if (!DatabaseBackupAgent::isRunnable(status)) {
 			throw backup_unneeded();
 		}
 
@@ -2189,7 +2183,7 @@ public:
 				logUid = _logUid;
 				logUidValue = BinaryWriter::toValue(logUid, Unversioned());
 
-				state Future<int> statusFuture= backupAgent->getStateValue(tr, logUid);
+				state Future<EBackupState> statusFuture = backupAgent->getStateValue(tr, logUid);
 				state Future<UID> destUidFuture = backupAgent->getDestUid(tr, logUid);
 				wait(success(statusFuture) && success(destUidFuture));
 
@@ -2197,8 +2191,8 @@ public:
 				if (destUid.isValid()) {
 					destUidValue = BinaryWriter::toValue(destUid, Unversioned());
 				}
-				int status = statusFuture.get();
-				if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) {
+				EBackupState status = statusFuture.get();
+				if (!backupAgent->isRunnable(status)) {
 					throw backup_unneeded();
 				}
 
@@ -2213,7 +2207,8 @@ public:
 
 				tr->clear(prefixRange(logUidValue.withPrefix(applyLogKeys.begin)));
 
-				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED)));
+				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)),
+				        StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_PARTIALLY_ABORTED)));
 
 				wait(tr->commit());
 				TraceEvent("DBA_Abort").detail("CommitVersion", tr->getCommittedVersion());
@@ -2286,7 +2281,8 @@ public:
 					}
 
 					if (abortOldBackup) {
-						srcTr->set( backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_ABORTED) ));
+						srcTr->set(backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus),
+						           StringRef(BackupAgentBase::getStateText(EBackupState::STATE_ABORTED)));
 						srcTr->set( backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid );
 						srcTr->clear(prefixRange(logUidValue.withPrefix(backupLogKeys.begin)));
 						srcTr->clear(prefixRange(logUidValue.withPrefix(logRangesRange.begin)));
@@ -2307,7 +2303,8 @@ public:
 						break;
 					}
 
-					srcTr->set( backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED) ));
+					srcTr->set(backupAgent->sourceStates.pack(DatabaseBackupAgent::keyStateStatus),
+					           StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_PARTIALLY_ABORTED)));
 					srcTr->set( backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId), backupUid );
 
 					wait( eraseLogData(srcTr, logUidValue, destUidValue) || partialTimeout );
@@ -2341,7 +2338,8 @@ public:
 					return Void();
 				}
 
-				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_ABORTED)));
+				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)),
+				        StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_ABORTED)));
 
 				wait(tr->commit());
 
@@ -2382,13 +2380,11 @@ public:
 				state Future<Optional<Value>> fStopVersionKey = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop));
 				state Future<Optional<Key>> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges));
 
-				int backupStateInt = wait(backupAgent->getStateValue(tr, logUid));
-				state BackupAgentBase::enumState backupState = (BackupAgentBase::enumState)backupStateInt;
-				
-				if (backupState == DatabaseBackupAgent::STATE_NEVERRAN) {
+				state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid));
+
+				if (backupState == EBackupState::STATE_NEVERRAN) {
 					statusText += "No previous backups found.\n";
-				}
-				else {
+				} else {
 					state std::string tagNameDisplay;
 					Optional<Key> tagName = wait(fTagName);
 
@@ -2408,23 +2404,20 @@ public:
 					}
 
 					switch (backupState) {
-					case BackupAgentBase::STATE_SUBMITTED:
+					case EBackupState::STATE_SUBMITTED:
 						statusText += "The DR on tag `" + tagNameDisplay + "' is NOT a complete copy of the primary database (just started).\n";
 						break;
-					case BackupAgentBase::STATE_RUNNING:
+					case EBackupState::STATE_RUNNING:
 						statusText += "The DR on tag `" + tagNameDisplay + "' is NOT a complete copy of the primary database.\n";
 						break;
-					case BackupAgentBase::STATE_RUNNING_DIFFERENTIAL:
+					case EBackupState::STATE_RUNNING_DIFFERENTIAL:
 						statusText += "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database.\n";
 						break;
-					case BackupAgentBase::STATE_COMPLETED:
-					{
+					case EBackupState::STATE_COMPLETED: {
 						Version stopVersion = stopVersionKey.present() ? BinaryReader::fromStringRef<Version>(stopVersionKey.get(), Unversioned()) : -1;
 						statusText += "The previous DR on tag `" + tagNameDisplay + "' completed at version " + format("%lld", stopVersion) + ".\n";
-					}
-						break;
-					case BackupAgentBase::STATE_PARTIALLY_ABORTED:
-					{
+					} break;
+					case EBackupState::STATE_PARTIALLY_ABORTED: {
 						statusText += "The previous DR on tag `" + tagNameDisplay + "' " + BackupAgentBase::getStateText(backupState) + ".\n";
 						statusText += "Abort the DR with --cleanup before starting a new DR.\n";
 						break;
@@ -2485,13 +2478,15 @@ public:
 		return statusText;
 	}
 
-	ACTOR static Future<int> getStateValue(DatabaseBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot) {
+	ACTOR static Future<EBackupState> getStateValue(DatabaseBackupAgent* backupAgent,
+	                                                Reference<ReadYourWritesTransaction> tr, UID logUid,
+	                                                bool snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus);
 		Optional<Value> status = wait(tr->get(statusKey, snapshot));
 
-		return (!status.present()) ? DatabaseBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString());
+		return (!status.present()) ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString());
 	}
 
 	ACTOR static Future<UID> getDestUid(DatabaseBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot) {
@@ -2536,7 +2531,8 @@ Future<std::string> DatabaseBackupAgent::getStatus(Database cx, int errorLimit,
 	return DatabaseBackupAgentImpl::getStatus(this, cx, errorLimit, tagName);
 }
 
-Future<int> DatabaseBackupAgent::getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot) {
+Future<EBackupState> DatabaseBackupAgent::getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid,
+                                                        bool snapshot) {
 	return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot);
 }
 
@@ -2552,11 +2548,11 @@ Future<Void> DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key
 	return DatabaseBackupAgentImpl::waitUpgradeToLatestDrVersion(this, cx, tagName);
 }
 
-Future<int> DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) {
+Future<EBackupState> DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) {
 	return DatabaseBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone);
 }
 
-Future<int> DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName) {
+Future<EBackupState> DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName) {
 	return DatabaseBackupAgentImpl::waitSubmitted(this, cx, tagName);
 }
 

fdbclient/DatabaseConfiguration.cpp

@@ -29,12 +29,12 @@ DatabaseConfiguration::DatabaseConfiguration()
 void DatabaseConfiguration::resetInternal() {
 	// does NOT reset rawConfiguration
 	initialized = false;
-	proxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
+	commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
 	    storageTeamSize = desiredLogRouterCount = -1;
 	tLogVersion = TLogVersion::DEFAULT;
 	tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
 	tLogSpillType = TLogSpillType::DEFAULT;
-	autoProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
+	autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;
 	autoGrvProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES;
 	autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS;
 	autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS;
@@ -165,40 +165,39 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() {
 
 bool DatabaseConfiguration::isValid() const {
 	if( !(initialized &&
-		tLogWriteAntiQuorum >= 0 &&
-		tLogWriteAntiQuorum <= tLogReplicationFactor/2 &&
-		tLogReplicationFactor >= 1 &&
-		storageTeamSize >= 1 &&
-		getDesiredProxies() >= 1 &&
-		getDesiredGrvProxies() >= 1 &&
-		getDesiredLogs() >= 1 &&
-		getDesiredResolvers() >= 1 &&
-		tLogVersion != TLogVersion::UNSET &&
-		tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
-		tLogVersion <= TLogVersion::MAX_SUPPORTED &&
-		tLogDataStoreType != KeyValueStoreType::END &&
-		tLogSpillType != TLogSpillType::UNSET &&
-		!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
-		storageServerStoreType != KeyValueStoreType::END &&
-		autoProxyCount >= 1 &&
-		autoGrvProxyCount >= 1 &&
-		autoResolverCount >= 1 &&
-		autoDesiredTLogCount >= 1 &&
-		storagePolicy &&
-		tLogPolicy &&
-		getDesiredRemoteLogs() >= 1 &&
-		remoteTLogReplicationFactor >= 0 &&
-		repopulateRegionAntiQuorum >= 0 &&
-		repopulateRegionAntiQuorum <= 1 &&
-		usableRegions >= 1 &&
-		usableRegions <= 2 &&
-		regions.size() <= 2 &&
-		( usableRegions == 1 || regions.size() == 2 ) &&
-		( regions.size() == 0 || regions[0].priority >= 0 ) &&
-		( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
+ 		tLogWriteAntiQuorum >= 0 &&
+ 		tLogWriteAntiQuorum <= tLogReplicationFactor/2 &&
+ 		tLogReplicationFactor >= 1 &&
+ 		storageTeamSize >= 1 &&
+ 		getDesiredCommitProxies() >= 1 &&
+ 		getDesiredGrvProxies() >= 1 &&
+ 		getDesiredLogs() >= 1 &&
+ 		getDesiredResolvers() >= 1 &&
+ 		tLogVersion != TLogVersion::UNSET &&
+ 		tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
+ 		tLogVersion <= TLogVersion::MAX_SUPPORTED &&
+ 		tLogDataStoreType != KeyValueStoreType::END &&
+ 		tLogSpillType != TLogSpillType::UNSET &&
+ 		!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
+ 		storageServerStoreType != KeyValueStoreType::END &&
+ 		autoCommitProxyCount >= 1 &&
+ 		autoGrvProxyCount >= 1 &&
+ 		autoResolverCount >= 1 &&
+ 		autoDesiredTLogCount >= 1 &&
+ 		storagePolicy &&
+ 		tLogPolicy &&
+ 		getDesiredRemoteLogs() >= 1 &&
+ 		remoteTLogReplicationFactor >= 0 &&
+ 		repopulateRegionAntiQuorum >= 0 &&
+ 		repopulateRegionAntiQuorum <= 1 &&
+ 		usableRegions >= 1 &&
+ 		usableRegions <= 2 &&
+ 		regions.size() <= 2 &&
+ 		( usableRegions == 1 || regions.size() == 2 ) &&
+ 		( regions.size() == 0 || regions[0].priority >= 0 ) &&
+ 		( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
 		return false;
 	}
-
 	std::set<Key> dcIds;
 	dcIds.insert(Key());
 	for(auto& r : regions) {
@@ -318,11 +317,11 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	if (desiredTLogCount != -1 || isOverridden("logs")) {
 		result["logs"] = desiredTLogCount;
 	}
-	if (proxyCount != -1 || isOverridden("proxies")) {
-		result["proxies"] = proxyCount;
+	if (commitProxyCount != -1 || isOverridden("commit_proxies")) {
+		result["commit_proxies"] = commitProxyCount;
 	}
 	if (grvProxyCount != -1 || isOverridden("grv_proxies")) {
-		result["grv_proxies"] = proxyCount;
+		result["grv_proxies"] = grvProxyCount;
 	}
 	if (resolverCount != -1 || isOverridden("resolvers")) {
 		result["resolvers"] = resolverCount;
@@ -336,8 +335,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	if (repopulateRegionAntiQuorum != 0 || isOverridden("repopulate_anti_quorum")) {
 		result["repopulate_anti_quorum"] = repopulateRegionAntiQuorum;
 	}
-	if (autoProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_PROXIES || isOverridden("auto_proxies")) {
-		result["auto_proxies"] = autoProxyCount;
+	if (autoCommitProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES || isOverridden("auto_commit_proxies")) {
+		result["auto_commit_proxies"] = autoCommitProxyCount;
 	}
 	if (autoGrvProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES || isOverridden("auto_grv_proxies")) {
 		result["auto_grv_proxies"] = autoGrvProxyCount;
@@ -419,8 +418,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 
 	if (ck == LiteralStringRef("initialized")) {
 		initialized = true;
-	} else if (ck == LiteralStringRef("proxies")) {
-		parse(&proxyCount, value);
+	} else if (ck == LiteralStringRef("commit_proxies")) {
+		parse(&commitProxyCount, value);
 	} else if (ck == LiteralStringRef("grv_proxies")) {
 		parse(&grvProxyCount, value);
 	} else if (ck == LiteralStringRef("resolvers")) {
@@ -459,8 +458,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	} else if (ck == LiteralStringRef("storage_engine")) {
 		parse((&type), value);
 		storageServerStoreType = (KeyValueStoreType::StoreType)type;
-	} else if (ck == LiteralStringRef("auto_proxies")) {
-		parse(&autoProxyCount, value);
+	} else if (ck == LiteralStringRef("auto_commit_proxies")) {
+		parse(&autoCommitProxyCount, value);
 	} else if (ck == LiteralStringRef("auto_grv_proxies")) {
 		parse(&autoGrvProxyCount, value);
 	} else if (ck == LiteralStringRef("auto_resolvers")) {

fdbclient/DatabaseConfiguration.h

@@ -133,15 +133,19 @@ struct DatabaseConfiguration {
 	}
 
 	//Killing an entire datacenter counts as killing one zone in modes that support it
-	int32_t maxZoneFailuresTolerated() const {
+	int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const {
 		int worstSatellite = regions.size() ? std::numeric_limits<int>::max() : 0;
+		int regionsWithNonNegativePriority = 0;
 		for(auto& r : regions) {
+			if(r.priority >= 0) {
+				regionsWithNonNegativePriority++;
+			}
 			worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
 			if(r.satelliteTLogUsableDcsFallback > 0) {
 				worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
 			}
 		}
-		if(usableRegions > 1 && worstSatellite > 0) {
+		if(usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && (!forAvailability || regionsWithNonNegativePriority > 1)) {
 			return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1), storageTeamSize - 1);
 		} else if(worstSatellite > 0) {
 			return std::min(tLogReplicationFactor + worstSatellite - 2 - tLogWriteAntiQuorum, storageTeamSize - 1);
@@ -149,9 +153,9 @@ struct DatabaseConfiguration {
 		return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1);
 	}
 
-	// Proxy Servers
-	int32_t proxyCount;
-	int32_t autoProxyCount;
+	// CommitProxy Servers
+	int32_t commitProxyCount;
+	int32_t autoCommitProxyCount;
 	int32_t grvProxyCount;
 	int32_t autoGrvProxyCount;
 
@@ -192,7 +196,10 @@ struct DatabaseConfiguration {
 	bool isExcludedServer( NetworkAddressList ) const;
 	std::set<AddressExclusion> getExcludedServers() const;
 
-	int32_t getDesiredProxies() const { if(proxyCount == -1) return autoProxyCount; return proxyCount; }
+	int32_t getDesiredCommitProxies() const {
+		if (commitProxyCount == -1) return autoCommitProxyCount;
+		return commitProxyCount;
+	}
 	int32_t getDesiredGrvProxies() const {
 		if (grvProxyCount == -1) return autoGrvProxyCount;
 		return grvProxyCount;

fdbclient/DatabaseContext.h

@@ -29,7 +29,7 @@
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/MasterProxyInterface.h"
+#include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
 #include "fdbrpc/QueueModel.h"
 #include "fdbrpc/MultiInterface.h"
@@ -68,7 +68,7 @@ struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>
 	}
 };
 
-using ProxyInfo = ModelInterface<MasterProxyInterface>;
+using CommitProxyInfo = ModelInterface<CommitProxyInterface>;
 using GrvProxyInfo = ModelInterface<GrvProxyInterface>;
 
 class ClientTagThrottleData : NonCopyable {
@@ -165,8 +165,8 @@ public:
 	bool sampleOnCost(uint64_t cost) const;
 
 	void updateProxies();
-	Reference<ProxyInfo> getMasterProxies(bool useProvisionalProxies);
-	Future<Reference<ProxyInfo>> getMasterProxiesFuture(bool useProvisionalProxies);
+	Reference<CommitProxyInfo> getCommitProxies(bool useProvisionalProxies);
+	Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(bool useProvisionalProxies);
 	Reference<GrvProxyInfo> getGrvProxies(bool useProvisionalProxies);
 	Future<Void> onProxiesChanged();
 	Future<HealthMetrics> getHealthMetrics(bool detailed);
@@ -223,9 +223,9 @@ public:
 	Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile;
 	AsyncTrigger proxiesChangeTrigger;
 	Future<Void> monitorProxiesInfoChange;
-	Reference<ProxyInfo> masterProxies;
+	Reference<CommitProxyInfo> commitProxies;
 	Reference<GrvProxyInfo> grvProxies;
-	bool proxyProvisional;
+	bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time.
 	UID proxiesLastChange;
 	LocalityData clientLocality;
 	QueueModel queueModel;

fdbclient/FDBTypes.h

@@ -257,6 +257,7 @@ struct Traceable<std::set<T>> : std::true_type {
 std::string printable( const StringRef& val );
 std::string printable( const std::string& val );
 std::string printable( const KeyRangeRef& range );
+std::string printable( const VectorRef<KeyRangeRef>& val);
 std::string printable( const VectorRef<StringRef>& val );
 std::string printable( const VectorRef<KeyValueRef>& val );
 std::string printable( const KeyValueRef& val );
@@ -289,6 +290,14 @@ struct KeyRangeRef {
 	bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
 	bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
 	bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
+	bool intersects(const VectorRef<KeyRangeRef>& keysVec) const {
+		for (const auto& keys : keysVec) {
+			if (intersects(keys)) {
+				return true;
+			}
+		}
+		return false;
+	}
 	bool empty() const { return begin == end; }
 	bool singleKeyRange() const { return equalsKeyAfter(begin, end); }
 
@@ -745,14 +754,16 @@ struct TLogVersion {
 		// V3 was the introduction of spill by reference;
 		// V4 changed how data gets written to satellite TLogs so that we can peek from them;
 		// V5 merged reference and value spilling
+		// V6 added span context to list of serialized mutations sent from proxy to tlogs
 		// V1 = 1,  // 4.6 is dispatched to via 6.0
 		V2 = 2, // 6.0
 		V3 = 3, // 6.1
 		V4 = 4, // 6.2
 		V5 = 5, // 6.3
+		V6 = 6, // 7.0
 		MIN_SUPPORTED = V2,
-		MAX_SUPPORTED = V5,
-		MIN_RECRUITABLE = V4,
+		MAX_SUPPORTED = V6,
+		MIN_RECRUITABLE = V5,
 		DEFAULT = V5,
 	} version;
 
@@ -775,6 +786,7 @@ struct TLogVersion {
 		if (s == LiteralStringRef("3")) return V3;
 		if (s == LiteralStringRef("4")) return V4;
 		if (s == LiteralStringRef("5")) return V5;
+		if (s == LiteralStringRef("6")) return V6;
 		return default_error_or();
 	}
 };

fdbclient/FileBackupAgent.actor.cpp

@@ -623,8 +623,6 @@ namespace fileBackup {
 	// Very simple format compared to KeyRange files.
 	// Header, [Key, Value]... Key len
 	struct LogFileWriter {
-		static const std::string &FFs;
-
 		LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
 		  : file(file), blockSize(blockSize), blockEnd(0) {}
 
@@ -748,9 +746,10 @@ namespace fileBackup {
 		state Subspace newConfigSpace = uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(fileBackupPrefixRange.begin), uid);
 
 		Optional<Value> statusStr = wait(tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus)));
-		state EBackupState status = !statusStr.present() ? FileBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
+	    state EBackupState status =
+	        !statusStr.present() ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
 
-		TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup")
+	    TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup")
 				.detail("TagName", tagName.c_str())
 				.detail("Status", BackupAgentBase::getStateText(status));
 
@@ -770,9 +769,9 @@ namespace fileBackup {
 
 		// Set old style state key to Aborted if it was Runnable
 		if(backupAgent->isRunnable(status))
-			tr->set(statusKey, StringRef(FileBackupAgent::getStateText(BackupAgentBase::STATE_ABORTED)));
+		    tr->set(statusKey, StringRef(FileBackupAgent::getStateText(EBackupState::STATE_ABORTED)));
 
-		return Void();
+	    return Void();
 	}
 
 	struct AbortFiveZeroBackupTask : TaskFuncBase {
@@ -792,13 +791,13 @@ namespace fileBackup {
 			return Void();
 		}
 
-		virtual StringRef getName() const {
-			TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called");
+	    StringRef getName() const override {
+		    TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called");
 			ASSERT(false);
 			return StringRef();
-		}
+	    }
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
+	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortFiveZeroBackupTask::name = LiteralStringRef("abort_legacy_backup");
@@ -822,11 +821,11 @@ namespace fileBackup {
 		state BackupConfig config(current.first);
 		EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
 
-		if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) {
-			throw backup_unneeded();
-		}
+	    if (!backupAgent->isRunnable(status)) {
+		    throw backup_unneeded();
+	    }
 
-		TraceEvent(SevInfo, "FBA_AbortFileOneBackup")
+	    TraceEvent(SevInfo, "FBA_AbortFileOneBackup")
 				.detail("TagName", tagName.c_str())
 				.detail("Status", BackupAgentBase::getStateText(status));
 
@@ -862,13 +861,13 @@ namespace fileBackup {
 			return Void();
 		}
 
-		virtual StringRef getName() const {
-			TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called");
+	    StringRef getName() const override {
+		    TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called");
 			ASSERT(false);
 			return StringRef();
-		}
+	    }
 
-		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
+	    Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef AbortFiveOneBackupTask::name = LiteralStringRef("abort_legacy_backup_5.2");
@@ -939,24 +938,18 @@ namespace fileBackup {
 	// Backup and Restore taskFunc definitions will inherit from one of the following classes which
 	// servers to catch and log to the appropriate config any error that execute/finish didn't catch and log.
 	struct RestoreTaskFuncBase : TaskFuncBase {
-		virtual Future<Void> handleError(Database cx, Reference<Task> task, Error const &error) {
-			return RestoreConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
-		}
-		virtual std::string toString(Reference<Task> task)
-		{
-			return "";
-		}
-	};
+	    Future<Void> handleError(Database cx, Reference<Task> task, Error const& error) final {
+		    return RestoreConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
+	    }
+	    virtual std::string toString(Reference<Task> task) const { return ""; }
+    };
 
 	struct BackupTaskFuncBase : TaskFuncBase {
-		virtual Future<Void> handleError(Database cx, Reference<Task> task, Error const &error) {
-			return BackupConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
-		}
-		virtual std::string toString(Reference<Task> task)
-		{
-			return "";
-		}
-	};
+	    Future<Void> handleError(Database cx, Reference<Task> task, Error const& error) final {
+		    return BackupConfig(task).logError(cx, error, format("'%s' on '%s'", error.what(), task->params[Task::reservedTaskParamKeyType].printable().c_str()));
+	    }
+	    virtual std::string toString(Reference<Task> task) const { return ""; }
+    };
 
 	ACTOR static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr, Key beginKey, Key endKey, int limit) {
 
@@ -975,9 +968,9 @@ namespace fileBackup {
 
 	struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<Key> beginKey() {
 				return LiteralStringRef(__FUNCTION__);
 			}
@@ -989,15 +982,15 @@ namespace fileBackup {
 			}
 		} Params;
 
-		std::string toString(Reference<Task> task) {
-			return format("beginKey '%s' endKey '%s' addTasks %d",
+	    std::string toString(Reference<Task> task) const override {
+		    return format("beginKey '%s' endKey '%s' addTasks %d",
 				Params.beginKey().get(task).printable().c_str(),
 				Params.endKey().get(task).printable().c_str(),
 				Params.addBackupRangeTasks().get(task)
 			);
-		}
+	    }
 
-		StringRef getName() const { return name; };
+	    StringRef getName() const { return name; };
 
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
@@ -1270,14 +1263,13 @@ namespace fileBackup {
 
 	};
 	StringRef BackupRangeTaskFunc::name = LiteralStringRef("file_backup_write_range_5.2");
-	const uint32_t BackupRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(BackupRangeTaskFunc);
 
 	struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			// Set by Execute, used by Finish
 			static TaskParam<int64_t> shardsBehind() {
 				return LiteralStringRef(__FUNCTION__);
@@ -1791,14 +1783,13 @@ namespace fileBackup {
 
 	};
 	StringRef BackupSnapshotDispatchTask::name = LiteralStringRef("file_backup_dispatch_ranges_5.2");
-	const uint32_t BackupSnapshotDispatchTask::version = 1;
 	REGISTER_TASKFUNC(BackupSnapshotDispatchTask);
 
 	struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<bool> addBackupLogRangeTasks() {
 				return LiteralStringRef(__FUNCTION__);
 			}
@@ -1993,14 +1984,13 @@ namespace fileBackup {
 	};
 
 	StringRef BackupLogRangeTaskFunc::name = LiteralStringRef("file_backup_write_logs_5.2");
-	const uint32_t BackupLogRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(BackupLogRangeTaskFunc);
 
 	//This task stopped being used in 6.2, however the code remains here to handle upgrades.
 	struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
-		StringRef getName() const { return name; };
+	    static constexpr uint32_t version = 1;
+	    StringRef getName() const { return name; };
 
 		static struct {
 			static TaskParam<Version> beginVersion() {
@@ -2050,16 +2040,15 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("file_backup_erase_logs_5.2");
-	const uint32_t EraseLogRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
 
 
 
 	struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<Version> prevBeginVersion() {
 				return LiteralStringRef(__FUNCTION__);
 			}
@@ -2100,10 +2089,10 @@ namespace fileBackup {
 			}
 
 			// If the backup is restorable but the state is not differential then set state to differential
-			if(restorableVersion.present() && backupState != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL)
-				config.stateEnum().set(tr, BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
+		    if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL)
+			    config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL);
 
-			// If stopWhenDone is set and there is a restorable version, set the done future and do not create further tasks.
+		    // If stopWhenDone is set and there is a restorable version, set the done future and do not create further tasks.
 			if(stopWhenDone && restorableVersion.present()) {
 				wait(onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
 
@@ -2178,14 +2167,13 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs_5.2");
-	const uint32_t BackupLogsDispatchTask::version = 1;
 	REGISTER_TASKFUNC(BackupLogsDispatchTask);
 
 	struct FileBackupFinishedTask : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		StringRef getName() const { return name; };
+	    StringRef getName() const { return name; };
 
 		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version));
@@ -2219,13 +2207,12 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished_5.2");
-	const uint32_t FileBackupFinishedTask::version = 1;
 	REGISTER_TASKFUNC(FileBackupFinishedTask);
 
 	struct BackupSnapshotManifest : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
-		static struct {
+	    static constexpr uint32_t version = 1;
+	    static struct {
 			static TaskParam<Version> endVersion() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -2350,10 +2337,10 @@ namespace fileBackup {
 			}
 
 			// If the backup is restorable and the state isn't differential the set state to differential
-			if(restorableVersion.present() && backupState != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL)
-				config.stateEnum().set(tr, BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
+		    if (restorableVersion.present() && backupState != EBackupState::STATE_RUNNING_DIFFERENTIAL)
+			    config.stateEnum().set(tr, EBackupState::STATE_RUNNING_DIFFERENTIAL);
 
-			// Unless we are to stop, start the next snapshot using the default interval
+		    // Unless we are to stop, start the next snapshot using the default interval
 			Reference<TaskFuture> snapshotDoneFuture = task->getDoneFuture(futureBucket);
 			if(!stopWhenDone) {
 				wait(config.initNewSnapshot(tr) && success(BackupSnapshotDispatchTask::addTask(tr, taskBucket, task, 1, TaskCompletionKey::signal(snapshotDoneFuture))));
@@ -2380,7 +2367,6 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest_5.2");
-	const uint32_t BackupSnapshotManifest::version = 1;
 	REGISTER_TASKFUNC(BackupSnapshotManifest);
 
 	Future<Key> BackupSnapshotDispatchTask::addSnapshotManifestTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor) {
@@ -2389,9 +2375,9 @@ namespace fileBackup {
 
 	struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -2533,7 +2519,6 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start_5.2");
-	const uint32_t StartFullBackupTaskFunc::version = 1;
 	REGISTER_TASKFUNC(StartFullBackupTaskFunc);
 
 	struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
@@ -2576,15 +2561,14 @@ namespace fileBackup {
 		}
 
 		static StringRef name;
-		static const uint32_t version;
-		StringRef getName() const { return name; };
+	    static constexpr uint32_t version = 1;
+	    StringRef getName() const { return name; };
 
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 
 	};
 	StringRef RestoreCompleteTaskFunc::name = LiteralStringRef("restore_complete");
-	const uint32_t RestoreCompleteTaskFunc::version = 1;
 	REGISTER_TASKFUNC(RestoreCompleteTaskFunc);
 
 	struct RestoreFileTaskFuncBase : RestoreTaskFuncBase {
@@ -2594,13 +2578,13 @@ namespace fileBackup {
 			static TaskParam<int64_t> readLen() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
-		std::string toString(Reference<Task> task) {
-			return format("fileName '%s' readLen %lld readOffset %lld",
+	    std::string toString(Reference<Task> task) const override {
+		    return format("fileName '%s' readLen %lld readOffset %lld",
 				Params.inputFile().get(task).fileName.c_str(),
 				Params.readLen().get(task),
 				Params.readOffset().get(task));
-		}
-	};
+	    }
+    };
 
 	struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		static struct : InputParams {
@@ -2621,14 +2605,14 @@ namespace fileBackup {
 			}
 		} Params;
 
-		std::string toString(Reference<Task> task) {
-			std::string returnStr = RestoreFileTaskFuncBase::toString(task);
+	    std::string toString(Reference<Task> task) const override {
+		    std::string returnStr = RestoreFileTaskFuncBase::toString(task);
 			for(auto &range : Params.getOriginalFileRanges(task))
 				returnStr += format("  originalFileRange '%s'", printable(range).c_str());
 			return returnStr;
-		}
+	    }
 
-		ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+	    ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
 			state RestoreConfig restore(task);
 
 			state RestoreFile rangeFile = Params.inputFile().get(task);
@@ -2840,20 +2824,19 @@ namespace fileBackup {
 		}
 
 		static StringRef name;
-		static const uint32_t version;
-		StringRef getName() const { return name; };
+	    static constexpr uint32_t version = 1;
+	    StringRef getName() const { return name; };
 
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreRangeTaskFunc::name = LiteralStringRef("restore_range_data");
-	const uint32_t RestoreRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(RestoreRangeTaskFunc);
 
 	struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
-		StringRef getName() const { return name; };
+	    static constexpr uint32_t version = 1;
+	    StringRef getName() const { return name; };
 
 		static struct : InputParams {
 		} Params;
@@ -2995,13 +2978,12 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreLogDataTaskFunc::name = LiteralStringRef("restore_log_data");
-	const uint32_t RestoreLogDataTaskFunc::version = 1;
 	REGISTER_TASKFUNC(RestoreLogDataTaskFunc);
 
 	struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
-		StringRef getName() const { return name; };
+	    static constexpr uint32_t version = 1;
+	    StringRef getName() const { return name; };
 
 		static struct {
 			static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
@@ -3308,7 +3290,6 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef RestoreDispatchTaskFunc::name = LiteralStringRef("restore_dispatch");
-	const uint32_t RestoreDispatchTaskFunc::version = 1;
 	REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
 
 	ACTOR Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -3402,9 +3383,9 @@ namespace fileBackup {
 
 	struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		static StringRef name;
-		static const uint32_t version;
+	    static constexpr uint32_t version = 1;
 
-		static struct {
+	    static struct {
 			static TaskParam<Version> firstVersion() { return LiteralStringRef(__FUNCTION__); }
 		} Params;
 
@@ -3469,12 +3450,13 @@ namespace fileBackup {
 			if (beginVersion == invalidVersion) {
 				beginVersion = 0;
 			}
-			Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(restoreVersion, incremental, beginVersion));
-			if (!incremental) {
-				beginVersion = restorable.get().snapshot.beginVersion;
-			}
+		    Optional<RestorableFileSet> restorable =
+		        wait(bc->getRestoreSet(restoreVersion, VectorRef<KeyRangeRef>(), incremental, beginVersion));
+		    if (!incremental) {
+			    beginVersion = restorable.get().snapshot.beginVersion;
+		    }
 
-			if(!restorable.present())
+		    if(!restorable.present())
 				throw restore_missing_data();
 
 			// First version for which log data should be applied
@@ -3596,7 +3578,6 @@ namespace fileBackup {
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
 	StringRef StartFullRestoreTaskFunc::name = LiteralStringRef("restore_start");
-	const uint32_t StartFullRestoreTaskFunc::version = 1;
 	REGISTER_TASKFUNC(StartFullRestoreTaskFunc);
 }
 
@@ -3612,7 +3593,7 @@ struct LogInfo : public ReferenceCounted<LogInfo> {
 
 class FileBackupAgentImpl {
 public:
-	static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
+	static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8;
 
 	// Parallel restore
 	ACTOR static Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true) {
@@ -3746,7 +3727,9 @@ public:
 
 	// This method will return the final status of the backup at tag, and return the URL that was used on the tag
 	// when that status value was read.
-	ACTOR static Future<int> waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, bool stopWhenDone, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr) {
+	ACTOR static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName,
+	                                             bool stopWhenDone, Reference<IBackupContainer>* pContainer = nullptr,
+	                                             UID* pUID = nullptr) {
 		state std::string backTrace;
 		state KeyBackedTag tag = makeBackupTag(tagName);
 
@@ -3767,7 +3750,8 @@ public:
 				// Break, if one of the following is true
 				//  - no longer runnable
 				//  - in differential mode (restorable) and stopWhenDone is not enabled
-				if( !FileBackupAgent::isRunnable(status) || ((!stopWhenDone) && (BackupAgentBase::STATE_RUNNING_DIFFERENTIAL == status) )) {
+				if (!FileBackupAgent::isRunnable(status) ||
+				    ((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status))) {
 
 					if(pContainer != nullptr) {
 						Reference<IBackupContainer> c = wait(config.backupContainer().getOrThrow(tr, false, backup_invalid_info()));
@@ -4103,7 +4087,7 @@ public:
 		state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
 		EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));
 
-		if (!backupAgent->isRunnable((BackupAgentBase::enumState)status)) {
+		if (!backupAgent->isRunnable(status)) {
 			throw backup_unneeded();
 		}
 
@@ -4206,13 +4190,13 @@ public:
 					JsonBuilderObject statusDoc;
 					statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState));
 					statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState));
-					statusDoc.setKey("Completed", backupState == BackupAgentBase::STATE_COMPLETED);
+					statusDoc.setKey("Completed", backupState == EBackupState::STATE_COMPLETED);
 					statusDoc.setKey("Running", BackupAgentBase::isRunnable(backupState));
 					doc.setKey("Status", statusDoc);
 
 					state Future<Void> done = Void();
 
-					if(backupState != BackupAgentBase::STATE_NEVERRAN) {
+					if (backupState != EBackupState::STATE_NEVERRAN) {
 						state Reference<IBackupContainer> bc;
 						state TimestampedVersion latestRestorable;
 
@@ -4224,7 +4208,7 @@ public:
 
 						if(latestRestorable.present()) {
 							JsonBuilderObject o = latestRestorable.toJSON();
-							if(backupState != BackupAgentBase::STATE_COMPLETED) {
+							if (backupState != EBackupState::STATE_COMPLETED) {
 								o.setKey("LagSeconds", (recentReadVersion - latestRestorable.version.get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
 							}
 							doc.setKey("LatestRestorablePoint", o);
@@ -4232,7 +4216,8 @@ public:
 						doc.setKey("DestinationURL", bc->getURL());
 					}
 
-					if(backupState == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || backupState == BackupAgentBase::STATE_RUNNING) {
+					if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL ||
+					    backupState == EBackupState::STATE_RUNNING) {
 						state int64_t snapshotInterval;
 						state int64_t logBytesWritten;
 						state int64_t rangeBytesWritten;
@@ -4355,23 +4340,28 @@ public:
 					bool snapshotProgress = false;
 
 					switch (backupState) {
-						case BackupAgentBase::STATE_SUBMITTED:
-							statusText += "The backup on tag `" + tagName + "' is in progress (just started) to " + bc->getURL() + ".\n";
-							break;
-						case BackupAgentBase::STATE_RUNNING:
-							statusText += "The backup on tag `" + tagName + "' is in progress to " + bc->getURL() + ".\n";
-							snapshotProgress = true;
-							break;
-						case BackupAgentBase::STATE_RUNNING_DIFFERENTIAL:
-							statusText += "The backup on tag `" + tagName + "' is restorable but continuing to " + bc->getURL() + ".\n";
-							snapshotProgress = true;
-							break;
-						case BackupAgentBase::STATE_COMPLETED:
-							statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " completed at version " + format("%lld", latestRestorableVersion.orDefault(-1)) + ".\n";
-							break;
-						default:
-							statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " " + backupStatus + ".\n";
-							break;
+					case EBackupState::STATE_SUBMITTED:
+						statusText += "The backup on tag `" + tagName + "' is in progress (just started) to " +
+						              bc->getURL() + ".\n";
+						break;
+					case EBackupState::STATE_RUNNING:
+						statusText += "The backup on tag `" + tagName + "' is in progress to " + bc->getURL() + ".\n";
+						snapshotProgress = true;
+						break;
+					case EBackupState::STATE_RUNNING_DIFFERENTIAL:
+						statusText += "The backup on tag `" + tagName + "' is restorable but continuing to " +
+						              bc->getURL() + ".\n";
+						snapshotProgress = true;
+						break;
+					case EBackupState::STATE_COMPLETED:
+						statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() +
+						              " completed at version " + format("%lld", latestRestorableVersion.orDefault(-1)) +
+						              ".\n";
+						break;
+					default:
+						statusText += "The previous backup on tag `" + tagName + "' at " + bc->getURL() + " " +
+						              backupStatus + ".\n";
+						break;
 					}
 					statusText += format("BackupUID: %s\n", uidAndAbortedFlag.get().first.toString().c_str());
 					statusText += format("BackupURL: %s\n", bc->getURL().c_str());
@@ -4407,7 +4397,7 @@ public:
 									);
 
 						statusText += format("Snapshot interval is %lld seconds.  ", snapshotInterval);
-						if(backupState == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL)
+						if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL)
 							statusText += format("Current snapshot progress target is %3.2f%% (>100%% means the snapshot is supposed to be done)\n", 100.0 * (recentReadVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion)) ;
 						else
 							statusText += "The initial snapshot is still running.\n";
@@ -4495,7 +4485,7 @@ public:
 	                                     Version beginVersion, UID randomUid) {
 		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(url.toString());
 
-		state BackupDescription desc = wait(bc->describeBackup());
+		state BackupDescription desc = wait(bc->describeBackup(true));
 		if(cxOrig.present()) {
 			wait(desc.resolveVersionTimes(cxOrig.get()));
 		}
@@ -4509,11 +4499,12 @@ public:
 		}
 
 		Optional<RestorableFileSet> restoreSet =
-		    wait(bc->getRestoreSet(targetVersion, incrementalBackupOnly, beginVersion));
+		    wait(bc->getRestoreSet(targetVersion, VectorRef<KeyRangeRef>(), incrementalBackupOnly, beginVersion));
 
 		if(!restoreSet.present()) {
 			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
 				.detail("BackupContainer", bc->getURL())
+				.detail("BeginVersion", beginVersion)
 				.detail("TargetVersion", targetVersion);
 			fprintf(stderr, "ERROR: Restore version %" PRId64 " is not possible from %s\n", targetVersion, bc->getURL().c_str());
 			throw restore_invalid_version();
@@ -4565,7 +4556,7 @@ public:
 				backupConfig = BackupConfig(uidFlag.first);
 				state EBackupState status = wait(backupConfig.stateEnum().getOrThrow(ryw_tr));
 
-				if (status != BackupAgentBase::STATE_RUNNING_DIFFERENTIAL ) {
+				if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL) {
 					throw backup_duplicate();
 				}
 
@@ -4766,7 +4757,8 @@ void FileBackupAgent::setLastRestorable(Reference<ReadYourWritesTransaction> tr,
 	tr->set(lastRestorable.pack(tagName), BinaryWriter::toValue<Version>(version, Unversioned()));
 }
 
-Future<int> FileBackupAgent::waitBackup(Database cx, std::string tagName, bool stopWhenDone, Reference<IBackupContainer> *pContainer, UID *pUID) {
+Future<EBackupState> FileBackupAgent::waitBackup(Database cx, std::string tagName, bool stopWhenDone,
+                                                 Reference<IBackupContainer>* pContainer, UID* pUID) {
 	return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID);
 }
 
@@ -5029,4 +5021,4 @@ void simulateBlobFailure() {
 			throw lookup_failed();
 		}
 	}
-}
+}

fdbclient/GrvProxyInterface.h

@@ -27,6 +27,8 @@
 // with RateKeeper to gather health information of the cluster.
 struct GrvProxyInterface {
 	constexpr static FileIdentifier file_identifier = 8743216;
+	enum { LocationAwareLoadBalance = 1 };
+	enum { AlwaysFresh = 1 };
 
 	Optional<Key> processId;
 	bool provisional;

fdbclient/HTTP.actor.cpp

@@ -72,7 +72,7 @@ namespace HTTP {
 	}
 
 	PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) {
-		PacketWriter writer(dest, NULL, Unversioned());
+		PacketWriter writer(dest, nullptr, Unversioned());
 		writer.serializeBytes(verb);
 		writer.serializeBytes(" ", 1);
 		writer.serializeBytes(resource);
@@ -238,7 +238,7 @@ namespace HTTP {
 				{
 					// Read the line that contains the chunk length as text in hex
 					size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
-					state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), NULL, 16);
+					state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
 
 					// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer
 					r->content.erase(pos, lineLen + 2);
@@ -301,7 +301,7 @@ namespace HTTP {
 		state TraceEvent event(SevDebug, "HTTPRequest");
 
 		state UnsentPacketQueue empty;
-		if(pContent == NULL)
+		if(pContent == nullptr)
 			pContent = ∅
 
 		// There is no standard http request id header field, so either a global default can be set via a knob

fdbclient/IClientApi.h

@@ -49,6 +49,8 @@ public:
 
 	virtual void addReadConflictRange(const KeyRangeRef& keys) = 0;
 	virtual ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) = 0;
+	virtual ThreadFuture<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRangeRef& range,
+	                                                                        int64_t chunkSize) = 0;
 
 	virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0;
 	virtual void set(const KeyRef& key, const ValueRef& value) = 0;

fdbclient/JSONDoc.h

@@ -67,11 +67,11 @@
 //    // The following would throw if a.b.c did not exist, or if it was not an int.
 //    int x = r["a.b.c"].get_int();
 struct JSONDoc {
-	JSONDoc() : pObj(NULL) {}
+	JSONDoc() : pObj(nullptr) {}
 
 	// Construction from const json_spirit::mObject, trivial and will never throw.
 	// Resulting JSONDoc will not allow modifications.
-	JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(NULL) {}
+	JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(nullptr) {}
 
 	// Construction from json_spirit::mObject.  Allows modifications.
 	JSONDoc(json_spirit::mObject &o) : pObj(&o), wpObj(&o) {}
@@ -79,7 +79,7 @@ struct JSONDoc {
 	// Construction from const json_spirit::mValue (which is a Variant type) which will try to
 	// convert it to an mObject.  This will throw if that fails, just as it would
 	// if the caller called get_obj() itself and used the previous constructor instead.
-	JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(NULL) {}
+	JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(nullptr) {}
 
 	// Construction from non-const json_spirit::mValue - will convert the mValue to
 	// an object if it isn't already and then attach to it.
@@ -98,13 +98,13 @@ struct JSONDoc {
 	// path into on the "dot" character.
 	// When a path is found, pLast is updated.
 	bool has(std::string path, bool split=true) {
-		if (pObj == NULL)
+		if (pObj == nullptr)
 			return false;
 
 		if (path.empty())
 			return false;
 		size_t start = 0;
-		const json_spirit::mValue *curVal = NULL;
+		const json_spirit::mValue *curVal = nullptr;
 		while (start < path.size())
 		{
 			// If a path segment is found then curVal must be an object
@@ -140,7 +140,7 @@ struct JSONDoc {
 	// Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there)
 	// and returns a reference to the Value at that location.
 	json_spirit::mValue & create(std::string path, bool split=true) {
-		if (wpObj == NULL || path.empty())
+		if (wpObj == nullptr || path.empty())
 			throw std::runtime_error("JSON Object not writable or bad JSON path");
 
 		size_t start = 0;
@@ -280,7 +280,7 @@ struct JSONDoc {
 	}
 
 	const json_spirit::mValue & last() const { return *pLast; }
-	bool valid() const { return pObj != NULL; }
+	bool valid() const { return pObj != nullptr; }
 
 	const json_spirit::mObject & obj() {
 		// This dummy object is necessary to make working with obj() easier when this does not currently
@@ -304,7 +304,7 @@ struct JSONDoc {
 	static uint64_t expires_reference_version;
 private:
 	const json_spirit::mObject *pObj;
-	// Writeable pointer to the same object.  Will be NULL if initialized from a const object.
+	// Writeable pointer to the same object.  Will be nullptr if initialized from a const object.
 	json_spirit::mObject *wpObj;
 	const json_spirit::mValue *pLast;
 };

fdbclient/Knobs.cpp

@@ -52,7 +52,7 @@ void ClientKnobs::initialize(bool randomize) {
 	init( COORDINATOR_RECONNECTION_DELAY,          1.0 );
 	init( CLIENT_EXAMPLE_AMOUNT,                    20 );
 	init( MAX_CLIENT_STATUS_AGE,                   1.0 );
-	init( MAX_MASTER_PROXY_CONNECTIONS,              5 ); if( randomize && BUGGIFY ) MAX_MASTER_PROXY_CONNECTIONS = 1;
+	init( MAX_COMMIT_PROXY_CONNECTIONS,              5 ); if( randomize && BUGGIFY ) MAX_COMMIT_PROXY_CONNECTIONS = 1;
 	init( MAX_GRV_PROXY_CONNECTIONS,                 3 ); if( randomize && BUGGIFY ) MAX_GRV_PROXY_CONNECTIONS = 1;
 	init( STATUS_IDLE_TIMEOUT,                   120.0 );
 
@@ -104,7 +104,7 @@ void ClientKnobs::initialize(bool randomize) {
 	init( WATCH_POLLING_TIME,                      1.0 ); if( randomize && BUGGIFY ) WATCH_POLLING_TIME = 5.0;
 	init( NO_RECENT_UPDATES_DURATION,             20.0 ); if( randomize && BUGGIFY ) NO_RECENT_UPDATES_DURATION = 0.1;
 	init( FAST_WATCH_TIMEOUT,                     20.0 ); if( randomize && BUGGIFY ) FAST_WATCH_TIMEOUT = 1.0;
-	init( WATCH_TIMEOUT,                         900.0 ); if( randomize && BUGGIFY ) WATCH_TIMEOUT = 20.0;
+	init( WATCH_TIMEOUT,                          30.0 ); if( randomize && BUGGIFY ) WATCH_TIMEOUT = 20.0;
 
 	// Core
 	init( CORE_VERSIONSPERSECOND,		           1e6 );
@@ -171,10 +171,12 @@ void ClientKnobs::initialize(bool randomize) {
 	init( MIN_CLEANUP_SECONDS,                  3600.0 );
 
 	// Configuration
-	init( DEFAULT_AUTO_PROXIES,                      3 );
+	init( DEFAULT_AUTO_COMMIT_PROXIES,               3 );
 	init( DEFAULT_AUTO_GRV_PROXIES,                  1 );
 	init( DEFAULT_AUTO_RESOLVERS,                    1 );
 	init( DEFAULT_AUTO_LOGS,                         3 );
+	init( DEFAULT_COMMIT_GRV_PROXIES_RATIO,          3 );
+	init( DEFAULT_MAX_GRV_PROXIES,                   4 );
 
 	init( IS_ACCEPTABLE_DELAY,                     1.5 );
 

fdbclient/Knobs.h

@@ -46,7 +46,7 @@ public:
 	double COORDINATOR_RECONNECTION_DELAY;
 	int CLIENT_EXAMPLE_AMOUNT;
 	double MAX_CLIENT_STATUS_AGE;
-	int MAX_MASTER_PROXY_CONNECTIONS;
+	int MAX_COMMIT_PROXY_CONNECTIONS;
 	int MAX_GRV_PROXY_CONNECTIONS;
 	double STATUS_IDLE_TIMEOUT;
 
@@ -167,8 +167,10 @@ public:
 	double MIN_CLEANUP_SECONDS;
 
 	// Configuration
-	int32_t DEFAULT_AUTO_PROXIES;
+	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
 	int32_t DEFAULT_AUTO_GRV_PROXIES;
+	int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO;
+	int32_t DEFAULT_MAX_GRV_PROXIES;
 	int32_t DEFAULT_AUTO_RESOLVERS;
 	int32_t DEFAULT_AUTO_LOGS;
 

fdbclient/ManagementAPI.actor.cpp

@@ -19,8 +19,10 @@
  */
 
 #include <cinttypes>
+#include <string>
 #include <vector>
 
+#include "fdbclient/Knobs.h"
 #include "flow/Arena.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
@@ -33,6 +35,7 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/StatusClient.h"
+#include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include "fdbrpc/ReplicationPolicy.h"
 #include "fdbrpc/Replication.h"
@@ -78,8 +81,42 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 		std::string key = mode.substr(0, pos);
 		std::string value = mode.substr(pos+1);
 
-		if ((key == "logs" || key == "proxies" || key == "grv_proxies" || key == "resolvers" || key == "remote_logs" ||
-		     key == "log_routers" || key == "usable_regions" || key == "repopulate_anti_quorum") &&
+		if (key == "proxies" && isInteger(value)) {
+			printf("Warning: Proxy role is being split into GRV Proxy and Commit Proxy, now prefer configuring "
+			       "'grv_proxies' and 'commit_proxies' separately. Generally we should follow that 'commit_proxies'"
+			       " is three times of 'grv_proxies' count and 'grv_proxies' should be not more than 4.\n");
+			int proxiesCount = atoi(value.c_str());
+			if (proxiesCount == -1) {
+				proxiesCount = CLIENT_KNOBS->DEFAULT_AUTO_GRV_PROXIES + CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;
+				ASSERT_WE_THINK(proxiesCount >= 2);
+			}
+
+			if (proxiesCount < 2) {
+				printf("Error: At least 2 proxies (1 GRV proxy and 1 Commit proxy) are required.\n");
+				return out;
+			}
+
+			int grvProxyCount =
+			    std::max(1, std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES,
+			                         proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1)));
+			int commitProxyCount = proxiesCount - grvProxyCount;
+			ASSERT_WE_THINK(grvProxyCount >= 1 && commitProxyCount >= 1);
+
+			out[p + "grv_proxies"] = std::to_string(grvProxyCount);
+			out[p + "commit_proxies"] = std::to_string(commitProxyCount);
+			printf("%d proxies are automatically converted into %d GRV proxies and %d Commit proxies.\n", proxiesCount,
+			       grvProxyCount, commitProxyCount);
+
+			TraceEvent("DatabaseConfigurationProxiesSpecified")
+			    .detail("SpecifiedProxies", atoi(value.c_str()))
+			    .detail("EffectiveSpecifiedProxies", proxiesCount)
+			    .detail("ConvertedGrvProxies", grvProxyCount)
+			    .detail("ConvertedCommitProxies", commitProxyCount);
+		}
+
+		if ((key == "logs" || key == "commit_proxies" || key == "grv_proxies" || key == "resolvers" ||
+		     key == "remote_logs" || key == "log_routers" || key == "usable_regions" ||
+		     key == "repopulate_anti_quorum") &&
 		    isInteger(value)) {
 			out[p+key] = value;
 		}
@@ -229,7 +266,8 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 	return out;
 }
 
-ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& modeTokens, std::map<std::string, std::string>& outConf ) {
+ConfigurationResult buildConfiguration(std::vector<StringRef> const& modeTokens,
+                                       std::map<std::string, std::string>& outConf) {
 	for(auto it : modeTokens) {
 		std::string mode = it.toString();
 		auto m = configForToken( mode );
@@ -265,7 +303,7 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
 	return ConfigurationResult::SUCCESS;
 }
 
-ConfigurationResult::Type buildConfiguration( std::string const& configMode, std::map<std::string, std::string>& outConf ) {
+ConfigurationResult buildConfiguration(std::string const& configMode, std::map<std::string, std::string>& outConf) {
 	std::vector<StringRef> modes;
 
 	int p = 0;
@@ -305,7 +343,7 @@ ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration( Database cx ) {
 	}
 }
 
-ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std::string, std::string> m, bool force ) {
+ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string, std::string> m, bool force) {
 	state StringRef initIdKey = LiteralStringRef( "\xff/init_id" );
 	state Transaction tr(cx);
 
@@ -656,7 +694,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 			}
 
 			if (processClass.classType() == ProcessClass::TransactionClass ||
-			    processClass.classType() == ProcessClass::ProxyClass ||
+			    processClass.classType() == ProcessClass::CommitProxyClass ||
 			    processClass.classType() == ProcessClass::GrvProxyClass ||
 			    processClass.classType() == ProcessClass::ResolutionClass ||
 			    processClass.classType() == ProcessClass::StatelessClass ||
@@ -701,7 +739,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 			if (proc.second == ProcessClass::StatelessClass) {
 				existingStatelessCount++;
 			}
-			if(proc.second == ProcessClass::ProxyClass) {
+			if (proc.second == ProcessClass::CommitProxyClass) {
 				existingProxyCount++;
 			}
 			if (proc.second == ProcessClass::GrvProxyClass) {
@@ -734,19 +772,18 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 		resolverCount = result.old_resolvers;
 	}
 
-	result.desired_proxies = std::max(std::min(12, processCount / 15), 1);
+	result.desired_commit_proxies = std::max(std::min(12, processCount / 15), 1);
 	int proxyCount;
-	if (!statusObjConfig.get("proxies", result.old_proxies)) {
-		result.old_proxies = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
-		statusObjConfig.get("auto_proxies", result.old_proxies);
-		result.auto_proxies = result.desired_proxies;
-		proxyCount = result.auto_proxies;
+	if (!statusObjConfig.get("commit_proxies", result.old_commit_proxies)) {
+		result.old_commit_proxies = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;
+		statusObjConfig.get("auto_commit_proxies", result.old_commit_proxies);
+		result.auto_commit_proxies = result.desired_commit_proxies;
+		proxyCount = result.auto_commit_proxies;
 	} else {
-		result.auto_proxies = result.old_proxies;
-		proxyCount = result.old_proxies;
+		result.auto_commit_proxies = result.old_commit_proxies;
+		proxyCount = result.old_commit_proxies;
 	}
 
-	// Need to configure a good number.
 	result.desired_grv_proxies = std::max(std::min(4, processCount / 20), 1);
 	int grvProxyCount;
 	if (!statusObjConfig.get("grv_proxies", result.old_grv_proxies)) {
@@ -823,7 +860,7 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 	return result;
 }
 
-ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoResult conf ) {
+ACTOR Future<ConfigurationResult> autoConfig(Database cx, ConfigureAutoResult conf) {
 	state Transaction tr(cx);
 	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(),Unversioned());
 
@@ -857,8 +894,8 @@ ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoRe
 			if (conf.auto_logs != conf.old_logs)
 				tr.set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs));
 
-			if(conf.auto_proxies != conf.old_proxies)
-				tr.set(configKeysPrefix.toString() + "auto_proxies", format("%d", conf.auto_proxies));
+			if (conf.auto_commit_proxies != conf.old_commit_proxies)
+				tr.set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies));
 
 			if (conf.auto_grv_proxies != conf.old_grv_proxies)
 				tr.set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies));
@@ -890,7 +927,8 @@ ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoRe
 	}
 }
 
-Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ) {
+Future<ConfigurationResult> changeConfig(Database const& cx, std::vector<StringRef> const& modes,
+                                         Optional<ConfigureAutoResult> const& conf, bool force) {
 	if( modes.size() && modes[0] == LiteralStringRef("auto") && conf.present() ) {
 		return autoConfig(cx, conf.get());
 	}
@@ -902,7 +940,7 @@ Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<
 	return changeConfig(cx, m, force);
 }
 
-Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& modes, bool force ) {
+Future<ConfigurationResult> changeConfig(Database const& cx, std::string const& modes, bool force) {
 	TraceEvent("ChangeConfig").detail("Mode", modes);
 	std::map<std::string,std::string> m;
 	auto r = buildConfiguration( modes, m );
@@ -971,7 +1009,7 @@ ACTOR Future<std::vector<NetworkAddress>> getCoordinators( Database cx ) {
 	}
 }
 
-ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuorumChange> change ) {
+ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change) {
 	state Transaction tr(cx);
 	state int retries = 0;
 	state std::vector<NetworkAddress> desiredCoordinators;
@@ -991,7 +1029,7 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
 			if ( cx->getConnectionFile() && old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName() )
 				return CoordinatorsResult::BAD_DATABASE_STATE;  // Someone changed the "name" of the database??
 
-			state CoordinatorsResult::Type result = CoordinatorsResult::SUCCESS;
+			state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 			if(!desiredCoordinators.size()) {
 				std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) );
 				desiredCoordinators = _desiredCoordinators;
@@ -1058,42 +1096,48 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
 	}
 }
 
-struct SpecifiedQuorumChange : IQuorumChange {
+struct SpecifiedQuorumChange final : IQuorumChange {
 	vector<NetworkAddress> desired;
 	explicit SpecifiedQuorumChange( vector<NetworkAddress> const& desired ) : desired(desired) {}
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) {
+	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators,
+	                                                      Reference<ClusterConnectionFile>,
+	                                                      CoordinatorsResult&) override {
 		return desired;
 	}
 };
 Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) { return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses)); }
 
-struct NoQuorumChange : IQuorumChange {
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) {
+struct NoQuorumChange final : IQuorumChange {
+	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators,
+	                                                      Reference<ClusterConnectionFile>,
+	                                                      CoordinatorsResult&) override {
 		return oldCoordinators;
 	}
 };
 Reference<IQuorumChange> noQuorumChange() { return Reference<IQuorumChange>(new NoQuorumChange); }
 
-struct NameQuorumChange : IQuorumChange {
+struct NameQuorumChange final : IQuorumChange {
 	std::string newName;
 	Reference<IQuorumChange> otherChange;
 	explicit NameQuorumChange( std::string const& newName, Reference<IQuorumChange> const& otherChange ) : newName(newName), otherChange(otherChange) {}
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> cf, CoordinatorsResult::Type& t ) {
+	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators,
+	                                                      Reference<ClusterConnectionFile> cf,
+	                                                      CoordinatorsResult& t) override {
 		return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t);
 	}
-	virtual std::string getDesiredClusterKeyName() {
-		return newName;
-	}
+	std::string getDesiredClusterKeyName() const override { return newName; }
 };
 Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other) {
 	return Reference<IQuorumChange>(new NameQuorumChange( name, other ));
 }
 
-struct AutoQuorumChange : IQuorumChange {
+struct AutoQuorumChange final : IQuorumChange {
 	int desired;
 	explicit AutoQuorumChange( int desired ) : desired(desired) {}
 
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, CoordinatorsResult::Type& err ) {
+	Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators,
+	                                                      Reference<ClusterConnectionFile> ccf,
+	                                                      CoordinatorsResult& err) override {
 		return getDesired( this, tr, oldCoordinators, ccf, &err );
 	}
 
@@ -1145,7 +1189,10 @@ struct AutoQuorumChange : IQuorumChange {
 		return true; // The status quo seems fine
 	}
 
-	ACTOR static Future<vector<NetworkAddress>> getDesired( AutoQuorumChange* self, Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, CoordinatorsResult::Type* err ) {
+	ACTOR static Future<vector<NetworkAddress>> getDesired(AutoQuorumChange* self, Transaction* tr,
+	                                                       vector<NetworkAddress> oldCoordinators,
+	                                                       Reference<ClusterConnectionFile> ccf,
+	                                                       CoordinatorsResult* err) {
 		state int desiredCount = self->desired;
 
 		if(desiredCount == -1) {

fdbclient/ManagementAPI.actor.h

@@ -43,41 +43,35 @@ standard API and some knowledge of the contents of the system key space.
 
 // ConfigurationResult enumerates normal outcomes of changeConfig() and various error
 // conditions specific to it.  changeConfig may also throw an Error to report other problems.
-class ConfigurationResult {
-public:
-	enum Type {
-		NO_OPTIONS_PROVIDED,
-		CONFLICTING_OPTIONS,
-		UNKNOWN_OPTION,
-		INCOMPLETE_CONFIGURATION,
-		INVALID_CONFIGURATION,
-		DATABASE_ALREADY_CREATED,
-		DATABASE_CREATED,
-		DATABASE_UNAVAILABLE,
-		STORAGE_IN_UNKNOWN_DCID,
-		REGION_NOT_FULLY_REPLICATED,
-		MULTIPLE_ACTIVE_REGIONS,
-		REGIONS_CHANGED,
-		NOT_ENOUGH_WORKERS,
-		REGION_REPLICATION_MISMATCH,
-		DCID_MISSING,
-		LOCKED_NOT_NEW,
-		SUCCESS,
-	};
+enum class ConfigurationResult {
+	NO_OPTIONS_PROVIDED,
+	CONFLICTING_OPTIONS,
+	UNKNOWN_OPTION,
+	INCOMPLETE_CONFIGURATION,
+	INVALID_CONFIGURATION,
+	DATABASE_ALREADY_CREATED,
+	DATABASE_CREATED,
+	DATABASE_UNAVAILABLE,
+	STORAGE_IN_UNKNOWN_DCID,
+	REGION_NOT_FULLY_REPLICATED,
+	MULTIPLE_ACTIVE_REGIONS,
+	REGIONS_CHANGED,
+	NOT_ENOUGH_WORKERS,
+	REGION_REPLICATION_MISMATCH,
+	DCID_MISSING,
+	LOCKED_NOT_NEW,
+	SUCCESS,
 };
 
-class CoordinatorsResult {
-public:
-	enum Type {
-		INVALID_NETWORK_ADDRESSES,
-		SAME_NETWORK_ADDRESSES,
-		NOT_COORDINATORS, //FIXME: not detected
-		DATABASE_UNREACHABLE, //FIXME: not detected
-		BAD_DATABASE_STATE,
-		COORDINATOR_UNREACHABLE,
-		NOT_ENOUGH_MACHINES,
-		SUCCESS
-	};
+enum class CoordinatorsResult {
+	INVALID_NETWORK_ADDRESSES,
+	SAME_NETWORK_ADDRESSES,
+	NOT_COORDINATORS, // FIXME: not detected
+	DATABASE_UNREACHABLE, // FIXME: not detected
+	BAD_DATABASE_STATE,
+	COORDINATOR_UNREACHABLE,
+	NOT_ENOUGH_MACHINES,
+	SUCCESS
 };
 
 struct ConfigureAutoResult {
@@ -86,7 +80,7 @@ struct ConfigureAutoResult {
 	int32_t machines;
 
 	std::string old_replication;
-	int32_t old_proxies;
+	int32_t old_commit_proxies;
 	int32_t old_grv_proxies;
 	int32_t old_resolvers;
 	int32_t old_logs;
@@ -94,38 +88,46 @@ struct ConfigureAutoResult {
 	int32_t old_machines_with_transaction;
 
 	std::string auto_replication;
-	int32_t auto_proxies;
+	int32_t auto_commit_proxies;
 	int32_t auto_grv_proxies;
 	int32_t auto_resolvers;
 	int32_t auto_logs;
 	int32_t auto_processes_with_transaction;
 	int32_t auto_machines_with_transaction;
 
-	int32_t desired_proxies;
+	int32_t desired_commit_proxies;
 	int32_t desired_grv_proxies;
 	int32_t desired_resolvers;
 	int32_t desired_logs;
 
 	ConfigureAutoResult()
-	  : processes(-1), machines(-1), old_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1),
-	    old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_proxies(-1), auto_grv_proxies(-1),
-	    auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1),
-	    desired_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1), desired_logs(-1) {}
+	  : processes(-1), machines(-1), old_commit_proxies(-1), old_grv_proxies(-1), old_resolvers(-1), old_logs(-1),
+	    old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_commit_proxies(-1),
+	    auto_grv_proxies(-1), auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1),
+	    auto_machines_with_transaction(-1), desired_commit_proxies(-1), desired_grv_proxies(-1), desired_resolvers(-1),
+	    desired_logs(-1) {}
 
 	bool isValid() const { return processes != -1; }
 };
 
-ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& modeTokens, std::map<std::string, std::string>& outConf );  // Accepts a vector of configuration tokens
-ConfigurationResult::Type buildConfiguration( std::string const& modeString, std::map<std::string, std::string>& outConf );				// Accepts tokens separated by spaces in a single string
+ConfigurationResult buildConfiguration(
+    std::vector<StringRef> const& modeTokens,
+    std::map<std::string, std::string>& outConf); // Accepts a vector of configuration tokens
+ConfigurationResult buildConfiguration(
+    std::string const& modeString,
+    std::map<std::string, std::string>& outConf); // Accepts tokens separated by spaces in a single string
 
 bool isCompleteConfiguration( std::map<std::string, std::string> const& options );
 
 // All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error).
-Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& configMode, bool force );  // Accepts tokens separated by spaces in a single string
+Future<ConfigurationResult> changeConfig(Database const& cx, std::string const& configMode,
+                                         bool force); // Accepts tokens separated by spaces in a single string
 
 ConfigureAutoResult parseConfig( StatusObject const& status );
-Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force );  // Accepts a vector of configuration tokens
-ACTOR Future<ConfigurationResult::Type> changeConfig(
+Future<ConfigurationResult> changeConfig(Database const& cx, std::vector<StringRef> const& modes,
+                                         Optional<ConfigureAutoResult> const& conf,
+                                         bool force); // Accepts a vector of configuration tokens
+ACTOR Future<ConfigurationResult> changeConfig(
     Database cx, std::map<std::string, std::string> m,
     bool force); // Accepts a full configuration in key/value format (from buildConfiguration)
 
@@ -134,12 +136,15 @@ ACTOR Future<Void> waitForFullReplication(Database cx);
 
 struct IQuorumChange : ReferenceCounted<IQuorumChange> {
 	virtual ~IQuorumChange() {}
-	virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) = 0;
-	virtual std::string getDesiredClusterKeyName() { return std::string(); }
+	virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                              vector<NetworkAddress> oldCoordinators,
+	                                                              Reference<ClusterConnectionFile>,
+	                                                              CoordinatorsResult&) = 0;
+	virtual std::string getDesiredClusterKeyName() const { return std::string(); }
 };
 
 // Change to use the given set of coordination servers
-ACTOR Future<CoordinatorsResult::Type> changeQuorum(Database cx, Reference<IQuorumChange> change);
+ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
 Reference<IQuorumChange> autoQuorumChange(int desired = -1);
 Reference<IQuorumChange> noQuorumChange();
 Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&);

fdbclient/MetricLogger.actor.cpp

@@ -171,7 +171,7 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri
 // Implementation of IMetricDB
 class MetricDB : public IMetricDB {
 public:
-	MetricDB(ReadYourWritesTransaction *tr = NULL) : tr(tr) {}
+	MetricDB(ReadYourWritesTransaction *tr = nullptr) : tr(tr) {}
 	~MetricDB() {}
 
 	// levelKey is the prefix for the entire level, no timestamp at the end

fdbclient/MonitorLeader.actor.cpp

@@ -624,8 +624,8 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
 		choose {
 			when( ClientDBInfo ni = wait( brokenPromiseToNever( knownLeader->get().get().clientInterface.openDatabase.getReply( req ) ) ) ) {
 				TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id())
-				    .detail("MasterProxy0", ni.masterProxies.size() ? ni.masterProxies[0].id() : UID())
-					.detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID())
+				    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID())
+				    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID())
 				    .detail("ClientID", ni.id);
 				clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
 			}
@@ -681,24 +681,25 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
 	}
 }
 
-void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastMasterProxyUIDs, std::vector<MasterProxyInterface>& lastMasterProxies,
-					  std::vector<UID>& lastGrvProxyUIDs, std::vector<GrvProxyInterface>& lastGrvProxies) {
-	if(ni.masterProxies.size() > CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS) {
-		std::vector<UID> masterProxyUIDs;
-		for(auto& masterProxy : ni.masterProxies) {
-			masterProxyUIDs.push_back(masterProxy.id());
+void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastCommitProxyUIDs,
+                     std::vector<CommitProxyInterface>& lastCommitProxies, std::vector<UID>& lastGrvProxyUIDs,
+                     std::vector<GrvProxyInterface>& lastGrvProxies) {
+	if (ni.commitProxies.size() > CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS) {
+		std::vector<UID> commitProxyUIDs;
+		for (auto& commitProxy : ni.commitProxies) {
+			commitProxyUIDs.push_back(commitProxy.id());
 		}
-		if(masterProxyUIDs != lastMasterProxyUIDs) {
-			lastMasterProxyUIDs.swap(masterProxyUIDs);
-			lastMasterProxies = ni.masterProxies;
-			deterministicRandom()->randomShuffle(lastMasterProxies);
-			lastMasterProxies.resize(CLIENT_KNOBS->MAX_MASTER_PROXY_CONNECTIONS);
-			for(int i = 0; i < lastMasterProxies.size(); i++) {
-				TraceEvent("ConnectedMasterProxy").detail("MasterProxy", lastMasterProxies[i].id());
+		if (commitProxyUIDs != lastCommitProxyUIDs) {
+			lastCommitProxyUIDs.swap(commitProxyUIDs);
+			lastCommitProxies = ni.commitProxies;
+			deterministicRandom()->randomShuffle(lastCommitProxies);
+			lastCommitProxies.resize(CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS);
+			for (int i = 0; i < lastCommitProxies.size(); i++) {
+				TraceEvent("ConnectedCommitProxy").detail("CommitProxy", lastCommitProxies[i].id());
 			}
 		}
-		ni.firstProxy = ni.masterProxies[0];
-		ni.masterProxies = lastMasterProxies;
+		ni.firstCommitProxy = ni.commitProxies[0];
+		ni.commitProxies = lastCommitProxies;
 	}
 	if(ni.grvProxies.size() > CLIENT_KNOBS->MAX_GRV_PROXY_CONNECTIONS) {
 		std::vector<UID> grvProxyUIDs;
@@ -719,14 +720,16 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastMasterProxyUIDs, s
 }
 
 // Leader is the process that will be elected by coordinators as the cluster controller
-ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, MonitorLeaderInfo info, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup) {
+ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
+    Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, MonitorLeaderInfo info,
+    Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup) {
 	state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
 	state vector<NetworkAddress> addrs = cs.coordinators();
 	state int idx = 0;
 	state int successIdx = 0;
 	state Optional<double> incorrectTime;
-	state std::vector<UID> lastProxyUIDs;
-	state std::vector<MasterProxyInterface> lastProxies;
+	state std::vector<UID> lastCommitProxyUIDs;
+	state std::vector<CommitProxyInterface> lastCommitProxies;
 	state std::vector<UID> lastGrvProxyUIDs;
 	state std::vector<GrvProxyInterface> lastGrvProxies;
 
@@ -780,7 +783,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
 			connFile->notifyConnected();
 
 			auto& ni = rep.get().mutate();
-			shrinkProxyList(ni, lastProxyUIDs, lastProxies, lastGrvProxyUIDs, lastGrvProxies);
+			shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
 			clientInfo->set( ni );
 			successIdx = idx;
 		} else {

fdbclient/MonitorLeader.h

@@ -25,7 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/ClusterInterface.h"
-#include "fdbclient/MasterProxyInterface.h"
+#include "fdbclient/CommitProxyInterface.h"
 
 #define CLUSTER_FILE_ENV_VAR_NAME "FDB_CLUSTER_FILE"
 
@@ -67,8 +67,9 @@ Future<Void> monitorLeaderForProxies( Value const& key, vector<NetworkAddress> c
 
 Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile, Reference<AsyncVar<ClientDBInfo>> const& clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions, Key const& traceLogGroup );
 
-void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastMasterProxyUIDs, std::vector<MasterProxyInterface>& lastMasterProxies,
-					  std::vector<UID>& lastGrvProxyUIDs, std::vector<GrvProxyInterface>& lastGrvProxies);
+void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastCommitProxyUIDs,
+                     std::vector<CommitProxyInterface>& lastCommitProxies, std::vector<UID>& lastGrvProxyUIDs,
+                     std::vector<GrvProxyInterface>& lastGrvProxies);
 
 #ifndef __INTEL_COMPILER
 #pragma region Implementation