diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c
index aca4a8d413..dc429aa78c 100644
--- a/bindings/c/test/mako/mako.c
+++ b/bindings/c/test/mako/mako.c
@@ -2,6 +2,7 @@
 #include <fcntl.h>
 #include <getopt.h>
 #include <math.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -1474,6 +1475,7 @@ int init_args(mako_args_t* args) {
 	}
 	args->client_threads_per_version = 0;
 	args->disable_ryw = 0;
+	args->json_output_path[0] = '\0';
 	return 0;
 }
 
@@ -1640,6 +1642,7 @@ void usage() {
 	printf("%-24s %s\n", "    --flatbuffers", "Use flatbuffers");
 	printf("%-24s %s\n", "    --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
 	printf("%-24s %s\n", "    --disable_ryw", "Disable snapshot read-your-writes");
+	printf("%-24s %s\n", "    --json_report=PATH", "Output stats to the specified json file (Default: mako.json)");
 }
 
 /* parse benchmark paramters */
@@ -1648,12 +1651,11 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 	int c;
 	int idx;
 	while (1) {
-		const char* short_options = "a:c:d:p:t:r:s:i:x:v:m:hjz";
+		const char* short_options = "a:c:p:t:r:s:i:x:v:m:hz";
 		static struct option long_options[] = {
 			/* name, has_arg, flag, val */
 			{ "api_version", required_argument, NULL, 'a' },
 			{ "cluster", required_argument, NULL, 'c' },
-			{ "num_databases", optional_argument, NULL, 'd' },
 			{ "procs", required_argument, NULL, 'p' },
 			{ "threads", required_argument, NULL, 't' },
 			{ "rows", required_argument, NULL, 'r' },
@@ -1678,7 +1680,6 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 			{ "txntrace", required_argument, NULL, ARG_TXNTRACE },
 			/* no args */
 			{ "help", no_argument, NULL, 'h' },
-			{ "json", no_argument, NULL, 'j' },
 			{ "zipf", no_argument, NULL, 'z' },
 			{ "commitget", no_argument, NULL, ARG_COMMITGET },
 			{ "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS },
@@ -1689,6 +1690,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 			{ "version", no_argument, NULL, ARG_VERSION },
 			{ "client_threads_per_version", required_argument, NULL, ARG_CLIENT_THREADS_PER_VERSION },
 			{ "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW },
+			{ "json_report", optional_argument, NULL, ARG_JSON_REPORT },
 			{ NULL, 0, NULL, 0 }
 		};
 		idx = 0;
@@ -1859,6 +1861,16 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 		case ARG_DISABLE_RYW:
 			args->disable_ryw = 1;
 			break;
+		case ARG_JSON_REPORT:
+			if (optarg == NULL && (argv[optind] == NULL || (argv[optind] != NULL && argv[optind][0] == '-'))) {
+				// if --report_json is the last option and no file is specified
+				// or --report_json is followed by another option
+				char default_file[] = "mako.json";
+				strncpy(args->json_output_path, default_file, strlen(default_file));
+			} else {
+				strncpy(args->json_output_path, optarg, strlen(optarg) + 1);
+			}
+			break;
 		}
 	}
 
@@ -1885,6 +1897,41 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
 	return 0;
 }
 
+char* get_ops_name(int ops_code) {
+	switch (ops_code) {
+	case OP_GETREADVERSION:
+		return "GRV";
+	case OP_GET:
+		return "GET";
+	case OP_GETRANGE:
+		return "GETRANGE";
+	case OP_SGET:
+		return "SGET";
+	case OP_SGETRANGE:
+		return "SGETRANGE";
+	case OP_UPDATE:
+		return "UPDATE";
+	case OP_INSERT:
+		return "INSERT";
+	case OP_INSERTRANGE:
+		return "INSERTRANGE";
+	case OP_CLEAR:
+		return "CLEAR";
+	case OP_SETCLEAR:
+		return "SETCLEAR";
+	case OP_CLEARRANGE:
+		return "CLEARRANGE";
+	case OP_SETCLEARRANGE:
+		return "SETCLEARRANGE";
+	case OP_COMMIT:
+		return "COMMIT";
+	case OP_TRANSACTION:
+		return "TRANSACTION";
+	default:
+		return "";
+	}
+}
+
 int validate_args(mako_args_t* args) {
 	if (args->mode == MODE_INVALID) {
 		fprintf(stderr, "ERROR: --mode has to be set\n");
@@ -1954,7 +2001,7 @@ int validate_args(mako_args_t* args) {
 #define STATS_TITLE_WIDTH 12
 #define STATS_FIELD_WIDTH 12
 
-void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, struct timespec* prev) {
+void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, struct timespec* prev, FILE* fp) {
 	int i, j;
 	int op;
 	int print_err;
@@ -1967,7 +2014,7 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 	uint64_t totalxacts = 0;
 	static uint64_t conflicts_prev = 0;
 	uint64_t conflicts = 0;
-	double durationns = (now->tv_sec - prev->tv_sec) * 1000000000.0 + (now->tv_nsec - prev->tv_nsec);
+	double duration_nsec = (now->tv_sec - prev->tv_sec) * 1000000000.0 + (now->tv_nsec - prev->tv_nsec);
 
 	for (i = 0; i < args->num_processes; i++) {
 		for (j = 0; j < args->num_threads; j++) {
@@ -1979,10 +2026,18 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 			}
 		}
 	}
+
+	if (fp) {
+		fwrite("{", 1, 1, fp);
+	}
 	printf("%" STR(STATS_TITLE_WIDTH) "s ", "OPS");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0) {
-			printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op] - ops_total_prev[op]);
+			uint64_t ops_total_diff = ops_total[op] - ops_total_prev[op];
+			printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total_diff);
+			if (fp) {
+				fprintf(fp, "\"%s\": %lld,", get_ops_name(op), ops_total_diff);
+			}
 			errors_diff[op] = errors_total[op] - errors_total_prev[op];
 			print_err = (errors_diff[op] > 0);
 			ops_total_prev[op] = ops_total[op];
@@ -1990,11 +2045,19 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 		}
 	}
 	/* TPS */
-	printf("%" STR(STATS_FIELD_WIDTH) ".2f ", (totalxacts - totalxacts_prev) * 1000000000.0 / durationns);
+	double tps = (totalxacts - totalxacts_prev) * 1000000000.0 / duration_nsec;
+	printf("%" STR(STATS_FIELD_WIDTH) ".2f ", tps);
+	if (fp) {
+		fprintf(fp, "\"tps\": %.2f,", tps);
+	}
 	totalxacts_prev = totalxacts;
 
 	/* Conflicts */
-	printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", (conflicts - conflicts_prev) * 1000000000.0 / durationns);
+	double conflicts_diff = (conflicts - conflicts_prev) * 1000000000.0 / duration_nsec;
+	printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", conflicts_diff);
+	if (fp) {
+		fprintf(fp, "\"conflictsPerSec\": %.2f},", conflicts_diff);
+	}
 	conflicts_prev = conflicts;
 
 	if (print_err) {
@@ -2002,10 +2065,14 @@ void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, s
 		for (op = 0; op < MAX_OP; op++) {
 			if (args->txnspec.ops[op][OP_COUNT] > 0) {
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]);
+				if (fp) {
+					fprintf(fp, "\"errors\": %.2f", conflicts_diff);
+				}
 			}
 		}
 		printf("\n");
 	}
+
 	return;
 }
 
@@ -2019,44 +2086,7 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
 			printf(" ");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0) {
-			switch (op) {
-			case OP_GETREADVERSION:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "GRV");
-				break;
-			case OP_GET:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "GET");
-				break;
-			case OP_GETRANGE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "GETRANGE");
-				break;
-			case OP_SGET:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGET");
-				break;
-			case OP_SGETRANGE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGETRANGE");
-				break;
-			case OP_UPDATE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "UPDATE");
-				break;
-			case OP_INSERT:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERT");
-				break;
-			case OP_INSERTRANGE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERTRANGE");
-				break;
-			case OP_CLEAR:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEAR");
-				break;
-			case OP_SETCLEAR:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLEAR");
-				break;
-			case OP_CLEARRANGE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEARRANGE");
-				break;
-			case OP_SETCLEARRANGE:
-				printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLRRANGE");
-				break;
-			}
+			printf("%" STR(STATS_FIELD_WIDTH) "s ", get_ops_name(op));
 		}
 	}
 
@@ -2109,7 +2139,8 @@ void print_report(mako_args_t* args,
                   mako_stats_t* stats,
                   struct timespec* timer_now,
                   struct timespec* timer_start,
-                  pid_t* pid_main) {
+                  pid_t* pid_main,
+                  FILE* fp) {
 	int i, j, k, op, index;
 	uint64_t totalxacts = 0;
 	uint64_t conflicts = 0;
@@ -2121,7 +2152,7 @@ void print_report(mako_args_t* args,
 	uint64_t lat_samples[MAX_OP] = { 0 };
 	uint64_t lat_max[MAX_OP] = { 0 };
 
-	uint64_t durationns =
+	uint64_t duration_nsec =
 	    (timer_now->tv_sec - timer_start->tv_sec) * 1000000000 + (timer_now->tv_nsec - timer_start->tv_nsec);
 
 	for (op = 0; op < MAX_OP; op++) {
@@ -2155,7 +2186,8 @@ void print_report(mako_args_t* args,
 	}
 
 	/* overall stats */
-	printf("\n====== Total Duration %6.3f sec ======\n\n", (double)durationns / 1000000000);
+	double total_duration = duration_nsec * 1.0 / 1000000000;
+	printf("\n====== Total Duration %6.3f sec ======\n\n", total_duration);
 	printf("Total Processes:  %8d\n", args->num_processes);
 	printf("Total Threads:    %8d\n", args->num_threads);
 	if (args->tpsmax == args->tpsmin)
@@ -2180,32 +2212,62 @@ void print_report(mako_args_t* args,
 	printf("Total Xacts:      %8lld\n", totalxacts);
 	printf("Total Conflicts:  %8lld\n", conflicts);
 	printf("Total Errors:     %8lld\n", totalerrors);
-	printf("Overall TPS:      %8lld\n\n", totalxacts * 1000000000 / durationns);
+	printf("Overall TPS:      %8lld\n\n", totalxacts * 1000000000 / duration_nsec);
+
+	if (fp) {
+		fprintf(fp, "\"results\": {");
+		fprintf(fp, "\"totalDuration\": %6.3f,", total_duration);
+		fprintf(fp, "\"totalProcesses\": %d,", args->num_processes);
+		fprintf(fp, "\"totalThreads\": %d,", args->num_threads);
+		fprintf(fp, "\"targetTPS\": %d,", args->tpsmax);
+		fprintf(fp, "\"totalXacts\": %lld,", totalxacts);
+		fprintf(fp, "\"totalConflicts\": %lld,", conflicts);
+		fprintf(fp, "\"totalErrors\": %lld,", totalerrors);
+		fprintf(fp, "\"overallTPS\": %lld,", totalxacts * 1000000000 / duration_nsec);
+	}
 
 	/* per-op stats */
 	print_stats_header(args, true, true, false);
 
 	/* OPS */
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Total OPS");
+	if (fp) {
+		fprintf(fp, "\"totalOps\": {");
+	}
 	for (op = 0; op < MAX_OP; op++) {
 		if ((args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
 			printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]);
+			if (fp) {
+				fprintf(fp, "\"%s\": %lld,", get_ops_name(op), ops_total[op]);
+			}
 		}
 	}
 
 	/* TPS */
-	printf("%" STR(STATS_FIELD_WIDTH) ".2f ", totalxacts * 1000000000.0 / durationns);
+	double tps = totalxacts * 1000000000.0 / duration_nsec;
+	printf("%" STR(STATS_FIELD_WIDTH) ".2f ", tps);
 
 	/* Conflicts */
-	printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", conflicts * 1000000000.0 / durationns);
+	double conflicts_rate = conflicts * 1000000000.0 / duration_nsec;
+	printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", conflicts_rate);
+
+	if (fp) {
+		fprintf(fp, "}, \"tps\": %.2f, \"conflictsPerSec\": %.2f, \"errors\": {", tps, conflicts_rate);
+	}
 
 	/* Errors */
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) {
 			printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]);
+			if (fp) {
+				fprintf(fp, "\"%s\": %lld,", get_ops_name(op), errors_total[op]);
+			}
 		}
 	}
+	if (fp) {
+		fprintf(fp, "}, \"numSamples\": {");
+	}
 	printf("\n\n");
 
 	printf("%s", "Latency (us)");
@@ -2220,11 +2282,17 @@ void print_report(mako_args_t* args,
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
+			if (fp) {
+				fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_samples[op]);
+			}
 		}
 	}
 	printf("\n");
 
 	/* Min Latency */
+	if (fp) {
+		fprintf(fp, "}, \"minLatency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Min");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
@@ -2232,17 +2300,26 @@ void print_report(mako_args_t* args,
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_min[op]);
+				}
 			}
 		}
 	}
 	printf("\n");
 
 	/* Avg Latency */
+	if (fp) {
+		fprintf(fp, "}, \"avgLatency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Avg");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
 			if (lat_total[op]) {
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_total[op] / lat_samples[op]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_total[op] / lat_samples[op]);
+				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2251,6 +2328,9 @@ void print_report(mako_args_t* args,
 	printf("\n");
 
 	/* Max Latency */
+	if (fp) {
+		fprintf(fp, "}, \"maxLatency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Max");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
@@ -2258,6 +2338,9 @@ void print_report(mako_args_t* args,
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), lat_max[op]);
+				}
 			}
 		}
 	}
@@ -2268,6 +2351,9 @@ void print_report(mako_args_t* args,
 	int point_99_9pct, point_99pct, point_95pct;
 
 	/* Median Latency */
+	if (fp) {
+		fprintf(fp, "}, \"medianLatency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Median");
 	int num_points[MAX_OP] = { 0 };
 	for (op = 0; op < MAX_OP; op++) {
@@ -2304,6 +2390,9 @@ void print_report(mako_args_t* args,
 					median = (dataPoints[op][num_points[op] / 2] + dataPoints[op][num_points[op] / 2 - 1]) >> 1;
 				}
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", median);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), median);
+				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2312,6 +2401,9 @@ void print_report(mako_args_t* args,
 	printf("\n");
 
 	/* 95%ile Latency */
+	if (fp) {
+		fprintf(fp, "}, \"p95Latency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "95.0 pctile");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
@@ -2322,6 +2414,9 @@ void print_report(mako_args_t* args,
 			if (lat_total[op]) {
 				point_95pct = ((float)(num_points[op]) * 0.95) - 1;
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_95pct]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_95pct]);
+				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2330,6 +2425,9 @@ void print_report(mako_args_t* args,
 	printf("\n");
 
 	/* 99%ile Latency */
+	if (fp) {
+		fprintf(fp, "}, \"p99Latency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "99.0 pctile");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
@@ -2340,6 +2438,9 @@ void print_report(mako_args_t* args,
 			if (lat_total[op]) {
 				point_99pct = ((float)(num_points[op]) * 0.99) - 1;
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_99pct]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_99pct]);
+				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
@@ -2348,6 +2449,9 @@ void print_report(mako_args_t* args,
 	printf("\n");
 
 	/* 99.9%ile Latency */
+	if (fp) {
+		fprintf(fp, "}, \"p99.9Latency\": {");
+	}
 	printf("%-" STR(STATS_TITLE_WIDTH) "s ", "99.9 pctile");
 	for (op = 0; op < MAX_OP; op++) {
 		if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION || op == OP_COMMIT) {
@@ -2358,12 +2462,18 @@ void print_report(mako_args_t* args,
 			if (lat_total[op]) {
 				point_99_9pct = ((float)(num_points[op]) * 0.999) - 1;
 				printf("%" STR(STATS_FIELD_WIDTH) "lld ", dataPoints[op][point_99_9pct]);
+				if (fp) {
+					fprintf(fp, "\"%s\": %lld,", get_ops_name(op), dataPoints[op][point_99_9pct]);
+				}
 			} else {
 				printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A");
 			}
 		}
 	}
 	printf("\n");
+	if (fp) {
+		fprintf(fp, "}}");
+	}
 
 	char command_remove[NAME_MAX] = { '\0' };
 	sprintf(command_remove, "rm -rf %s%d", TEMP_DATA_STORE, *pid_main);
@@ -2394,6 +2504,44 @@ int stats_process_main(mako_args_t* args,
 	if (args->verbose >= VERBOSE_DEFAULT)
 		print_stats_header(args, false, true, false);
 
+	FILE* fp = NULL;
+	if (args->json_output_path[0] != '\0') {
+		fp = fopen(args->json_output_path, "w");
+		fprintf(fp, "{\"makoArgs\": {");
+		fprintf(fp, "\"api_version\": %d,", args->api_version);
+		fprintf(fp, "\"json\": %d,", args->json);
+		fprintf(fp, "\"num_processes\": %d,", args->num_processes);
+		fprintf(fp, "\"num_threads\": %d,", args->num_threads);
+		fprintf(fp, "\"mode\": %d,", args->mode);
+		fprintf(fp, "\"rows\": %d,", args->rows);
+		fprintf(fp, "\"seconds\": %d,", args->seconds);
+		fprintf(fp, "\"iteration\": %d,", args->iteration);
+		fprintf(fp, "\"tpsmax\": %d,", args->tpsmax);
+		fprintf(fp, "\"tpsmin\": %d,", args->tpsmin);
+		fprintf(fp, "\"tpsinterval\": %d,", args->tpsinterval);
+		fprintf(fp, "\"tpschange\": %d,", args->tpschange);
+		fprintf(fp, "\"sampling\": %d,", args->sampling);
+		fprintf(fp, "\"key_length\": %d,", args->key_length);
+		fprintf(fp, "\"value_length\": %d,", args->value_length);
+		fprintf(fp, "\"commit_get\": %d,", args->commit_get);
+		fprintf(fp, "\"verbose\": %d,", args->verbose);
+		fprintf(fp, "\"cluster_file\": \"%s\",", args->cluster_files);
+		fprintf(fp, "\"log_group\": \"%s\",", args->log_group);
+		fprintf(fp, "\"prefixpadding\": %d,", args->prefixpadding);
+		fprintf(fp, "\"trace\": %d,", args->trace);
+		fprintf(fp, "\"tracepath\": \"%s\",", args->tracepath);
+		fprintf(fp, "\"traceformat\": %d,", args->traceformat);
+		fprintf(fp, "\"knobs\": \"%s\",", args->knobs);
+		fprintf(fp, "\"flatbuffers\": %d,", args->flatbuffers);
+		fprintf(fp, "\"txntrace\": %d,", args->txntrace);
+		fprintf(fp, "\"txntagging\": %d,", args->txntagging);
+		fprintf(fp, "\"txntagging_prefix\": \"%s\",", args->txntagging_prefix);
+		fprintf(fp, "\"streaming_mode\": %d,", args->streaming_mode);
+		fprintf(fp, "\"disable_ryw\": %d,", args->disable_ryw);
+		fprintf(fp, "\"json_output_path\": \"%s\",", args->json_output_path);
+		fprintf(fp, "},\"samples\": [");
+	}
+
 	clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
 	timer_prev.tv_sec = timer_start.tv_sec;
 	timer_prev.tv_nsec = timer_start.tv_nsec;
@@ -2435,19 +2583,28 @@ int stats_process_main(mako_args_t* args,
 			}
 
 			if (args->verbose >= VERBOSE_DEFAULT)
-				print_stats(args, stats, &timer_now, &timer_prev);
+				print_stats(args, stats, &timer_now, &timer_prev, fp);
 			timer_prev.tv_sec = timer_now.tv_sec;
 			timer_prev.tv_nsec = timer_now.tv_nsec;
 		}
 	}
 
+	if (fp) {
+		fprintf(fp, "],");
+	}
+
 	/* print report */
 	if (args->verbose >= VERBOSE_DEFAULT) {
 		clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now);
 		while (*stopcount < args->num_threads * args->num_processes) {
 			usleep(10000); /* 10ms */
 		}
-		print_report(args, stats, &timer_now, &timer_start, pid_main);
+		print_report(args, stats, &timer_now, &timer_start, pid_main, fp);
+	}
+
+	if (fp) {
+		fprintf(fp, "}");
+		fclose(fp);
 	}
 
 	return 0;
diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h
index 7d839f3221..66a8039dcf 100644
--- a/bindings/c/test/mako/mako.h
+++ b/bindings/c/test/mako/mako.h
@@ -81,8 +81,9 @@ enum Arguments {
 	ARG_TXNTAGGING,
 	ARG_TXNTAGGINGPREFIX,
 	ARG_STREAMING_MODE,
+	ARG_DISABLE_RYW,
 	ARG_CLIENT_THREADS_PER_VERSION,
-	ARG_DISABLE_RYW
+	ARG_JSON_REPORT
 };
 
 enum TPSChangeTypes { TPS_SIN, TPS_SQUARE, TPS_PULSE };
@@ -144,6 +145,7 @@ typedef struct {
 	FDBStreamingMode streaming_mode;
 	uint32_t client_threads_per_version;
 	int disable_ryw;
+	char json_output_path[PATH_MAX];
 } mako_args_t;
 
 /* shared memory */
diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake
index e89ef3ce82..176c88c9b6 100644
--- a/cmake/Jemalloc.cmake
+++ b/cmake/Jemalloc.cmake
@@ -35,7 +35,7 @@ else()
     BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
     "${JEMALLOC_DIR}/lib/libjemalloc.a"
     "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
-    CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx
+    CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
     BUILD_IN_SOURCE ON
     BUILD_COMMAND make
     INSTALL_DIR "${JEMALLOC_DIR}"
diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst
index cacfccf56a..6d2243f02f 100644
--- a/documentation/sphinx/source/command-line-interface.rst
+++ b/documentation/sphinx/source/command-line-interface.rst
@@ -164,6 +164,10 @@ If the ``failed`` keyword is specified, the address is marked as failed and adde
 
 For more information on excluding servers, see :ref:`removing-machines-from-a-cluster`.
 
+Warning about potential dataloss ``failed`` option: if a server is the last one in some team(s), excluding it with ``failed`` will lose all data in the team(s), and hence ``failed`` should only be set when the server(s) have permanently failed.
+
+In the case all servers of a team have failed permanently, excluding all the servers will clean up the corresponding keyrange, and fix the invalid metadata. The keyrange will be assigned to a new team as an empty shard.
+
 exit
 ----
 
diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst
index b54258baca..735f3714c1 100644
--- a/documentation/sphinx/source/release-notes/release-notes-630.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-630.rst
@@ -2,6 +2,13 @@
 Release Notes
 #############
 
+6.3.22
+======
+* Added histograms to client GRV batcher. `(PR #5760) <https://github.com/apple/foundationdb/pull/5760>`_
+* Added FastAlloc memory utilization trace. `(PR #5759) <https://github.com/apple/foundationdb/pull/5759>`_
+* Added locality cache size to TransactionMetrics. `(PR #5771) <https://github.com/apple/foundationdb/pull/5771>`_
+* Added a new feature that allows FDB to failover to remote DC when the primary is experiencing massive grey failure. This feature is turned off by default. `(PR #5774) <https://github.com/apple/foundationdb/pull/5774>`_
+
 6.3.21
 ======
 * Added a ThreadID field to all trace events for the purpose of multi-threaded client debugging. `(PR #5665) <https://github.com/apple/foundationdb/pull/5665>`_
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index f8833c3d78..61a734ecb6 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -37,6 +37,7 @@
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/IKnobCollection.h"
 #include "fdbclient/RunTransaction.actor.h"
@@ -3095,7 +3096,7 @@ Optional<Database> connectToCluster(std::string const& clusterFile,
 	} catch (Error& e) {
 		if (!quiet) {
 			fprintf(stderr, "ERROR: %s\n", e.what());
-			fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str());
+			fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str());
 		}
 		return db;
 	}
diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp
index 8580e6b063..9296f9ef88 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.actor.cpp
@@ -393,5 +393,11 @@ CommandFactory excludeFactory(
         "command returns \nimmediately without checking if the exclusions have completed successfully.\n"
         "If 'FORCE' is set, the command does not perform safety checks before excluding.\n"
         "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n"
-        "for data movement to finish and the server cannot be included again."));
+        "for data movement to finish and the server cannot be included again."
+        "\n\nWARNING of potential dataloss\n:"
+        "If a to-be-excluded server is the last server of some team(s), and 'failed' is set, the data in the team(s) "
+        "will be lost. 'failed' should be set only if the server(s) have permanently failed."
+        "In the case all servers of a team have failed permanently and dataloss has been a fact, excluding all the "
+        "servers will clean up the corresponding keyrange, and fix the invalid metadata. The keyrange will be "
+        "assigned to a new team as an empty shard."));
 } // namespace fdb_cli
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 9aaa8a8311..31ac1a4418 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "boost/lexical_cast.hpp"
+#include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IClientApi.h"
@@ -1035,8 +1036,8 @@ ACTOR Future<bool> exclude(Database db,
 			    locality.c_str());
 		}
 
+		ClusterConnectionString ccs = wait(ccf->getStoredConnectionString());
 		bool foundCoordinator = false;
-		auto ccs = ClusterConnectionFile(ccf->getFilename()).getConnectionString();
 		for (const auto& c : ccs.coordinators()) {
 			if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
 			    std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
@@ -1586,12 +1587,12 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	try {
 		localDb = Database::createDatabase(ccf, opt.api_version, IsInternal::False);
 		if (!opt.exec.present()) {
-			printf("Using cluster file `%s'.\n", ccf->getFilename().c_str());
+			printf("Using cluster file `%s'.\n", ccf->getLocation().c_str());
 		}
 		db = API->createDatabase(opt.clusterFile.c_str());
 	} catch (Error& e) {
 		fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code());
-		printf("Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str());
+		printf("Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str());
 		return 1;
 	}
 
@@ -1602,7 +1603,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 		    .detail("Version", FDB_VT_VERSION)
 		    .detail("PackageName", FDB_VT_PACKAGE_NAME)
 		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
-		    .detail("ClusterFile", ccf->getFilename().c_str())
+		    .detail("ClusterFile", ccf->toString())
 		    .detail("ConnectionString", ccf->getConnectionString().toString())
 		    .setMaxFieldLength(10000)
 		    .detail("CommandLine", opt.commandLine)
diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt
index 8a7b69dcd5..11690e8d02 100644
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@@ -31,6 +31,12 @@ set(FDBCLIENT_SRCS
   ClientKnobs.h
   ClientLogEvents.h
   ClientWorkerInterface.h
+  ClusterConnectionFile.actor.cpp
+  ClusterConnectionFile.h
+  ClusterConnectionKey.actor.cpp
+  ClusterConnectionKey.actor.h
+  ClusterConnectionMemoryRecord.actor.cpp
+  ClusterConnectionMemoryRecord.h
   ClusterInterface.h
   CommitProxyInterface.h
   CommitTransaction.h
diff --git a/fdbclient/ClusterConnectionFile.actor.cpp b/fdbclient/ClusterConnectionFile.actor.cpp
new file mode 100644
index 0000000000..82431de62e
--- /dev/null
+++ b/fdbclient/ClusterConnectionFile.actor.cpp
@@ -0,0 +1,178 @@
+/*
+ * ClusterConnectionFile.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClusterConnectionFile.h"
+#include "fdbclient/MonitorLeader.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// Loads and parses the file at 'filename', throwing errors if the file cannot be read or the format is invalid.
+ClusterConnectionFile::ClusterConnectionFile(std::string const& filename)
+  : IClusterConnectionRecord(ConnectionStringNeedsPersisted::False) {
+	if (!fileExists(filename)) {
+		throw no_cluster_file_found();
+	}
+
+	cs = ClusterConnectionString(readFileBytes(filename, MAX_CLUSTER_FILE_BYTES));
+	this->filename = filename;
+}
+
+// Creates a cluster file with a given connection string and saves it to the specified file.
+ClusterConnectionFile::ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents)
+  : IClusterConnectionRecord(ConnectionStringNeedsPersisted::True) {
+	this->filename = filename;
+	cs = contents;
+}
+
+// Returns the connection string currently held in this object. This may not match the string on disk if it hasn't
+// been persisted or if the file has been modified externally.
+ClusterConnectionString const& ClusterConnectionFile::getConnectionString() const {
+	return cs;
+}
+
+// Sets the connections string held by this object and persists it.
+Future<Void> ClusterConnectionFile::setConnectionString(ClusterConnectionString const& conn) {
+	ASSERT(filename.size());
+	cs = conn;
+	return success(persist());
+}
+
+// Get the connection string stored in the file.
+Future<ClusterConnectionString> ClusterConnectionFile::getStoredConnectionString() {
+	try {
+		return ClusterConnectionFile(filename).cs;
+	} catch (Error& e) {
+		return e;
+	}
+}
+
+// Checks whether the connection string in the file matches the connection string stored in memory. The cluster
+// string stored in the file is returned via the reference parameter connectionString.
+Future<bool> ClusterConnectionFile::upToDate(ClusterConnectionString& fileConnectionString) {
+	try {
+		// the cluster file hasn't been created yet so there's nothing to check
+		if (needsToBePersisted())
+			return true;
+
+		ClusterConnectionFile temp(filename);
+		fileConnectionString = temp.getConnectionString();
+		return fileConnectionString.toString() == cs.toString();
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename);
+		return false; // Swallow the error and report that the file is out of date
+	}
+}
+
+// Returns the specified path of the cluster file.
+std::string ClusterConnectionFile::getLocation() const {
+	return filename;
+}
+
+// Creates a copy of this object with a modified connection string but that isn't persisted.
+Reference<IClusterConnectionRecord> ClusterConnectionFile::makeIntermediateRecord(
+    ClusterConnectionString const& connectionString) const {
+	return makeReference<ClusterConnectionFile>(filename, connectionString);
+}
+
+// Returns a string representation of this cluster connection record. This will include the type of record and the
+// filename of the cluster file.
+std::string ClusterConnectionFile::toString() const {
+	// This is a fairly naive attempt to generate a URI-like string. It will not account for characters like spaces, it
+	// may use backslashes in windows paths, etc.
+	// SOMEDAY: we should encode this string as a proper URI.
+	return "file://" + filename;
+}
+
+// returns <resolved name, was default file>
+std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
+	if (filename.length())
+		return std::make_pair(filename, false);
+
+	std::string f;
+	bool isDefaultFile = true;
+	if (platform::getEnvironmentVar(CLUSTER_FILE_ENV_VAR_NAME, f)) {
+		// If this is set but points to a file that does not
+		// exist, we will not fallback to any other methods
+		isDefaultFile = false;
+	} else if (fileExists("fdb.cluster"))
+		f = "fdb.cluster";
+	else
+		f = platform::getDefaultClusterFilePath();
+
+	return std::make_pair(f, isDefaultFile);
+}
+
+// get a human readable error message describing the error returned from the constructor
+std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> const& resolvedClusterFile,
+                                                  Error const& e) {
+	bool isDefault = resolvedClusterFile.second;
+	if (e.code() == error_code_connection_string_invalid) {
+		return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
+	} else if (e.code() == error_code_no_cluster_file_found) {
+		if (isDefault)
+			return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s",
+			              platform::getDefaultClusterFilePath().c_str(),
+			              CLUSTER_FILE_ENV_VAR_NAME,
+			              e.code(),
+			              e.what());
+		else
+			return format(
+			    "Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
+	} else {
+		return format(
+		    "Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
+	}
+}
+
+// Writes the connection string to the cluster file
+Future<bool> ClusterConnectionFile::persist() {
+	setPersisted();
+
+	if (filename.size()) {
+		try {
+			atomicReplace(filename,
+			              "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
+			                  cs.toString().append("\n"));
+
+			Future<bool> isUpToDate = IClusterConnectionRecord::upToDate();
+
+			// The implementation of upToDate in this class is synchronous
+			ASSERT(isUpToDate.isReady());
+
+			if (!isUpToDate.get()) {
+				// This should only happen in rare scenarios where multiple processes are updating the same file to
+				// different values simultaneously In that case, we don't have any guarantees about which file will
+				// ultimately be written
+				TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace")
+				    .detail("Filename", filename)
+				    .detail("ConnectionString", cs.toString());
+				return false;
+			}
+
+			return true;
+		} catch (Error& e) {
+			TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile")
+			    .error(e)
+			    .detail("Filename", filename)
+			    .detail("ConnectionString", cs.toString());
+		}
+	}
+
+	return false;
+}
diff --git a/fdbclient/ClusterConnectionFile.h b/fdbclient/ClusterConnectionFile.h
new file mode 100644
index 0000000000..b12f6baf9e
--- /dev/null
+++ b/fdbclient/ClusterConnectionFile.h
@@ -0,0 +1,81 @@
+/*
+ * ClusterConnectionFile.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef FDBCLIENT_CLUSTERCONNECTIONFILE_H
+#define FDBCLIENT_CLUSTERCONNECTIONFILE_H
+
+#include "fdbclient/CoordinationInterface.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// An implementation of IClusterConnectionRecord backed by a file.
+class ClusterConnectionFile : public IClusterConnectionRecord, ReferenceCounted<ClusterConnectionFile>, NonCopyable {
+public:
+	// Loads and parses the file at 'filename', throwing errors if the file cannot be read or the format is invalid.
+	explicit ClusterConnectionFile(std::string const& filename);
+
+	// Creates a cluster file with a given connection string and saves it to the specified file.
+	explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents);
+
+	// Returns the connection string currently held in this object. This may not match the string on disk if it hasn't
+	// been persisted or if the file has been modified externally.
+	ClusterConnectionString const& getConnectionString() const override;
+
+	// Sets the connections string held by this object and persists it.
+	Future<Void> setConnectionString(ClusterConnectionString const&) override;
+
+	// Get the connection string stored in the file.
+	Future<ClusterConnectionString> getStoredConnectionString() override;
+
+	// Checks whether the connection string in the file matches the connection string stored in memory. The cluster
+	// string stored in the file is returned via the reference parameter connectionString.
+	Future<bool> upToDate(ClusterConnectionString& fileConnectionString) override;
+
+	// Returns the specified path of the cluster file.
+	std::string getLocation() const override;
+
+	// Creates a copy of this object with a modified connection string but that isn't persisted.
+	Reference<IClusterConnectionRecord> makeIntermediateRecord(
+	    ClusterConnectionString const& connectionString) const override;
+
+	// Returns a string representation of this cluster connection record. This will include the type of record and the
+	// filename of the cluster file.
+	std::string toString() const override;
+
+	void addref() override { ReferenceCounted<ClusterConnectionFile>::addref(); }
+	void delref() override { ReferenceCounted<ClusterConnectionFile>::delref(); }
+
+	// returns <resolved name, was default file>
+	static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);
+
+	// get a human readable error message describing the error returned from the constructor
+	static std::string getErrorString(std::pair<std::string, bool> const& resolvedFile, Error const& e);
+
+protected:
+	// Writes the connection string to the cluster file
+	Future<bool> persist() override;
+
+private:
+	ClusterConnectionString cs;
+	std::string filename;
+};
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbclient/ClusterConnectionKey.actor.cpp b/fdbclient/ClusterConnectionKey.actor.cpp
new file mode 100644
index 0000000000..0a29d9d075
--- /dev/null
+++ b/fdbclient/ClusterConnectionKey.actor.cpp
@@ -0,0 +1,172 @@
+/*
+ * ClusterConnectionKey.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClusterConnectionKey.actor.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// Creates a cluster connection record with a given connection string and saves it to the specified key. Needs to be
+// persisted should be set to true unless this ClusterConnectionKey is being created with the value read from the
+// key.
+ClusterConnectionKey::ClusterConnectionKey(Database db,
+                                           Key connectionStringKey,
+                                           ClusterConnectionString const& contents,
+                                           ConnectionStringNeedsPersisted needsToBePersisted)
+  : IClusterConnectionRecord(needsToBePersisted), db(db), cs(contents), connectionStringKey(connectionStringKey) {
+	if (!needsToBePersisted) {
+		lastPersistedConnectionString = ValueRef(contents.toString());
+	}
+}
+
+// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
+// format is invalid.
+ACTOR Future<Reference<ClusterConnectionKey>> ClusterConnectionKey::loadClusterConnectionKey(Database db,
+                                                                                             Key connectionStringKey) {
+	state Transaction tr(db);
+	loop {
+		try {
+			Optional<Value> v = wait(tr.get(connectionStringKey));
+			if (!v.present()) {
+				throw connection_string_invalid();
+			}
+			return makeReference<ClusterConnectionKey>(db,
+			                                           connectionStringKey,
+			                                           ClusterConnectionString(v.get().toString()),
+			                                           ConnectionStringNeedsPersisted::False);
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+// Returns the connection string currently held in this object. This may not match the string in the database if it
+// hasn't been persisted or if the key has been modified externally.
+ClusterConnectionString const& ClusterConnectionKey::getConnectionString() const {
+	return cs;
+}
+
+// Sets the connections string held by this object and persists it.
+Future<Void> ClusterConnectionKey::setConnectionString(ClusterConnectionString const& connectionString) {
+	cs = connectionString;
+	return success(persist());
+}
+
+// Get the connection string stored in the database.
+ACTOR Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionStringImpl(
+    Reference<ClusterConnectionKey> self) {
+	Reference<ClusterConnectionKey> cck =
+	    wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
+	return cck->cs;
+}
+
+Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionString() {
+	return getStoredConnectionStringImpl(Reference<ClusterConnectionKey>::addRef(this));
+}
+
+ACTOR Future<bool> ClusterConnectionKey::upToDateImpl(Reference<ClusterConnectionKey> self,
+                                                      ClusterConnectionString* connectionString) {
+	try {
+		// the cluster file hasn't been created yet so there's nothing to check
+		if (self->needsToBePersisted())
+			return true;
+
+		Reference<ClusterConnectionKey> temp =
+		    wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
+		*connectionString = temp->getConnectionString();
+		return connectionString->toString() == self->cs.toString();
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "ClusterKeyError").error(e).detail("Key", self->connectionStringKey);
+		return false; // Swallow the error and report that the file is out of date
+	}
+}
+
+// Checks whether the connection string in the database matches the connection string stored in memory. The cluster
+// string stored in the database is returned via the reference parameter connectionString.
+Future<bool> ClusterConnectionKey::upToDate(ClusterConnectionString& connectionString) {
+	return upToDateImpl(Reference<ClusterConnectionKey>::addRef(this), &connectionString);
+}
+
+// Returns the key where the connection string is stored.
+std::string ClusterConnectionKey::getLocation() const {
+	return printable(connectionStringKey);
+}
+
+// Creates a copy of this object with a modified connection string but that isn't persisted.
+Reference<IClusterConnectionRecord> ClusterConnectionKey::makeIntermediateRecord(
+    ClusterConnectionString const& connectionString) const {
+	return makeReference<ClusterConnectionKey>(db, connectionStringKey, connectionString);
+}
+
+// Returns a string representation of this cluster connection record. This will include the type of record and the
+// key where the record is stored.
+std::string ClusterConnectionKey::toString() const {
+	return "fdbkey://" + printable(connectionStringKey);
+}
+
+ACTOR Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnectionKey> self) {
+	self->setPersisted();
+	state Value newConnectionString = ValueRef(self->cs.toString());
+
+	try {
+		state Transaction tr(self->db);
+		loop {
+			try {
+				Optional<Value> existingConnectionString = wait(tr.get(self->connectionStringKey));
+				// Someone has already updated the connection string to what we want
+				if (existingConnectionString.present() && existingConnectionString.get() == newConnectionString) {
+					self->lastPersistedConnectionString = newConnectionString;
+					return true;
+				}
+				// Someone has updated the connection string to something we didn't expect, in which case we leave it
+				// alone. It's possible this could result in the stored string getting stuck if the connection string
+				// changes twice and only the first change is recorded. If the process that wrote the first change dies
+				// and no other process attempts to write the intermediate state, then only a newly opened connection
+				// key would be able to update the state.
+				else if (existingConnectionString.present() &&
+				         existingConnectionString != self->lastPersistedConnectionString) {
+					TraceEvent(SevWarnAlways, "UnableToChangeConnectionKeyDueToMismatch")
+					    .detail("ConnectionKey", self->connectionStringKey)
+					    .detail("NewConnectionString", newConnectionString)
+					    .detail("ExpectedStoredConnectionString", self->lastPersistedConnectionString)
+					    .detail("ActualStoredConnectionString", existingConnectionString);
+					return false;
+				}
+				tr.set(self->connectionStringKey, newConnectionString);
+				wait(tr.commit());
+
+				self->lastPersistedConnectionString = newConnectionString;
+				return true;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "UnableToChangeConnectionKey")
+		    .error(e)
+		    .detail("ConnectionKey", self->connectionStringKey)
+		    .detail("ConnectionString", self->cs.toString());
+	}
+
+	return false;
+};
+
+// Writes the connection string to the database
+Future<bool> ClusterConnectionKey::persist() {
+	return persistImpl(Reference<ClusterConnectionKey>::addRef(this));
+}
\ No newline at end of file
diff --git a/fdbclient/ClusterConnectionKey.actor.h b/fdbclient/ClusterConnectionKey.actor.h
new file mode 100644
index 0000000000..e60ebf185a
--- /dev/null
+++ b/fdbclient/ClusterConnectionKey.actor.h
@@ -0,0 +1,97 @@
+/*
+ * ClusterConnectionKey.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
+// version.
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_G_H)
+#define FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_G_H
+#include "fdbclient/ClusterConnectionKey.actor.g.h"
+#elif !defined(FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_H)
+#define FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_H
+
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// An implementation of IClusterConnectionRecord backed by a key in a FoundationDB database.
+class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<ClusterConnectionKey>, NonCopyable {
+public:
+	// Creates a cluster connection record with a given connection string and saves it to the specified key. Needs to be
+	// persisted should be set to true unless this ClusterConnectionKey is being created with the value read from the
+	// key.
+	ClusterConnectionKey(Database db,
+	                     Key connectionStringKey,
+	                     ClusterConnectionString const& contents,
+	                     ConnectionStringNeedsPersisted needsToBePersisted = ConnectionStringNeedsPersisted::True);
+
+	// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
+	// format is invalid.
+	ACTOR static Future<Reference<ClusterConnectionKey>> loadClusterConnectionKey(Database db, Key connectionStringKey);
+
+	// Returns the connection string currently held in this object. This may not match the string in the database if it
+	// hasn't been persisted or if the key has been modified externally.
+	ClusterConnectionString const& getConnectionString() const override;
+
+	// Sets the connections string held by this object and persists it.
+	Future<Void> setConnectionString(ClusterConnectionString const&) override;
+
+	// Get the connection string stored in the database.
+	Future<ClusterConnectionString> getStoredConnectionString() override;
+
+	// Checks whether the connection string in the database matches the connection string stored in memory. The cluster
+	// string stored in the database is returned via the reference parameter connectionString.
+	Future<bool> upToDate(ClusterConnectionString& connectionString) override;
+
+	// Returns the key where the connection string is stored.
+	std::string getLocation() const override;
+
+	// Creates a copy of this object with a modified connection string but that isn't persisted.
+	Reference<IClusterConnectionRecord> makeIntermediateRecord(
+	    ClusterConnectionString const& connectionString) const override;
+
+	// Returns a string representation of this cluster connection record. This will include the type of record and the
+	// key where the record is stored.
+	std::string toString() const override;
+
+	void addref() override { ReferenceCounted<ClusterConnectionKey>::addref(); }
+	void delref() override { ReferenceCounted<ClusterConnectionKey>::delref(); }
+
+protected:
+	// Writes the connection string to the database
+	Future<bool> persist() override;
+
+private:
+	ACTOR static Future<ClusterConnectionString> getStoredConnectionStringImpl(Reference<ClusterConnectionKey> self);
+	ACTOR static Future<bool> upToDateImpl(Reference<ClusterConnectionKey> self,
+	                                       ClusterConnectionString* connectionString);
+	ACTOR static Future<bool> persistImpl(Reference<ClusterConnectionKey> self);
+
+	// The database where the connection key is stored. Note that this does not need to be the same database as the one
+	// that the connection string would connect to.
+	Database db;
+	ClusterConnectionString cs;
+	Key connectionStringKey;
+	Optional<Value> lastPersistedConnectionString;
+};
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbclient/ClusterConnectionMemoryRecord.actor.cpp b/fdbclient/ClusterConnectionMemoryRecord.actor.cpp
new file mode 100644
index 0000000000..b3afc0ef96
--- /dev/null
+++ b/fdbclient/ClusterConnectionMemoryRecord.actor.cpp
@@ -0,0 +1,68 @@
+/*
+ * ClusterConnectionMemoryRecord.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbclient/MonitorLeader.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+// Returns the connection string currently held in this object.
+ClusterConnectionString const& ClusterConnectionMemoryRecord::getConnectionString() const {
+	return cs;
+}
+
+// Sets the connections string held by this object.
+Future<Void> ClusterConnectionMemoryRecord::setConnectionString(ClusterConnectionString const& conn) {
+	cs = conn;
+	return Void();
+}
+
+// Returns the connection string currently held in this object (there is no persistent storage).
+Future<ClusterConnectionString> ClusterConnectionMemoryRecord::getStoredConnectionString() {
+	return cs;
+}
+
+// Because the memory record is not persisted, it is always up to date and this returns true. The connection string
+// is returned via the reference parameter connectionString.
+Future<bool> ClusterConnectionMemoryRecord::upToDate(ClusterConnectionString& fileConnectionString) {
+	fileConnectionString = cs;
+	return true;
+}
+
+// Returns the ID of the memory record.
+std::string ClusterConnectionMemoryRecord::getLocation() const {
+	return id.toString();
+}
+
+// Returns a copy of this object with a modified connection string.
+Reference<IClusterConnectionRecord> ClusterConnectionMemoryRecord::makeIntermediateRecord(
+    ClusterConnectionString const& connectionString) const {
+	return makeReference<ClusterConnectionMemoryRecord>(connectionString);
+}
+
+// Returns a string representation of this cluster connection record. This will include the type and id of the
+// record.
+std::string ClusterConnectionMemoryRecord::toString() const {
+	return "memory://" + id.toString();
+}
+
+// This is a no-op for memory records. Returns true to indicate success.
+Future<bool> ClusterConnectionMemoryRecord::persist() {
+	return true;
+}
\ No newline at end of file
diff --git a/fdbclient/ClusterConnectionMemoryRecord.h b/fdbclient/ClusterConnectionMemoryRecord.h
new file mode 100644
index 0000000000..2d30855f01
--- /dev/null
+++ b/fdbclient/ClusterConnectionMemoryRecord.h
@@ -0,0 +1,74 @@
+/*
+ * ClusterConnectionMemoryRecord.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef FDBCLIENT_CLUSTERCONNECTIONMEMORYRECORD_H
+#define FDBCLIENT_CLUSTERCONNECTIONMEMORYRECORD_H
+
+#include "fdbclient/CoordinationInterface.h"
+
+// An implementation of IClusterConnectionRecord that is stored in memory only and not persisted.
+class ClusterConnectionMemoryRecord : public IClusterConnectionRecord,
+                                      ReferenceCounted<ClusterConnectionMemoryRecord>,
+                                      NonCopyable {
+public:
+	// Creates a cluster file with a given connection string.
+	explicit ClusterConnectionMemoryRecord(ClusterConnectionString const& cs)
+	  : IClusterConnectionRecord(ConnectionStringNeedsPersisted::False), id(deterministicRandom()->randomUniqueID()),
+	    cs(cs) {}
+
+	// Returns the connection string currently held in this object.
+	ClusterConnectionString const& getConnectionString() const override;
+
+	// Sets the connections string held by this object.
+	Future<Void> setConnectionString(ClusterConnectionString const&) override;
+
+	// Returns the connection string currently held in this object (there is no persistent storage).
+	Future<ClusterConnectionString> getStoredConnectionString() override;
+
+	// Because the memory record is not persisted, it is always up to date and this returns true. The connection string
+	// is returned via the reference parameter connectionString.
+	Future<bool> upToDate(ClusterConnectionString& fileConnectionString) override;
+
+	// Returns a location string for the memory record that includes its ID.
+	std::string getLocation() const override;
+
+	// Returns a copy of this object with a modified connection string.
+	Reference<IClusterConnectionRecord> makeIntermediateRecord(
+	    ClusterConnectionString const& connectionString) const override;
+
+	// Returns a string representation of this cluster connection record. This will include the type and id of the
+	// record.
+	std::string toString() const override;
+
+	void addref() override { ReferenceCounted<ClusterConnectionMemoryRecord>::addref(); }
+	void delref() override { ReferenceCounted<ClusterConnectionMemoryRecord>::delref(); }
+
+protected:
+	// This is a no-op for memory records. Returns true to indicate success.
+	Future<bool> persist() override;
+
+private:
+	// A unique ID for the record
+	UID id;
+	ClusterConnectionString cs;
+};
+
+#endif
\ No newline at end of file
diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h
index 2ef30ce4f9..bdd4a1c4d6 100644
--- a/fdbclient/CommitTransaction.h
+++ b/fdbclient/CommitTransaction.h
@@ -219,8 +219,8 @@ struct CommitTransactionRef {
 
 struct MutationsAndVersionRef {
 	VectorRef<MutationRef> mutations;
-	Version version;
-	Version knownCommittedVersion;
+	Version version = invalidVersion;
+	Version knownCommittedVersion = invalidVersion;
 
 	MutationsAndVersionRef() {}
 	explicit MutationsAndVersionRef(Version version, Version knownCommittedVersion)
diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index c1873a0b92..b5e3b50bad 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -45,11 +45,23 @@ struct ClientLeaderRegInterface {
 	}
 };
 
+// A string containing the information necessary to connect to a cluster.
+//
+// The format of the connection string is: description:id@[addrs]+
+// The description and id together are called the "key"
+//
+// The following is enforced about the format of the file:
+//  - The key must contain one (and only one) ':' character
+//  - The description contains only allowed characters (a-z, A-Z, 0-9, _)
+//  - The ID contains only allowed characters (a-z, A-Z, 0-9)
+//  - At least one address is specified
+//  - There is no address present more than once
 class ClusterConnectionString {
 public:
 	ClusterConnectionString() {}
 	ClusterConnectionString(std::string const& connectionString);
 	ClusterConnectionString(std::vector<NetworkAddress>, Key);
+
 	std::vector<NetworkAddress> const& coordinators() const { return coord; }
 	Key clusterKey() const { return key; }
 	Key clusterKeyName() const {
@@ -65,45 +77,70 @@ private:
 	Key key, keyDesc;
 };
 
-class ClusterConnectionFile : NonCopyable, public ReferenceCounted<ClusterConnectionFile> {
+FDB_DECLARE_BOOLEAN_PARAM(ConnectionStringNeedsPersisted);
+
+// A record that stores the connection string used to connect to a cluster. This record can be updated when a cluster
+// notifies a connected party that the connection string has changed.
+//
+// The typically used cluster connection record is a cluster file (implemented in ClusterConnectionFile). This interface
+// provides an abstraction over the cluster file so that we can persist the connection string in other locations or have
+// one that is only stored in memory.
+class IClusterConnectionRecord {
 public:
-	ClusterConnectionFile() {}
-	// Loads and parses the file at 'path', throwing errors if the file cannot be read or the format is invalid.
-	//
-	// The format of the file is: description:id@[addrs]+
-	//  The description and id together are called the "key"
-	//
-	// The following is enforced about the format of the file:
-	//  - The key must contain one (and only one) ':' character
-	//  - The description contains only allowed characters (a-z, A-Z, 0-9, _)
-	//  - The ID contains only allowed characters (a-z, A-Z, 0-9)
-	//  - At least one address is specified
-	//  - There is no address present more than once
-	explicit ClusterConnectionFile(std::string const& path);
-	explicit ClusterConnectionFile(ClusterConnectionString const& cs) : cs(cs), setConn(false) {}
-	explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents);
+	IClusterConnectionRecord(ConnectionStringNeedsPersisted connectionStringNeedsPersisted)
+	  : connectionStringNeedsPersisted(connectionStringNeedsPersisted) {}
+	virtual ~IClusterConnectionRecord() {}
 
-	// returns <resolved name, was default file>
-	static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);
-	// get a human readable error message describing the error returned from the constructor
-	static std::string getErrorString(std::pair<std::string, bool> const& resolvedFile, Error const& e);
+	// Returns the connection string currently held in this object. This may not match the stored record if it hasn't
+	// been persisted or if the persistent storage for the record has been modified externally.
+	virtual ClusterConnectionString const& getConnectionString() const = 0;
 
-	ClusterConnectionString const& getConnectionString() const;
-	bool writeFile();
-	void setConnectionString(ClusterConnectionString const&);
-	std::string const& getFilename() const {
-		ASSERT(filename.size());
-		return filename;
-	}
-	bool canGetFilename() const { return filename.size() != 0; }
-	bool fileContentsUpToDate() const;
-	bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
+	// Sets the connections string held by this object and persists it.
+	virtual Future<Void> setConnectionString(ClusterConnectionString const&) = 0;
+
+	// If this record is backed by persistent storage, get the connection string from that storage. Otherwise, return
+	// the connection string stored in memory.
+	virtual Future<ClusterConnectionString> getStoredConnectionString() = 0;
+
+	// Checks whether the connection string in persisten storage matches the connection string stored in memory.
+	Future<bool> upToDate();
+
+	// Checks whether the connection string in persisten storage matches the connection string stored in memory. The
+	// cluster string stored in persistent storage is returned via the reference parameter connectionString.
+	virtual Future<bool> upToDate(ClusterConnectionString& connectionString) = 0;
+
+	// Returns a string representing the location of the cluster record. For example, this could be the filename or key
+	// that stores the connection string.
+	virtual std::string getLocation() const = 0;
+
+	// Creates a copy of this object with a modified connection string but that isn't persisted.
+	virtual Reference<IClusterConnectionRecord> makeIntermediateRecord(
+	    ClusterConnectionString const& connectionString) const = 0;
+
+	// Returns a string representation of this cluster connection record. This will include the type and location of the
+	// record.
+	virtual std::string toString() const = 0;
+
+	// Signals to the connection record that it was successfully used to connect to a cluster.
 	void notifyConnected();
 
+	virtual void addref() = 0;
+	virtual void delref() = 0;
+
+protected:
+	// Writes the connection string to the backing persistent storage, if applicable.
+	virtual Future<bool> persist() = 0;
+
+	// Returns whether the connection record contains a connection string that needs to be persisted upon connection.
+	bool needsToBePersisted() const;
+
+	// Clears the flag needs persisted flag.
+	void setPersisted();
+
 private:
-	ClusterConnectionString cs;
-	std::string filename;
-	bool setConn;
+	// A flag that indicates whether this connection record needs to be persisted when it succesfully establishes a
+	// connection.
+	bool connectionStringNeedsPersisted;
 };
 
 struct LeaderInfo {
@@ -199,9 +236,9 @@ class ClientCoordinators {
 public:
 	std::vector<ClientLeaderRegInterface> clientLeaderServers;
 	Key clusterKey;
-	Reference<ClusterConnectionFile> ccf;
+	Reference<IClusterConnectionRecord> ccr;
 
-	explicit ClientCoordinators(Reference<ClusterConnectionFile> ccf);
+	explicit ClientCoordinators(Reference<IClusterConnectionRecord> ccr);
 	explicit ClientCoordinators(Key clusterKey, std::vector<NetworkAddress> coordinators);
 	ClientCoordinators() {}
 };
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 3f4425c3aa..837d4ec793 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -167,7 +167,7 @@ public:
 
 	// Constructs a new copy of this DatabaseContext from the parameters of this DatabaseContext
 	Database clone() const {
-		return Database(new DatabaseContext(connectionFile,
+		return Database(new DatabaseContext(connectionRecord,
 		                                    clientInfo,
 		                                    coordinator,
 		                                    clientInfoMonitor,
@@ -231,16 +231,16 @@ public:
 
 	Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
 	                            // leader. The cluster file therefore is valid, but the database might be unavailable.
-	Reference<ClusterConnectionFile> getConnectionFile();
+	Reference<IClusterConnectionRecord> getConnectionRecord();
 
 	// Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
 	//
 	// Meant to be used as part of a 'hot standby' solution to switch to the standby. A correct switch will involve
 	// advancing the version on the new cluster sufficiently far that any transaction begun with a read version from the
 	// old cluster will fail to commit. Assuming the above version-advancing is done properly, a call to
-	// switchConnectionFile guarantees that any read with a version from the old cluster will not be attempted on the
+	// switchConnectionRecord guarantees that any read with a version from the old cluster will not be attempted on the
 	// new cluster.
-	Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby);
+	Future<Void> switchConnectionRecord(Reference<IClusterConnectionRecord> standby);
 	Future<Void> connectionFileChanged();
 	IsSwitchable switchable{ false };
 
@@ -268,7 +268,7 @@ public:
 	                                    Optional<Version> end);
 
 	// private:
-	explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
+	explicit DatabaseContext(Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connectionRecord,
 	                         Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
 	                         Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
 	                         Future<Void> clientInfoMonitor,
@@ -285,7 +285,7 @@ public:
 	void expireThrottles();
 
 	// Key DB-specific information
-	Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile;
+	Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connectionRecord;
 	AsyncTrigger proxiesChangeTrigger;
 	Future<Void> monitorProxiesInfoChange;
 	Future<Void> monitorTssInfoChange;
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index fa9525154e..53e9dba1a8 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -24,6 +24,7 @@
 
 #include "fdbclient/Knobs.h"
 #include "flow/Arena.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
@@ -778,15 +779,18 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
 
 	state ClusterConnectionString old(currentKey.get().toString());
-	if (tr->getDatabase()->getConnectionFile() &&
+	if (tr->getDatabase()->getConnectionRecord() &&
 	    old.clusterKeyName().toString() !=
-	        tr->getDatabase()->getConnectionFile()->getConnectionString().clusterKeyName())
+	        tr->getDatabase()->getConnectionRecord()->getConnectionString().clusterKeyName())
 		return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
 
 	state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 	if (!desiredCoordinators->size()) {
 		std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
-		    tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
+		    tr,
+		    old.coordinators(),
+		    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
+		    result));
 		*desiredCoordinators = _desiredCoordinators;
 	}
 
@@ -821,7 +825,7 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 	}
 
 	std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-	ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
+	ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(conn)));
 
 	leaderServers.reserve(coord.clientLeaderServers.size());
 	for (int i = 0; i < coord.clientLeaderServers.size(); i++)
@@ -854,14 +858,17 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
 
 			state ClusterConnectionString old(currentKey.get().toString());
-			if (cx->getConnectionFile() &&
-			    old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName())
+			if (cx->getConnectionRecord() &&
+			    old.clusterKeyName().toString() != cx->getConnectionRecord()->getConnectionString().clusterKeyName())
 				return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
 
 			state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 			if (!desiredCoordinators.size()) {
 				std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
-				    &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
+				    &tr,
+				    old.coordinators(),
+				    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
+				    result));
 				desiredCoordinators = _desiredCoordinators;
 			}
 
@@ -907,7 +914,8 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 			TEST(old.clusterKeyName() == conn.clusterKeyName()); // Quorum change with unchanged name
 
 			state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-			state ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
+			state ClientCoordinators coord(
+			    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(conn)));
 			// check if allowed to modify the cluster descriptor
 			if (!change->getDesiredClusterKeyName().empty()) {
 				CheckDescriptorMutableReply mutabilityReply =
@@ -942,7 +950,7 @@ struct SpecifiedQuorumChange final : IQuorumChange {
 	explicit SpecifiedQuorumChange(std::vector<NetworkAddress> const& desired) : desired(desired) {}
 	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
 	                                                           std::vector<NetworkAddress> oldCoordinators,
-	                                                           Reference<ClusterConnectionFile>,
+	                                                           Reference<IClusterConnectionRecord>,
 	                                                           CoordinatorsResult&) override {
 		return desired;
 	}
@@ -954,7 +962,7 @@ Reference<IQuorumChange> specifiedQuorumChange(std::vector<NetworkAddress> const
 struct NoQuorumChange final : IQuorumChange {
 	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
 	                                                           std::vector<NetworkAddress> oldCoordinators,
-	                                                           Reference<ClusterConnectionFile>,
+	                                                           Reference<IClusterConnectionRecord>,
 	                                                           CoordinatorsResult&) override {
 		return oldCoordinators;
 	}
@@ -970,9 +978,9 @@ struct NameQuorumChange final : IQuorumChange {
 	  : newName(newName), otherChange(otherChange) {}
 	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
 	                                                           std::vector<NetworkAddress> oldCoordinators,
-	                                                           Reference<ClusterConnectionFile> cf,
+	                                                           Reference<IClusterConnectionRecord> ccr,
 	                                                           CoordinatorsResult& t) override {
-		return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t);
+		return otherChange->getDesiredCoordinators(tr, oldCoordinators, ccr, t);
 	}
 	std::string getDesiredClusterKeyName() const override { return newName; }
 };
@@ -986,9 +994,9 @@ struct AutoQuorumChange final : IQuorumChange {
 
 	Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
 	                                                           std::vector<NetworkAddress> oldCoordinators,
-	                                                           Reference<ClusterConnectionFile> ccf,
+	                                                           Reference<IClusterConnectionRecord> ccr,
 	                                                           CoordinatorsResult& err) override {
-		return getDesired(Reference<AutoQuorumChange>::addRef(this), tr, oldCoordinators, ccf, &err);
+		return getDesired(Reference<AutoQuorumChange>::addRef(this), tr, oldCoordinators, ccr, &err);
 	}
 
 	ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
@@ -1006,7 +1014,7 @@ struct AutoQuorumChange final : IQuorumChange {
 	ACTOR static Future<bool> isAcceptable(AutoQuorumChange* self,
 	                                       Transaction* tr,
 	                                       std::vector<NetworkAddress> oldCoordinators,
-	                                       Reference<ClusterConnectionFile> ccf,
+	                                       Reference<IClusterConnectionRecord> ccr,
 	                                       int desiredCount,
 	                                       std::set<AddressExclusion>* excluded) {
 		// Are there enough coordinators for the redundancy level?
@@ -1016,7 +1024,7 @@ struct AutoQuorumChange final : IQuorumChange {
 			return false;
 
 		// Check availability
-		ClientCoordinators coord(ccf);
+		ClientCoordinators coord(ccr);
 		std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 		leaderServers.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
@@ -1054,7 +1062,7 @@ struct AutoQuorumChange final : IQuorumChange {
 	ACTOR static Future<std::vector<NetworkAddress>> getDesired(Reference<AutoQuorumChange> self,
 	                                                            Transaction* tr,
 	                                                            std::vector<NetworkAddress> oldCoordinators,
-	                                                            Reference<ClusterConnectionFile> ccf,
+	                                                            Reference<IClusterConnectionRecord> ccr,
 	                                                            CoordinatorsResult* err) {
 		state int desiredCount = self->desired;
 
@@ -1088,7 +1096,7 @@ struct AutoQuorumChange final : IQuorumChange {
 		}
 
 		if (checkAcceptable) {
-			bool ok = wait(isAcceptable(self.getPtr(), tr, oldCoordinators, ccf, desiredCount, &excluded));
+			bool ok = wait(isAcceptable(self.getPtr(), tr, oldCoordinators, ccr, desiredCount, &excluded));
 			if (ok)
 				return oldCoordinators;
 		}
@@ -2177,7 +2185,7 @@ ACTOR Future<Void> advanceVersion(Database cx, Version v) {
 	}
 }
 
-ACTOR Future<Void> forceRecovery(Reference<ClusterConnectionFile> clusterFile, Key dcId) {
+ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Key dcId) {
 	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
 	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
 
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index 74dbefce37..64e1e9c6b9 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -130,7 +130,7 @@ struct IQuorumChange : ReferenceCounted<IQuorumChange> {
 	virtual ~IQuorumChange() {}
 	virtual Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
 	                                                                   std::vector<NetworkAddress> oldCoordinators,
-	                                                                   Reference<ClusterConnectionFile>,
+	                                                                   Reference<IClusterConnectionRecord>,
 	                                                                   CoordinatorsResult&) = 0;
 	virtual std::string getDesiredClusterKeyName() const { return std::string(); }
 };
@@ -218,7 +218,7 @@ ACTOR Future<Void> advanceVersion(Database cx, Version v);
 
 ACTOR Future<int> setDDMode(Database cx, int mode);
 
-ACTOR Future<Void> forceRecovery(Reference<ClusterConnectionFile> clusterFile, Standalone<StringRef> dcId);
+ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
 
 ACTOR Future<Void> printHealthyZone(Database cx);
 ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 7047935b0c..b2b4069cc9 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -18,8 +18,10 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/MonitorLeader.h"
 #include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/NativeAPI.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/UnitTest.h"
 #include "fdbrpc/genericactors.actor.h"
@@ -48,124 +50,25 @@ std::string trim(std::string const& connectionString) {
 
 } // namespace
 
-std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
-	if (filename.length())
-		return std::make_pair(filename, false);
+FDB_DEFINE_BOOLEAN_PARAM(ConnectionStringNeedsPersisted);
 
-	std::string f;
-	bool isDefaultFile = true;
-	if (platform::getEnvironmentVar(CLUSTER_FILE_ENV_VAR_NAME, f)) {
-		// If this is set but points to a file that does not
-		// exist, we will not fallback to any other methods
-		isDefaultFile = false;
-	} else if (fileExists("fdb.cluster"))
-		f = "fdb.cluster";
-	else
-		f = platform::getDefaultClusterFilePath();
-
-	return std::make_pair(f, isDefaultFile);
-}
-
-std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> const& resolvedClusterFile,
-                                                  Error const& e) {
-	bool isDefault = resolvedClusterFile.second;
-	if (e.code() == error_code_connection_string_invalid) {
-		return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
-	} else if (e.code() == error_code_no_cluster_file_found) {
-		if (isDefault)
-			return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s",
-			              platform::getDefaultClusterFilePath().c_str(),
-			              CLUSTER_FILE_ENV_VAR_NAME,
-			              e.code(),
-			              e.what());
-		else
-			return format(
-			    "Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
-	} else {
-		return format(
-		    "Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
-	}
-}
-
-ClusterConnectionFile::ClusterConnectionFile(std::string const& filename) {
-	if (!fileExists(filename)) {
-		throw no_cluster_file_found();
-	}
-
-	cs = ClusterConnectionString(readFileBytes(filename, MAX_CLUSTER_FILE_BYTES));
-	this->filename = filename;
-	setConn = false;
-}
-
-ClusterConnectionFile::ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents) {
-	this->filename = filename;
-	cs = contents;
-	setConn = true;
-}
-
-ClusterConnectionString const& ClusterConnectionFile::getConnectionString() const {
-	return cs;
-}
-
-void ClusterConnectionFile::notifyConnected() {
-	if (setConn) {
-		this->writeFile();
-	}
-}
-
-bool ClusterConnectionFile::fileContentsUpToDate() const {
+Future<bool> IClusterConnectionRecord::upToDate() {
 	ClusterConnectionString temp;
-	return fileContentsUpToDate(temp);
+	return upToDate(temp);
 }
 
-bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const {
-	try {
-		// the cluster file hasn't been created yet so there's nothing to check
-		if (setConn)
-			return true;
-
-		ClusterConnectionFile temp(filename);
-		fileConnectionString = temp.getConnectionString();
-		return fileConnectionString.toString() == cs.toString();
-	} catch (Error& e) {
-		TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename);
-		return false; // Swallow the error and report that the file is out of date
+void IClusterConnectionRecord::notifyConnected() {
+	if (connectionStringNeedsPersisted) {
+		this->persist();
 	}
 }
 
-bool ClusterConnectionFile::writeFile() {
-	setConn = false;
-	if (filename.size()) {
-		try {
-			atomicReplace(filename,
-			              "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
-			                  cs.toString().append("\n"));
-			if (!fileContentsUpToDate()) {
-				// This should only happen in rare scenarios where multiple processes are updating the same file to
-				// different values simultaneously In that case, we don't have any guarantees about which file will
-				// ultimately be written
-				TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace")
-				    .detail("Filename", filename)
-				    .detail("ConnStr", cs.toString());
-				return false;
-			}
-
-			return true;
-		} catch (Error& e) {
-			TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile")
-			    .error(e)
-			    .detail("Filename", filename)
-			    .detail("ConnStr", cs.toString());
-		}
-	}
-
-	return false;
+bool IClusterConnectionRecord::needsToBePersisted() const {
+	return connectionStringNeedsPersisted;
 }
 
-void ClusterConnectionFile::setConnectionString(ClusterConnectionString const& conn) {
-	ASSERT(filename.size());
-	cs = conn;
-	writeFile();
+void IClusterConnectionRecord::setPersisted() {
+	connectionStringNeedsPersisted = false;
 }
 
 std::string ClusterConnectionString::getErrorString(std::string const& source, Error const& e) {
@@ -367,8 +270,8 @@ std::string ClusterConnectionString::toString() const {
 	return s;
 }
 
-ClientCoordinators::ClientCoordinators(Reference<ClusterConnectionFile> ccf) : ccf(ccf) {
-	ClusterConnectionString cs = ccf->getConnectionString();
+ClientCoordinators::ClientCoordinators(Reference<IClusterConnectionRecord> ccr) : ccr(ccr) {
+	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
 		clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
 	clusterKey = cs.clusterKey();
@@ -379,7 +282,7 @@ ClientCoordinators::ClientCoordinators(Key clusterKey, std::vector<NetworkAddres
 	for (const auto& coord : coordinators) {
 		clientLeaderServers.push_back(ClientLeaderRegInterface(coord));
 	}
-	ccf = makeReference<ClusterConnectionFile>(ClusterConnectionString(coordinators, clusterKey));
+	ccr = makeReference<ClusterConnectionMemoryRecord>(ClusterConnectionString(coordinators, clusterKey));
 }
 
 ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote)
@@ -476,10 +379,10 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 }
 
 // Leader is the process that will be elected by coordinators as the cluster controller
-ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterConnectionRecord> connRecord,
                                                            Reference<AsyncVar<Value>> outSerializedLeaderInfo,
                                                            MonitorLeaderInfo info) {
-	state ClientCoordinators coordinators(info.intermediateConnFile);
+	state ClientCoordinators coordinators(info.intermediateConnRecord);
 	state AsyncTrigger nomineeChange;
 	state std::vector<Optional<LeaderInfo>> nominees;
 	state Future<Void> allActors;
@@ -502,25 +405,26 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConn
 			if (leader.get().first.forward) {
 				TraceEvent("MonitorLeaderForwarding")
 				    .detail("NewConnStr", leader.get().first.serializedInfo.toString())
-				    .detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString())
+				    .detail("OldConnStr", info.intermediateConnRecord->getConnectionString().toString())
 				    .trackLatest("MonitorLeaderForwarding");
-				info.intermediateConnFile = makeReference<ClusterConnectionFile>(
-				    connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString()));
+				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
+				    ClusterConnectionString(leader.get().first.serializedInfo.toString()));
 				return info;
 			}
-			if (connFile != info.intermediateConnFile) {
+			if (connRecord != info.intermediateConnRecord) {
 				if (!info.hasConnected) {
 					TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
-					    .detail("Filename", connFile->getFilename())
-					    .detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
-					    .detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
+					    .detail("ClusterFile", connRecord->toString())
+					    .detail("StoredConnectionString", connRecord->getConnectionString().toString())
+					    .detail("CurrentConnectionString",
+					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				connFile->setConnectionString(info.intermediateConnFile->getConnectionString());
-				info.intermediateConnFile = connFile;
+				connRecord->setConnectionString(info.intermediateConnRecord->getConnectionString());
+				info.intermediateConnRecord = connRecord;
 			}
 
 			info.hasConnected = true;
-			connFile->notifyConnected();
+			connRecord->notifyConnected();
 
 			outSerializedLeaderInfo->set(leader.get().first.serializedInfo);
 		}
@@ -528,11 +432,11 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConn
 	}
 }
 
-ACTOR Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> connRecord,
                                          Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
-	state MonitorLeaderInfo info(connFile);
+	state MonitorLeaderInfo info(connRecord);
 	loop {
-		MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connFile, outSerializedLeaderInfo, info));
+		MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connRecord, outSerializedLeaderInfo, info));
 		info = _info;
 	}
 }
@@ -750,13 +654,13 @@ void shrinkProxyList(ClientDBInfo& ni,
 }
 
 ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
-    Reference<ClusterConnectionFile> connFile,
+    Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<ClientDBInfo>> clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     MonitorLeaderInfo info,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup) {
-	state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
+	state ClusterConnectionString cs = info.intermediateConnRecord->getConnectionString();
 	state std::vector<NetworkAddress> addrs = cs.coordinators();
 	state int idx = 0;
 	state int successIndex = 0;
@@ -779,20 +683,24 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		req.supportedVersions = supportedVersions->get();
 		req.traceLogGroup = traceLogGroup;
 
-		ClusterConnectionString fileConnectionString;
-		if (connFile && !connFile->fileContentsUpToDate(fileConnectionString)) {
-			req.issues.push_back_deep(req.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
-			std::string connectionString = connFile->getConnectionString().toString();
-			if (!incorrectTime.present()) {
-				incorrectTime = now();
-			}
-			if (connFile->canGetFilename()) {
-				// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the
-				// file right before us)
+		state ClusterConnectionString storedConnectionString;
+		if (connRecord) {
+			bool upToDate = wait(connRecord->upToDate(storedConnectionString));
+			if (!upToDate) {
+				req.issues.push_back_deep(req.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
+				std::string connectionString = connRecord->getConnectionString().toString();
+				if (!incorrectTime.present()) {
+					incorrectTime = now();
+				}
+
+				// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing
+				// the file right before us)
 				TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
-				    .detail("Filename", connFile->getFilename())
-				    .detail("ConnectionStringFromFile", fileConnectionString.toString())
+				    .detail("ClusterFile", connRecord->toString())
+				    .detail("StoredConnectionString", storedConnectionString.toString())
 				    .detail("CurrentConnectionString", connectionString);
+			} else {
+				incorrectTime = Optional<double>();
 			}
 		} else {
 			incorrectTime = Optional<double>();
@@ -804,24 +712,25 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			if (rep.get().read().forward.present()) {
 				TraceEvent("MonitorProxiesForwarding")
 				    .detail("NewConnStr", rep.get().read().forward.get().toString())
-				    .detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
-				info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
-				    connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
+				    .detail("OldConnStr", info.intermediateConnRecord->getConnectionString().toString());
+				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
+				    ClusterConnectionString(rep.get().read().forward.get().toString()));
 				return info;
 			}
-			if (connFile != info.intermediateConnFile) {
+			if (connRecord != info.intermediateConnRecord) {
 				if (!info.hasConnected) {
 					TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
-					    .detail("Filename", connFile->getFilename())
-					    .detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
-					    .detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
+					    .detail("ClusterFile", connRecord->toString())
+					    .detail("StoredConnectionString", connRecord->getConnectionString().toString())
+					    .detail("CurrentConnectionString",
+					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				connFile->setConnectionString(info.intermediateConnFile->getConnectionString());
-				info.intermediateConnFile = connFile;
+				connRecord->setConnectionString(info.intermediateConnRecord->getConnectionString());
+				info.intermediateConnRecord = connRecord;
 			}
 
 			info.hasConnected = true;
-			connFile->notifyConnected();
+			connRecord->notifyConnected();
 
 			auto& ni = rep.get().mutate();
 			shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
@@ -838,21 +747,21 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 }
 
 ACTOR Future<Void> monitorProxies(
-    Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile,
+    Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connRecord,
     Reference<AsyncVar<ClientDBInfo>> clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup) {
-	state MonitorLeaderInfo info(connFile->get());
+	state MonitorLeaderInfo info(connRecord->get());
 	loop {
 		choose {
 			when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
-			         connFile->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
+			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
 				info = _info;
 			}
-			when(wait(connFile->onChange())) {
+			when(wait(connRecord->onChange())) {
 				info.hasConnected = false;
-				info.intermediateConnFile = connFile->get();
+				info.intermediateConnRecord = connRecord->get();
 			}
 		}
 	}
diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h
index b9c58de422..2903ff51b9 100644
--- a/fdbclient/MonitorLeader.h
+++ b/fdbclient/MonitorLeader.h
@@ -54,11 +54,11 @@ struct ClientData {
 
 struct MonitorLeaderInfo {
 	bool hasConnected;
-	Reference<ClusterConnectionFile> intermediateConnFile;
+	Reference<IClusterConnectionRecord> intermediateConnRecord;
 
 	MonitorLeaderInfo() : hasConnected(false) {}
-	explicit MonitorLeaderInfo(Reference<ClusterConnectionFile> intermediateConnFile)
-	  : hasConnected(false), intermediateConnFile(intermediateConnFile) {}
+	explicit MonitorLeaderInfo(Reference<IClusterConnectionRecord> intermediateConnRecord)
+	  : hasConnected(false), intermediateConnRecord(intermediateConnRecord) {}
 };
 
 Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<LeaderInfo>>& nominees);
@@ -68,7 +68,7 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 // If a leader is elected for long enough and communication with a quorum of coordinators is possible, eventually
 // outKnownLeader will be that leader's interface.
 template <class LeaderInterface>
-Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
+Future<Void> monitorLeader(Reference<IClusterConnectionRecord> const& connFile,
                            Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader);
 
 // This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect
@@ -80,7 +80,7 @@ Future<Void> monitorLeaderAndGetClientInfo(Value const& key,
                                            Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
 
 Future<Void> monitorProxies(
-    Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile,
+    Reference<AsyncVar<Reference<IClusterConnectionRecord>>> const& connRecord,
     Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> const& coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
@@ -96,7 +96,7 @@ void shrinkProxyList(ClientDBInfo& ni,
 #pragma region Implementation
 #endif
 
-Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> const& connFile,
+Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> const& connRecord,
                                    Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
 
 template <class LeaderInterface>
@@ -119,11 +119,11 @@ struct LeaderDeserializer<ClusterInterface> {
 };
 
 template <class LeaderInterface>
-Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
+Future<Void> monitorLeader(Reference<IClusterConnectionRecord> const& connRecord,
                            Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
 	LeaderDeserializer<LeaderInterface> deserializer;
 	auto serializedInfo = makeReference<AsyncVar<Value>>();
-	Future<Void> m = monitorLeaderInternal(connFile, serializedInfo);
+	Future<Void> m = monitorLeaderInternal(connRecord, serializedInfo);
 	return m || deserializer(serializedInfo, outKnownLeader);
 }
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 14d2dda485..117524a43a 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -37,6 +37,7 @@
 #include "fdbclient/Atomic.h"
 #include "fdbclient/BlobGranuleCommon.h"
 #include "fdbclient/ClusterInterface.h"
+#include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.actor.h"
@@ -377,8 +378,9 @@ ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
 
 		ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)
 		    .detail("Cluster",
-		            cx->getConnectionFile() ? cx->getConnectionFile()->getConnectionString().clusterKeyName().toString()
-		                                    : "")
+		            cx->getConnectionRecord()
+		                ? cx->getConnectionRecord()->getConnectionString().clusterKeyName().toString()
+		                : "")
 		    .detail("Internal", cx->internal);
 
 		cx->cc.logToTraceEvent(ev);
@@ -1028,14 +1030,14 @@ void DatabaseContext::registerSpecialKeySpaceModule(SpecialKeySpace::MODULE modu
 	specialKeySpaceModules.push_back(std::move(impl));
 }
 
-ACTOR Future<RangeResult> getWorkerInterfaces(Reference<ClusterConnectionFile> clusterFile);
+ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
 ACTOR Future<Optional<Value>> getJSON(Database db);
 
 struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl {
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override {
-		if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionFile()) {
+		if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
 			Key prefix = Key(getKeyRange().begin);
-			return map(getWorkerInterfaces(ryw->getDatabase()->getConnectionFile()),
+			return map(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()),
 			           [prefix = prefix, kr = KeyRange(kr)](const RangeResult& in) {
 				           RangeResult result;
 				           for (const auto& [k_, v] : in) {
@@ -1167,7 +1169,7 @@ Future<RangeResult> HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction*
 	return healthMetricsGetRangeActor(ryw, kr);
 }
 
-DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
+DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connectionRecord,
                                  Reference<AsyncVar<ClientDBInfo>> clientInfo,
                                  Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
                                  Future<Void> clientInfoMonitor,
@@ -1178,7 +1180,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
                                  IsInternal internal,
                                  int apiVersion,
                                  IsSwitchable switchable)
-  : lockAware(lockAware), switchable(switchable), connectionFile(connectionFile), proxyProvisional(false),
+  : lockAware(lockAware), switchable(switchable), connectionRecord(connectionRecord), proxyProvisional(false),
     clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), internal(internal),
     cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc),
     transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
@@ -1384,7 +1386,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 		    std::make_unique<SingleSpecialKeyImpl>(LiteralStringRef("\xff\xff/status/json"),
 		                                           [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			                                           if (ryw->getDatabase().getPtr() &&
-			                                               ryw->getDatabase()->getConnectionFile()) {
+			                                               ryw->getDatabase()->getConnectionRecord()) {
 				                                           ++ryw->getDatabase()->transactionStatusRequests;
 				                                           return getJSON(ryw->getDatabase());
 			                                           } else {
@@ -1398,8 +1400,9 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 		        LiteralStringRef("\xff\xff/cluster_file_path"),
 		        [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			        try {
-				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionFile()) {
-					        Optional<Value> output = StringRef(ryw->getDatabase()->getConnectionFile()->getFilename());
+				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
+					        Optional<Value> output =
+					            StringRef(ryw->getDatabase()->getConnectionRecord()->getLocation());
 					        return output;
 				        }
 			        } catch (Error& e) {
@@ -1415,8 +1418,8 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
 		        LiteralStringRef("\xff\xff/connection_string"),
 		        [](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
 			        try {
-				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionFile()) {
-					        Reference<ClusterConnectionFile> f = ryw->getDatabase()->getConnectionFile();
+				        if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
+					        Reference<IClusterConnectionRecord> f = ryw->getDatabase()->getConnectionRecord();
 					        Optional<Value> output = StringRef(f->getConnectionString().toString());
 					        return output;
 				        }
@@ -1476,7 +1479,7 @@ Database DatabaseContext::create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
                                  LockAware lockAware,
                                  int apiVersion,
                                  IsSwitchable switchable) {
-	return Database(new DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>>(),
+	return Database(new DatabaseContext(Reference<AsyncVar<Reference<IClusterConnectionRecord>>>(),
 	                                    clientInfo,
 	                                    makeReference<AsyncVar<Optional<ClientLeaderRegInterface>>>(),
 	                                    clientInfoMonitor,
@@ -1700,11 +1703,12 @@ Future<Void> DatabaseContext::onConnected() {
 	return connected;
 }
 
-ACTOR static Future<Void> switchConnectionFileImpl(Reference<ClusterConnectionFile> connFile, DatabaseContext* self) {
+ACTOR static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectionRecord> connRecord,
+                                                     DatabaseContext* self) {
 	TEST(true); // Switch connection file
-	TraceEvent("SwitchConnectionFile")
-	    .detail("ConnectionFile", connFile->canGetFilename() ? connFile->getFilename() : "")
-	    .detail("ConnectionString", connFile->getConnectionString().toString());
+	TraceEvent("SwitchConnectionRecord")
+	    .detail("ClusterFile", connRecord->toString())
+	    .detail("ConnectionString", connRecord->getConnectionString().toString());
 
 	// Reset state from former cluster.
 	self->commitProxies.clear();
@@ -1717,38 +1721,38 @@ ACTOR static Future<Void> switchConnectionFileImpl(Reference<ClusterConnectionFi
 	clearedClientInfo.grvProxies.clear();
 	clearedClientInfo.id = deterministicRandom()->randomUniqueID();
 	self->clientInfo->set(clearedClientInfo);
-	self->connectionFile->set(connFile);
+	self->connectionRecord->set(connRecord);
 
 	state Database db(Reference<DatabaseContext>::addRef(self));
 	state Transaction tr(db);
 	loop {
 		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		try {
-			TraceEvent("SwitchConnectionFileAttemptingGRV").log();
+			TraceEvent("SwitchConnectionRecordAttemptingGRV").log();
 			Version v = wait(tr.getReadVersion());
-			TraceEvent("SwitchConnectionFileGotRV")
+			TraceEvent("SwitchConnectionRecordGotRV")
 			    .detail("ReadVersion", v)
 			    .detail("MinAcceptableReadVersion", self->minAcceptableReadVersion);
 			ASSERT(self->minAcceptableReadVersion != std::numeric_limits<Version>::max());
 			self->connectionFileChangedTrigger.trigger();
 			return Void();
 		} catch (Error& e) {
-			TraceEvent("SwitchConnectionFileError").detail("Error", e.what());
+			TraceEvent("SwitchConnectionRecordError").detail("Error", e.what());
 			wait(tr.onError(e));
 		}
 	}
 }
 
-Reference<ClusterConnectionFile> DatabaseContext::getConnectionFile() {
-	if (connectionFile) {
-		return connectionFile->get();
+Reference<IClusterConnectionRecord> DatabaseContext::getConnectionRecord() {
+	if (connectionRecord) {
+		return connectionRecord->get();
 	}
-	return Reference<ClusterConnectionFile>();
+	return Reference<IClusterConnectionRecord>();
 }
 
-Future<Void> DatabaseContext::switchConnectionFile(Reference<ClusterConnectionFile> standby) {
+Future<Void> DatabaseContext::switchConnectionRecord(Reference<IClusterConnectionRecord> standby) {
 	ASSERT(switchable);
-	return switchConnectionFileImpl(standby, this);
+	return switchConnectionRecordImpl(standby, this);
 }
 
 Future<Void> DatabaseContext::connectionFileChanged() {
@@ -1773,7 +1777,7 @@ extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& c
 // Creates a database object that represents a connection to a cluster
 // This constructor uses a preallocated DatabaseContext that may have been created
 // on another thread
-Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
+Database Database::createDatabase(Reference<IClusterConnectionRecord> connRecord,
                                   int apiVersion,
                                   IsInternal internal,
                                   LocalityData const& clientLocality,
@@ -1781,13 +1785,13 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
 	if (!g_network)
 		throw network_not_setup();
 
-	if (connFile) {
+	if (connRecord) {
 		if (networkOptions.traceDirectory.present() && !traceFileIsOpen()) {
 			g_network->initMetrics();
 			FlowTransport::transport().initMetrics();
 			initTraceEventMetrics();
 
-			auto publicIP = determinePublicIPAutomatically(connFile->getConnectionString());
+			auto publicIP = determinePublicIPAutomatically(connRecord->getConnectionString());
 			selectTraceFormatter(networkOptions.traceFormat);
 			selectTraceClockSource(networkOptions.traceClockSource);
 			openTraceFile(NetworkAddress(publicIP, ::getpid()),
@@ -1803,8 +1807,8 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
 			    .detail("SourceVersion", getSourceVersion())
 			    .detail("Version", FDB_VT_VERSION)
 			    .detail("PackageName", FDB_VT_PACKAGE_NAME)
-			    .detail("ClusterFile", connFile->getFilename().c_str())
-			    .detail("ConnectionString", connFile->getConnectionString().toString())
+			    .detail("ClusterFile", connRecord->toString())
+			    .detail("ConnectionString", connRecord->getConnectionString().toString())
 			    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 			    .detail("ApiVersion", apiVersion)
 			    .detailf("ImageOffset", "%p", platform::getImageOffset())
@@ -1821,9 +1825,9 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
 
 	auto clientInfo = makeReference<AsyncVar<ClientDBInfo>>();
 	auto coordinator = makeReference<AsyncVar<Optional<ClientLeaderRegInterface>>>();
-	auto connectionFile = makeReference<AsyncVar<Reference<ClusterConnectionFile>>>();
-	connectionFile->set(connFile);
-	Future<Void> clientInfoMonitor = monitorProxies(connectionFile,
+	auto connectionRecord = makeReference<AsyncVar<Reference<IClusterConnectionRecord>>>();
+	connectionRecord->set(connRecord);
+	Future<Void> clientInfoMonitor = monitorProxies(connectionRecord,
 	                                                clientInfo,
 	                                                coordinator,
 	                                                networkOptions.supportedVersions,
@@ -1831,7 +1835,7 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
 
 	DatabaseContext* db;
 	if (preallocatedDb) {
-		db = new (preallocatedDb) DatabaseContext(connectionFile,
+		db = new (preallocatedDb) DatabaseContext(connectionRecord,
 		                                          clientInfo,
 		                                          coordinator,
 		                                          clientInfoMonitor,
@@ -1843,7 +1847,7 @@ Database Database::createDatabase(Reference<ClusterConnectionFile> connFile,
 		                                          apiVersion,
 		                                          IsSwitchable::True);
 	} else {
-		db = new DatabaseContext(connectionFile,
+		db = new DatabaseContext(connectionRecord,
 		                         clientInfo,
 		                         coordinator,
 		                         clientInfoMonitor,
@@ -1868,9 +1872,9 @@ Database Database::createDatabase(std::string connFileName,
                                   int apiVersion,
                                   IsInternal internal,
                                   LocalityData const& clientLocality) {
-	Reference<ClusterConnectionFile> rccf = Reference<ClusterConnectionFile>(
+	Reference<IClusterConnectionRecord> rccr = Reference<IClusterConnectionRecord>(
 	    new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFileName).first));
-	return Database::createDatabase(rccf, apiVersion, internal, clientLocality);
+	return Database::createDatabase(rccr, apiVersion, internal, clientLocality);
 }
 
 Reference<WatchMetadata> DatabaseContext::getWatchMetadata(KeyRef key) const {
@@ -2832,7 +2836,7 @@ ACTOR Future<Version> watchValue(Future<Version> version,
 				                          TaskPriority::DefaultPromiseEndpoint))) {
 					resp = r;
 				}
-				when(wait(cx->connectionFile ? cx->connectionFile->onChange() : Never())) { wait(Never()); }
+				when(wait(cx->connectionRecord ? cx->connectionRecord->onChange() : Never())) { wait(Never()); }
 			}
 			if (info.debugID.present()) {
 				g_traceBatch.addEvent("WatchValueDebug",
@@ -6538,7 +6542,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 		throw;
 	}
 	TraceEvent("ExclusionSafetyCheckCoordinators").log();
-	state ClientCoordinators coordinatorList(cx->getConnectionFile());
+	state ClientCoordinators coordinatorList(cx->getConnectionRecord());
 	state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	leaderServers.reserve(coordinatorList.clientLeaderServers.size());
 	for (int i = 0; i < coordinatorList.clientLeaderServers.size(); i++) {
@@ -6617,9 +6621,9 @@ ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef add
 		duration = 0;
 	// fetch the addresses of all workers
 	state std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
-	if (!cx->getConnectionFile())
+	if (!cx->getConnectionRecord())
 		return 0;
-	RangeResult kvs = wait(getWorkerInterfaces(cx->getConnectionFile()));
+	RangeResult kvs = wait(getWorkerInterfaces(cx->getConnectionRecord()));
 	ASSERT(!kvs.more);
 	// Note: reuse this knob from fdbcli, change it if necessary
 	Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
@@ -6641,7 +6645,7 @@ Future<int64_t> DatabaseContext::rebootWorker(StringRef addr, bool check, int du
 }
 
 Future<Void> DatabaseContext::forceRecoveryWithDataLoss(StringRef dcId) {
-	return forceRecovery(getConnectionFile(), dcId);
+	return forceRecovery(getConnectionRecord(), dcId);
 }
 
 ACTOR static Future<Void> createSnapshotActor(DatabaseContext* cx, UID snapUID, StringRef snapCmd) {
@@ -6751,7 +6755,15 @@ ACTOR Future<Void> mergeChangeFeedStream(std::vector<std::pair<StorageServerInte
 			}
 			checkVersion = nextStream.next.version;
 		}
-		nextOut.push_back_deep(nextOut.arena(), nextStream.next);
+		if (nextOut.size() && nextStream.next.version == nextOut.back().version) {
+			if (nextStream.next.mutations.size() &&
+			    nextStream.next.mutations.front().param1 != lastEpochEndPrivateKey) {
+				nextOut.back().mutations.append_deep(
+				    nextOut.arena(), nextStream.next.mutations.begin(), nextStream.next.mutations.size());
+			}
+		} else {
+			nextOut.push_back_deep(nextOut.arena(), nextStream.next);
+		}
 		try {
 			Standalone<MutationsAndVersionRef> res = waitNext(nextStream.results.getFuture());
 			nextStream.next = res;
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index 6175205481..6bd5ab892e 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -83,7 +83,7 @@ public:
 	// Creates a database object that represents a connection to a cluster
 	// This constructor uses a preallocated DatabaseContext that may have been created
 	// on another thread
-	static Database createDatabase(Reference<ClusterConnectionFile> connFile,
+	static Database createDatabase(Reference<IClusterConnectionRecord> connRecord,
 	                               int apiVersion,
 	                               IsInternal internal = IsInternal::True,
 	                               LocalityData const& clientLocality = LocalityData(),
diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp
index 1b22cf0d5a..0c887ae67c 100644
--- a/fdbclient/PaxosConfigTransaction.actor.cpp
+++ b/fdbclient/PaxosConfigTransaction.actor.cpp
@@ -317,7 +317,7 @@ public:
 	Future<Void> commit() { return commit(this); }
 
 	PaxosConfigTransactionImpl(Database const& cx) : cx(cx) {
-		auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators();
+		auto coordinators = cx->getConnectionRecord()->getConnectionString().coordinators();
 		ctis.reserve(coordinators.size());
 		for (const auto& coordinator : coordinators) {
 			ctis.emplace_back(coordinator);
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index dffb84db62..f156a36c85 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -1336,9 +1336,9 @@ ACTOR Future<Optional<Value>> getJSON(Database db) {
 	return getValueFromJSON(statusObj);
 }
 
-ACTOR Future<RangeResult> getWorkerInterfaces(Reference<ClusterConnectionFile> clusterFile) {
+ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> connRecord) {
 	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
-	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
+	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(connRecord, clusterInterface);
 
 	loop {
 		choose {
@@ -1371,7 +1371,7 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot
 		}
 	} else {
 		if (key == LiteralStringRef("\xff\xff/status/json")) {
-			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
+			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
 				++tr.getDatabase()->transactionStatusRequests;
 				return getJSON(tr.getDatabase());
 			} else {
@@ -1381,8 +1381,8 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot
 
 		if (key == LiteralStringRef("\xff\xff/cluster_file_path")) {
 			try {
-				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
-					Optional<Value> output = StringRef(tr.getDatabase()->getConnectionFile()->getFilename());
+				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
+					Optional<Value> output = StringRef(tr.getDatabase()->getConnectionRecord()->getLocation());
 					return output;
 				}
 			} catch (Error& e) {
@@ -1393,8 +1393,8 @@ Future<Optional<Value>> ReadYourWritesTransaction::get(const Key& key, Snapshot
 
 		if (key == LiteralStringRef("\xff\xff/connection_string")) {
 			try {
-				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
-					Reference<ClusterConnectionFile> f = tr.getDatabase()->getConnectionFile();
+				if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
+					Reference<IClusterConnectionRecord> f = tr.getDatabase()->getConnectionRecord();
 					Optional<Value> output = StringRef(f->getConnectionString().toString());
 					return output;
 				}
@@ -1454,8 +1454,8 @@ Future<RangeResult> ReadYourWritesTransaction::getRange(KeySelector begin,
 		}
 	} else {
 		if (begin.getKey() == LiteralStringRef("\xff\xff/worker_interfaces")) {
-			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
-				return getWorkerInterfaces(tr.getDatabase()->getConnectionFile());
+			if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionRecord()) {
+				return getWorkerInterfaces(tr.getDatabase()->getConnectionRecord());
 			} else {
 				return RangeResult();
 			}
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 757608633f..dc96419ab3 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -55,7 +55,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BUGGIFY_RECOVER_MEMORY_LIMIT,                          1e6 );
 	init( BUGGIFY_WORKER_REMOVED_MAX_LAG,                         30 );
 	init( UPDATE_STORAGE_BYTE_LIMIT,                             1e6 );
-	init( TLOG_PEEK_DELAY,                                   0.00005 );
+	init( TLOG_PEEK_DELAY,                                    0.0005 );
 	init( LEGACY_TLOG_UPGRADE_ENTRIES_PER_VERSION,               100 );
 	init( VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS,             1072 ); // Based on a naive interpretation of the gcc version of std::deque, we would expect this to be 16 bytes overhead per 512 bytes data. In practice, it seems to be 24 bytes overhead per 512.
 	init( VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD, std::ceil(16.0 * VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS / 1024) );
@@ -64,7 +64,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( TLOG_MESSAGE_BLOCK_BYTES,                             10e6 );
 	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) ); //1.0121466709838096006362758832473
 	init( PEEK_TRACKER_EXPIRATION_TIME,                          600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = deterministicRandom()->coinflip() ? 0.1 : 120;
-	init( PEEK_USING_STREAMING,                                 true );
+	init( PEEK_USING_STREAMING,                                 true ); if( randomize && BUGGIFY ) PEEK_USING_STREAMING = false;
 	init( PARALLEL_GET_MORE_REQUESTS,                             32 ); if( randomize && BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2;
 	init( MULTI_CURSOR_PRE_FETCH_LIMIT,                           10 );
 	init( MAX_QUEUE_COMMIT_BYTES,                               15e6 ); if( randomize && BUGGIFY ) MAX_QUEUE_COMMIT_BYTES = 5000;
@@ -537,6 +537,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( TARGET_BYTES_PER_STORAGE_SERVER_BATCH,               750e6 ); if( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER_BATCH = 1500e3;
 	init( SPRING_BYTES_STORAGE_SERVER_BATCH,                   100e6 ); if( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER_BATCH = 150e3;
 	init( STORAGE_HARD_LIMIT_BYTES,                           1500e6 ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES = 4500e3;
+	init( STORAGE_HARD_LIMIT_BYTES_OVERAGE,                   5000e3 ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_OVERAGE = 100e3; // byte+version overage ensures storage server makes enough progress on freeing up storage queue memory at hard limit by ensuring it advances desiredOldestVersion enough per commit cycle.
+	init( STORAGE_HARD_LIMIT_VERSION_OVERAGE, VERSIONS_PER_SECOND / 4.0 );
 	init( STORAGE_DURABILITY_LAG_HARD_MAX,                    2000e6 ); if( smallStorageTarget ) STORAGE_DURABILITY_LAG_HARD_MAX = 100e6;
 	init( STORAGE_DURABILITY_LAG_SOFT_MAX,                     250e6 ); if( smallStorageTarget ) STORAGE_DURABILITY_LAG_SOFT_MAX = 10e6;
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 2537fc64be..97ff86a92f 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -474,6 +474,8 @@ public:
 	int64_t TARGET_BYTES_PER_STORAGE_SERVER_BATCH;
 	int64_t SPRING_BYTES_STORAGE_SERVER_BATCH;
 	int64_t STORAGE_HARD_LIMIT_BYTES;
+	int64_t STORAGE_HARD_LIMIT_BYTES_OVERAGE;
+	int64_t STORAGE_HARD_LIMIT_VERSION_OVERAGE;
 	int64_t STORAGE_DURABILITY_LAG_HARD_MAX;
 	int64_t STORAGE_DURABILITY_LAG_SOFT_MAX;
 
diff --git a/fdbclient/SimpleConfigTransaction.actor.cpp b/fdbclient/SimpleConfigTransaction.actor.cpp
index 77f72fb9f2..f1bc1aebb9 100644
--- a/fdbclient/SimpleConfigTransaction.actor.cpp
+++ b/fdbclient/SimpleConfigTransaction.actor.cpp
@@ -126,7 +126,7 @@ class SimpleConfigTransactionImpl {
 
 public:
 	SimpleConfigTransactionImpl(Database const& cx) : cx(cx) {
-		auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators();
+		auto coordinators = cx->getConnectionRecord()->getConnectionString().coordinators();
 		std::sort(coordinators.begin(), coordinators.end());
 		cti = ConfigTransactionInterface(coordinators[0]);
 	}
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index d2fcbf96ec..bea850dea5 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -27,6 +27,7 @@
 #include <exception>
 
 #include "fdbclient/ActorLineageProfiler.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/ProcessInterface.h"
 #include "fdbclient/GlobalConfig.actor.h"
@@ -1590,8 +1591,7 @@ CoordinatorsImpl::CoordinatorsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {
 Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
 	RangeResult result;
 	KeyRef prefix(getKeyRange().begin);
-	// the constructor of ClusterConnectionFile already checks whether the file is valid
-	auto cs = ClusterConnectionFile(ryw->getDatabase()->getConnectionFile()->getFilename()).getConnectionString();
+	auto cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString();
 	auto coordinator_processes = cs.coordinators();
 	Key cluster_decription_key = prefix.withSuffix(LiteralStringRef("cluster_description"));
 	if (kr.contains(cluster_decription_key)) {
@@ -1737,7 +1737,10 @@ ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransac
 	state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 
 	std::vector<NetworkAddress> _desiredCoordinators = wait(autoQuorumChange()->getDesiredCoordinators(
-	    &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
+	    &tr,
+	    old.coordinators(),
+	    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
+	    result));
 
 	if (result == CoordinatorsResult::NOT_ENOUGH_MACHINES) {
 		// we could get not_enough_machines if we happen to see the database while the cluster controller is updating
diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp
index 01f8bedf0b..2829771ba8 100644
--- a/fdbclient/StatusClient.actor.cpp
+++ b/fdbclient/StatusClient.actor.cpp
@@ -302,11 +302,11 @@ void JSONDoc::mergeValueInto(json_spirit::mValue& dst, const json_spirit::mValue
 
 // Check if a quorum of coordination servers is reachable
 // Will not throw, will just return non-present Optional if error
-ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<ClusterConnectionFile> f,
+ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
                                                                      bool* quorum_reachable,
                                                                      int* coordinatorsFaultTolerance) {
 	try {
-		state ClientCoordinators coord(f);
+		state ClientCoordinators coord(connRecord);
 		state StatusObject statusObj;
 
 		state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
@@ -365,14 +365,16 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<C
 
 // Client section of the json output
 // Will NOT throw, errors will be put into messages array
-ACTOR Future<StatusObject> clientStatusFetcher(Reference<ClusterConnectionFile> f,
+ACTOR Future<StatusObject> clientStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
                                                StatusArray* messages,
                                                bool* quorum_reachable,
                                                int* coordinatorsFaultTolerance) {
 	state StatusObject statusObj;
 
-	Optional<StatusObject> coordsStatusObj =
-	    wait(clientCoordinatorsStatusFetcher(f, quorum_reachable, coordinatorsFaultTolerance));
+	state Optional<StatusObject> coordsStatusObj =
+	    wait(clientCoordinatorsStatusFetcher(connRecord, quorum_reachable, coordinatorsFaultTolerance));
+	state bool contentsUpToDate = wait(connRecord->upToDate());
+
 	if (coordsStatusObj.present()) {
 		statusObj["coordinators"] = coordsStatusObj.get();
 		if (!*quorum_reachable)
@@ -381,17 +383,17 @@ ACTOR Future<StatusObject> clientStatusFetcher(Reference<ClusterConnectionFile>
 		messages->push_back(makeMessage("status_incomplete_coordinators", "Could not fetch coordinator info."));
 
 	StatusObject statusObjClusterFile;
-	statusObjClusterFile["path"] = f->getFilename();
-	bool contentsUpToDate = f->fileContentsUpToDate();
+	statusObjClusterFile["path"] = connRecord->getLocation();
 	statusObjClusterFile["up_to_date"] = contentsUpToDate;
 	statusObj["cluster_file"] = statusObjClusterFile;
 
 	if (!contentsUpToDate) {
+		ClusterConnectionString storedConnectionString = wait(connRecord->getStoredConnectionString());
 		std::string description = "Cluster file contents do not match current cluster connection string.";
 		description += "\nThe file contains the connection string: ";
-		description += ClusterConnectionFile(f->getFilename()).getConnectionString().toString().c_str();
+		description += storedConnectionString.toString().c_str();
 		description += "\nThe current connection string is: ";
-		description += f->getConnectionString().toString().c_str();
+		description += connRecord->getConnectionString().toString().c_str();
 		description += "\nVerify the cluster file and its parent directory are writable and that the cluster file has "
 		               "not been overwritten externally. To change coordinators without manual intervention, the "
 		               "cluster file and its containing folder must be writable by all servers and clients. If a "
@@ -491,7 +493,7 @@ StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectRead
 	return databaseStatus;
 }
 
-ACTOR Future<StatusObject> statusFetcherImpl(Reference<ClusterConnectionFile> f,
+ACTOR Future<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord> connRecord,
                                              Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface) {
 	if (!g_network)
 		throw network_not_setup();
@@ -508,7 +510,7 @@ ACTOR Future<StatusObject> statusFetcherImpl(Reference<ClusterConnectionFile> f,
 		state int64_t clientTime = g_network->timer();
 
 		StatusObject _statusObjClient =
-		    wait(clientStatusFetcher(f, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance));
+		    wait(clientStatusFetcher(connRecord, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance));
 		statusObjClient = _statusObjClient;
 
 		if (clientTime != -1)
@@ -598,7 +600,7 @@ ACTOR Future<StatusObject> statusFetcherImpl(Reference<ClusterConnectionFile> f,
 }
 
 ACTOR Future<Void> timeoutMonitorLeader(Database db) {
-	state Future<Void> leadMon = monitorLeader<ClusterInterface>(db->getConnectionFile(), db->statusClusterInterface);
+	state Future<Void> leadMon = monitorLeader<ClusterInterface>(db->getConnectionRecord(), db->statusClusterInterface);
 	loop {
 		wait(delay(CLIENT_KNOBS->STATUS_IDLE_TIMEOUT + 0.00001 + db->lastStatusFetch - now()));
 		if (now() - db->lastStatusFetch > CLIENT_KNOBS->STATUS_IDLE_TIMEOUT) {
@@ -615,5 +617,5 @@ Future<StatusObject> StatusClient::statusFetcher(Database db) {
 		db->statusLeaderMon = timeoutMonitorLeader(db);
 	}
 
-	return statusFetcherImpl(db->getConnectionFile(), db->statusClusterInterface);
+	return statusFetcherImpl(db->getConnectionRecord(), db->statusClusterInterface);
 }
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index ee7e0b75e2..3810d08191 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/versions.h"
diff --git a/fdbrpc/FailureMonitor.h b/fdbrpc/FailureMonitor.h
index 04154fa103..795a6e9953 100644
--- a/fdbrpc/FailureMonitor.h
+++ b/fdbrpc/FailureMonitor.h
@@ -157,7 +157,7 @@ public:
 private:
 	std::unordered_map<NetworkAddress, FailureStatus> addressStatus;
 	YieldedAsyncMap<Endpoint, bool> endpointKnownFailed;
-	YieldedAsyncMap<NetworkAddress, bool> disconnectTriggers;
+	AsyncMap<NetworkAddress, bool> disconnectTriggers;
 	std::unordered_set<Endpoint> failedEndpoints;
 
 	friend class OnStateChangedActorActor;
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 8f664c0877..cb596e8931 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -175,6 +175,7 @@ set(FDBSERVER_SRCS
   workloads/Cycle.actor.cpp
   workloads/ChangeFeeds.actor.cpp
   workloads/DataDistributionMetrics.actor.cpp
+  workloads/DataLossRecovery.actor.cpp
   workloads/DDBalance.actor.cpp
   workloads/DDMetrics.actor.cpp
   workloads/DDMetricsExclude.actor.cpp
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 6abd87e126..298b7867f5 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -27,6 +27,7 @@
 #include "fdbrpc/FailureMonitor.h"
 #include "flow/ActorCollection.h"
 #include "flow/SystemMonitor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/BackupInterface.h"
 #include "fdbserver/CoordinationInterface.h"
@@ -5591,7 +5592,7 @@ ACTOR Future<Void> clusterController(ServerCoordinators coordinators,
 	}
 }
 
-ACTOR Future<Void> clusterController(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> connRecord,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
                                      Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
                                      Future<Void> recoveredDiskFiles,
@@ -5601,7 +5602,7 @@ ACTOR Future<Void> clusterController(Reference<ClusterConnectionFile> connFile,
 	state bool hasConnected = false;
 	loop {
 		try {
-			ServerCoordinators coordinators(connFile);
+			ServerCoordinators coordinators(connRecord);
 			wait(clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, configDBType));
 		} catch (Error& e) {
 			if (e.code() != error_code_coordinators_changed)
@@ -5620,7 +5621,8 @@ TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") {
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	state ClusterControllerData data(ClusterControllerFullInterface(),
 	                                 LocalityData(),
-	                                 ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                                 ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                                     new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 	state NetworkAddress workerAddress(IPAddress(0x01010101), 1);
 	state NetworkAddress badPeer1(IPAddress(0x02020202), 1);
 	state NetworkAddress badPeer2(IPAddress(0x03030303), 1);
@@ -5678,7 +5680,8 @@ TEST_CASE("/fdbserver/clustercontroller/updateRecoveredWorkers") {
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	ClusterControllerData data(ClusterControllerFullInterface(),
 	                           LocalityData(),
-	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 	NetworkAddress worker1(IPAddress(0x01010101), 1);
 	NetworkAddress worker2(IPAddress(0x11111111), 1);
 	NetworkAddress badPeer1(IPAddress(0x02020202), 1);
@@ -5714,7 +5717,8 @@ TEST_CASE("/fdbserver/clustercontroller/getServersWithDegradedLink") {
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	ClusterControllerData data(ClusterControllerFullInterface(),
 	                           LocalityData(),
-	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 	NetworkAddress worker(IPAddress(0x01010101), 1);
 	NetworkAddress badPeer1(IPAddress(0x02020202), 1);
 	NetworkAddress badPeer2(IPAddress(0x03030303), 1);
@@ -5816,7 +5820,8 @@ TEST_CASE("/fdbserver/clustercontroller/recentRecoveryCountDueToHealth") {
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	ClusterControllerData data(ClusterControllerFullInterface(),
 	                           LocalityData(),
-	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 
 	ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 0);
 
@@ -5836,7 +5841,8 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	ClusterControllerData data(ClusterControllerFullInterface(),
 	                           LocalityData(),
-	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 	NetworkAddress master(IPAddress(0x01010101), 1);
 	NetworkAddress tlog(IPAddress(0x02020202), 1);
 	NetworkAddress satelliteTlog(IPAddress(0x03030303), 1);
@@ -5940,7 +5946,8 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer
 	// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
 	ClusterControllerData data(ClusterControllerFullInterface(),
 	                           LocalityData(),
-	                           ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))));
 	NetworkAddress master(IPAddress(0x01010101), 1);
 	NetworkAddress tlog(IPAddress(0x02020202), 1);
 	NetworkAddress satelliteTlog(IPAddress(0x03030303), 1);
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/CoordinatedState.actor.cpp
index 02a3529132..4ffdd057a2 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/CoordinatedState.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbserver/CoordinatedState.h"
 #include "fdbserver/CoordinationInterface.h"
 #include "fdbserver/Knobs.h"
@@ -288,8 +289,7 @@ struct MovableCoordinatedStateImpl {
 		// reached the point where a leader elected by the new coordinators should be doing the rest of the work
 		// (and therefore the caller should die).
 		state CoordinatedState cs(self->coordinators);
-		state CoordinatedState nccs(
-		    ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile(nc))));
+		state CoordinatedState nccs(ServerCoordinators(makeReference<ClusterConnectionMemoryRecord>(nc)));
 		state Future<Void> creationTimeout = delay(30);
 		ASSERT(self->lastValue.present() && self->lastCSValue.present());
 		TraceEvent("StartMove").detail("ConnectionString", nc.toString());
@@ -306,7 +306,7 @@ struct MovableCoordinatedStateImpl {
 			when(wait(nccs.setExclusive(
 			    BinaryWriter::toValue(MovableValue(self->lastValue.get(),
 			                                       MovableValue::MovingFrom,
-			                                       self->coordinators.ccf->getConnectionString().toString()),
+			                                       self->coordinators.ccr->getConnectionString().toString()),
 			                          IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()))))) {}
 		}
 
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index 9d5455b477..a4fe8b4b1a 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -95,8 +95,8 @@ LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : Client
 	forward.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_FORWARD, TaskPriority::Coordination);
 }
 
-ServerCoordinators::ServerCoordinators(Reference<ClusterConnectionFile> cf) : ClientCoordinators(cf) {
-	ClusterConnectionString cs = ccf->getConnectionString();
+ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
+	ClusterConnectionString cs = ccr->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s) {
 		leaderElectionServers.emplace_back(*s);
 		stateServers.emplace_back(*s);
@@ -588,7 +588,7 @@ StringRef getClusterDescriptor(Key key) {
 ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
                                 OnDemandStore* pStore,
                                 UID id,
-                                Reference<ClusterConnectionFile> ccf) {
+                                Reference<IClusterConnectionRecord> ccr) {
 	state LeaderRegisterCollection regs(pStore);
 	state ActorCollection forwarders(false);
 
@@ -609,12 +609,12 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				info.forward = forward.get().serializedInfo;
 				req.reply.send(CachedSerialization<ClientDBInfo>(info));
 			} else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT &&
 				    getClusterDescriptor(req.clusterKey).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "OpenDatabaseCoordRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.clusterKey)
 					    .detail("IncomingCoordinators", describeList(req.coordinators, req.coordinators.size()));
 					req.reply.sendError(wrong_connection_file());
@@ -628,13 +628,13 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			if (forward.present()) {
 				req.reply.send(forward.get());
 			} else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "ElectionResultRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.key)
-					    .detail("ClusterKey", ccf->getConnectionString().clusterKey())
+					    .detail("ClusterKey", ccr->getConnectionString().clusterKey())
 					    .detail("IncomingCoordinators", describeList(req.coordinators, req.coordinators.size()));
 					req.reply.sendError(wrong_connection_file());
 				} else {
@@ -647,13 +647,13 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			if (forward.present())
 				req.reply.send(forward.get());
 			else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "GetLeaderRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.key)
-					    .detail("ClusterKey", ccf->getConnectionString().clusterKey());
+					    .detail("ClusterKey", ccr->getConnectionString().clusterKey());
 					req.reply.sendError(wrong_connection_file());
 				} else {
 					regs.getInterface(req.key, id).getLeader.send(req);
@@ -665,11 +665,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			if (forward.present())
 				req.reply.send(forward.get());
 			else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "CandidacyRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.key);
 					req.reply.sendError(wrong_connection_file());
 				} else {
@@ -682,11 +682,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			if (forward.present())
 				req.reply.send(LeaderHeartbeatReply{ false });
 			else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "LeaderHeartbeatRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.key);
 					req.reply.sendError(wrong_connection_file());
 				} else {
@@ -699,11 +699,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			if (forward.present())
 				req.reply.send(Void());
 			else {
-				StringRef clusterName = ccf->getConnectionString().clusterKeyName();
+				StringRef clusterName = ccr->getConnectionString().clusterKeyName();
 				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT && getClusterDescriptor(req.key).compare(clusterName)) {
-					TraceEvent(SevWarn, "CCFMismatch")
+					TraceEvent(SevWarn, "CCRMismatch")
 					    .detail("RequestType", "ForwardRequest")
-					    .detail("LocalCS", ccf->getConnectionString().toString())
+					    .detail("LocalCS", ccr->getConnectionString().toString())
 					    .detail("IncomingClusterKey", req.key);
 					req.reply.sendError(wrong_connection_file());
 				} else {
@@ -721,7 +721,7 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 }
 
 ACTOR Future<Void> coordinationServer(std::string dataFolder,
-                                      Reference<ClusterConnectionFile> ccf,
+                                      Reference<IClusterConnectionRecord> ccr,
                                       ConfigDBType configDBType) {
 	state UID myID = deterministicRandom()->randomUniqueID();
 	state LeaderElectionRegInterface myLeaderInterface(g_network);
@@ -744,7 +744,7 @@ ACTOR Future<Void> coordinationServer(std::string dataFolder,
 	}
 
 	try {
-		wait(localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store, myID, ccf) ||
+		wait(localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store, myID, ccr) ||
 		     store.getError() || configDatabaseServer);
 		throw internal_error();
 	} catch (Error& e) {
diff --git a/fdbserver/CoordinationInterface.h b/fdbserver/CoordinationInterface.h
index 3631495dd6..f6238cc208 100644
--- a/fdbserver/CoordinationInterface.h
+++ b/fdbserver/CoordinationInterface.h
@@ -214,7 +214,7 @@ struct ForwardRequest {
 
 class ServerCoordinators : public ClientCoordinators {
 public:
-	explicit ServerCoordinators(Reference<ClusterConnectionFile>);
+	explicit ServerCoordinators(Reference<IClusterConnectionRecord>);
 
 	std::vector<LeaderElectionRegInterface> leaderElectionServers;
 	std::vector<GenerationRegInterface> stateServers;
@@ -222,7 +222,7 @@ public:
 };
 
 Future<Void> coordinationServer(std::string const& dataFolder,
-                                Reference<ClusterConnectionFile> const& ccf,
+                                Reference<IClusterConnectionRecord> const& ccf,
                                 ConfigDBType const&);
 
 #endif
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 93eb45b552..c853b0110f 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -922,14 +922,19 @@ struct DDTeamCollection : ReferenceCounted<DDTeamCollection> {
 
 		// Prefer a healthy team not containing excludeServer.
 		if (candidates.size() > 0) {
-			return teams[deterministicRandom()->randomInt(0, candidates.size())]->getServerIDs();
-		}
-
-		// The backup choice is a team with at least one server besides excludeServer, in this
-		// case, the team  will be possibily relocated to a healthy destination later by DD.
-		if (backup.size() > 0) {
-			std::vector<UID> res = teams[deterministicRandom()->randomInt(0, backup.size())]->getServerIDs();
-			std::remove(res.begin(), res.end(), excludeServer);
+			return teams[candidates[deterministicRandom()->randomInt(0, candidates.size())]]->getServerIDs();
+		} else if (backup.size() > 0) {
+			// The backup choice is a team with at least one server besides excludeServer, in this
+			// case, the team  will be possibily relocated to a healthy destination later by DD.
+			std::vector<UID> servers =
+			    teams[backup[deterministicRandom()->randomInt(0, backup.size())]]->getServerIDs();
+			std::vector<UID> res;
+			for (const UID& id : servers) {
+				if (id != excludeServer) {
+					res.push_back(id);
+				}
+			}
+			TraceEvent("FoundNonoptimalTeamForDroppedShard", excludeServer).detail("Team", describe(res));
 			return res;
 		}
 
diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp
index 40136fd45b..94476569ee 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/LeaderElection.actor.cpp
@@ -137,21 +137,21 @@ ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators,
 
 				if (!hasConnected) {
 					TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
-					    .detail("Filename", coordinators.ccf->getFilename())
-					    .detail("ConnectionStringFromFile", coordinators.ccf->getConnectionString().toString())
+					    .detail("ClusterFile", coordinators.ccr->toString())
+					    .detail("StoredConnectionString", coordinators.ccr->getConnectionString().toString())
 					    .detail("CurrentConnectionString", leader.get().first.serializedInfo.toString());
 				}
-				coordinators.ccf->setConnectionString(
+				coordinators.ccr->setConnectionString(
 				    ClusterConnectionString(leader.get().first.serializedInfo.toString()));
 				TraceEvent("LeaderForwarding")
-				    .detail("ConnStr", coordinators.ccf->getConnectionString().toString())
+				    .detail("ConnStr", coordinators.ccr->getConnectionString().toString())
 				    .trackLatest("LeaderForwarding");
 				throw coordinators_changed();
 			}
 
 			if (leader.present() && leader.get().second) {
 				hasConnected = true;
-				coordinators.ccf->notifyConnected();
+				coordinators.ccr->notifyConnected();
 			}
 
 			if (leader.present() && leader.get().second && leader.get().first.equalInternalId(myInfo)) {
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index 39c688da72..b3f6fc1c7a 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -663,6 +663,11 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 	if (g_network->isSimulated())
 		wait(delay(5.0));
 
+	TraceEvent("QuietDatabaseWaitingOnFullRecovery").log();
+	while (dbInfo->get().recoveryState != RecoveryState::FULLY_RECOVERED) {
+		wait(dbInfo->onChange());
+	}
+
 	// The quiet database check (which runs at the end of every test) will always time out due to active data movement.
 	// To get around this, quiet Database will disable the perpetual wiggle in the setup phase.
 
diff --git a/fdbserver/RestoreWorker.actor.cpp b/fdbserver/RestoreWorker.actor.cpp
index 43224fe318..7bf4010999 100644
--- a/fdbserver/RestoreWorker.actor.cpp
+++ b/fdbserver/RestoreWorker.actor.cpp
@@ -406,11 +406,11 @@ ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
 	return Void();
 }
 
-ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> restoreWorker(Reference<IClusterConnectionRecord> connRecord,
                                  LocalityData locality,
                                  std::string coordFolder) {
 	try {
-		Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST, IsInternal::True, locality);
+		Database cx = Database::createDatabase(connRecord, Database::API_VERSION_LATEST, IsInternal::True, locality);
 		wait(reportErrors(_restoreWorker(cx, locality), "RestoreWorker"));
 	} catch (Error& e) {
 		TraceEvent("FastRestoreWorker").detail("Error", e.what());
diff --git a/fdbserver/RestoreWorkerInterface.actor.h b/fdbserver/RestoreWorkerInterface.actor.h
index d2d681a0e4..02b84af808 100644
--- a/fdbserver/RestoreWorkerInterface.actor.h
+++ b/fdbserver/RestoreWorkerInterface.actor.h
@@ -711,7 +711,9 @@ std::string getRoleStr(RestoreRole role);
 
 ////--- Interface functions
 ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality);
-ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> ccf, LocalityData locality, std::string coordFolder);
+ACTOR Future<Void> restoreWorker(Reference<IClusterConnectionRecord> ccr,
+                                 LocalityData locality,
+                                 std::string coordFolder);
 
 extern const KeyRef restoreLeaderKey;
 extern const KeyRangeRef restoreWorkersKeys;
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 8fcd33bb2d..9062c20b58 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -26,6 +26,8 @@
 #include <toml.hpp>
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/simulator.h"
+#include "fdbclient/ClusterConnectionFile.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
@@ -387,7 +389,7 @@ T simulate(const T& in) {
 	return out;
 }
 
-ACTOR Future<Void> runBackup(Reference<ClusterConnectionFile> connFile) {
+ACTOR Future<Void> runBackup(Reference<IClusterConnectionRecord> connRecord) {
 	state std::vector<Future<Void>> agentFutures;
 
 	while (g_simulator.backupAgents == ISimulator::BackupAgentType::WaitForType) {
@@ -395,7 +397,7 @@ ACTOR Future<Void> runBackup(Reference<ClusterConnectionFile> connFile) {
 	}
 
 	if (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) {
-		Database cx = Database::createDatabase(connFile, -1);
+		Database cx = Database::createDatabase(connRecord, -1);
 
 		state FileBackupAgent fileAgent;
 		agentFutures.push_back(fileAgent.run(
@@ -414,7 +416,7 @@ ACTOR Future<Void> runBackup(Reference<ClusterConnectionFile> connFile) {
 	throw internal_error();
 }
 
-ACTOR Future<Void> runDr(Reference<ClusterConnectionFile> connFile) {
+ACTOR Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
 	state std::vector<Future<Void>> agentFutures;
 
 	while (g_simulator.drAgents == ISimulator::BackupAgentType::WaitForType) {
@@ -422,13 +424,13 @@ ACTOR Future<Void> runDr(Reference<ClusterConnectionFile> connFile) {
 	}
 
 	if (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) {
-		Database cx = Database::createDatabase(connFile, -1);
+		Database cx = Database::createDatabase(connRecord, -1);
 
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		state Database extraDB = Database::createDatabase(extraFile, -1);
 
 		TraceEvent("StartingDrAgents")
-		    .detail("ConnFile", connFile->getConnectionString().toString())
+		    .detail("ConnectionString", connRecord->getConnectionString().toString())
 		    .detail("ExtraString", extraFile->getConnectionString().toString());
 
 		state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
@@ -459,7 +461,7 @@ enum AgentMode { AgentNone = 0, AgentOnly = 1, AgentAddition = 2 };
 // SOMEDAY: when a process can be rebooted in isolation from the other on that machine,
 //  a loop{} will be needed around the waiting on simulatedFDBD(). For now this simply
 //  takes care of house-keeping such as context switching and file closing.
-ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConnectionRecord> connRecord,
                                                          IPAddress ip,
                                                          bool sslEnabled,
                                                          uint16_t port,
@@ -525,7 +527,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 			    .detail("Version", FDB_VT_VERSION)
 			    .detail("PackageName", FDB_VT_PACKAGE_NAME)
 			    .detail("DataFolder", *dataFolder)
-			    .detail("ConnectionString", connFile ? connFile->getConnectionString().toString() : "")
+			    .detail("ConnectionString", connRecord ? connRecord->getConnectionString().toString() : "")
 			    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 			    .detail("CommandLine", "fdbserver -r simulation")
 			    .detail("BuggifyEnabled", isBuggifyEnabled(BuggifyType::General))
@@ -546,7 +548,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 					futures.push_back(FlowTransport::transport().bind(n, n));
 				}
 				if (runBackupAgents != AgentOnly) {
-					futures.push_back(fdbd(connFile,
+					futures.push_back(fdbd(connRecord,
 					                       localities,
 					                       processClass,
 					                       *dataFolder,
@@ -561,8 +563,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 					                       configDBType));
 				}
 				if (runBackupAgents != AgentNone) {
-					futures.push_back(runBackup(connFile));
-					futures.push_back(runDr(connFile));
+					futures.push_back(runBackup(connRecord));
+					futures.push_back(runDr(connRecord));
 				}
 
 				futures.push_back(success(onShutdown));
@@ -666,9 +668,9 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 
 			if (!useSeedFile) {
 				writeFile(joinPath(*dataFolder, "fdb.cluster"), connStr.toString());
-				connFile = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"));
+				connRecord = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"));
 			} else {
-				connFile =
+				connRecord =
 				    makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"), connStr.toString());
 			}
 		} else {
@@ -747,9 +749,9 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			state std::vector<Future<ISimulator::KillType>> processes;
 			for (int i = 0; i < ips.size(); i++) {
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
-				Reference<ClusterConnectionFile> clusterFile(useSeedFile
-				                                                 ? new ClusterConnectionFile(path, connStr.toString())
-				                                                 : new ClusterConnectionFile(path));
+				Reference<IClusterConnectionRecord> clusterFile(
+				    useSeedFile ? new ClusterConnectionFile(path, connStr.toString())
+				                : new ClusterConnectionFile(path));
 				const int listenPort = i * listenPerProcess + 1;
 				AgentMode agentMode =
 				    runBackupAgents == AgentOnly ? (i == ips.size() - 1 ? AgentOnly : AgentNone) : runBackupAgents;
@@ -2196,7 +2198,7 @@ ACTOR void setupAndRun(std::string dataFolder,
                        bool restoring,
                        std::string whitelistBinPaths) {
 	state std::vector<Future<Void>> systemActors;
-	state Optional<ClusterConnectionString> connFile;
+	state Optional<ClusterConnectionString> connectionString;
 	state Standalone<StringRef> startingConfiguration;
 	state int testerCount = 1;
 	state TestConfig testConfig;
@@ -2258,7 +2260,7 @@ ACTOR void setupAndRun(std::string dataFolder,
 			wait(timeoutError(restartSimulatedSystem(&systemActors,
 			                                         dataFolder,
 			                                         &testerCount,
-			                                         &connFile,
+			                                         &connectionString,
 			                                         &startingConfiguration,
 			                                         testConfig,
 			                                         whitelistBinPaths,
@@ -2273,7 +2275,7 @@ ACTOR void setupAndRun(std::string dataFolder,
 			setupSimulatedSystem(&systemActors,
 			                     dataFolder,
 			                     &testerCount,
-			                     &connFile,
+			                     &connectionString,
 			                     &startingConfiguration,
 			                     whitelistBinPaths,
 			                     testConfig,
@@ -2282,7 +2284,7 @@ ACTOR void setupAndRun(std::string dataFolder,
 		}
 		std::string clusterFileDir = joinPath(dataFolder, deterministicRandom()->randomUniqueID().toString());
 		platform::createDirectory(clusterFileDir);
-		writeFile(joinPath(clusterFileDir, "fdb.cluster"), connFile.get().toString());
+		writeFile(joinPath(clusterFileDir, "fdb.cluster"), connectionString.get().toString());
 		wait(timeoutError(runTests(makeReference<ClusterConnectionFile>(joinPath(clusterFileDir, "fdb.cluster")),
 		                           TEST_TYPE_FROM_FILE,
 		                           TEST_ON_TESTERS,
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index d8a9563eb6..8176dac4bc 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -819,7 +819,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	for (auto& coordinator : coordinators.ccf->getConnectionString().coordinators()) {
+	for (auto& coordinator : coordinators.ccr->getConnectionString().coordinators()) {
 		roles.addCoordinatorRole(coordinator);
 	}
 
@@ -2436,7 +2436,7 @@ static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration confi
 		workerZones[worker.interf.address()] = worker.interf.locality.zoneId().orDefault(LiteralStringRef(""));
 	}
 	std::map<StringRef, int> coordinatorZoneCounts;
-	for (auto& coordinator : coordinators.ccf->getConnectionString().coordinators()) {
+	for (auto& coordinator : coordinators.ccr->getConnectionString().coordinators()) {
 		auto zone = workerZones[coordinator];
 		coordinatorZoneCounts[zone] += 1;
 	}
@@ -2820,7 +2820,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		state JsonBuilderObject data_overlay;
 
 		statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version());
-		statusObj["connection_string"] = coordinators.ccf->getConnectionString().toString();
+		statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString();
 		statusObj["bounce_impact"] = getBounceImpactInfo(statusCode);
 
 		state Optional<DatabaseConfiguration> configuration;
diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h
index 3874ff134d..0e73217a10 100644
--- a/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/TesterInterface.actor.h
@@ -113,14 +113,14 @@ struct TesterInterface {
 };
 
 ACTOR Future<Void> testerServerCore(TesterInterface interf,
-                                    Reference<ClusterConnectionFile> ccf,
+                                    Reference<IClusterConnectionRecord> ccr,
                                     Reference<AsyncVar<struct ServerDBInfo> const> serverDBInfo,
                                     LocalityData locality);
 
 enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
 enum test_type_t { TEST_TYPE_FROM_FILE, TEST_TYPE_CONSISTENCY_CHECK, TEST_TYPE_UNIT_TESTS };
 
-ACTOR Future<Void> runTests(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
                             test_type_t whatToRun,
                             test_location_t whereToRun,
                             int minTestersExpected,
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 5f789216ec..0deedd73c6 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -960,7 +960,7 @@ ACTOR Future<Void> extractClusterInterface(
     Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>> const> in,
     Reference<AsyncVar<Optional<struct ClusterInterface>>> out);
 
-ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> ccf,
+ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> ccr,
                         LocalityData localities,
                         ProcessClass processClass,
                         std::string dataFolder,
@@ -974,7 +974,7 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> ccf,
                         std::map<std::string, std::string> manualKnobOverrides,
                         ConfigDBType configDBType);
 
-ACTOR Future<Void> clusterController(Reference<ClusterConnectionFile> ccf,
+ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> ccr,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
                                      Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
                                      Future<Void> recoveredDiskFiles,
@@ -1002,8 +1002,8 @@ ACTOR Future<Void> storageServer(
     Reference<AsyncVar<ServerDBInfo> const> db,
     std::string folder,
     Promise<Void> recovered,
-    Reference<ClusterConnectionFile>
-        connFile); // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
+    Reference<IClusterConnectionRecord>
+        connRecord); // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
 ACTOR Future<Void> masterServer(MasterInterface mi,
                                 Reference<AsyncVar<ServerDBInfo> const> db,
                                 Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index fd753e82bf..1889b9525b 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -36,6 +36,7 @@
 #include <boost/interprocess/managed_shared_memory.hpp>
 
 #include "fdbclient/ActorLineageProfiler.h"
+#include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
@@ -805,7 +806,7 @@ Optional<bool> checkBuggifyOverride(const char* testFile) {
 // Takes a vector of public and listen address strings given via command line, and returns vector of NetworkAddress
 // objects.
 std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
-    const ClusterConnectionFile& connectionFile,
+    const IClusterConnectionRecord& connectionRecord,
     const std::vector<std::string>& publicAddressStrs,
     std::vector<std::string>& listenAddressStrs) {
 	if (listenAddressStrs.size() > 0 && publicAddressStrs.size() != listenAddressStrs.size()) {
@@ -823,7 +824,7 @@ std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
 	NetworkAddressList publicNetworkAddresses;
 	NetworkAddressList listenNetworkAddresses;
 
-	auto& coordinators = connectionFile.getConnectionString().coordinators();
+	auto& coordinators = connectionRecord.getConnectionString().coordinators();
 	ASSERT(coordinators.size() > 0);
 
 	for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
@@ -833,7 +834,7 @@ std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
 		if (autoPublicAddress) {
 			try {
 				const NetworkAddress& parsedAddress = NetworkAddress::parse("0.0.0.0:" + publicAddressStr.substr(5));
-				const IPAddress publicIP = determinePublicIPAutomatically(connectionFile.getConnectionString());
+				const IPAddress publicIP = determinePublicIPAutomatically(connectionRecord.getConnectionString());
 				currentPublicAddress = NetworkAddress(publicIP, parsedAddress.port, true, parsedAddress.isTLS());
 			} catch (Error& e) {
 				fprintf(stderr,
@@ -998,7 +999,7 @@ struct CLIOptions {
 	std::string configPath;
 	ConfigDBType configDBType{ ConfigDBType::DISABLED };
 
-	Reference<ClusterConnectionFile> connectionFile;
+	Reference<IClusterConnectionRecord> connectionFile;
 	Standalone<StringRef> machineId;
 	UnitTestParameters testParams;
 
@@ -1849,7 +1850,7 @@ int main(int argc, char* argv[]) {
 		    .detail("FileSystem", opts.fileSystemPath)
 		    .detail("DataFolder", opts.dataFolder)
 		    .detail("WorkingDirectory", cwd)
-		    .detail("ClusterFile", opts.connectionFile ? opts.connectionFile->getFilename().c_str() : "")
+		    .detail("ClusterFile", opts.connectionFile ? opts.connectionFile->toString() : "")
 		    .detail("ConnectionString",
 		            opts.connectionFile ? opts.connectionFile->getConnectionString().toString() : "")
 		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 428425e6c5..292a758f42 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -1867,7 +1867,7 @@ ACTOR Future<Void> masterCore(Reference<MasterData> self) {
 	tr.set(
 	    recoveryCommitRequest.arena, primaryLocalityKey, BinaryWriter::toValue(self->primaryLocality, Unversioned()));
 	tr.set(recoveryCommitRequest.arena, backupVersionKey, backupVersionValue);
-	tr.set(recoveryCommitRequest.arena, coordinatorsKey, self->coordinators.ccf->getConnectionString().toString());
+	tr.set(recoveryCommitRequest.arena, coordinatorsKey, self->coordinators.ccr->getConnectionString().toString());
 	tr.set(recoveryCommitRequest.arena, logsKey, self->logSystem->getLogsValue());
 	tr.set(recoveryCommitRequest.arena,
 	       primaryDatacenterKey,
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 97847ecce2..fcd299abe7 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -691,6 +691,9 @@ public:
 	bool debug_inApplyUpdate;
 	double debug_lastValidateTime;
 
+	int64_t lastBytesInputEBrake;
+	Version lastDurableVersionEBrake;
+
 	int maxQueryQueue;
 	int getAndResetMaxQueryQueueSize() {
 		int val = maxQueryQueue;
@@ -889,8 +892,8 @@ public:
 	    fetchKeysParallelismLock(SERVER_KNOBS->FETCH_KEYS_PARALLELISM),
 	    fetchKeysBytesBudget(SERVER_KNOBS->STORAGE_FETCH_BYTES), fetchKeysBudgetUsed(false),
 	    instanceID(deterministicRandom()->randomUniqueID().first()), shuttingDown(false), behind(false),
-	    versionBehind(false), debug_inApplyUpdate(false), debug_lastValidateTime(0), maxQueryQueue(0),
-	    transactionTagCounter(ssi.id()), counters(this),
+	    versionBehind(false), debug_inApplyUpdate(false), debug_lastValidateTime(0), lastBytesInputEBrake(0),
+	    lastDurableVersionEBrake(0), maxQueryQueue(0), transactionTagCounter(ssi.id()), counters(this),
 	    storageServerSourceTLogIDEventHolder(
 	        makeReference<EventCacheHolder>(ssi.id().toString() + "/StorageServerSourceTLogID")) {
 		version.initMetric(LiteralStringRef("StorageServer.Version"), counters.cc.id);
@@ -3285,41 +3288,12 @@ static const KeyRangeRef persistChangeFeedKeys =
     KeyRangeRef(LiteralStringRef(PERSIST_PREFIX "RF/"), LiteralStringRef(PERSIST_PREFIX "RF0"));
 // data keys are unmangled (but never start with PERSIST_PREFIX because they are always in allKeys)
 
-ACTOR Future<Void> fetchChangeFeed(StorageServer* data,
-                                   Key rangeId,
-                                   KeyRange range,
-                                   bool stopped,
-                                   Version fetchVersion) {
-	state Reference<ChangeFeedInfo> changeFeedInfo;
-	wait(delay(0)); // allow this actor to be cancelled by removals
-	bool existing = data->uidChangeFeed.count(rangeId);
-
-	TraceEvent(SevDebug, "FetchChangeFeed", data->thisServerID)
-	    .detail("RangeID", rangeId.printable())
-	    .detail("Range", range.toString())
-	    .detail("Existing", existing);
-
-	if (!existing) {
-		changeFeedInfo = Reference<ChangeFeedInfo>(new ChangeFeedInfo());
-		changeFeedInfo->range = range;
-		changeFeedInfo->id = rangeId;
-		changeFeedInfo->stopped = stopped;
-		data->uidChangeFeed[rangeId] = changeFeedInfo;
-		auto rs = data->keyChangeFeed.modify(range);
-		for (auto r = rs.begin(); r != rs.end(); ++r) {
-			r->value().push_back(changeFeedInfo);
-		}
-		data->keyChangeFeed.coalesce(range.contents());
-		auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
-		data->addMutationToMutationLog(
-		    mLV,
-		    MutationRef(MutationRef::SetValue,
-		                persistChangeFeedKeys.begin.toString() + rangeId.toString(),
-		                changeFeedValue(range, invalidVersion, ChangeFeedStatus::CHANGE_FEED_CREATE)));
-	} else {
-		changeFeedInfo = data->uidChangeFeed[rangeId];
-	}
-
+ACTOR Future<Void> fetchChangeFeedApplier(StorageServer* data,
+                                          Reference<ChangeFeedInfo> changeFeedInfo,
+                                          Key rangeId,
+                                          KeyRange range,
+                                          Version fetchVersion,
+                                          bool existing) {
 	state PromiseStream<Standalone<VectorRef<MutationsAndVersionRef>>> feedResults;
 	state Future<Void> feed = data->cx->getChangeFeedStream(
 	    feedResults, rangeId, 0, existing ? fetchVersion + 1 : data->version.get() + 1, range);
@@ -3340,7 +3314,7 @@ ACTOR Future<Void> fetchChangeFeed(StorageServer* data,
 				wait(yield());
 			}
 		} catch (Error& e) {
-			if (e.code() != error_code_end_of_stream && e.code() != error_code_change_feed_not_registered) {
+			if (e.code() != error_code_end_of_stream) {
 				throw;
 			}
 			return Void();
@@ -3396,13 +3370,61 @@ ACTOR Future<Void> fetchChangeFeed(StorageServer* data,
 			wait(yield());
 		}
 	} catch (Error& e) {
-		if (e.code() != error_code_end_of_stream && e.code() != error_code_change_feed_not_registered) {
+		if (e.code() != error_code_end_of_stream) {
 			throw;
 		}
 	}
 	return Void();
 }
 
+ACTOR Future<Void> fetchChangeFeed(StorageServer* data,
+                                   Key rangeId,
+                                   KeyRange range,
+                                   bool stopped,
+                                   Version fetchVersion) {
+	state Reference<ChangeFeedInfo> changeFeedInfo;
+	wait(delay(0)); // allow this actor to be cancelled by removals
+	state bool existing = data->uidChangeFeed.count(rangeId);
+
+	TraceEvent(SevDebug, "FetchChangeFeed", data->thisServerID)
+	    .detail("RangeID", rangeId.printable())
+	    .detail("Range", range.toString())
+	    .detail("Existing", existing);
+
+	if (!existing) {
+		changeFeedInfo = Reference<ChangeFeedInfo>(new ChangeFeedInfo());
+		changeFeedInfo->range = range;
+		changeFeedInfo->id = rangeId;
+		changeFeedInfo->stopped = stopped;
+		data->uidChangeFeed[rangeId] = changeFeedInfo;
+		auto rs = data->keyChangeFeed.modify(range);
+		for (auto r = rs.begin(); r != rs.end(); ++r) {
+			r->value().push_back(changeFeedInfo);
+		}
+		data->keyChangeFeed.coalesce(range.contents());
+		auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
+		data->addMutationToMutationLog(
+		    mLV,
+		    MutationRef(MutationRef::SetValue,
+		                persistChangeFeedKeys.begin.toString() + rangeId.toString(),
+		                changeFeedValue(range, invalidVersion, ChangeFeedStatus::CHANGE_FEED_CREATE)));
+	} else {
+		changeFeedInfo = data->uidChangeFeed[rangeId];
+	}
+
+	loop {
+		try {
+			wait(fetchChangeFeedApplier(data, changeFeedInfo, rangeId, range, fetchVersion, existing));
+			return Void();
+		} catch (Error& e) {
+			if (e.code() != error_code_change_feed_not_registered) {
+				throw;
+			}
+		}
+		wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+	}
+}
+
 ACTOR Future<Void> dispatchChangeFeeds(StorageServer* data, UID fetchKeysID, KeyRange keys, Version fetchVersion) {
 	// find overlapping range feeds
 	state std::map<Key, Future<Void>> feedFetches;
@@ -3871,7 +3893,7 @@ void ShardInfo::addMutation(Version version, bool fromFetch, MutationRef const&
 	}
 }
 
-enum ChangeServerKeysContext { CSK_UPDATE, CSK_RESTORE };
+enum ChangeServerKeysContext { CSK_UPDATE, CSK_RESTORE, CSK_ASSIGN_EMPTY };
 const char* changeServerKeysContextName[] = { "Update", "Restore" };
 
 void changeServerKeys(StorageServer* data,
@@ -3939,6 +3961,7 @@ void changeServerKeys(StorageServer* data,
 	auto vr = data->newestAvailableVersion.intersectingRanges(keys);
 	std::vector<std::pair<KeyRange, Version>> changeNewestAvailable;
 	std::vector<KeyRange> removeRanges;
+	std::vector<KeyRange> newEmptyRanges;
 	for (auto r = vr.begin(); r != vr.end(); ++r) {
 		KeyRangeRef range = keys & r->range();
 		bool dataAvailable = r->value() == latestVersion || r->value() >= version;
@@ -3949,7 +3972,14 @@ void changeServerKeys(StorageServer* data,
 		//     .detail("NowAssigned", nowAssigned)
 		//     .detail("NewestAvailable", r->value())
 		//     .detail("ShardState0", data->shards[range.begin]->debugDescribeState());
-		if (!nowAssigned) {
+		if (context == CSK_ASSIGN_EMPTY && !dataAvailable) {
+			ASSERT(nowAssigned);
+			TraceEvent("ChangeServerKeysAddEmptyRange", data->thisServerID)
+			    .detail("Begin", range.begin)
+			    .detail("End", range.end);
+			newEmptyRanges.push_back(range);
+			data->addShard(ShardInfo::newReadWrite(range, data));
+		} else if (!nowAssigned) {
 			if (dataAvailable) {
 				ASSERT(r->value() ==
 				       latestVersion); // Not that we care, but this used to be checked instead of dataAvailable
@@ -3962,7 +3992,7 @@ void changeServerKeys(StorageServer* data,
 		} else if (!dataAvailable) {
 			// SOMEDAY: Avoid restarting adding/transferred shards
 			if (version == 0) { // bypass fetchkeys; shard is known empty at version 0
-				TraceEvent("ChangeServerKeysAddEmptyRange", data->thisServerID)
+				TraceEvent("ChangeServerKeysInitialRange", data->thisServerID)
 				    .detail("Begin", range.begin)
 				    .detail("End", range.end);
 				changeNewestAvailable.emplace_back(range, latestVersion);
@@ -3996,6 +4026,14 @@ void changeServerKeys(StorageServer* data,
 		removeDataRange(data, data->addVersionToMutationLog(data->data().getLatestVersion()), data->shards, *r);
 		setAvailableStatus(data, *r, false);
 	}
+
+	// Clear the moving-in empty range, and set it available at the latestVersion.
+	for (const auto& range : newEmptyRanges) {
+		MutationRef clearRange(MutationRef::ClearRange, range.begin, range.end);
+		data->addMutation(data->data().getLatestVersion(), true, clearRange, range, data->updateEagerReads);
+		data->newestAvailableVersion.insert(range, latestVersion);
+		setAvailableStatus(data, range, true);
+	}
 	validate(data);
 }
 
@@ -4115,8 +4153,8 @@ private:
 				// the data for change.version-1 (changes from versions < change.version)
 				// If emptyRange, treat the shard as empty, see removeKeysFromFailedServer() for more details about this
 				// scenario.
-				const Version shardVersion = (emptyRange && nowAssigned) ? 0 : currentVersion - 1;
-				changeServerKeys(data, keys, nowAssigned, shardVersion, CSK_UPDATE);
+				const ChangeServerKeysContext context = emptyRange ? CSK_ASSIGN_EMPTY : CSK_UPDATE;
+				changeServerKeys(data, keys, nowAssigned, currentVersion - 1, context);
 			}
 
 			processedStartKey = false;
@@ -4339,18 +4377,36 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
 	try {
 		// If we are disk bound and durableVersion is very old, we need to block updates or we could run out of memory
 		// This is often referred to as the storage server e-brake (emergency brake)
-		state double waitStartT = 0;
-		while (data->queueSize() >= SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES &&
-		       data->durableVersion.get() < data->desiredOldestVersion.get()) {
-			if (now() - waitStartT >= 1) {
-				TraceEvent(SevWarn, "StorageServerUpdateLag", data->thisServerID)
-				    .detail("Version", data->version.get())
-				    .detail("DurableVersion", data->durableVersion.get());
-				waitStartT = now();
-			}
 
-			data->behind = true;
-			wait(delayJittered(.005, TaskPriority::TLogPeekReply));
+		// We allow the storage server to make some progress between e-brake periods, referreed to as "overage", in
+		// order to ensure that it advances desiredOldestVersion enough for updateStorage to make enough progress on
+		// freeing up queue size.
+		state double waitStartT = 0;
+		if (data->queueSize() >= SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES &&
+		    data->durableVersion.get() < data->desiredOldestVersion.get() &&
+		    ((data->desiredOldestVersion.get() - SERVER_KNOBS->STORAGE_HARD_LIMIT_VERSION_OVERAGE >
+		      data->lastDurableVersionEBrake) ||
+		     (data->counters.bytesInput.getValue() - SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE >
+		      data->lastBytesInputEBrake))) {
+
+			while (data->queueSize() >= SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES &&
+			       data->durableVersion.get() < data->desiredOldestVersion.get()) {
+				if (now() - waitStartT >= 1) {
+					TraceEvent(SevWarn, "StorageServerUpdateLag", data->thisServerID)
+					    .detail("Version", data->version.get())
+					    .detail("DurableVersion", data->durableVersion.get())
+					    .detail("DesiredOldestVersion", data->desiredOldestVersion.get())
+					    .detail("QueueSize", data->queueSize())
+					    .detail("LastBytesInputEBrake", data->lastBytesInputEBrake)
+					    .detail("LastDurableVersionEBrake", data->lastDurableVersionEBrake);
+					waitStartT = now();
+				}
+
+				data->behind = true;
+				wait(delayJittered(.005, TaskPriority::TLogPeekReply));
+			}
+			data->lastBytesInputEBrake = data->counters.bytesInput.getValue();
+			data->lastDurableVersionEBrake = data->durableVersion.get();
 		}
 
 		if (g_network->isSimulated() && data->isTss() && g_simulator.tssMode == ISimulator::TSSMode::EnabledAddDelay &&
@@ -5801,7 +5857,7 @@ ACTOR Future<Void> reportStorageServerState(StorageServer* self) {
 			level = SevWarnAlways;
 		}
 
-		TraceEvent(level, "FetchKeyCurrentStatus")
+		TraceEvent(level, "FetchKeysCurrentStatus", self->thisServerID)
 		    .detail("Timestamp", now())
 		    .detail("LongestRunningTime", longestRunningFetchKeys.first)
 		    .detail("StartKey", longestRunningFetchKeys.second.begin)
@@ -5956,13 +6012,13 @@ bool storageServerTerminated(StorageServer& self, IKeyValueStore* persistentData
 		return false;
 }
 
-ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<ClusterConnectionFile> connFile, UID id) {
-	if (store->getType() != KeyValueStoreType::MEMORY || connFile.getPtr() == nullptr) {
+ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterConnectionRecord> connRecord, UID id) {
+	if (store->getType() != KeyValueStoreType::MEMORY || connRecord.getPtr() == nullptr) {
 		return Never();
 	}
 
 	// create a temp client connect to DB
-	Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST);
+	Database cx = Database::createDatabase(connRecord, Database::API_VERSION_LATEST);
 
 	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 	state int noCanRemoveCount = 0;
@@ -6196,7 +6252,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
                                  Reference<AsyncVar<ServerDBInfo> const> db,
                                  std::string folder,
                                  Promise<Void> recovered,
-                                 Reference<ClusterConnectionFile> connFile) {
+                                 Reference<IClusterConnectionRecord> connRecord) {
 	state StorageServer self(persistentData, db, ssi);
 	self.folder = folder;
 
@@ -6210,7 +6266,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 			// for memory storage engine type, wait until recovery is done before commit
 			when(wait(self.storage.commit())) {}
 
-			when(wait(memoryStoreRecover(persistentData, connFile, self.thisServerID))) {
+			when(wait(memoryStoreRecover(persistentData, connRecord, self.thisServerID))) {
 				TraceEvent("DisposeStorageServer", self.thisServerID).log();
 				throw worker_removed();
 			}
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 3184ba3ef4..4bfaf2c722 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -599,7 +599,7 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 }
 
 ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
-                                        Reference<ClusterConnectionFile> ccf,
+                                        Reference<IClusterConnectionRecord> ccr,
                                         Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
                                         LocalityData locality) {
 	state WorkloadInterface workIface;
@@ -614,7 +614,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 		startRole(Role::TESTER, workIface.id(), UID(), details);
 
 		if (work.useDatabase) {
-			cx = Database::createDatabase(ccf, -1, IsInternal::True, locality);
+			cx = Database::createDatabase(ccr, -1, IsInternal::True, locality);
 			wait(delay(1.0));
 		}
 
@@ -658,7 +658,7 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 }
 
 ACTOR Future<Void> testerServerCore(TesterInterface interf,
-                                    Reference<ClusterConnectionFile> ccf,
+                                    Reference<IClusterConnectionRecord> ccr,
                                     Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
                                     LocalityData locality) {
 	state PromiseStream<Future<Void>> addWorkload;
@@ -668,7 +668,7 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 	loop choose {
 		when(wait(workerFatalError)) {}
 		when(WorkloadRequest work = waitNext(interf.recruitments.getFuture())) {
-			addWorkload.send(testerServerWorkload(work, ccf, dbInfo, locality));
+			addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality));
 		}
 	}
 }
@@ -1583,8 +1583,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
  * functionality. Its main purpose is to generate the test specification from passed arguments and then call into the
  * correct actor which will orchestrate the actual test.
  *
- * \param connFile A cluster connection file. Not all tests require a functional cluster but all tests require
- * a cluster file.
+ * \param connRecord A cluster connection record. Not all tests require a functional cluster but all tests require
+ * a cluster record.
  * \param whatToRun TEST_TYPE_FROM_FILE to read the test description from a passed toml file or
  * TEST_TYPE_CONSISTENCY_CHECK to generate a test spec for consistency checking
  * \param at TEST_HERE: this process will act as a test client and execute the given workload. TEST_ON_SERVERS: Run a
@@ -1600,7 +1600,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
  *
  * \returns A future which will be set after all tests finished.
  */
-ACTOR Future<Void> runTests(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
                             test_type_t whatToRun,
                             test_location_t at,
                             int minTestersExpected,
@@ -1612,8 +1612,8 @@ ACTOR Future<Void> runTests(Reference<ClusterConnectionFile> connFile,
 	auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
 	auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
 	std::vector<Future<Void>> actors;
-	if (connFile) {
-		actors.push_back(reportErrors(monitorLeader(connFile, cc), "MonitorLeader"));
+	if (connRecord) {
+		actors.push_back(reportErrors(monitorLeader(connRecord, cc), "MonitorLeader"));
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
 	}
 
@@ -1688,7 +1688,7 @@ ACTOR Future<Void> runTests(Reference<ClusterConnectionFile> connFile,
 		std::vector<TesterInterface> iTesters(1);
 		actors.push_back(
 		    reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
-		actors.push_back(reportErrors(testerServerCore(iTesters[0], connFile, db, locality), "TesterServerCore"));
+		actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
 		tests = runTests(cc, ci, iTesters, testSpecs, startingConfiguration, locality);
 	} else {
 		tests = reportErrors(runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality),
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index af2b8e2beb..c42fcff082 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -520,7 +520,7 @@ ACTOR Future<Void> registrationClient(Reference<AsyncVar<Optional<ClusterControl
                                       Reference<AsyncVar<Optional<RatekeeperInterface>> const> rkInterf,
                                       Reference<AsyncVar<Optional<BlobManagerInterface>> const> bmInterf,
                                       Reference<AsyncVar<bool> const> degraded,
-                                      Reference<ClusterConnectionFile> connFile,
+                                      Reference<IClusterConnectionRecord> connRecord,
                                       Reference<AsyncVar<std::set<std::string>> const> issues,
                                       Reference<LocalConfiguration> localConfig) {
 	// Keeps the cluster controller (as it may be re-elected) informed that this worker exists
@@ -534,6 +534,16 @@ ACTOR Future<Void> registrationClient(Reference<AsyncVar<Optional<ClusterControl
 	state Future<Void> cacheErrorsFuture;
 	state Optional<double> incorrectTime;
 	loop {
+		state ClusterConnectionString storedConnectionString;
+		state bool upToDate = true;
+		if (connRecord) {
+			bool upToDateResult = wait(connRecord->upToDate(storedConnectionString));
+			upToDate = upToDateResult;
+		}
+		if (upToDate) {
+			incorrectTime = Optional<double>();
+		}
+
 		RegisterWorkerRequest request(interf,
 		                              initialClass,
 		                              processClass,
@@ -545,28 +555,25 @@ ACTOR Future<Void> registrationClient(Reference<AsyncVar<Optional<ClusterControl
 		                              degraded->get(),
 		                              localConfig->lastSeenVersion(),
 		                              localConfig->configClassSet());
+
 		for (auto const& i : issues->get()) {
 			request.issues.push_back_deep(request.issues.arena(), i);
 		}
-		ClusterConnectionString fileConnectionString;
-		if (connFile && !connFile->fileContentsUpToDate(fileConnectionString)) {
+
+		if (!upToDate) {
 			request.issues.push_back_deep(request.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
-			std::string connectionString = connFile->getConnectionString().toString();
+			std::string connectionString = connRecord->getConnectionString().toString();
 			if (!incorrectTime.present()) {
 				incorrectTime = now();
 			}
-			if (connFile->canGetFilename()) {
-				// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the
-				// file right before us)
-				TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
-				    .detail("Filename", connFile->getFilename())
-				    .detail("ConnectionStringFromFile", fileConnectionString.toString())
-				    .detail("CurrentConnectionString", connectionString);
-			}
-		} else {
-			incorrectTime = Optional<double>();
-		}
 
+			// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing
+			// the file right before us)
+			TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
+			    .detail("ClusterFile", connRecord->toString())
+			    .detail("StoredConnectionString", storedConnectionString.toString())
+			    .detail("CurrentConnectionString", connectionString);
+		}
 		auto peers = FlowTransport::transport().getIncompatiblePeers();
 		for (auto it = peers->begin(); it != peers->end();) {
 			if (now() - it->second.second > FLOW_KNOBS->INCOMPATIBLE_PEER_DELAY_BEFORE_LOGGING) {
@@ -1103,7 +1110,7 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
 		DUMPTOKEN(recruited.getKeyValuesStream);
 
 		prevStorageServer =
-		    storageServer(store, recruited, db, folder, Promise<Void>(), Reference<ClusterConnectionFile>(nullptr));
+		    storageServer(store, recruited, db, folder, Promise<Void>(), Reference<IClusterConnectionRecord>(nullptr));
 		prevStorageServer = handleIOErrors(prevStorageServer, store, id, store->onClosed());
 	}
 }
@@ -1316,7 +1323,7 @@ struct SharedLogsValue {
 	  : actor(actor), uid(uid), requests(requests) {}
 };
 
-ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
                                 Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
                                 LocalityData locality,
                                 Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
@@ -1392,7 +1399,7 @@ ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
 	errorForwarders.add(loadedPonger(interf.debugPing.getFuture()));
 	errorForwarders.add(waitFailureServer(interf.waitFailure.getFuture()));
 	errorForwarders.add(monitorTraceLogIssues(issues));
-	errorForwarders.add(testerServerCore(interf.testerInterface, connFile, dbInfo, locality));
+	errorForwarders.add(testerServerCore(interf.testerInterface, connRecord, dbInfo, locality));
 	errorForwarders.add(monitorHighMemory(memoryProfileThreshold));
 
 	filesClosed.add(stopping.getFuture());
@@ -1473,7 +1480,7 @@ ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
 				DUMPTOKEN(recruited.getKeyValuesStream);
 
 				Promise<Void> recovery;
-				Future<Void> f = storageServer(kv, recruited, dbInfo, folder, recovery, connFile);
+				Future<Void> f = storageServer(kv, recruited, dbInfo, folder, recovery, connRecord);
 				recoveries.push_back(recovery.getFuture());
 				f = handleIOErrors(f, kv, s.storeID, kvClosed);
 				f = storageServerRollbackRebooter(&runningStorages,
@@ -1600,7 +1607,7 @@ ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
 		                                       rkInterf,
 		                                       bmInterf,
 		                                       degraded,
-		                                       connFile,
+		                                       connRecord,
 		                                       issues,
 		                                       localConfig));
 
@@ -1717,7 +1724,7 @@ ACTOR Future<Void> workerServer(Reference<ClusterConnectionFile> connFile,
 
 				// printf("Recruited as masterServer\n");
 				Future<Void> masterProcess = masterServer(
-				    recruited, dbInfo, ccInterface, ServerCoordinators(connFile), req.lifetime, req.forceRecovery);
+				    recruited, dbInfo, ccInterface, ServerCoordinators(connRecord), req.lifetime, req.forceRecovery);
 				errorForwarders.add(
 				    zombie(recruited, forwardError(errors, Role::MASTER, recruited.id(), masterProcess)));
 				req.reply.send(recruited);
@@ -2339,10 +2346,10 @@ ACTOR Future<UID> createAndLockProcessIdFile(std::string folder) {
 }
 
 ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGeneration(
-    Reference<ClusterConnectionFile> connFile,
+    Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<Value>> result,
     MonitorLeaderInfo info) {
-	state ClusterConnectionString ccf = info.intermediateConnFile->getConnectionString();
+	state ClusterConnectionString ccf = info.intermediateConnRecord->getConnectionString();
 	state std::vector<NetworkAddress> addrs = ccf.coordinators();
 	state ElectionResultRequest request;
 	state int index = 0;
@@ -2360,24 +2367,24 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 		if (leader.present()) {
 			if (leader.get().present()) {
 				if (leader.get().get().forward) {
-					info.intermediateConnFile = makeReference<ClusterConnectionFile>(
-					    connFile->getFilename(), ClusterConnectionString(leader.get().get().serializedInfo.toString()));
+					info.intermediateConnRecord = connRecord->makeIntermediateRecord(
+					    ClusterConnectionString(leader.get().get().serializedInfo.toString()));
 					return info;
 				}
-				if (connFile != info.intermediateConnFile) {
+				if (connRecord != info.intermediateConnRecord) {
 					if (!info.hasConnected) {
 						TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
-						    .detail("Filename", connFile->getFilename())
-						    .detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
+						    .detail("ClusterFile", connRecord->toString())
+						    .detail("StoredConnectionString", connRecord->getConnectionString().toString())
 						    .detail("CurrentConnectionString",
-						            info.intermediateConnFile->getConnectionString().toString());
+						            info.intermediateConnRecord->getConnectionString().toString());
 					}
-					connFile->setConnectionString(info.intermediateConnFile->getConnectionString());
-					info.intermediateConnFile = connFile;
+					connRecord->setConnectionString(info.intermediateConnRecord->getConnectionString());
+					info.intermediateConnRecord = connRecord;
 				}
 
 				info.hasConnected = true;
-				connFile->notifyConnected();
+				connRecord->notifyConnected();
 				request.knownLeader = leader.get().get().changeID;
 
 				ClusterControllerPriorityInfo info = leader.get().get().getPriorityInfo();
@@ -2400,35 +2407,35 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 	}
 }
 
-ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClusterConnectionRecord> connRecord,
                                                                  Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
-	state MonitorLeaderInfo info(connFile);
+	state MonitorLeaderInfo info(connRecord);
 	loop {
 		MonitorLeaderInfo _info =
-		    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connFile, outSerializedLeaderInfo, info));
+		    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info));
 		info = _info;
 	}
 }
 
 template <class LeaderInterface>
 Future<Void> monitorLeaderWithDelayedCandidacyImpl(
-    Reference<ClusterConnectionFile> const& connFile,
+    Reference<IClusterConnectionRecord> const& connRecord,
     Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
 	LeaderDeserializer<LeaderInterface> deserializer;
 	auto serializedInfo = makeReference<AsyncVar<Value>>();
-	Future<Void> m = monitorLeaderWithDelayedCandidacyImplInternal(connFile, serializedInfo);
+	Future<Void> m = monitorLeaderWithDelayedCandidacyImplInternal(connRecord, serializedInfo);
 	return m || deserializer(serializedInfo, outKnownLeader);
 }
 
 ACTOR Future<Void> monitorLeaderWithDelayedCandidacy(
-    Reference<ClusterConnectionFile> connFile,
+    Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
     Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
     Future<Void> recoveredDiskFiles,
     LocalityData locality,
     Reference<AsyncVar<ServerDBInfo>> dbInfo,
     ConfigDBType configDBType) {
-	state Future<Void> monitor = monitorLeaderWithDelayedCandidacyImpl(connFile, currentCC);
+	state Future<Void> monitor = monitorLeaderWithDelayedCandidacyImpl(connRecord, currentCC);
 	state Future<Void> timeout;
 
 	wait(recoveredDiskFiles);
@@ -2454,7 +2461,7 @@ ACTOR Future<Void> monitorLeaderWithDelayedCandidacy(
 			when(wait(timeout.isValid() ? timeout : Never())) {
 				monitor.cancel();
 				wait(clusterController(
-				    connFile, currentCC, asyncPriorityInfo, recoveredDiskFiles, locality, configDBType));
+				    connRecord, currentCC, asyncPriorityInfo, recoveredDiskFiles, locality, configDBType));
 				return Void();
 			}
 		}
@@ -2504,7 +2511,7 @@ ACTOR Future<Void> serveProcess() {
 	}
 }
 
-ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
+ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
                         LocalityData localities,
                         ProcessClass processClass,
                         std::string dataFolder,
@@ -2535,7 +2542,7 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
 	actors.push_back(serveProcess());
 
 	try {
-		ServerCoordinators coordinators(connFile);
+		ServerCoordinators coordinators(connRecord);
 		if (g_network->isSimulated()) {
 			whitelistBinPaths = ",, random_path,  /bin/snap_create.sh,,";
 		}
@@ -2552,7 +2559,7 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
 		if (coordFolder.size()) {
 			// SOMEDAY: remove the fileNotFound wrapper and make DiskQueue construction safe from errors setting up
 			// their files
-			actors.push_back(fileNotFoundToNever(coordinationServer(coordFolder, coordinators.ccf, configDBType)));
+			actors.push_back(fileNotFoundToNever(coordinationServer(coordFolder, coordinators.ccr, configDBType)));
 		}
 
 		state UID processIDUid = wait(createAndLockProcessIdFile(dataFolder));
@@ -2569,21 +2576,25 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
 		actors.push_back(reportErrors(monitorAndWriteCCPriorityInfo(fitnessFilePath, asyncPriorityInfo),
 		                              "MonitorAndWriteCCPriorityInfo"));
 		if (processClass.machineClassFitness(ProcessClass::ClusterController) == ProcessClass::NeverAssign) {
-			actors.push_back(reportErrors(monitorLeader(connFile, cc), "ClusterController"));
+			actors.push_back(reportErrors(monitorLeader(connRecord, cc), "ClusterController"));
 		} else if (processClass.machineClassFitness(ProcessClass::ClusterController) == ProcessClass::WorstFit &&
 		           SERVER_KNOBS->MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS > 0) {
-			actors.push_back(reportErrors(
-			    monitorLeaderWithDelayedCandidacy(
-			        connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, dbInfo, configDBType),
-			    "ClusterController"));
+			actors.push_back(reportErrors(monitorLeaderWithDelayedCandidacy(connRecord,
+			                                                                cc,
+			                                                                asyncPriorityInfo,
+			                                                                recoveredDiskFiles.getFuture(),
+			                                                                localities,
+			                                                                dbInfo,
+			                                                                configDBType),
+			                              "ClusterController"));
 		} else {
 			actors.push_back(reportErrors(
 			    clusterController(
-			        connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, configDBType),
+			        connRecord, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, configDBType),
 			    "ClusterController"));
 		}
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
-		actors.push_back(reportErrorsExcept(workerServer(connFile,
+		actors.push_back(reportErrorsExcept(workerServer(connRecord,
 		                                                 cc,
 		                                                 localities,
 		                                                 asyncPriorityInfo,
diff --git a/fdbserver/workloads/ApiWorkload.h b/fdbserver/workloads/ApiWorkload.h
index b3992f33f3..c345f95226 100644
--- a/fdbserver/workloads/ApiWorkload.h
+++ b/fdbserver/workloads/ApiWorkload.h
@@ -23,6 +23,7 @@
 #pragma once
 
 #include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbserver/workloads/MemoryKeyValueStore.h"
@@ -239,7 +240,7 @@ struct ApiWorkload : TestWorkload {
 
 		useExtraDB = g_simulator.extraDB != nullptr;
 		if (useExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 			extraDB = Database::createDatabase(extraFile, -1);
 		}
 	}
diff --git a/fdbserver/workloads/AtomicSwitchover.actor.cpp b/fdbserver/workloads/AtomicSwitchover.actor.cpp
index f60156ddf7..aedd4f0dae 100644
--- a/fdbserver/workloads/AtomicSwitchover.actor.cpp
+++ b/fdbserver/workloads/AtomicSwitchover.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -38,7 +39,7 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		extraDB = Database::createDatabase(extraFile, -1);
 	}
 
diff --git a/fdbserver/workloads/BackupToDBAbort.actor.cpp b/fdbserver/workloads/BackupToDBAbort.actor.cpp
index 1cad65dc7b..1a6a23eca6 100644
--- a/fdbserver/workloads/BackupToDBAbort.actor.cpp
+++ b/fdbserver/workloads/BackupToDBAbort.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -35,7 +36,7 @@ struct BackupToDBAbort : TestWorkload {
 
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		extraDB = Database::createDatabase(extraFile, -1);
 
 		lockid = UID(0xbeeffeed, 0xdecaf00d);
diff --git a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
index c2d0ecd1ed..b80952f352 100644
--- a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -127,7 +128,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 		}
 
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		extraDB = Database::createDatabase(extraFile, -1);
 
 		TraceEvent("BARW_Start").detail("Locked", locked);
diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
index 414bcfabb3..91c56166a1 100644
--- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
+++ b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
@@ -75,7 +76,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			}
 		}
 
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		extraDB = Database::createDatabase(extraFile, -1);
 
 		TraceEvent("DRU_Start").log();
diff --git a/fdbserver/workloads/ChangeConfig.actor.cpp b/fdbserver/workloads/ChangeConfig.actor.cpp
index 50073f29a0..c19a97782d 100644
--- a/fdbserver/workloads/ChangeConfig.actor.cpp
+++ b/fdbserver/workloads/ChangeConfig.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
@@ -56,7 +57,7 @@ struct ChangeConfigWorkload : TestWorkload {
 	// for the extra cluster.
 	ACTOR Future<Void> extraDatabaseConfigure(ChangeConfigWorkload* self) {
 		if (g_network->isSimulated() && g_simulator.extraDB) {
-			auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 			state Database extraDB = Database::createDatabase(extraFile, -1);
 
 			wait(delay(5 * deterministicRandom()->random01()));
diff --git a/fdbserver/workloads/DataLossRecovery.actor.cpp b/fdbserver/workloads/DataLossRecovery.actor.cpp
new file mode 100644
index 0000000000..8169b6ecf0
--- /dev/null
+++ b/fdbserver/workloads/DataLossRecovery.actor.cpp
@@ -0,0 +1,256 @@
+/*
+ * DataLossRecovery.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include <limits>
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/QuietDatabase.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/flow.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+std::string printValue(const ErrorOr<Optional<Value>>& value) {
+	if (value.isError()) {
+		return value.getError().name();
+	}
+	return value.get().present() ? value.get().get().toString() : "Value Not Found.";
+}
+} // namespace
+
+struct DataLossRecoveryWorkload : TestWorkload {
+	FlowLock startMoveKeysParallelismLock;
+	FlowLock finishMoveKeysParallelismLock;
+	const bool enabled;
+	bool pass;
+	NetworkAddress addr;
+
+	DataLossRecoveryWorkload(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), startMoveKeysParallelismLock(1), finishMoveKeysParallelismLock(1), enabled(!clientId),
+	    pass(true) {}
+
+	void validationFailed(ErrorOr<Optional<Value>> expectedValue, ErrorOr<Optional<Value>> actualValue) {
+		TraceEvent(SevError, "TestFailed")
+		    .detail("ExpectedValue", printValue(expectedValue))
+		    .detail("ActualValue", printValue(actualValue));
+		pass = false;
+	}
+
+	std::string description() const override { return "DataLossRecovery"; }
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+
+	Future<Void> start(Database const& cx) override {
+		if (!enabled) {
+			return Void();
+		}
+		return _start(this, cx);
+	}
+
+	ACTOR Future<Void> _start(DataLossRecoveryWorkload* self, Database cx) {
+		state Key key = "TestKey"_sr;
+		state Key endKey = "TestKey0"_sr;
+		state Value oldValue = "TestValue"_sr;
+		state Value newValue = "TestNewValue"_sr;
+
+		wait(self->writeAndVerify(self, cx, key, oldValue));
+
+		// Move [key, endKey) to team: {address}.
+		state NetworkAddress address = wait(self->disableDDAndMoveShard(self, cx, KeyRangeRef(key, endKey)));
+		wait(self->readAndVerify(self, cx, key, oldValue));
+
+		// Kill team {address}, and expect read to timeout.
+		self->killProcess(self, address);
+		wait(self->readAndVerify(self, cx, key, timed_out()));
+
+		// Reenable DD and exclude address as fail, so that [key, endKey) will be dropped and moved to a new team.
+		// Expect read to return 'value not found'.
+		int ignore = wait(setDDMode(cx, 1));
+		wait(self->exclude(cx, address));
+		wait(self->readAndVerify(self, cx, key, Optional<Value>()));
+
+		// Write will scceed.
+		wait(self->writeAndVerify(self, cx, key, newValue));
+
+		return Void();
+	}
+
+	ACTOR Future<Void> readAndVerify(DataLossRecoveryWorkload* self,
+	                                 Database cx,
+	                                 Key key,
+	                                 ErrorOr<Optional<Value>> expectedValue) {
+		state Transaction tr(cx);
+
+		loop {
+			try {
+				state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
+				const bool equal = !expectedValue.isError() && res == expectedValue.get();
+				if (!equal) {
+					self->validationFailed(expectedValue, ErrorOr<Optional<Value>>(res));
+				}
+				break;
+			} catch (Error& e) {
+				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
+					break;
+				}
+				wait(tr.onError(e));
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR Future<Void> writeAndVerify(DataLossRecoveryWorkload* self, Database cx, Key key, Optional<Value> value) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				if (value.present()) {
+					tr.set(key, value.get());
+				} else {
+					tr.clear(key);
+				}
+				wait(timeoutError(tr.commit(), 30.0));
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		wait(self->readAndVerify(self, cx, key, value));
+
+		return Void();
+	}
+
+	ACTOR Future<Void> exclude(Database cx, NetworkAddress addr) {
+		state Transaction tr(cx);
+		state std::vector<AddressExclusion> servers;
+		servers.push_back(AddressExclusion(addr.ip, addr.port));
+		loop {
+			try {
+				excludeServers(tr, servers, true);
+				wait(tr.commit());
+				break;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+
+		// Wait until all data are moved out of servers.
+		std::set<NetworkAddress> inProgress = wait(checkForExcludingServers(cx, servers, true));
+		ASSERT(inProgress.empty());
+
+		TraceEvent("ExcludedFailedServer").detail("Address", addr.toString());
+		return Void();
+	}
+
+	// Move keys to a random selected team consisting of a single SS, after disabling DD, so that keys won't be
+	// kept in the new team until DD is enabled.
+	// Returns the address of the single SS of the new team.
+	ACTOR Future<NetworkAddress> disableDDAndMoveShard(DataLossRecoveryWorkload* self, Database cx, KeyRange keys) {
+		// Disable DD to avoid DD undoing of our move.
+		state int ignore = wait(setDDMode(cx, 0));
+		state NetworkAddress addr;
+
+		// Pick a random SS as the dest, keys will reside on a single server after the move.
+		state std::vector<UID> dest;
+		while (dest.empty()) {
+			std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
+			if (!interfs.empty()) {
+				const auto& interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
+				if (g_simulator.protectedAddresses.count(interf.address()) == 0) {
+					dest.push_back(interf.uniqueID);
+					addr = interf.address();
+				}
+			}
+		}
+
+		state UID owner = deterministicRandom()->randomUniqueID();
+		state DDEnabledState ddEnabledState;
+
+		state Transaction tr(cx);
+
+		loop {
+			try {
+				BinaryWriter wrMyOwner(Unversioned());
+				wrMyOwner << owner;
+				tr.set(moveKeysLockOwnerKey, wrMyOwner.toValue());
+				wait(tr.commit());
+
+				MoveKeysLock moveKeysLock;
+				moveKeysLock.myOwner = owner;
+
+				wait(moveKeys(cx,
+				              keys,
+				              dest,
+				              dest,
+				              moveKeysLock,
+				              Promise<Void>(),
+				              &self->startMoveKeysParallelismLock,
+				              &self->finishMoveKeysParallelismLock,
+				              false,
+				              UID(), // for logging only
+				              &ddEnabledState));
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_movekeys_conflict) {
+					// Conflict on moveKeysLocks with the current running DD is expected, just retry.
+					tr.reset();
+				} else {
+					wait(tr.onError(e));
+				}
+			}
+		}
+
+		TraceEvent("TestKeyMoved").detail("NewTeam", describe(dest)).detail("Address", addr.toString());
+
+		state Transaction validateTr(cx);
+		loop {
+			try {
+				Standalone<VectorRef<const char*>> addresses = wait(validateTr.getAddressesForKey(keys.begin));
+				// The move function is not what we are testing here, crash the test if the move fails.
+				ASSERT(addresses.size() == 1);
+				ASSERT(std::string(addresses[0]) == addr.toString());
+				break;
+			} catch (Error& e) {
+				wait(validateTr.onError(e));
+			}
+		}
+
+		return addr;
+	}
+
+	void killProcess(DataLossRecoveryWorkload* self, const NetworkAddress& addr) {
+		ISimulator::ProcessInfo* process = g_simulator.getProcessByAddress(addr);
+		ASSERT(process->addresses.contains(addr));
+		g_simulator.killProcess(process, ISimulator::KillInstantly);
+		TraceEvent("TestTeamKilled").detail("Address", addr);
+	}
+
+	Future<bool> check(Database const& cx) override { return pass; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<DataLossRecoveryWorkload> DataLossRecoveryWorkloadFactory("DataLossRecovery");
\ No newline at end of file
diff --git a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
index de5fd7549b..197f924b91 100644
--- a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
+++ b/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/RunTransaction.actor.h"
 #include "fdbrpc/simulator.h"
@@ -37,7 +38,7 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 
 	DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		ASSERT(g_simulator.extraDB != nullptr);
-		auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+		auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 		extraDB = Database::createDatabase(extraFile, -1);
 		testDuration = getOption(options, LiteralStringRef("testDuration"), 100.0);
 		switchAfter = getOption(options, LiteralStringRef("switchAfter"), 50.0);
@@ -53,7 +54,7 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 		if (clientId != 0) {
 			return Void();
 		}
-		auto switchConnFileDb = Database::createDatabase(cx->getConnectionFile(), -1);
+		auto switchConnFileDb = Database::createDatabase(cx->getConnectionRecord(), -1);
 		originalDB = cx;
 		std::vector<Future<Void>> clients = { readerClientSeparateDBs(cx, this),
 			                                  doSwitch(switchConnFileDb, this),
@@ -141,8 +142,8 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 		TraceEvent("DifferentClusters_CopiedDatabase").log();
 		wait(advanceVersion(self->extraDB, rv));
 		TraceEvent("DifferentClusters_AdvancedVersion").log();
-		wait(cx->switchConnectionFile(
-		    makeReference<ClusterConnectionFile>(self->extraDB->getConnectionFile()->getConnectionString())));
+		wait(cx->switchConnectionRecord(
+		    makeReference<ClusterConnectionMemoryRecord>(self->extraDB->getConnectionRecord()->getConnectionString())));
 		TraceEvent("DifferentClusters_SwitchedConnectionFile").log();
 		state Transaction tr(cx);
 		tr.setVersion(rv);
@@ -156,7 +157,7 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 			TraceEvent("DifferentClusters_ReadError").error(e);
 			wait(tr.onError(e));
 		}
-		// In an actual switch we would call switchConnectionFile after unlocking the database. But it's possible
+		// In an actual switch we would call switchConnectionRecord after unlocking the database. But it's possible
 		// that a storage server serves a read at |rv| even after the recovery caused by unlocking the database, and we
 		// want to make that more likely for this test. So read at |rv| then unlock.
 		wait(unlockDatabase(self->extraDB, lockUid));
diff --git a/fdbserver/workloads/KillRegion.actor.cpp b/fdbserver/workloads/KillRegion.actor.cpp
index a6b9f10618..6490c2c4bd 100644
--- a/fdbserver/workloads/KillRegion.actor.cpp
+++ b/fdbserver/workloads/KillRegion.actor.cpp
@@ -101,7 +101,7 @@ struct KillRegionWorkload : TestWorkload {
 
 		TraceEvent("ForceRecovery_Begin").log();
 
-		wait(forceRecovery(cx->getConnectionFile(), LiteralStringRef("1")));
+		wait(forceRecovery(cx->getConnectionRecord(), LiteralStringRef("1")));
 
 		TraceEvent("ForceRecovery_UsableRegions").log();
 
diff --git a/fdbserver/workloads/VersionStamp.actor.cpp b/fdbserver/workloads/VersionStamp.actor.cpp
index d2576002fa..a5055d2e4d 100644
--- a/fdbserver/workloads/VersionStamp.actor.cpp
+++ b/fdbserver/workloads/VersionStamp.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbrpc/ContinuousSample.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/BulkSetup.actor.h"
@@ -150,7 +151,7 @@ struct VersionStampWorkload : TestWorkload {
 
 	ACTOR Future<bool> _check(Database cx, VersionStampWorkload* self) {
 		if (self->validateExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 			cx = Database::createDatabase(extraFile, -1);
 		}
 		state ReadYourWritesTransaction tr(cx);
@@ -309,7 +310,7 @@ struct VersionStampWorkload : TestWorkload {
 		state Database extraDB;
 
 		if (g_simulator.extraDB != nullptr) {
-			auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 			extraDB = Database::createDatabase(extraFile, -1);
 		}
 
diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp
index 79dd30eb48..e0d47d68e4 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.actor.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/ReadYourWrites.h"
@@ -81,7 +82,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 
 		useExtraDB = g_simulator.extraDB != nullptr;
 		if (useExtraDB) {
-			auto extraFile = makeReference<ClusterConnectionFile>(*g_simulator.extraDB);
+			auto extraFile = makeReference<ClusterConnectionMemoryRecord>(*g_simulator.extraDB);
 			extraDB = Database::createDatabase(extraFile, -1);
 			useSystemKeys = false;
 		}
diff --git a/flow/actorcompiler/ActorCompiler.cs b/flow/actorcompiler/ActorCompiler.cs
index 3cf1853ab6..3961ead11b 100644
--- a/flow/actorcompiler/ActorCompiler.cs
+++ b/flow/actorcompiler/ActorCompiler.cs
@@ -110,7 +110,8 @@ namespace actorcompiler
                     name = name,
                     returnType = returnType,
                     endIsUnreachable = endIsUnreachable,
-                    formalParameters = formalParameters
+                    formalParameters = formalParameters,
+                    indentation = indentation
                 }
             );
         }
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 95176f1320..796fc141ac 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -134,6 +134,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/CycleAndLock.toml)
   add_fdb_test(TEST_FILES fast/CycleTest.toml)
   add_fdb_test(TEST_FILES fast/ChangeFeeds.toml)
+  add_fdb_test(TEST_FILES fast/DataLossRecovery.toml)
   add_fdb_test(TEST_FILES fast/FuzzApiCorrectness.toml)
   add_fdb_test(TEST_FILES fast/FuzzApiCorrectnessClean.toml)
   add_fdb_test(TEST_FILES fast/IncrementalBackup.toml)
diff --git a/tests/fast/DataLossRecovery.toml b/tests/fast/DataLossRecovery.toml
new file mode 100644
index 0000000000..6cebd91b97
--- /dev/null
+++ b/tests/fast/DataLossRecovery.toml
@@ -0,0 +1,13 @@
+[configuration]
+config = 'triple'
+storageEngineType = 0
+processesPerMachine = 2
+coordinators = 3
+machineCount = 45
+
+[[test]]
+testTitle = 'DataLossRecovery'
+useDB = true
+
+    [[test.workload]]
+    testName = 'DataLossRecovery'